ssl_kex: ensure the public key uses uncompressed encoding

EC_POINT_oct2point() does most of the validation we need it to do, but it
has to accept the point at infinity, compressed and hybrid encodings for
historic reasons. So exclude these cases: the point at infinity makes no
sense here and will be caught later in ECDH_compute_key(), the compressed
and hybrid encodings MUST NOT be supported per RFC 8422 section 5.1.2.

This is implemented using the strategy already used in ec_convert.c since
the point_conversion_form_t is completely unfit for anything.

Set decode_error to ensure we send that alert. We may make some effort to
use illegal_parameter later.

Issue about the missing alert and the point at infinity raised by Lucca
Hirschi et al.

ok jsing kenjiro

1 files changed, 22 insertions, 2 deletions

diff --git a/src/lib/libssl/ssl_kex.c b/src/lib/libssl/ssl_kex.c
index ad69b2f485..a7d02892bc 100644
--- a/src/lib/libssl/ssl_kex.c
+++ b/src/lib/libssl/ssl_kex.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_kex.c,v 1.14 2026/06/08 11:38:04 tb Exp $ */
+/* $OpenBSD: ssl_kex.c,v 1.15 2026/06/08 11:52:43 tb Exp $ */
 /*
  * Copyright (c) 2020, 2021 Joel Sing <jsing@openbsd.org>
  *
@@ -355,14 +355,32 @@ ssl_kex_public_ecdhe_ecp(EC_KEY *ecdh, CBB *cbb)
         return ret;
 }
 
+#define EC_POINT_UNCOMPRESSED           0x04
+#define EC_POINT_CONVERSION_MASK        0x06
+
 int
 ssl_kex_peer_public_ecdhe_ecp(EC_KEY *ecdh, int nid, CBS *cbs,
     int *decode_error)
 {
         EC_GROUP *group = NULL;
         EC_POINT *point = NULL;
+        uint8_t form;
         int ret = 0;
 
+        /*
+         * Check that the peer's public key uses uncompressed encoding.
+         * This ensures that the public key is not the point at infinity
+         * and enforces correct point encoding via EC_POINT_oct2point().
+         */
+        if (!CBS_peek_u8(cbs, &form)) {
+                *decode_error = 1;
+                goto err;
+        }
+        if ((form & EC_POINT_CONVERSION_MASK) != EC_POINT_UNCOMPRESSED) {
+                *decode_error = 1;
+                goto err;
+        }
+
         if ((group = EC_GROUP_new_by_curve_name(nid)) == NULL)
                 goto err;
 
@@ -372,8 +390,10 @@ ssl_kex_peer_public_ecdhe_ecp(EC_KEY *ecdh, int nid, CBS *cbs,
         if ((point = EC_POINT_new(group)) == NULL)
                 goto err;
         if (EC_POINT_oct2point(group, point, CBS_data(cbs), CBS_len(cbs),
-            NULL) == 0)
+            NULL) == 0) {
+                *decode_error = 1;
                 goto err;
+        }
         if (!EC_KEY_set_public_key(ecdh, point))
                 goto err;