libssl: rework cert signature security level

This switches to using the X509_get_signature_info() API instead of hand
rolling a part of it. This is slightly tangly since the security level API
is strange. In particular, some failures are passed to the security level
callback so that applications can override them.

This makes the security level API handle RSA-PSS and EdDSA certificates
correctly and the handshake with such can progress a bit further. Of note,
we check that the certs are actually suitable for use in TLS per RFC 8446
contrary to what OpenSSL does.

ok beck jsing

1 files changed, 28 insertions, 24 deletions

diff --git a/src/lib/libssl/ssl_seclevel.c b/src/lib/libssl/ssl_seclevel.c
index 6a5d16bfaa..1448368e71 100644
--- a/src/lib/libssl/ssl_seclevel.c
+++ b/src/lib/libssl/ssl_seclevel.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ssl_seclevel.c,v 1.28 2024/05/09 07:12:03 tb Exp $ */
+/*      $OpenBSD: ssl_seclevel.c,v 1.29 2024/10/17 06:19:06 tb Exp $ */
 /*
  * Copyright (c) 2020-2022 Theo Buehler <tb@openbsd.org>
  *
@@ -331,45 +331,49 @@ ssl_security_cert_key(const SSL_CTX *ctx, const SSL *ssl, X509 *x509, int secop)
 }
 
 static int
-ssl_cert_signature_md_nid(X509 *x509)
+ssl_security_cert_sig_security_bits(X509 *x509, int *out_md_nid)
 {
-        int md_nid, signature_nid;
+        int pkey_nid, security_bits;
+        uint32_t flags;
 
-        if ((signature_nid = X509_get_signature_nid(x509)) == NID_undef)
-                return NID_undef;
+        *out_md_nid = NID_undef;
 
-        if (!OBJ_find_sigid_algs(signature_nid, &md_nid, NULL))
-                return NID_undef;
-
-        return md_nid;
-}
-
-static int
-ssl_cert_md_nid_security_bits(int md_nid)
-{
-        const EVP_MD *md;
-
-        if (md_nid == NID_undef)
+        /*
+         * Returning -1 security bits makes the default security callback fail
+         * to match bonkers behavior in OpenSSL. This in turn lets a security
+         * callback override such failures.
+         */
+        if (!X509_get_signature_info(x509, out_md_nid, &pkey_nid, &security_bits,
+            &flags))
                 return -1;
-
-        if ((md = EVP_get_digestbynid(md_nid)) == NULL)
+        /*
+         * OpenSSL doesn't check flags. Test RSA-PSS certs we were provided have
+         * a salt length distinct from hash length and thus fail this check.
+         */
+        if ((flags & X509_SIG_INFO_TLS) == 0)
                 return -1;
 
-        /* Assume 4 bits of collision resistance for each hash octet. */
-        return EVP_MD_size(md) * 4;
+        /* Weird OpenSSL behavior only relevant for EdDSA certs in LibreSSL. */
+        if (*out_md_nid == NID_undef)
+                *out_md_nid = pkey_nid;
+
+        return security_bits;
 }
 
 static int
 ssl_security_cert_sig(const SSL_CTX *ctx, const SSL *ssl, X509 *x509, int secop)
 {
-        int md_nid, security_bits;
+        int md_nid = NID_undef, security_bits = -1;
 
         /* Don't check signature if self signed. */
         if ((X509_get_extension_flags(x509) & EXFLAG_SS) != 0)
                 return 1;
 
-        md_nid = ssl_cert_signature_md_nid(x509);
-        security_bits = ssl_cert_md_nid_security_bits(md_nid);
+        /*
+         * The default security callback fails on -1 security bits. It ignores
+         * the md_nid (aka version) argument we pass from here.
+         */
+        security_bits = ssl_security_cert_sig_security_bits(x509, &md_nid);
 
         if (ssl != NULL)
                 return ssl_security(ssl, secop, security_bits, md_nid, x509);