1 files changed, 28 insertions, 24 deletions

diff --git a/src/lib/libssl/ssl_seclevel.c b/src/lib/libssl/ssl_seclevel.c
index 6a5d16bfaa..1448368e71 100644
--- a/src/lib/libssl/ssl_seclevel.c
+++ b/src/lib/libssl/ssl_seclevel.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ssl_seclevel.c,v 1.28 2024/05/09 07:12:03 tb Exp $ */
+/*      $OpenBSD: ssl_seclevel.c,v 1.29 2024/10/17 06:19:06 tb Exp $ */
 /*
  * Copyright (c) 2020-2022 Theo Buehler <tb@openbsd.org>
  *
@@ -331,45 +331,49 @@ ssl_security_cert_key(const SSL_CTX *ctx, const SSL *ssl, X509 *x509, int secop)
 }
 
 static int
-ssl_cert_signature_md_nid(X509 *x509)
+ssl_security_cert_sig_security_bits(X509 *x509, int *out_md_nid)
 {
-        int md_nid, signature_nid;
+        int pkey_nid, security_bits;
+        uint32_t flags;
 
-        if ((signature_nid = X509_get_signature_nid(x509)) == NID_undef)
-                return NID_undef;
+        *out_md_nid = NID_undef;
 
-        if (!OBJ_find_sigid_algs(signature_nid, &md_nid, NULL))
-                return NID_undef;
-
-        return md_nid;
-}
-
-static int
-ssl_cert_md_nid_security_bits(int md_nid)
-{
-        const EVP_MD *md;
-
-        if (md_nid == NID_undef)
+        /*
+         * Returning -1 security bits makes the default security callback fail
+         * to match bonkers behavior in OpenSSL. This in turn lets a security
+         * callback override such failures.
+         */
+        if (!X509_get_signature_info(x509, out_md_nid, &pkey_nid, &security_bits,
+            &flags))
                 return -1;
-
-        if ((md = EVP_get_digestbynid(md_nid)) == NULL)
+        /*
+         * OpenSSL doesn't check flags. Test RSA-PSS certs we were provided have
+         * a salt length distinct from hash length and thus fail this check.
+         */
+        if ((flags & X509_SIG_INFO_TLS) == 0)
                 return -1;
 
-        /* Assume 4 bits of collision resistance for each hash octet. */
-        return EVP_MD_size(md) * 4;
+        /* Weird OpenSSL behavior only relevant for EdDSA certs in LibreSSL. */
+        if (*out_md_nid == NID_undef)
+                *out_md_nid = pkey_nid;
+
+        return security_bits;
 }
 
 static int
 ssl_security_cert_sig(const SSL_CTX *ctx, const SSL *ssl, X509 *x509, int secop)
 {
-        int md_nid, security_bits;
+        int md_nid = NID_undef, security_bits = -1;
 
         /* Don't check signature if self signed. */
         if ((X509_get_extension_flags(x509) & EXFLAG_SS) != 0)
                 return 1;
 
-        md_nid = ssl_cert_signature_md_nid(x509);
-        security_bits = ssl_cert_md_nid_security_bits(md_nid);
+        /*
+         * The default security callback fails on -1 security bits. It ignores
+         * the md_nid (aka version) argument we pass from here.
+         */
+        security_bits = ssl_security_cert_sig_security_bits(x509, &md_nid);
 
         if (ssl != NULL)
                 return ssl_security(ssl, secop, security_bits, md_nid, x509);