Implement X509_get_signature_info()

This is a slightly strange combination of OBJ_find_sigid_algs() and the
security level API necessary because OBJ_find_sigid_algs() on its own
isn't smart enough for the special needs of RSA-PSS and EdDSA.

The API extracts the hash's NID and the pubkey's NID from the certificate's
signatureAlgorithm and invokes special handlers for RSA-PSS and EdDSA
for retrieving the corresponding information. This isn't entirely free
for RSA-PSS, but for now we don't cache this information.

The security bits calculation is a bit hand-wavy, but that's something
that comes along with this sort of numerology.

ok jsing

1 files changed, 55 insertions, 1 deletions

diff --git a/src/lib/libcrypto/rsa/rsa_ameth.c b/src/lib/libcrypto/rsa/rsa_ameth.c
index c722188c43..d7ce931733 100644
--- a/src/lib/libcrypto/rsa/rsa_ameth.c
+++ b/src/lib/libcrypto/rsa/rsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_ameth.c,v 1.58 2024/03/17 07:10:00 tb Exp $ */
+/* $OpenBSD: rsa_ameth.c,v 1.59 2024/08/28 07:15:04 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -845,6 +845,58 @@ rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,
         return 1;
 }
 
+static int
+rsa_pss_signature_info(const X509_ALGOR *alg, int *out_md_nid,
+    int *out_pkey_nid, int *out_security_bits, uint32_t *out_flags)
+{
+        RSA_PSS_PARAMS *pss = NULL;
+        const ASN1_OBJECT *aobj;
+        const EVP_MD *md, *mgf1md;
+        int md_len, salt_len;
+        int md_nid = NID_undef, pkey_nid = NID_undef;
+        int security_bits = -1;
+        uint32_t flags = 0;
+
+        X509_ALGOR_get0(&aobj, NULL, NULL, alg);
+        if (OBJ_obj2nid(aobj) != EVP_PKEY_RSA_PSS)
+                goto err;
+
+        if ((pss = rsa_pss_decode(alg)) == NULL)
+                goto err;
+        if (!rsa_pss_get_param(pss, &md, &mgf1md, &salt_len))
+                goto err;
+
+        if ((md_nid = EVP_MD_type(md)) == NID_undef)
+                goto err;
+        if ((md_len = EVP_MD_size(md)) <= 0)
+                goto err;
+
+        /*
+         * RFC 8446, section 4.2.3 - restricts the digest algorithm:
+         * - it must be one of SHA256, SHA384, and SHA512;
+         * - the same digest must be used in the mask generation function;
+         * - the salt length must match the output length of the digest.
+         * XXX - consider separate flags for these checks.
+         */
+        if (md_nid == NID_sha256 || md_nid == NID_sha384 || md_nid == NID_sha512) {
+                if (md_nid == EVP_MD_type(mgf1md) && salt_len == md_len)
+                        flags |= X509_SIG_INFO_TLS;
+        }
+
+        security_bits = md_len * 4;
+        flags |= X509_SIG_INFO_VALID;
+
+        *out_md_nid = md_nid;
+        *out_pkey_nid = pkey_nid;
+        *out_security_bits = security_bits;
+        *out_flags = flags;
+
+ err:
+        RSA_PSS_PARAMS_free(pss);
+
+        return (flags & X509_SIG_INFO_VALID) != 0;
+}
+
 #ifndef OPENSSL_NO_CMS
 static int
 rsa_cms_verify(CMS_SignerInfo *si)
@@ -1216,6 +1268,8 @@ const EVP_PKEY_ASN1_METHOD rsa_pss_asn1_meth = {
         .pkey_bits = rsa_bits,
         .pkey_security_bits = rsa_security_bits,
 
+        .signature_info = rsa_pss_signature_info,
+
         .sig_print = rsa_sig_print,
 
         .pkey_free = rsa_free,