Implement Ed25519 signatures for CMS (RFC 8419)

This adds support for Edwards curve digital signature algorithms in the
cryptographic message syntax, as specified in RFC 8419. Only Ed25519 is
supported since that is the only EdDSA algorithm that LibreSSL supports
(this is unlikely to change ever, but, as they say - never is a very
long time).

This has the usual curly interactions between EVP and CMS with poorly
documented interfaces and lots of confusing magic return values and
controls. This improves upon existing control handlers by documenting
what is being done and why. Unlike other (draft) implementations we
also happen to use the correct hashing algorithm.

There are no plans to implement RFC 8418.

joint work with job at p2k23

ok jsing

1 files changed, 64 insertions, 20 deletions

diff --git a/src/lib/libcrypto/cms/cms_sd.c b/src/lib/libcrypto/cms/cms_sd.c
index b644717bbb..5a38bf59aa 100644
--- a/src/lib/libcrypto/cms/cms_sd.c
+++ b/src/lib/libcrypto/cms/cms_sd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: cms_sd.c,v 1.30 2024/02/02 14:13:11 tb Exp $ */
+/* $OpenBSD: cms_sd.c,v 1.31 2024/03/29 06:41:58 tb Exp $ */
 /*
  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project.
@@ -277,6 +277,64 @@ cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd)
         return 1;
 }
 
+static const EVP_MD *
+cms_SignerInfo_default_digest_md(const CMS_SignerInfo *si)
+{
+        int rv, nid;
+
+        if (si->pkey == NULL) {
+                CMSerror(CMS_R_NO_PUBLIC_KEY);
+                return NULL;
+        }
+
+        /* On failure or unsupported operation, give up. */
+        if ((rv = EVP_PKEY_get_default_digest_nid(si->pkey, &nid)) <= 0)
+                return NULL;
+        if (rv > 2)
+                return NULL;
+
+        /*
+         * XXX - we need to identify EdDSA in a better way. Figure out where
+         * and how. This mimics EdDSA checks in openssl/ca.c and openssl/req.c.
+         */
+
+        /* The digest md is required to be EVP_sha512() (EdDSA). */
+        if (rv == 2 && nid == NID_undef)
+                return EVP_sha512();
+
+        /* Use mandatory or default digest. */
+        return EVP_get_digestbynid(nid);
+}
+
+static const EVP_MD *
+cms_SignerInfo_signature_md(const CMS_SignerInfo *si)
+{
+        int rv, nid;
+
+        if (si->pkey == NULL) {
+                CMSerror(CMS_R_NO_PUBLIC_KEY);
+                return NULL;
+        }
+
+        /* Fall back to digestAlgorithm unless pkey has a mandatory digest. */
+        if ((rv = EVP_PKEY_get_default_digest_nid(si->pkey, &nid)) <= 1)
+                return EVP_get_digestbyobj(si->digestAlgorithm->algorithm);
+        if (rv > 2)
+                return NULL;
+
+        /*
+         * XXX - we need to identify EdDSA in a better way. Figure out where
+         * and how. This mimics EdDSA checks in openssl/ca.c and openssl/req.c.
+         */
+
+        /* The signature md is required to be EVP_md_null() (EdDSA). */
+        if (nid == NID_undef)
+                return EVP_md_null();
+
+        /* Use mandatory digest. */
+        return EVP_get_digestbynid(nid);
+}
+
 CMS_SignerInfo *
 CMS_add1_signer(CMS_ContentInfo *cms, X509 *signer, EVP_PKEY *pk,
     const EVP_MD *md, unsigned int flags)
@@ -325,19 +383,10 @@ CMS_add1_signer(CMS_ContentInfo *cms, X509 *signer, EVP_PKEY *pk,
         if (!cms_set1_SignerIdentifier(si->sid, signer, type))
                 goto err;
 
+        if (md == NULL)
+                md = cms_SignerInfo_default_digest_md(si);
         if (md == NULL) {
-                int def_nid;
-                if (EVP_PKEY_get_default_digest_nid(pk, &def_nid) <= 0)
-                        goto err;
-                md = EVP_get_digestbynid(def_nid);
-                if (md == NULL) {
-                        CMSerror(CMS_R_NO_DEFAULT_DIGEST);
-                        goto err;
-                }
-        }
-
-        if (!md) {
-                CMSerror(CMS_R_NO_DIGEST_SET);
+                CMSerror(CMS_R_NO_DEFAULT_DIGEST);
                 goto err;
         }
 
@@ -735,7 +784,7 @@ CMS_SignerInfo_sign(CMS_SignerInfo *si)
         size_t sig_len = 0;
         int ret = 0;
 
-        if ((md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm)) == NULL)
+        if ((md = cms_SignerInfo_signature_md(si)) == NULL)
                 goto err;
 
         if (CMS_signed_get_attr_by_NID(si, NID_pkcs9_signingTime, -1) < 0) {
@@ -795,14 +844,9 @@ CMS_SignerInfo_verify(CMS_SignerInfo *si)
         int buf_len = 0;
         int ret = -1;
 
-        if ((md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm)) == NULL)
+        if ((md = cms_SignerInfo_signature_md(si)) == NULL)
                 goto err;
 
-        if (si->pkey == NULL) {
-                CMSerror(CMS_R_NO_PUBLIC_KEY);
-                goto err;
-        }
-
         if (si->mctx == NULL)
                 si->mctx = EVP_MD_CTX_new();
         if (si->mctx == NULL) {