Implement Ed25519 signatures for CMS (RFC 8419)

This adds support for Edwards curve digital signature algorithms in the
cryptographic message syntax, as specified in RFC 8419. Only Ed25519 is
supported since that is the only EdDSA algorithm that LibreSSL supports
(this is unlikely to change ever, but, as they say - never is a very
long time).

This has the usual curly interactions between EVP and CMS with poorly
documented interfaces and lots of confusing magic return values and
controls. This improves upon existing control handlers by documenting
what is being done and why. Unlike other (draft) implementations we
also happen to use the correct hashing algorithm.

There are no plans to implement RFC 8418.

joint work with job at p2k23

ok jsing

2 files changed, 126 insertions, 21 deletions

diff --git a/src/lib/libcrypto/cms/cms_sd.c b/src/lib/libcrypto/cms/cms_sd.c
index b644717bbb..5a38bf59aa 100644
--- a/src/lib/libcrypto/cms/cms_sd.c
+++ b/src/lib/libcrypto/cms/cms_sd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: cms_sd.c,v 1.30 2024/02/02 14:13:11 tb Exp $ */
+/* $OpenBSD: cms_sd.c,v 1.31 2024/03/29 06:41:58 tb Exp $ */
 /*
  * Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project.
@@ -277,6 +277,64 @@ cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd)
         return 1;
 }
 
+static const EVP_MD *
+cms_SignerInfo_default_digest_md(const CMS_SignerInfo *si)
+{
+        int rv, nid;
+
+        if (si->pkey == NULL) {
+                CMSerror(CMS_R_NO_PUBLIC_KEY);
+                return NULL;
+        }
+
+        /* On failure or unsupported operation, give up. */
+        if ((rv = EVP_PKEY_get_default_digest_nid(si->pkey, &nid)) <= 0)
+                return NULL;
+        if (rv > 2)
+                return NULL;
+
+        /*
+         * XXX - we need to identify EdDSA in a better way. Figure out where
+         * and how. This mimics EdDSA checks in openssl/ca.c and openssl/req.c.
+         */
+
+        /* The digest md is required to be EVP_sha512() (EdDSA). */
+        if (rv == 2 && nid == NID_undef)
+                return EVP_sha512();
+
+        /* Use mandatory or default digest. */
+        return EVP_get_digestbynid(nid);
+}
+
+static const EVP_MD *
+cms_SignerInfo_signature_md(const CMS_SignerInfo *si)
+{
+        int rv, nid;
+
+        if (si->pkey == NULL) {
+                CMSerror(CMS_R_NO_PUBLIC_KEY);
+                return NULL;
+        }
+
+        /* Fall back to digestAlgorithm unless pkey has a mandatory digest. */
+        if ((rv = EVP_PKEY_get_default_digest_nid(si->pkey, &nid)) <= 1)
+                return EVP_get_digestbyobj(si->digestAlgorithm->algorithm);
+        if (rv > 2)
+                return NULL;
+
+        /*
+         * XXX - we need to identify EdDSA in a better way. Figure out where
+         * and how. This mimics EdDSA checks in openssl/ca.c and openssl/req.c.
+         */
+
+        /* The signature md is required to be EVP_md_null() (EdDSA). */
+        if (nid == NID_undef)
+                return EVP_md_null();
+
+        /* Use mandatory digest. */
+        return EVP_get_digestbynid(nid);
+}
+
 CMS_SignerInfo *
 CMS_add1_signer(CMS_ContentInfo *cms, X509 *signer, EVP_PKEY *pk,
     const EVP_MD *md, unsigned int flags)
@@ -325,19 +383,10 @@ CMS_add1_signer(CMS_ContentInfo *cms, X509 *signer, EVP_PKEY *pk,
         if (!cms_set1_SignerIdentifier(si->sid, signer, type))
                 goto err;
 
+        if (md == NULL)
+                md = cms_SignerInfo_default_digest_md(si);
         if (md == NULL) {
-                int def_nid;
-                if (EVP_PKEY_get_default_digest_nid(pk, &def_nid) <= 0)
-                        goto err;
-                md = EVP_get_digestbynid(def_nid);
-                if (md == NULL) {
-                        CMSerror(CMS_R_NO_DEFAULT_DIGEST);
-                        goto err;
-                }
-        }
-
-        if (!md) {
-                CMSerror(CMS_R_NO_DIGEST_SET);
+                CMSerror(CMS_R_NO_DEFAULT_DIGEST);
                 goto err;
         }
 
@@ -735,7 +784,7 @@ CMS_SignerInfo_sign(CMS_SignerInfo *si)
         size_t sig_len = 0;
         int ret = 0;
 
-        if ((md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm)) == NULL)
+        if ((md = cms_SignerInfo_signature_md(si)) == NULL)
                 goto err;
 
         if (CMS_signed_get_attr_by_NID(si, NID_pkcs9_signingTime, -1) < 0) {
@@ -795,14 +844,9 @@ CMS_SignerInfo_verify(CMS_SignerInfo *si)
         int buf_len = 0;
         int ret = -1;
 
-        if ((md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm)) == NULL)
+        if ((md = cms_SignerInfo_signature_md(si)) == NULL)
                 goto err;
 
-        if (si->pkey == NULL) {
-                CMSerror(CMS_R_NO_PUBLIC_KEY);
-                goto err;
-        }
-
         if (si->mctx == NULL)
                 si->mctx = EVP_MD_CTX_new();
         if (si->mctx == NULL) {
diff --git a/src/lib/libcrypto/ec/ecx_methods.c b/src/lib/libcrypto/ec/ecx_methods.c
index cd512a447f..ab299a8d6b 100644
--- a/src/lib/libcrypto/ec/ecx_methods.c
+++ b/src/lib/libcrypto/ec/ecx_methods.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ecx_methods.c,v 1.11 2024/01/04 17:01:26 tb Exp $ */
+/*      $OpenBSD: ecx_methods.c,v 1.12 2024/03/29 06:41:58 tb Exp $ */
 /*
  * Copyright (c) 2022 Joel Sing <jsing@openbsd.org>
  *
@@ -17,6 +17,7 @@
 
 #include <string.h>
 
+#include <openssl/cms.h>
 #include <openssl/curve25519.h>
 #include <openssl/ec.h>
 #include <openssl/err.h>
@@ -530,10 +531,67 @@ ecx_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
         return -2;
 }
 
+#ifndef OPENSSL_NO_CMS
+static int
+ecx_cms_sign_or_verify(EVP_PKEY *pkey, long verify, CMS_SignerInfo *si)
+{
+        X509_ALGOR *digestAlgorithm, *signatureAlgorithm;
+        ASN1_OBJECT *aobj;
+
+        if (verify != 0 && verify != 1)
+                return -1;
+
+        /* Check that we have an Ed25519 public key. */
+        if (EVP_PKEY_id(pkey) != NID_ED25519)
+                return -1;
+
+        CMS_SignerInfo_get0_algs(si, NULL, NULL, &digestAlgorithm,
+            &signatureAlgorithm);
+
+        /* RFC 8419, section 2.3: digestAlgorithm MUST be SHA-512. */
+        if (digestAlgorithm == NULL)
+                return -1;
+        if (OBJ_obj2nid(digestAlgorithm->algorithm) != NID_sha512)
+                return -1;
+
+        /*
+         * RFC 8419, section 2.4: signatureAlgorithm MUST be Ed25519, and the
+         * parameters MUST be absent. For verification check that this is the
+         * case, for signing set the signatureAlgorithm accordingly.
+         */
+        if (verify) {
+                const ASN1_OBJECT *obj;
+                int param_type;
+
+                if (signatureAlgorithm == NULL)
+                        return -1;
+
+                X509_ALGOR_get0(&obj, &param_type, NULL, signatureAlgorithm);
+                if (OBJ_obj2nid(obj) != NID_ED25519)
+                        return -1;
+                if (param_type != V_ASN1_UNDEF)
+                        return -1;
+
+                return 1;
+        }
+
+        if ((aobj = OBJ_nid2obj(NID_ED25519)) == NULL)
+                return -1;
+        if (!X509_ALGOR_set0(signatureAlgorithm, aobj, V_ASN1_UNDEF, NULL))
+                return -1;
+
+        return 1;
+}
+#endif
+
 static int
 ecx_sign_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 {
         switch (op) {
+#ifndef OPENSSL_NO_CMS
+        case ASN1_PKEY_CTRL_CMS_SIGN:
+                return ecx_cms_sign_or_verify(pkey, arg1, arg2);
+#endif
         case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
                 /* PureEdDSA does its own hashing. */
                 *(int *)arg2 = NID_undef;
@@ -806,6 +864,9 @@ pkey_ecx_ed_ctrl(EVP_PKEY_CTX *pkey_ctx, int op, int arg1, void *arg2)
                 }
                 return 1;
 
+#ifndef OPENSSL_NO_CMS
+        case EVP_PKEY_CTRL_CMS_SIGN:
+#endif
         case EVP_PKEY_CTRL_DIGESTINIT:
                 return 1;
         }