Add internal version of X509_ALGOR_set_md()

X509_ALGOR_set_md() is a void function that cannot easily be error checked.
The caller has to jump through hoops to make sure this function doesn't
fail. Prepare replacing this internally with X509_ALGOR_set_evp_md(), which
allows error checking. There is one slight change of behavior: if the EVP_MD
object passed in does not have an OID known to the library, then this new
API fails.

It is unclear what the library should do with such an object and people
who use EVP_MD_meth_new() need to know what they are doing anyway and they
are better off teaching the lib about the OID if they're going to be
messing with certs.

Oh, and the prototype is in x509_local.h because the rest of this API is
in x509.h despite being implemented in asn1/.

ok jsing

2 files changed, 17 insertions, 7 deletions

diff --git a/src/lib/libcrypto/asn1/x_algor.c b/src/lib/libcrypto/asn1/x_algor.c
index a638337939..092ad80d2d 100644
--- a/src/lib/libcrypto/asn1/x_algor.c
+++ b/src/lib/libcrypto/asn1/x_algor.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: x_algor.c,v 1.26 2023/10/11 12:51:07 tb Exp $ */
+/* $OpenBSD: x_algor.c,v 1.27 2023/10/11 13:05:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -192,17 +192,25 @@ X509_ALGOR_get0(const ASN1_OBJECT **paobj, int *pptype, const void **ppval,
         }
 }
 
-/* Set up an X509_ALGOR DigestAlgorithmIdentifier from an EVP_MD */
-
-void
-X509_ALGOR_set_md(X509_ALGOR *alg, const EVP_MD *md)
+int
+X509_ALGOR_set_evp_md(X509_ALGOR *alg, const EVP_MD *md)
 {
+        ASN1_OBJECT *aobj;
         int param_type = V_ASN1_NULL;
 
         if ((EVP_MD_flags(md) & EVP_MD_FLAG_DIGALGID_ABSENT) != 0)
                 param_type = V_ASN1_UNDEF;
 
-        X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_MD_type(md)), param_type, NULL);
+        if ((aobj = OBJ_nid2obj(EVP_MD_type(md))) == NULL)
+                return 0;
+
+        return X509_ALGOR_set0(alg, aobj, param_type, NULL);
+}
+
+void
+X509_ALGOR_set_md(X509_ALGOR *alg, const EVP_MD *md)
+{
+        (void)X509_ALGOR_set_evp_md(alg, md);
 }
 
 int
diff --git a/src/lib/libcrypto/x509/x509_local.h b/src/lib/libcrypto/x509/x509_local.h
index f00a55bac8..44fe6ad805 100644
--- a/src/lib/libcrypto/x509/x509_local.h
+++ b/src/lib/libcrypto/x509/x509_local.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: x509_local.h,v 1.9 2023/05/28 05:25:24 tb Exp $ */
+/*      $OpenBSD: x509_local.h,v 1.10 2023/10/11 13:05:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2013.
  */
@@ -379,6 +379,8 @@ int x509_check_cert_time(X509_STORE_CTX *ctx, X509 *x, int quiet);
 
 int name_cmp(const char *name, const char *cmp);
 
+int X509_ALGOR_set_evp_md(X509_ALGOR *alg, const EVP_MD *md);
+
 int X509_policy_check(const STACK_OF(X509) *certs,
     const STACK_OF(ASN1_OBJECT) *user_policies, unsigned long flags,
     X509 **out_current_cert);