Prepare to provide EVP_MD_CTX{,_set}_pkey_ctx()

This API with very strange ownership handling is used by Ruby 3.1,
unfortunately.

For unclear reasons, it was decided that the caller retains ownership of
the pctx passed in.  EVP_PKEY_CTX aren't refcounted, so a flag was added to
make sure that md_ctx->pctx is not freed in EVP_MD_CTX_{cleanup,reset}().
Since EVP_MD_CTX_copy_ex() duplicates the md_ctx->pctx, the flag also needs
to be unset on the duplicated EVP_MD_CTX.

ok inoguchi jsing

4 files changed, 55 insertions, 5 deletions

diff --git a/src/lib/libcrypto/evp/digest.c b/src/lib/libcrypto/evp/digest.c
index 59c98b57b8..fd42318044 100644
--- a/src/lib/libcrypto/evp/digest.c
+++ b/src/lib/libcrypto/evp/digest.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: digest.c,v 1.32 2021/12/12 21:30:13 tb Exp $ */
+/* $OpenBSD: digest.c,v 1.33 2022/01/09 15:15:25 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -282,6 +282,12 @@ EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *in)
         EVP_MD_CTX_cleanup(out);
         memcpy(out, in, sizeof *out);
 
+        /*
+         * Because of the EVP_PKEY_CTX_dup() below, EVP_MD_CTX_cleanup() needs
+         * to free out->pctx in all cases (even if this flag is set on in).
+         */
+        EVP_MD_CTX_clear_flags(out, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX);
+
         if (in->md_data && out->digest->ctx_size) {
                 if (tmp_buf) {
                         out->md_data = tmp_buf;
@@ -383,7 +389,12 @@ EVP_MD_CTX_cleanup(EVP_MD_CTX *ctx)
         if (ctx->digest && ctx->digest->ctx_size && ctx->md_data &&
             !EVP_MD_CTX_test_flags(ctx, EVP_MD_CTX_FLAG_REUSE))
                 freezero(ctx->md_data, ctx->digest->ctx_size);
-        EVP_PKEY_CTX_free(ctx->pctx);
+        /*
+         * If EVP_MD_CTX_FLAG_KEEP_PKEY_CTX is set, EVP_MD_CTX_set_pkey() was
+         * called and its strange API contract implies we don't own ctx->pctx.
+         */
+        if (!EVP_MD_CTX_test_flags(ctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX))
+                EVP_PKEY_CTX_free(ctx->pctx);
 #ifndef OPENSSL_NO_ENGINE
         ENGINE_finish(ctx->engine);
 #endif
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index b5afa477da..aa5b35f67c 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.91 2022/01/07 21:58:17 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.92 2022/01/09 15:15:25 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -497,6 +497,10 @@ unsigned long EVP_MD_flags(const EVP_MD *md);
 
 const EVP_MD *EVP_MD_CTX_md(const EVP_MD_CTX *ctx);
 void *EVP_MD_CTX_md_data(const EVP_MD_CTX *ctx);
+#if defined(LIBRESSL_CRYPTO_INTERANL) || defined(LIBRESSL_NEXT_API)
+EVP_PKEY_CTX *EVP_MD_CTX_pkey_ctx(const EVP_MD_CTX *ctx);
+void EVP_MD_CTX_set_pkey_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pctx);
+#endif
 #define EVP_MD_CTX_size(e)              EVP_MD_size(EVP_MD_CTX_md(e))
 #define EVP_MD_CTX_block_size(e)        EVP_MD_block_size(EVP_MD_CTX_md(e))
 #define EVP_MD_CTX_type(e)              EVP_MD_type(EVP_MD_CTX_md(e))
diff --git a/src/lib/libcrypto/evp/evp_lib.c b/src/lib/libcrypto/evp/evp_lib.c
index 8070fa45ae..c96813987f 100644
--- a/src/lib/libcrypto/evp/evp_lib.c
+++ b/src/lib/libcrypto/evp/evp_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_lib.c,v 1.22 2022/01/07 11:13:54 tb Exp $ */
+/* $OpenBSD: evp_lib.c,v 1.23 2022/01/09 15:15:25 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -385,6 +385,35 @@ EVP_MD_CTX_md_data(const EVP_MD_CTX *ctx)
         return ctx->md_data;
 }
 
+EVP_PKEY_CTX *
+EVP_MD_CTX_pkey_ctx(const EVP_MD_CTX *ctx)
+{
+        return ctx->pctx;
+}
+
+void
+EVP_MD_CTX_set_pkey_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pctx)
+{
+        if (EVP_MD_CTX_test_flags(ctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX)) {
+                EVP_MD_CTX_clear_flags(ctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX);
+        } else {
+                EVP_PKEY_CTX_free(ctx->pctx);
+        }
+
+        ctx->pctx = pctx;
+
+        if (pctx != NULL) {
+                /*
+                 * For unclear reasons it was decided that the caller keeps
+                 * ownership of pctx. So a flag was invented to make sure we
+                 * don't free it in EVP_MD_CTX_cleanup(). We also need to
+                 * unset it in EVP_MD_CTX_copy_ex(). Fortunately, the flag
+                 * isn't public...
+                 */
+                EVP_MD_CTX_set_flags(ctx, EVP_MD_CTX_FLAG_KEEP_PKEY_CTX);
+        }
+}
+
 void
 EVP_MD_CTX_set_flags(EVP_MD_CTX *ctx, int flags)
 {
diff --git a/src/lib/libcrypto/evp/evp_locl.h b/src/lib/libcrypto/evp/evp_locl.h
index ec4cc6d63d..5eef0b244f 100644
--- a/src/lib/libcrypto/evp/evp_locl.h
+++ b/src/lib/libcrypto/evp/evp_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_locl.h,v 1.17 2021/12/12 21:21:58 tb Exp $ */
+/* $OpenBSD: evp_locl.h,v 1.18 2022/01/09 15:15:25 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -61,6 +61,12 @@
 
 __BEGIN_HIDDEN_DECLS
 
+/*
+ * Don't free md_ctx->pctx in EVP_MD_CTX_cleanup().  Needed for ownership
+ * handling in EVP_MD_CTX_set_pkey_ctx().
+ */
+#define EVP_MD_CTX_FLAG_KEEP_PKEY_CTX   0x0400
+
 /* Macros to code block cipher wrappers */
 
 /* Wrapper functions for each cipher mode */