Update RSA OAEP code.

This syncs the RSA OAEP code with OpenSSL 1.1.1d, correctly handling OAEP
padding and providing various OAEP related controls.

ok inoguchi@ tb@

2 files changed, 124 insertions, 21 deletions

diff --git a/src/lib/libcrypto/rsa/rsa.h b/src/lib/libcrypto/rsa/rsa.h
index db10049e0e..d3f9bee3a5 100644
--- a/src/lib/libcrypto/rsa/rsa.h
+++ b/src/lib/libcrypto/rsa/rsa.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa.h,v 1.43 2019/10/24 15:54:29 jsing Exp $ */
+/* $OpenBSD: rsa.h,v 1.44 2019/10/29 08:00:18 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -237,17 +237,39 @@ struct rsa_st {
                 EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_SIG,  \
                                 EVP_PKEY_CTRL_GET_RSA_MGF1_MD, 0, (void *)pmd)
 
-#define EVP_PKEY_CTRL_RSA_PADDING       (EVP_PKEY_ALG_CTRL + 1)
-#define EVP_PKEY_CTRL_RSA_PSS_SALTLEN   (EVP_PKEY_ALG_CTRL + 2)
+#define EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md) \
+        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \
+            EVP_PKEY_CTRL_RSA_OAEP_MD, 0, (void *)(md))
 
-#define EVP_PKEY_CTRL_RSA_KEYGEN_BITS   (EVP_PKEY_ALG_CTRL + 3)
-#define EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP (EVP_PKEY_ALG_CTRL + 4)
-#define EVP_PKEY_CTRL_RSA_MGF1_MD       (EVP_PKEY_ALG_CTRL + 5)
+#define EVP_PKEY_CTX_get_rsa_oaep_md(ctx, pmd) \
+        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \
+            EVP_PKEY_CTRL_GET_RSA_OAEP_MD, 0, (void *)(pmd))
+
+#define EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, l, llen) \
+        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \
+            EVP_PKEY_CTRL_RSA_OAEP_LABEL, llen, (void *)(l))
+
+#define EVP_PKEY_CTX_get0_rsa_oaep_label(ctx, l) \
+        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT, \
+            EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL, 0, (void *)(l))
+
+#define EVP_PKEY_CTRL_RSA_PADDING               (EVP_PKEY_ALG_CTRL + 1)
+#define EVP_PKEY_CTRL_RSA_PSS_SALTLEN           (EVP_PKEY_ALG_CTRL + 2)
+
+#define EVP_PKEY_CTRL_RSA_KEYGEN_BITS           (EVP_PKEY_ALG_CTRL + 3)
+#define EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP         (EVP_PKEY_ALG_CTRL + 4)
+#define EVP_PKEY_CTRL_RSA_MGF1_MD               (EVP_PKEY_ALG_CTRL + 5)
 
 #define EVP_PKEY_CTRL_GET_RSA_PADDING           (EVP_PKEY_ALG_CTRL + 6)
 #define EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN       (EVP_PKEY_ALG_CTRL + 7)
 #define EVP_PKEY_CTRL_GET_RSA_MGF1_MD           (EVP_PKEY_ALG_CTRL + 8)
 
+#define EVP_PKEY_CTRL_RSA_OAEP_MD               (EVP_PKEY_ALG_CTRL + 9)
+#define EVP_PKEY_CTRL_RSA_OAEP_LABEL            (EVP_PKEY_ALG_CTRL + 10)
+
+#define EVP_PKEY_CTRL_GET_RSA_OAEP_MD           (EVP_PKEY_ALG_CTRL + 11)
+#define EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL        (EVP_PKEY_ALG_CTRL + 12)
+
 #define RSA_PKCS1_PADDING       1
 #define RSA_SSLV23_PADDING      2
 #define RSA_NO_PADDING          3
diff --git a/src/lib/libcrypto/rsa/rsa_pmeth.c b/src/lib/libcrypto/rsa/rsa_pmeth.c
index d0cc50cd9f..a5dd86a5de 100644
--- a/src/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/src/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.22 2019/09/09 18:06:26 jsing Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.23 2019/10/29 08:00:18 jsing Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -68,7 +68,7 @@
 #include <openssl/evp.h>
 #include <openssl/rsa.h>
 #include <openssl/x509.h>
-
+#include <openssl/x509v3.h>
 
 #include "evp_locl.h"
 #include "rsa_locl.h"
@@ -87,10 +87,13 @@ typedef struct {
         const EVP_MD *md;
         /* message digest for MGF1 */
         const EVP_MD *mgf1md;
-        /* PSS/OAEP salt length */
+        /* PSS salt length */
         int saltlen;
         /* Temp buffer */
         unsigned char *tbuf;
+        /* OAEP label */
+        unsigned char *oaep_label;
+        size_t oaep_labellen;
 } RSA_PKEY_CTX;
 
 static int
@@ -98,15 +101,11 @@ pkey_rsa_init(EVP_PKEY_CTX *ctx)
 {
         RSA_PKEY_CTX *rctx;
 
-        rctx = malloc(sizeof(RSA_PKEY_CTX));
-        if (!rctx)
+        if ((rctx = calloc(1, sizeof(RSA_PKEY_CTX))) == NULL)
                 return 0;
+
         rctx->nbits = 2048;
-        rctx->pub_exp = NULL;
         rctx->pad_mode = RSA_PKCS1_PADDING;
-        rctx->md = NULL;
-        rctx->mgf1md = NULL;
-        rctx->tbuf = NULL;
 
         rctx->saltlen = -2;
 
@@ -124,6 +123,7 @@ pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
 
         if (!pkey_rsa_init(dst))
                 return 0;
+
         sctx = src->data;
         dctx = dst->data;
         dctx->nbits = sctx->nbits;
@@ -134,6 +134,15 @@ pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
         }
         dctx->pad_mode = sctx->pad_mode;
         dctx->md = sctx->md;
+        dctx->mgf1md = sctx->mgf1md;
+        if (sctx->oaep_label != NULL) {
+                free(dctx->oaep_label);
+                if ((dctx->oaep_label = calloc(1, sctx->oaep_labellen)) == NULL)
+                        return 0;
+                memcpy(dctx->oaep_label, sctx->oaep_label, sctx->oaep_labellen);
+                dctx->oaep_labellen = sctx->oaep_labellen;
+        }
+
         return 1;
 }
 
@@ -156,6 +165,7 @@ pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
         if (rctx) {
                 BN_free(rctx->pub_exp);
                 free(rctx->tbuf);
+                free(rctx->oaep_label);
                 free(rctx);
         }
 }
@@ -306,11 +316,23 @@ static int
 pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
     const unsigned char *in, size_t inlen)
 {
-        int ret;
         RSA_PKEY_CTX *rctx = ctx->data;
+        int ret;
 
-        ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
-            rctx->pad_mode);
+        if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
+                int klen = RSA_size(ctx->pkey->pkey.rsa);
+                if (!setup_tbuf(rctx, ctx))
+                        return -1;
+                if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
+                    in, inlen, rctx->oaep_label, rctx->oaep_labellen,
+                    rctx->md, rctx->mgf1md))
+                        return -1;
+                ret = RSA_public_encrypt(klen, rctx->tbuf, out,
+                    ctx->pkey->pkey.rsa, RSA_NO_PADDING);
+        } else {
+                ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
+                    rctx->pad_mode);
+        }
         if (ret < 0)
                 return ret;
         *outlen = ret;
@@ -324,8 +346,20 @@ pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
         int ret;
         RSA_PKEY_CTX *rctx = ctx->data;
 
-        ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
-            rctx->pad_mode);
+        if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
+                if (!setup_tbuf(rctx, ctx))
+                        return -1;
+                ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
+                    ctx->pkey->pkey.rsa, RSA_NO_PADDING);
+                if (ret <= 0)
+                        return ret;
+                ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf,
+                    ret, ret, rctx->oaep_label, rctx->oaep_labellen, rctx->md,
+                    rctx->mgf1md);
+        } else {
+                ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
+                    rctx->pad_mode);
+        }
         if (ret < 0)
                 return ret;
         *outlen = ret;
@@ -429,7 +463,8 @@ bad_pad:
 
         case EVP_PKEY_CTRL_RSA_MGF1_MD:
         case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
-                if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
+                if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING &&
+                    rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
                         RSAerror(RSA_R_INVALID_MGF1_MD);
                         return -2;
                 }
@@ -442,6 +477,29 @@ bad_pad:
                         rctx->mgf1md = p2;
                 return 1;
 
+        case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
+                if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
+                        RSAerror(RSA_R_INVALID_PADDING_MODE);
+                        return -2;
+                }
+                free(rctx->oaep_label);
+                if (p2 != NULL && p1 > 0) {
+                        rctx->oaep_label = p2;
+                        rctx->oaep_labellen = p1;
+                } else {
+                        rctx->oaep_label = NULL;
+                        rctx->oaep_labellen = 0;
+                }
+                return 1;
+
+        case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
+                if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
+                        RSAerror(RSA_R_INVALID_PADDING_MODE);
+                        return -2;
+                }
+                *(unsigned char **)p2 = rctx->oaep_label;
+                return rctx->oaep_labellen;
+
         case EVP_PKEY_CTRL_DIGESTINIT:
         case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
         case EVP_PKEY_CTRL_PKCS7_DECRYPT:
@@ -529,6 +587,29 @@ pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
                 return ret;
         }
 
+        if (strcmp(type, "rsa_mgf1_md") == 0)
+                return EVP_PKEY_CTX_md(ctx,
+                    EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT,
+                    EVP_PKEY_CTRL_RSA_MGF1_MD, value);
+
+        if (strcmp(type, "rsa_oaep_md") == 0)
+                return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_TYPE_CRYPT,
+                    EVP_PKEY_CTRL_RSA_OAEP_MD, value);
+
+        if (strcmp(type, "rsa_oaep_label") == 0) {
+                unsigned char *lab;
+                long lablen;
+                int ret;
+
+                if ((lab = string_to_hex(value, &lablen)) == NULL)
+                        return 0;
+                ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
+                if (ret <= 0)
+                        free(lab);
+
+                return ret;
+        }
+
 not_a_number:
 out_of_range:
         return -2;