Rework pkey_dsa_sign() and pkey_dsa_verify()

Since DSA_sign() and DSA_verify() ignore their type argument, don't bother
to determine it here. Check all size_t for overflow before passing them as
int arguments. Follow OpenSSL and add a check to see if the tbs blob's
length matches the one of the md, in case it is set on the EVP_PKEY_CTX.
Fix return value check of DSA_sign().

ok jsing

1 files changed, 25 insertions, 21 deletions

diff --git a/src/lib/libcrypto/dsa/dsa_pmeth.c b/src/lib/libcrypto/dsa/dsa_pmeth.c
index afc01a7791..b4ce58eef8 100644
--- a/src/lib/libcrypto/dsa/dsa_pmeth.c
+++ b/src/lib/libcrypto/dsa/dsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_pmeth.c,v 1.14 2022/11/19 06:33:00 tb Exp $ */
+/* $OpenBSD: dsa_pmeth.c,v 1.15 2022/11/19 11:53:38 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -128,24 +128,28 @@ pkey_dsa_cleanup(EVP_PKEY_CTX *ctx)
 }
 
 static int
-pkey_dsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
+pkey_dsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *out_siglen,
     const unsigned char *tbs, size_t tbslen)
 {
-        int ret, type;
-        unsigned int sltmp;
-        DSA_PKEY_CTX *dctx = ctx->data;
         DSA *dsa = ctx->pkey->pkey.dsa;
+        DSA_PKEY_CTX *dctx = ctx->data;
+        unsigned int siglen;
 
-        if (dctx->md)
-                type = EVP_MD_type(dctx->md);
-        else
-                type = NID_sha1;
+        *out_siglen = 0;
+
+        if (tbslen > INT_MAX)
+                 return 0;
 
-        ret = DSA_sign(type, tbs, tbslen, sig, &sltmp, dsa);
+        if (dctx->md != NULL) {
+                if (tbslen != EVP_MD_size(dctx->md))
+                        return 0;
+        }
+
+        if (!DSA_sign(0, tbs, tbslen, sig, &siglen, dsa))
+                return 0;
+
+        *out_siglen = siglen;
 
-        if (ret <= 0)
-                return ret;
-        *siglen = sltmp;
         return 1;
 }
 
@@ -153,18 +157,18 @@ static int
 pkey_dsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
     const unsigned char *tbs, size_t tbslen)
 {
-        int ret, type;
-        DSA_PKEY_CTX *dctx = ctx->data;
         DSA *dsa = ctx->pkey->pkey.dsa;
+        DSA_PKEY_CTX *dctx = ctx->data;
 
-        if (dctx->md)
-                type = EVP_MD_type(dctx->md);
-        else
-                type = NID_sha1;
+        if (tbslen > INT_MAX || siglen > INT_MAX)
+                 return 0;
 
-        ret = DSA_verify(type, tbs, tbslen, sig, siglen, dsa);
+        if (dctx->md != NULL) {
+                if (tbslen != EVP_MD_size(dctx->md))
+                        return 0;
+        }
 
-        return ret;
+        return DSA_verify(0, tbs, tbslen, sig, siglen, dsa);
 }
 
 static int