Mop up remaining uses of ASN1_bn_print()

This removes lots of silly buffers and will allow us to make this API
go away.

ok jsing

4 files changed, 38 insertions, 140 deletions

diff --git a/src/lib/libcrypto/dh/dh_ameth.c b/src/lib/libcrypto/dh/dh_ameth.c
index 3de0bb9333..61d3d14397 100644
--- a/src/lib/libcrypto/dh/dh_ameth.c
+++ b/src/lib/libcrypto/dh/dh_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_ameth.c,v 1.28 2023/04/17 05:57:17 tb Exp $ */
+/* $OpenBSD: dh_ameth.c,v 1.29 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -65,6 +65,7 @@
 #include <openssl/x509.h>
 
 #include "asn1_local.h"
+#include "bn_local.h"
 #include "dh_local.h"
 #include "evp_local.h"
 
@@ -280,17 +281,6 @@ err:
         return 0;
 }
 
-static void
-update_buflen(const BIGNUM *b, size_t *pbuflen)
-{
-        size_t i;
-
-        if (!b)
-                return;
-        if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
-                *pbuflen = i;
-}
-
 static int
 dh_param_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
 {
@@ -313,9 +303,7 @@ dh_param_encode(const EVP_PKEY *pkey, unsigned char **pder)
 static int
 do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 {
-        unsigned char *m = NULL;
         int reason = ERR_R_BUF_LIB, ret = 0;
-        size_t buf_len = 0;
         const char *ktype = NULL;
         BIGNUM *priv_key, *pub_key;
 
@@ -329,17 +317,6 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
         else
                 pub_key = NULL;
 
-        update_buflen(x->p, &buf_len);
-
-        if (buf_len == 0) {
-                reason = ERR_R_PASSED_NULL_PARAMETER;
-                goto err;
-        }
-
-        update_buflen(x->g, &buf_len);
-        update_buflen(pub_key, &buf_len);
-        update_buflen(priv_key, &buf_len);
-
         if (ptype == 2)
                 ktype = "PKCS#3 DH Private-Key";
         else if (ptype == 1)
@@ -347,9 +324,8 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
         else
                 ktype = "PKCS#3 DH Parameters";
 
-        m= malloc(buf_len + 10);
-        if (m == NULL) {
-                reason = ERR_R_MALLOC_FAILURE;
+        if (x->p == NULL) {
+                reason = ERR_R_PASSED_NULL_PARAMETER;
                 goto err;
         }
 
@@ -359,14 +335,14 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
                 goto err;
         indent += 4;
 
-        if (!ASN1_bn_print(bp, "private-key:", priv_key, m, indent))
+        if (!bn_printf(bp, priv_key, indent, "private-key:"))
                 goto err;
-        if (!ASN1_bn_print(bp, "public-key:", pub_key, m, indent))
+        if (!bn_printf(bp, pub_key, indent, "public-key:"))
                 goto err;
 
-        if (!ASN1_bn_print(bp, "prime:", x->p, m, indent))
+        if (!bn_printf(bp, x->p, indent, "prime:"))
                 goto err;
-        if (!ASN1_bn_print(bp, "generator:", x->g, m, indent))
+        if (!bn_printf(bp, x->g, indent, "generator:"))
                 goto err;
         if (x->length != 0) {
                 if (!BIO_indent(bp, indent, 128))
@@ -378,10 +354,9 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 
         ret = 1;
         if (0) {
-err:
+ err:
                 DHerror(reason);
         }
-        free(m);
         return(ret);
 }
 
diff --git a/src/lib/libcrypto/dsa/dsa_ameth.c b/src/lib/libcrypto/dsa/dsa_ameth.c
index f282caae06..5a0c3116aa 100644
--- a/src/lib/libcrypto/dsa/dsa_ameth.c
+++ b/src/lib/libcrypto/dsa/dsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_ameth.c,v 1.42 2023/03/04 21:42:49 tb Exp $ */
+/* $OpenBSD: dsa_ameth.c,v 1.43 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -385,25 +385,12 @@ int_dsa_free(EVP_PKEY *pkey)
         DSA_free(pkey->pkey.dsa);
 }
 
-static void
-update_buflen(const BIGNUM *b, size_t *pbuflen)
-{
-        size_t i;
-
-        if (!b)
-                return;
-        if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
-                *pbuflen = i;
-}
-
 static int
 do_dsa_print(BIO *bp, const DSA *x, int off, int ptype)
 {
-        unsigned char *m = NULL;
-        int ret = 0;
-        size_t buf_len = 0;
         const char *ktype = NULL;
         const BIGNUM *priv_key, *pub_key;
+        int ret = 0;
 
         if (ptype == 2)
                 priv_key = x->priv_key;
@@ -422,18 +409,6 @@ do_dsa_print(BIO *bp, const DSA *x, int off, int ptype)
         else
                 ktype = "DSA-Parameters";
 
-        update_buflen(x->p, &buf_len);
-        update_buflen(x->q, &buf_len);
-        update_buflen(x->g, &buf_len);
-        update_buflen(priv_key, &buf_len);
-        update_buflen(pub_key, &buf_len);
-
-        m = malloc(buf_len + 10);
-        if (m == NULL) {
-                DSAerror(ERR_R_MALLOC_FAILURE);
-                goto err;
-        }
-
         if (priv_key) {
                 if (!BIO_indent(bp, off, 128))
                         goto err;
@@ -442,19 +417,20 @@ do_dsa_print(BIO *bp, const DSA *x, int off, int ptype)
                         goto err;
         }
 
-        if (!ASN1_bn_print(bp, "priv:", priv_key, m, off))
+        if (!bn_printf(bp, priv_key, off, "priv:"))
                 goto err;
-        if (!ASN1_bn_print(bp, "pub: ", pub_key, m, off))
+        if (!bn_printf(bp, pub_key, off, "pub: "))
                 goto err;
-        if (!ASN1_bn_print(bp, "P:   ", x->p, m, off))
+        if (!bn_printf(bp, x->p, off, "P:   "))
                 goto err;
-        if (!ASN1_bn_print(bp, "Q:   ", x->q, m, off))
+        if (!bn_printf(bp, x->q, off, "Q:   "))
                 goto err;
-        if (!ASN1_bn_print(bp, "G:   ", x->g, m, off))
+        if (!bn_printf(bp, x->g, off, "G:   "))
                 goto err;
+
         ret = 1;
-err:
-        free(m);
+
+ err:
         return ret;
 }
 
@@ -594,27 +570,16 @@ dsa_sig_print(BIO *bp, const X509_ALGOR *sigalg, const ASN1_STRING *sig,
         dsa_sig = d2i_DSA_SIG(NULL, &p, sig->length);
         if (dsa_sig) {
                 int rv = 0;
-                size_t buf_len = 0;
-                unsigned char *m = NULL;
-
-                update_buflen(dsa_sig->r, &buf_len);
-                update_buflen(dsa_sig->s, &buf_len);
-                m = malloc(buf_len + 10);
-                if (m == NULL) {
-                        DSAerror(ERR_R_MALLOC_FAILURE);
-                        goto err;
-                }
 
                 if (BIO_write(bp, "\n", 1) != 1)
                         goto err;
 
-                if (!ASN1_bn_print(bp, "r:   ", dsa_sig->r, m, indent))
+                if (!bn_printf(bp, dsa_sig->r, indent, "r:   "))
                         goto err;
-                if (!ASN1_bn_print(bp, "s:   ", dsa_sig->s, m, indent))
+                if (!bn_printf(bp, dsa_sig->s, indent, "s:   "))
                         goto err;
                 rv = 1;
-err:
-                free(m);
+ err:
                 DSA_SIG_free(dsa_sig);
                 return rv;
         }
diff --git a/src/lib/libcrypto/ec/ec_ameth.c b/src/lib/libcrypto/ec/ec_ameth.c
index 8676ace9d8..49ae80494d 100644
--- a/src/lib/libcrypto/ec/ec_ameth.c
+++ b/src/lib/libcrypto/ec/ec_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ec_ameth.c,v 1.40 2023/07/03 09:25:44 tb Exp $ */
+/* $OpenBSD: ec_ameth.c,v 1.41 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -427,9 +427,7 @@ int_ec_free(EVP_PKEY *pkey)
 static int
 do_EC_KEY_print(BIO *bp, const EC_KEY *x, int off, int ktype)
 {
-        unsigned char *buffer = NULL;
         const char *ecstr;
-        size_t buf_len = 0, i;
         int ret = 0, reason = ERR_R_BIO_LIB;
         BIGNUM *pub_key = NULL;
         BN_CTX *ctx = NULL;
@@ -454,24 +452,13 @@ do_EC_KEY_print(BIO *bp, const EC_KEY *x, int off, int ktype)
                                 reason = ERR_R_EC_LIB;
                                 goto err;
                         }
-                        if (pub_key)
-                                buf_len = (size_t) BN_num_bytes(pub_key);
                 }
         }
         if (ktype == 2) {
                 priv_key = EC_KEY_get0_private_key(x);
-                if (priv_key && (i = (size_t) BN_num_bytes(priv_key)) > buf_len)
-                        buf_len = i;
         } else
                 priv_key = NULL;
 
-        if (ktype > 0) {
-                buf_len += 10;
-                if ((buffer = malloc(buf_len)) == NULL) {
-                        reason = ERR_R_MALLOC_FAILURE;
-                        goto err;
-                }
-        }
         if (ktype == 2)
                 ecstr = "Private-Key";
         else if (ktype == 1)
@@ -485,19 +472,21 @@ do_EC_KEY_print(BIO *bp, const EC_KEY *x, int off, int ktype)
             EC_GROUP_order_bits(group)) <= 0)
                 goto err;
 
-        if (!ASN1_bn_print(bp, "priv:", priv_key, buffer, off))
+        if (!bn_printf(bp, priv_key, off, "priv:"))
                 goto err;
-        if (!ASN1_bn_print(bp, "pub: ", pub_key, buffer, off))
+        if (!bn_printf(bp, pub_key, off, "pub: "))
                 goto err;
         if (!ECPKParameters_print(bp, group, off))
                 goto err;
+
         ret = 1;
+
  err:
         if (!ret)
                 ECerror(reason);
         BN_free(pub_key);
         BN_CTX_free(ctx);
-        free(buffer);
+
         return (ret);
 }
 
diff --git a/src/lib/libcrypto/rsa/rsa_ameth.c b/src/lib/libcrypto/rsa/rsa_ameth.c
index 1cf2069a18..825a9f4447 100644
--- a/src/lib/libcrypto/rsa/rsa_ameth.c
+++ b/src/lib/libcrypto/rsa/rsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_ameth.c,v 1.29 2023/05/19 17:31:20 tb Exp $ */
+/* $OpenBSD: rsa_ameth.c,v 1.30 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -68,6 +68,7 @@
 #include <openssl/x509.h>
 
 #include "asn1_local.h"
+#include "bn_local.h"
 #include "cryptlib.h"
 #include "evp_local.h"
 #include "rsa_local.h"
@@ -408,44 +409,13 @@ rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss, int indent)
 
 }
 
-static void
-update_buflen(const BIGNUM *b, size_t *pbuflen)
-{
-        size_t i;
-
-        if (!b)
-                return;
-        if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
-                *pbuflen = i;
-}
-
 static int
 pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
 {
         const RSA *x = pkey->pkey.rsa;
-        unsigned char *m = NULL;
         char *str;
         const char *s;
         int ret = 0, mod_len = 0;
-        size_t buf_len = 0;
-
-        update_buflen(x->n, &buf_len);
-        update_buflen(x->e, &buf_len);
-
-        if (priv) {
-                update_buflen(x->d, &buf_len);
-                update_buflen(x->p, &buf_len);
-                update_buflen(x->q, &buf_len);
-                update_buflen(x->dmp1, &buf_len);
-                update_buflen(x->dmq1, &buf_len);
-                update_buflen(x->iqmp, &buf_len);
-        }
-
-        m = malloc(buf_len + 10);
-        if (m == NULL) {
-                RSAerror(ERR_R_MALLOC_FAILURE);
-                goto err;
-        }
 
         if (x->n != NULL)
                 mod_len = BN_num_bits(x->n);
@@ -467,29 +437,28 @@ pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
                 str = "Modulus:";
                 s = "Exponent:";
         }
-        if (!ASN1_bn_print(bp, str, x->n, m, off))
+        if (!bn_printf(bp, x->n, off, "%s", str))
                 goto err;
-        if (!ASN1_bn_print(bp, s, x->e, m, off))
+        if (!bn_printf(bp, x->e, off, "%s", s))
                 goto err;
         if (priv) {
-                if (!ASN1_bn_print(bp, "privateExponent:", x->d, m, off))
+                if (!bn_printf(bp, x->d, off, "privateExponent:"))
                         goto err;
-                if (!ASN1_bn_print(bp, "prime1:", x->p, m, off))
+                if (!bn_printf(bp, x->p, off, "prime1:"))
                         goto err;
-                if (!ASN1_bn_print(bp, "prime2:", x->q, m, off))
+                if (!bn_printf(bp, x->q, off, "prime2:"))
                         goto err;
-                if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, m, off))
+                if (!bn_printf(bp, x->dmp1, off, "exponent1:"))
                         goto err;
-                if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, m, off))
+                if (!bn_printf(bp, x->dmq1, off, "exponent2:"))
                         goto err;
-                if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, m, off))
+                if (!bn_printf(bp, x->iqmp, off, "coefficient:"))
                         goto err;
         }
         if (pkey_is_pss(pkey) && !rsa_pss_param_print(bp, 1, x->pss, off))
                 goto err;
         ret = 1;
  err:
-        free(m);
         return ret;
 }