Convert openssl(1) to using BN_GENCB on the heap

This is three times the same thing while genrsa needs some extra steps
to deal with opaque BIGNUMs. We can also garbage collect some Win 3.1
contortions and use the conversion routines directly instead of doing
them manually.

ok jsing

4 files changed, 74 insertions, 51 deletions

diff --git a/src/usr.bin/openssl/dhparam.c b/src/usr.bin/openssl/dhparam.c
index b0dd510949..55263274b6 100644
--- a/src/usr.bin/openssl/dhparam.c
+++ b/src/usr.bin/openssl/dhparam.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dhparam.c,v 1.12 2019/07/14 03:30:45 guenther Exp $ */
+/* $OpenBSD: dhparam.c,v 1.13 2021/11/20 18:10:48 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -231,12 +231,13 @@ dhparam_usage()
         options_usage(dhparam_options);
 }
 
-static int dh_cb(int p, int n, BN_GENCB * cb);
+static int dh_cb(int p, int n, BN_GENCB *cb);
 
 int
 dhparam_main(int argc, char **argv)
 {
         BIO *in = NULL, *out = NULL;
+        BN_GENCB *cb = NULL;
         char *num_bits = NULL;
         DH *dh = NULL;
         int num = 0;
@@ -283,15 +284,19 @@ dhparam_main(int argc, char **argv)
         }
 
         if (num) {
+                if ((cb = BN_GENCB_new()) == NULL) {
+                        BIO_printf(bio_err,
+                            "Error allocating BN_GENCB object\n");
+                        goto end;
+                }
 
-                BN_GENCB cb;
-                BN_GENCB_set(&cb, dh_cb, bio_err);
+                BN_GENCB_set(cb, dh_cb, bio_err);
                 if (dhparam_config.dsaparam) {
                         DSA *dsa = DSA_new();
 
                         BIO_printf(bio_err, "Generating DSA parameters, %d bit long prime\n", num);
                         if (!dsa || !DSA_generate_parameters_ex(dsa, num,
-                                NULL, 0, NULL, NULL, &cb)) {
+                                NULL, 0, NULL, NULL, cb)) {
                                 DSA_free(dsa);
                                 ERR_print_errors(bio_err);
                                 goto end;
@@ -306,7 +311,7 @@ dhparam_main(int argc, char **argv)
                         dh = DH_new();
                         BIO_printf(bio_err, "Generating DH parameters, %d bit long safe prime, generator %d\n", num, dhparam_config.g);
                         BIO_printf(bio_err, "This is going to take a long time\n");
-                        if (!dh || !DH_generate_parameters_ex(dh, num, dhparam_config.g, &cb)) {
+                        if (!dh || !DH_generate_parameters_ex(dh, num, dhparam_config.g, cb)) {
                                 ERR_print_errors(bio_err);
                                 goto end;
                         }
@@ -469,6 +474,7 @@ dhparam_main(int argc, char **argv)
  end:
         BIO_free(in);
         BIO_free_all(out);
+        BN_GENCB_free(cb);
         DH_free(dh);
 
         return (ret);
@@ -476,7 +482,7 @@ dhparam_main(int argc, char **argv)
 
 /* dh_cb is identical to dsa_cb in apps/dsaparam.c */
 static int
-dh_cb(int p, int n, BN_GENCB * cb)
+dh_cb(int p, int n, BN_GENCB *cb)
 {
         char c = '*';
 
@@ -488,8 +494,8 @@ dh_cb(int p, int n, BN_GENCB * cb)
                 c = '*';
         if (p == 3)
                 c = '\n';
-        BIO_write(cb->arg, &c, 1);
-        (void) BIO_flush(cb->arg);
+        BIO_write(BN_GENCB_get_arg(cb), &c, 1);
+        (void) BIO_flush(BN_GENCB_get_arg(cb));
         return 1;
 }
 
diff --git a/src/usr.bin/openssl/dsaparam.c b/src/usr.bin/openssl/dsaparam.c
index 3c2ac89800..3a907fe620 100644
--- a/src/usr.bin/openssl/dsaparam.c
+++ b/src/usr.bin/openssl/dsaparam.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsaparam.c,v 1.11 2019/07/14 03:30:45 guenther Exp $ */
+/* $OpenBSD: dsaparam.c,v 1.12 2021/11/20 18:10:48 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -156,7 +156,7 @@ dsaparam_usage(void)
         options_usage(dsaparam_options);
 }
 
-static int dsa_cb(int p, int n, BN_GENCB * cb);
+static int dsa_cb(int p, int n, BN_GENCB *cb);
 
 int
 dsaparam_main(int argc, char **argv)
@@ -164,6 +164,7 @@ dsaparam_main(int argc, char **argv)
         DSA *dsa = NULL;
         int i;
         BIO *in = NULL, *out = NULL;
+        BN_GENCB *cb = NULL;
         int ret = 1;
         int numbits = -1;
         char *strbits = NULL;
@@ -218,8 +219,14 @@ dsaparam_main(int argc, char **argv)
         }
 
         if (numbits > 0) {
-                BN_GENCB cb;
-                BN_GENCB_set(&cb, dsa_cb, bio_err);
+                if ((cb = BN_GENCB_new()) == NULL) {
+                        BIO_printf(bio_err,
+                            "Error allocating BN_GENCB object\n");
+                        goto end;
+                }
+
+                BN_GENCB_set(cb, dsa_cb, bio_err);
+
                 dsa = DSA_new();
                 if (!dsa) {
                         BIO_printf(bio_err, "Error allocating DSA object\n");
@@ -227,7 +234,7 @@ dsaparam_main(int argc, char **argv)
                 }
                 BIO_printf(bio_err, "Generating DSA parameters, %d bit long prime\n", numbits);
                 BIO_printf(bio_err, "This could take some time\n");
-                if (!DSA_generate_parameters_ex(dsa, numbits, NULL, 0, NULL, NULL, &cb)) {
+                if (!DSA_generate_parameters_ex(dsa, numbits, NULL, 0, NULL, NULL, cb)) {
                         ERR_print_errors(bio_err);
                         BIO_printf(bio_err, "Error, DSA key generation failed\n");
                         goto end;
@@ -341,13 +348,14 @@ dsaparam_main(int argc, char **argv)
  end:
         BIO_free(in);
         BIO_free_all(out);
+        BN_GENCB_free(cb);
         DSA_free(dsa);
 
         return (ret);
 }
 
 static int
-dsa_cb(int p, int n, BN_GENCB * cb)
+dsa_cb(int p, int n, BN_GENCB *cb)
 {
         char c = '*';
 
@@ -359,8 +367,8 @@ dsa_cb(int p, int n, BN_GENCB * cb)
                 c = '*';
         if (p == 3)
                 c = '\n';
-        BIO_write(cb->arg, &c, 1);
-        (void) BIO_flush(cb->arg);
+        BIO_write(BN_GENCB_get_arg(cb), &c, 1);
+        (void) BIO_flush(BN_GENCB_get_arg(cb));
 #ifdef GENCB_TEST
         if (stop_keygen_flag)
                 return 0;
diff --git a/src/usr.bin/openssl/gendh.c b/src/usr.bin/openssl/gendh.c
index facc9248f3..c6564e047b 100644
--- a/src/usr.bin/openssl/gendh.c
+++ b/src/usr.bin/openssl/gendh.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: gendh.c,v 1.11 2019/07/14 03:30:45 guenther Exp $ */
+/* $OpenBSD: gendh.c,v 1.12 2021/11/20 18:10:48 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -84,7 +84,7 @@
 
 #define DEFBITS 512
 
-static int dh_cb(int p, int n, BN_GENCB * cb);
+static int dh_cb(int p, int n, BN_GENCB *cb);
 
 static struct {
         int g;
@@ -128,7 +128,7 @@ gendh_usage(void)
 int
 gendh_main(int argc, char **argv)
 {
-        BN_GENCB cb;
+        BN_GENCB *cb = NULL;
         DH *dh = NULL;
         int ret = 1, numbits = DEFBITS;
         BIO *out = NULL;
@@ -141,7 +141,12 @@ gendh_main(int argc, char **argv)
                 }
         }
 
-        BN_GENCB_set(&cb, dh_cb, bio_err);
+        if ((cb = BN_GENCB_new()) == NULL) {
+                BIO_printf(bio_err, "Error allocating BN_GENCB object\n");
+                goto end;
+        }
+
+        BN_GENCB_set(cb, dh_cb, bio_err);
 
         memset(&gendh_config, 0, sizeof(gendh_config));
 
@@ -180,7 +185,7 @@ gendh_main(int argc, char **argv)
         BIO_printf(bio_err, "This is going to take a long time\n");
 
         if (((dh = DH_new()) == NULL) ||
-            !DH_generate_parameters_ex(dh, numbits, gendh_config.g, &cb))
+            !DH_generate_parameters_ex(dh, numbits, gendh_config.g, cb))
                 goto end;
 
         if (!PEM_write_bio_DHparams(out, dh))
@@ -190,13 +195,14 @@ gendh_main(int argc, char **argv)
         if (ret != 0)
                 ERR_print_errors(bio_err);
         BIO_free_all(out);
+        BN_GENCB_free(cb);
         DH_free(dh);
 
         return (ret);
 }
 
 static int
-dh_cb(int p, int n, BN_GENCB * cb)
+dh_cb(int p, int n, BN_GENCB *cb)
 {
         char c = '*';
 
@@ -208,8 +214,8 @@ dh_cb(int p, int n, BN_GENCB * cb)
                 c = '*';
         if (p == 3)
                 c = '\n';
-        BIO_write(cb->arg, &c, 1);
-        (void) BIO_flush(cb->arg);
+        BIO_write(BN_GENCB_get_arg(cb), &c, 1);
+        (void) BIO_flush(BN_GENCB_get_arg(cb));
         return 1;
 }
 #endif
diff --git a/src/usr.bin/openssl/genrsa.c b/src/usr.bin/openssl/genrsa.c
index f0cea1f9b1..024fa88d26 100644
--- a/src/usr.bin/openssl/genrsa.c
+++ b/src/usr.bin/openssl/genrsa.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: genrsa.c,v 1.17 2019/07/24 14:23:25 inoguchi Exp $ */
+/* $OpenBSD: genrsa.c,v 1.18 2021/11/20 18:10:48 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -83,7 +83,7 @@
 
 #define DEFBITS 2048
 
-static int genrsa_cb(int p, int n, BN_GENCB * cb);
+static int genrsa_cb(int p, int n, BN_GENCB *cb);
 
 static struct {
         const EVP_CIPHER *enc;
@@ -270,15 +270,16 @@ genrsa_usage(void)
 int
 genrsa_main(int argc, char **argv)
 {
-        BN_GENCB cb;
+        BN_GENCB *cb = NULL;
         int ret = 1;
-        int i, num = DEFBITS;
-        char *numbits= NULL;
-        long l;
+        int num = DEFBITS;
+        char *numbits = NULL;
         char *passout = NULL;
         BIO *out = NULL;
-        BIGNUM *bn = BN_new();
+        BIGNUM *bn = NULL;
         RSA *rsa = NULL;
+        const BIGNUM *rsa_e = NULL;
+        char *rsa_e_hex = NULL, *rsa_e_dec = NULL;
 
         if (single_execution) {
                 if (pledge("stdio cpath wpath rpath tty", NULL) == -1) {
@@ -287,10 +288,15 @@ genrsa_main(int argc, char **argv)
                 }
         }
 
-        if (!bn)
+        if ((bn = BN_new()) == NULL)
                 goto err;
 
-        BN_GENCB_set(&cb, genrsa_cb, bio_err);
+        if ((cb = BN_GENCB_new()) == NULL) {
+                BIO_printf(bio_err, "Error allocating BN_GENCB object\n");
+                goto err;
+        }
+
+        BN_GENCB_set(cb, genrsa_cb, bio_err);
 
         if ((out = BIO_new(BIO_s_file())) == NULL) {
                 BIO_printf(bio_err, "unable to create BIO for output\n");
@@ -333,22 +339,16 @@ genrsa_main(int argc, char **argv)
                 goto err;
 
         if (!BN_set_word(bn, genrsa_config.f4) ||
-            !RSA_generate_key_ex(rsa, num, bn, &cb))
+            !RSA_generate_key_ex(rsa, num, bn, cb))
                 goto err;
 
-        /*
-         * We need to do the following for when the base number size is <
-         * long, esp windows 3.1 :-(.
-         */
-        l = 0L;
-        for (i = 0; i < rsa->e->top; i++) {
-#ifndef _LP64
-                l <<= BN_BITS4;
-                l <<= BN_BITS4;
-#endif
-                l += rsa->e->d[i];
-        }
-        BIO_printf(bio_err, "e is %ld (0x%lX)\n", l, l);
+        RSA_get0_key(rsa, NULL, &rsa_e, NULL);
+        if ((rsa_e_hex = BN_bn2hex(rsa_e)) == NULL)
+                goto err;
+        if ((rsa_e_dec = BN_bn2dec(rsa_e)) == NULL)
+                goto err;
+
+        BIO_printf(bio_err, "e is %s (0x%s)\n", rsa_e_hex, rsa_e_dec);
         {
                 PW_CB_DATA cb_data;
                 cb_data.password = passout;
@@ -361,8 +361,11 @@ genrsa_main(int argc, char **argv)
         ret = 0;
  err:
         BN_free(bn);
+        BN_GENCB_free(cb);
         RSA_free(rsa);
         BIO_free_all(out);
+        free(rsa_e_dec);
+        free(rsa_e_hex);
         free(passout);
 
         if (ret != 0)
@@ -372,7 +375,7 @@ genrsa_main(int argc, char **argv)
 }
 
 static int
-genrsa_cb(int p, int n, BN_GENCB * cb)
+genrsa_cb(int p, int n, BN_GENCB *cb)
 {
         char c = '*';
 
@@ -384,7 +387,7 @@ genrsa_cb(int p, int n, BN_GENCB * cb)
                 c = '*';
         if (p == 3)
                 c = '\n';
-        BIO_write(cb->arg, &c, 1);
-        (void) BIO_flush(cb->arg);
+        BIO_write(BN_GENCB_get_arg(cb), &c, 1);
+        (void) BIO_flush(BN_GENCB_get_arg(cb));
         return 1;
 }