openssl ca: use BN_bn2hex() rather than reimplementing it

ok jsing

1 files changed, 18 insertions, 28 deletions

diff --git a/src/usr.bin/openssl/ca.c b/src/usr.bin/openssl/ca.c
index 80561712ff..d69844620d 100644
--- a/src/usr.bin/openssl/ca.c
+++ b/src/usr.bin/openssl/ca.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ca.c,v 1.60 2024/07/08 05:56:17 tb Exp $ */
+/* $OpenBSD: ca.c,v 1.61 2025/02/25 09:49:33 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -148,7 +148,6 @@ static int do_revoke(X509 *x509, CA_DB *db, int ext, char *extval);
 static int get_certificate_status(const char *serial, CA_DB *db);
 static int do_updatedb(CA_DB *db);
 static int check_time_format(const char *str);
-static char *bin2hex(unsigned char *, size_t);
 char *make_revocation_str(int rev_type, char *rev_arg);
 int make_revoked(X509_REVOKED *rev, const char *str);
 int old_entry_print(BIO *bp, ASN1_OBJECT *obj, ASN1_STRING *str);
@@ -1254,22 +1253,30 @@ ca_main(int argc, char **argv)
                 if (cfg.verbose)
                         BIO_printf(bio_err, "writing new certificates\n");
                 for (i = 0; i < sk_X509_num(cert_sk); i++) {
-                        ASN1_INTEGER *serialNumber;
-                        int k;
+                        BIGNUM *bn;
                         char *serialstr;
-                        unsigned char *data;
                         char pempath[PATH_MAX];
+                        int k;
 
                         x = sk_X509_value(cert_sk, i);
 
-                        serialNumber = X509_get_serialNumber(x);
-                        j = ASN1_STRING_length(serialNumber);
-                        data = ASN1_STRING_data(serialNumber);
+                        if ((bn = ASN1_INTEGER_to_BN(X509_get_serialNumber(x),
+                            NULL)) == NULL)
+                                goto err;
 
-                        if (j > 0)
-                                serialstr = bin2hex(data, j);
-                        else
+                        if (BN_is_zero(bn)) {
+                                /* For consistency, BN_bn2hex(0) is 0, not 00. */
                                 serialstr = strdup("00");
+                        } else {
+                                /*
+                                 * Historical behavior is to ignore the sign
+                                 * that shouldn't be there anyway.
+                                 */
+                                BN_set_negative(bn, 0);
+                                serialstr = BN_bn2hex(bn);
+                        }
+                        BN_free(bn);
+
                         if (serialstr != NULL) {
                                 k = snprintf(pempath, sizeof(pempath),
                                     "%s/%s.pem", cfg.outdir, serialstr);
@@ -2817,20 +2824,3 @@ unpack_revinfo(ASN1_TIME **prevtm, int *preason, ASN1_OBJECT **phold,
 
         return ret;
 }
-
-static char *
-bin2hex(unsigned char *data, size_t len)
-{
-        char *ret = NULL;
-        char hex[] = "0123456789ABCDEF";
-        int i;
-
-        if ((ret = malloc(len * 2 + 1)) != NULL) {
-                for (i = 0; i < len; i++) {
-                        ret[i * 2 + 0] = hex[data[i] >> 4];
-                        ret[i * 2 + 1] = hex[data[i] & 0x0F];
-                }
-                ret[len * 2] = '\0';
-        }
-        return ret;
-}