Add bn_printf(), a replacement for ASN1_bn_print()

ASN1_bn_print() will be removed in an upcoming bump. This adds an internal
API that covers the same functionality but doesn't require that the caller
pass in a sufficiently large scratch space that ASN1_bn_print() may or may
not use. In addition, this takes a format string, which allows us to ditch
some extra dances.

ok jsing

3 files changed, 154 insertions, 2 deletions

diff --git a/src/lib/libcrypto/Makefile b/src/lib/libcrypto/Makefile
index 6e41dae162..f40ef656b7 100644
--- a/src/lib/libcrypto/Makefile
+++ b/src/lib/libcrypto/Makefile
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile,v 1.135 2023/07/05 12:31:14 tb Exp $
+# $OpenBSD: Makefile,v 1.136 2023/07/06 14:37:39 tb Exp $
 
 LIB=    crypto
 LIBREBUILD=y
@@ -195,6 +195,7 @@ SRCS+= bn_mont.c
 SRCS+= bn_mul.c
 SRCS+= bn_prime.c
 SRCS+= bn_primitives.c
+SRCS+= bn_print.c
 SRCS+= bn_rand.c
 SRCS+= bn_recp.c
 SRCS+= bn_shift.c
diff --git a/src/lib/libcrypto/bn/bn_local.h b/src/lib/libcrypto/bn/bn_local.h
index 17f5447bec..86aa972275 100644
--- a/src/lib/libcrypto/bn/bn_local.h
+++ b/src/lib/libcrypto/bn/bn_local.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_local.h,v 1.24 2023/06/24 16:01:43 jsing Exp $ */
+/* $OpenBSD: bn_local.h,v 1.25 2023/07/06 14:37:39 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -323,5 +323,9 @@ int bn_is_perfect_square(int *out_perfect, const BIGNUM *n, BN_CTX *ctx);
 
 int bn_is_prime_bpsw(int *is_prime, const BIGNUM *n, BN_CTX *ctx, size_t rounds);
 
+int bn_printf(BIO *bio, const BIGNUM *bn, int indent, const char *fmt, ...)
+    __attribute__((__format__ (printf, 4, 5)))
+    __attribute__((__nonnull__ (4)));
+
 __END_HIDDEN_DECLS
 #endif /* !HEADER_BN_LOCAL_H */
diff --git a/src/lib/libcrypto/bn/bn_print.c b/src/lib/libcrypto/bn/bn_print.c
new file mode 100644
index 0000000000..466aeb3d64
--- /dev/null
+++ b/src/lib/libcrypto/bn/bn_print.c
@@ -0,0 +1,147 @@
+/*      $OpenBSD: bn_print.c,v 1.40 2023/07/06 14:37:39 tb Exp $ */
+
+/*
+ * Copyright (c) 2023 Theo Buehler <tb@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <ctype.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <openssl/bio.h>
+#include <openssl/bn.h>
+
+#include "bytestring.h"
+
+static int
+bn_print_zero(BIO *bio, const BIGNUM *bn)
+{
+        if (!BN_is_zero(bn))
+                return 0;
+        if (BIO_printf(bio, " 0\n") <= 0)
+                return 0;
+        return 1;
+}
+
+static int
+bn_print_word(BIO *bio, const BIGNUM *bn)
+{
+        BN_ULONG word;
+        const char *neg = "";
+
+        if (BN_is_zero(bn) || BN_num_bytes(bn) > BN_BYTES)
+                return 0;
+
+        if (BN_is_negative(bn))
+                neg = "-";
+
+        word = BN_get_word(bn);
+        if (BIO_printf(bio, " %s%lu (%s0x%lx)\n", neg, word, neg, word) <= 0)
+                return 0;
+
+        return 1;
+}
+
+static int
+bn_print_bignum(BIO *bio, const BIGNUM *bn, int indent)
+{
+        CBS cbs;
+        char *hex = NULL;
+        size_t hex_len = 0;
+        size_t octets = 0;
+        uint8_t hi, lo;
+        const char *sep = ":";
+        int ret = 0;
+
+        if (BN_num_bytes(bn) <= BN_BYTES)
+                goto err;
+
+        /* Secondary indent is 4 spaces, capped at 128. */
+        if (indent > INT_MAX - 4)
+                goto err;
+        indent += 4;
+        if (indent > 128)
+                indent = 128;
+        if (indent < 0)
+                indent = 0;
+
+        if ((hex = BN_bn2hex(bn)) == NULL)
+                goto err;
+        hex_len = strlen(hex);
+
+        CBS_init(&cbs, hex, hex_len);
+
+        if (BN_is_negative(bn)) {
+                if (BIO_printf(bio, " (Negative)") <= 0)
+                        goto err;
+                if (!CBS_skip(&cbs, 1))
+                        goto err;
+        }
+
+        while (CBS_len(&cbs) > 0) {
+                if (!CBS_get_u8(&cbs, &hi))
+                        goto err;
+                if (!CBS_get_u8(&cbs, &lo))
+                        goto err;
+                if (octets++ % 15 == 0) {
+                        if (BIO_printf(bio, "\n%*s", indent, "") <= 0)
+                                goto err;
+                }
+                if (CBS_len(&cbs) == 0)
+                        sep = "";
+                if (BIO_printf(bio, "%c%c%s", tolower(hi), tolower(lo), sep) <= 0)
+                        goto err;
+        }
+
+        if (BIO_printf(bio, "\n") <= 0)
+                goto err;
+
+        ret = 1;
+
+ err:
+        freezero(hex, hex_len);
+
+        return ret;
+}
+
+int
+bn_printf(BIO *bio, const BIGNUM *bn, int indent, const char *fmt, ...)
+{
+        va_list ap;
+        int rv;
+
+        if (bn == NULL)
+                return 1;
+
+        if (!BIO_indent(bio, indent, 128))
+                return 0;
+
+        va_start(ap, fmt);
+        rv = BIO_vprintf(bio, fmt, ap);
+        va_end(ap);
+        if (rv < 0)
+                return 0;
+
+        if (BN_is_zero(bn))
+                return bn_print_zero(bio, bn);
+
+        if (BN_num_bytes(bn) <= BN_BYTES)
+                return bn_print_word(bio, bn);
+
+        return bn_print_bignum(bio, bn, indent);
+}