Use ASN1_INTEGER to encode/decode BIGNUM_it.

The current code simply shoves the unvalidated ASN.1 bytes into a BIGNUM on
the hope that other things will detect issues (such as negative values
being flipped to positive). Instead of doing this, decode and validate the
ASN.1 data using ASN1_INTEGER, then convert it to a BIGNUM. Similarly, for
encoding convert from BIGNUM to ASN1_INTEGER and use ASN1_INTEGER encoding.

ok tb@

1 files changed, 48 insertions, 34 deletions

diff --git a/src/lib/libcrypto/asn1/x_bignum.c b/src/lib/libcrypto/asn1/x_bignum.c
index 9ad7810306..d1f735b47d 100644
--- a/src/lib/libcrypto/asn1/x_bignum.c
+++ b/src/lib/libcrypto/asn1/x_bignum.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: x_bignum.c,v 1.11 2022/07/30 13:37:17 jsing Exp $ */
+/* $OpenBSD: x_bignum.c,v 1.12 2022/07/30 13:42:25 jsing Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -61,11 +61,11 @@
 #include <openssl/asn1t.h>
 #include <openssl/bn.h>
 
+#include "asn1_locl.h"
+#include "bytestring.h"
+
 /*
- * Custom primitive type for BIGNUM handling. This reads in an ASN1_INTEGER as a
- * BIGNUM directly. Currently it ignores the sign which isn't a problem since
- * all BIGNUMs used are non negative and anything that looks negative is
- * normally due to an encoding error.
+ * Custom primitive type for that reads an ASN.1 INTEGER into a BIGNUM.
  */
 
 static int bn_new(ASN1_VALUE **pval, const ASN1_ITEM *it);
@@ -136,51 +136,65 @@ bn_free(ASN1_VALUE **pval, const ASN1_ITEM *it)
 }
 
 static int
-bn_i2c(ASN1_VALUE **pval, unsigned char *cont, int *putype, const ASN1_ITEM *it)
+bn_i2c(ASN1_VALUE **pval, unsigned char *content, int *putype, const ASN1_ITEM *it)
 {
-        BIGNUM *bn;
-        int pad, len;
+        ASN1_INTEGER *aint = NULL;
+        unsigned char **pp = NULL;
+        const BIGNUM *bn;
+        int ret;
 
         if (*pval == NULL)
                 return -1;
-        bn = (BIGNUM *)*pval;
-        /* If MSB set in an octet we need a padding byte */
-        if (BN_num_bits(bn) & 0x7)
-                pad = 0;
-        else
-                pad = 1;
-        if (cont) {
-                if (pad)
-                        *cont++ = 0;
-                len = BN_bn2bin(bn, cont);
-        } else
-                len = BN_num_bytes(bn);
-        return pad + len;
+
+        bn = (const BIGNUM *)*pval;
+
+        if ((aint = BN_to_ASN1_INTEGER(bn, NULL)) == NULL)
+                return -1;
+
+        if (content != NULL)
+                pp = &content;
+
+        ret = i2c_ASN1_INTEGER(aint, pp);
+
+        ASN1_INTEGER_free(aint);
+
+        return ret;
 }
 
 static int
-bn_c2i(ASN1_VALUE **pval, const unsigned char *cont, int len, int utype,
-    char *free_cont, const ASN1_ITEM *it)
+bn_c2i(ASN1_VALUE **pval, const unsigned char *content, int len, int utype,
+    char *free_content, const ASN1_ITEM *it)
 {
+        ASN1_INTEGER *aint = NULL;
         BIGNUM *bn;
+        CBS cbs;
+        int ret = 0;
 
-        if (*pval == NULL) {
-                if (bn_new(pval, it) == 0)
-                        return 0;
-        }
-        bn = (BIGNUM *)*pval;
-        if (!BN_bin2bn(cont, len, bn)) {
-                bn_free(pval, it);
-                return 0;
-        }
-        return 1;
+        bn_clear(pval, it);
+
+        if (len < 0)
+                goto err;
+        CBS_init(&cbs, content, len);
+        if (!c2i_ASN1_INTEGER_cbs(&aint, &cbs))
+                goto err;
+
+        if ((bn = ASN1_INTEGER_to_BN(aint, NULL)) == NULL)
+                goto err;
+        *pval = (ASN1_VALUE *)bn;
+
+        ret = 1;
+
+ err:
+        ASN1_INTEGER_free(aint);
+
+        return ret;
 }
 
 static int
 bn_print(BIO *out, ASN1_VALUE **pval, const ASN1_ITEM *it, int indent,
     const ASN1_PCTX *pctx)
 {
-        BIGNUM *bn = (BIGNUM *)*pval;
+        const BIGNUM *bn = (BIGNUM *)*pval;
 
         if (!BN_print(out, bn))
                 return 0;