Inline use of bn_is_prime_bpsw()

Instead of using the BN_is_prime_fasttime_ex() API, use a direct call to
bn_is_prime_bpsw(). This increases readability and simplifies error
handling. Also put a division by two to the natural place now that we no
longer need to do Miller-Rabin rounds.

ok beck jsing

1 files changed, 20 insertions, 24 deletions

diff --git a/src/lib/libcrypto/bn/bn_prime.c b/src/lib/libcrypto/bn/bn_prime.c
index c3cf5b8986..f44fbdf79a 100644
--- a/src/lib/libcrypto/bn/bn_prime.c
+++ b/src/lib/libcrypto/bn/bn_prime.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_prime.c,v 1.24 2022/11/09 11:31:51 tb Exp $ */
+/* $OpenBSD: bn_prime.c,v 1.25 2022/11/09 15:33:13 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -160,11 +160,11 @@ int
 BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
     const BIGNUM *rem, BN_GENCB *cb)
 {
-        BIGNUM *t;
-        int found = 0;
-        int loops = 0;
-        int j;
         BN_CTX *ctx;
+        BIGNUM *p;
+        int is_prime;
+        int loops = 0;
+        int found = 0;
 
         if (bits < 2 || (bits == 2 && safe)) {
                 /*
@@ -178,11 +178,11 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
         if ((ctx = BN_CTX_new()) == NULL)
                 goto err;
         BN_CTX_start(ctx);
-        if ((t = BN_CTX_get(ctx)) == NULL)
+        if ((p = BN_CTX_get(ctx)) == NULL)
                 goto err;
 
  loop:
-        /* make a random number and set the top and bottom bits */
+        /* Make a random number and set the top and bottom bits. */
         if (add == NULL) {
                 if (!probable_prime(ret, bits))
                         goto err;
@@ -200,35 +200,31 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
                 goto err;
 
         if (!safe) {
-                j = BN_is_prime_fasttest_ex(ret, 1, ctx, 0, cb);
-                if (j == -1)
+                if (!bn_is_prime_bpsw(&is_prime, ret, ctx))
                         goto err;
-                if (j == 0)
+                if (!is_prime)
                         goto loop;
         } else {
-                /* for "safe prime" generation,
-                 * check that (p-1)/2 is prime.
-                 * Since a prime is odd, We just
-                 * need to divide by 2 */
-                if (!BN_rshift1(t, ret))
+                if (!bn_is_prime_bpsw(&is_prime, ret, ctx))
                         goto err;
+                if (!is_prime)
+                        goto loop;
 
-                j = BN_is_prime_fasttest_ex(ret, 1, ctx, 0, cb);
-                if (j == -1)
+                /*
+                 * For safe prime generation, check that p = (ret-1)/2 is prime.
+                 * Since this prime has >= 3 bits, it is odd, and we can simply
+                 * divide by 2.
+                 */
+                if (!BN_rshift1(p, ret))
                         goto err;
-                if (j == 0)
-                        goto loop;
 
-                j = BN_is_prime_fasttest_ex(t, 1, ctx, 0, cb);
-                if (j == -1)
+                if (!bn_is_prime_bpsw(&is_prime, p, ctx))
                         goto err;
-                if (j == 0)
+                if (!is_prime)
                         goto loop;
 
                 if (!BN_GENCB_call(cb, 2, loops - 1))
                         goto err;
-
-                /* We have a safe prime test pass */
         }
 
         found = 1;