Fix BN_is_prime_* calls in libcrypto, the API returns -1 on error.

From BoringSSL's commit 53409ee3d7595ed37da472bc73b010cd2c8a5ffd
by David Benjamin.

ok djm, jsing

3 files changed, 35 insertions, 18 deletions

diff --git a/src/lib/libcrypto/bn/bn_x931p.c b/src/lib/libcrypto/bn/bn_x931p.c
index 45b61c9128..55ca21c08c 100644
--- a/src/lib/libcrypto/bn/bn_x931p.c
+++ b/src/lib/libcrypto/bn/bn_x931p.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_x931p.c,v 1.10 2017/01/25 06:15:44 beck Exp $ */
+/* $OpenBSD: bn_x931p.c,v 1.11 2019/01/20 01:56:59 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2005.
  */
@@ -71,7 +71,7 @@
 static int
 bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb)
 {
-        int i = 0;
+        int i = 0, is_prime;
 
         if (!BN_copy(pi, Xpi))
                 return 0;
@@ -81,7 +81,10 @@ bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb)
                 i++;
                 BN_GENCB_call(cb, 0, i);
                 /* NB 27 MR is specificed in X9.31 */
-                if (BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb))
+                is_prime = BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb);
+                if (is_prime < 0)
+                        return 0;
+                if (is_prime == 1)
                         break;
                 if (!BN_add_word(pi, 2))
                         return 0;
@@ -173,13 +176,20 @@ BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2, const BIGNUM *Xp,
                         goto err;
                 if (!BN_gcd_ct(t, pm1, e, ctx))
                         goto err;
-                if (BN_is_one(t)
-                /* X9.31 specifies 8 MR and 1 Lucas test or any prime test
-                 * offering similar or better guarantees 50 MR is considerably
-                 * better.
-                 */
-                    && BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb))
-                        break;
+                if (BN_is_one(t)) {
+                        int r;
+
+                        /*
+                         * X9.31 specifies 8 MR and 1 Lucas test or any prime
+                         * test offering similar or better guarantees 50 MR
+                         * is considerably better.
+                         */
+                        r = BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb);
+                        if (r < 0)
+                                goto err;
+                        if (r == 1)
+                                break;
+                }
                 if (!BN_add(p, p, p1p2))
                         goto err;
         }
diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index a6010f0a6d..a8227d31ca 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.16 2016/07/05 02:54:35 bcook Exp $ */
+/* $OpenBSD: dh_check.c,v 1.17 2019/01/20 01:56:59 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -74,7 +74,7 @@
 int
 DH_check(const DH *dh, int *ret)
 {
-        int ok = 0;
+        int is_prime, ok = 0;
         BN_CTX *ctx = NULL;
         BN_ULONG l;
         BIGNUM *q = NULL;
@@ -102,16 +102,23 @@ DH_check(const DH *dh, int *ret)
         } else
                 *ret |= DH_UNABLE_TO_CHECK_GENERATOR;
 
-        if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL))
+        is_prime = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
+        if (is_prime < 0)
+                goto err;
+        if (is_prime == 0)
                 *ret |= DH_CHECK_P_NOT_PRIME;
         else {
                 if (!BN_rshift1(q, dh->p))
                         goto err;
-                if (!BN_is_prime_ex(q, BN_prime_checks, ctx, NULL))
+                is_prime = BN_is_prime_ex(q, BN_prime_checks, ctx, NULL);
+                if (is_prime < 0)
+                        goto err;
+                if (is_prime == 0)
                         *ret |= DH_CHECK_P_NOT_SAFE_PRIME;
         }
         ok = 1;
-err:
+
+ err:
         BN_CTX_free(ctx);
         BN_free(q);
         return ok;
diff --git a/src/lib/libcrypto/dsa/dsa_ameth.c b/src/lib/libcrypto/dsa/dsa_ameth.c
index 26d81eed7b..85ef234bb9 100644
--- a/src/lib/libcrypto/dsa/dsa_ameth.c
+++ b/src/lib/libcrypto/dsa/dsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_ameth.c,v 1.26 2018/08/24 20:22:15 tb Exp $ */
+/* $OpenBSD: dsa_ameth.c,v 1.27 2019/01/20 01:56:59 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -515,7 +515,7 @@ old_dsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
          * Check that q is not a composite number.
          */
 
-        if (BN_is_prime_ex(dsa->q, BN_prime_checks, ctx, NULL) == 0) {
+        if (BN_is_prime_ex(dsa->q, BN_prime_checks, ctx, NULL) <= 0) {
                 DSAerror(DSA_R_BAD_Q_VALUE);
                 goto err;
         }
@@ -525,7 +525,7 @@ old_dsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
         EVP_PKEY_assign_DSA(pkey, dsa);
         return 1;
 
-err:
+ err:
         BN_CTX_free(ctx);
         DSA_free(dsa);
         return 0;