From 4d359baaefb1597abf730cf3a09574e1e03d620b Mon Sep 17 00:00:00 2001
From: tb <>
Date: Sun, 20 Jan 2019 01:56:59 +0000
Subject: Fix BN_is_prime_* calls in libcrypto, the API returns -1 on error.

From BoringSSL's commit 53409ee3d7595ed37da472bc73b010cd2c8a5ffd
by David Benjamin.

ok djm, jsing
---
 src/lib/libcrypto/bn/bn_x931p.c | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

(limited to 'src/lib/libcrypto/bn')

diff --git a/src/lib/libcrypto/bn/bn_x931p.c b/src/lib/libcrypto/bn/bn_x931p.c
index 45b61c9128..55ca21c08c 100644
--- a/src/lib/libcrypto/bn/bn_x931p.c
+++ b/src/lib/libcrypto/bn/bn_x931p.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_x931p.c,v 1.10 2017/01/25 06:15:44 beck Exp $ */
+/* $OpenBSD: bn_x931p.c,v 1.11 2019/01/20 01:56:59 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2005.
  */
@@ -71,7 +71,7 @@
 static int
 bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb)
 {
-	int i = 0;
+	int i = 0, is_prime;
 
 	if (!BN_copy(pi, Xpi))
 		return 0;
@@ -81,7 +81,10 @@ bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb)
 		i++;
 		BN_GENCB_call(cb, 0, i);
 		/* NB 27 MR is specificed in X9.31 */
-		if (BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb))
+		is_prime = BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb);
+		if (is_prime < 0)
+			return 0;
+		if (is_prime == 1)
 			break;
 		if (!BN_add_word(pi, 2))
 			return 0;
@@ -173,13 +176,20 @@ BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2, const BIGNUM *Xp,
 			goto err;
 		if (!BN_gcd_ct(t, pm1, e, ctx))
 			goto err;
-		if (BN_is_one(t)
-		/* X9.31 specifies 8 MR and 1 Lucas test or any prime test
-		 * offering similar or better guarantees 50 MR is considerably
-		 * better.
-		 */
-		    && BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb))
-			break;
+		if (BN_is_one(t)) {
+			int r;
+
+			/*
+			 * X9.31 specifies 8 MR and 1 Lucas test or any prime
+			 * test offering similar or better guarantees 50 MR
+			 * is considerably better.
+			 */
+			r = BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb);
+			if (r < 0)
+				goto err;
+			if (r == 1)
+				break;
+		}
 		if (!BN_add(p, p, p1p2))
 			goto err;
 	}
-- 
cgit v1.2.3-55-g6feb