From d5dab9ba75c9c829a61151d8cec3e00e899ad4b6 Mon Sep 17 00:00:00 2001
From: tb <>
Date: Sun, 20 Jan 2019 01:56:59 +0000
Subject: Fix BN_is_prime_* calls in libcrypto, the API returns -1 on error.

From BoringSSL's commit 53409ee3d7595ed37da472bc73b010cd2c8a5ffd
by David Benjamin.

ok djm, jsing
---
 src/lib/libcrypto/dh/dh_check.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

(limited to 'src/lib/libcrypto/dh/dh_check.c')

diff --git a/src/lib/libcrypto/dh/dh_check.c b/src/lib/libcrypto/dh/dh_check.c
index a6010f0a6d..a8227d31ca 100644
--- a/src/lib/libcrypto/dh/dh_check.c
+++ b/src/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.16 2016/07/05 02:54:35 bcook Exp $ */
+/* $OpenBSD: dh_check.c,v 1.17 2019/01/20 01:56:59 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -74,7 +74,7 @@
 int
 DH_check(const DH *dh, int *ret)
 {
-	int ok = 0;
+	int is_prime, ok = 0;
 	BN_CTX *ctx = NULL;
 	BN_ULONG l;
 	BIGNUM *q = NULL;
@@ -102,16 +102,23 @@ DH_check(const DH *dh, int *ret)
 	} else
 		*ret |= DH_UNABLE_TO_CHECK_GENERATOR;
 
-	if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL))
+	is_prime = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
+	if (is_prime < 0)
+		goto err;
+	if (is_prime == 0)
 		*ret |= DH_CHECK_P_NOT_PRIME;
 	else {
 		if (!BN_rshift1(q, dh->p))
 			goto err;
-		if (!BN_is_prime_ex(q, BN_prime_checks, ctx, NULL))
+		is_prime = BN_is_prime_ex(q, BN_prime_checks, ctx, NULL);
+		if (is_prime < 0)
+			goto err;
+		if (is_prime == 0)
 			*ret |= DH_CHECK_P_NOT_SAFE_PRIME;
 	}
 	ok = 1;
-err:
+
+ err:
 	BN_CTX_free(ctx);
 	BN_free(q);
 	return ok;
-- 
cgit v1.2.3-55-g6feb