From 76fac483ab33603d9382e483db0e7bc158578397 Mon Sep 17 00:00:00 2001
From: tb <>
Date: Tue, 13 Jun 2023 09:28:13 +0000
Subject: Disallow aliasing of return value and modulus

All the functions changed in this commit would silently misbehave if the
return value aliases the modulus, most of the time they would succeed and
return an incorrect result of 0 in that situation. This adjusts all the
functions in BN_mod.c, others and documentation will follow later.

Prompted by a bug report about BN_mod_inverse() by Guido Vranken.

ok jsing
---
 src/lib/libcrypto/bn/bn_mod.c | 45 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/lib/libcrypto/bn/bn_mod.c b/src/lib/libcrypto/bn/bn_mod.c
index 868ef5bc5b..79766d0036 100644
--- a/src/lib/libcrypto/bn/bn_mod.c
+++ b/src/lib/libcrypto/bn/bn_mod.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_mod.c,v 1.20 2023/03/27 10:21:23 tb Exp $ */
+/* $OpenBSD: bn_mod.c,v 1.21 2023/06/13 09:28:13 tb Exp $ */
 /* Includes code written by Lenka Fibikova <fibikova@exp-math.uni-essen.de>
  * for the OpenSSL project. */
 /* ====================================================================
@@ -136,6 +136,10 @@ BN_mod_nonct(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 int
 BN_nnmod(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_mod_ct(r, a, m, ctx))
 		return 0;
 	if (BN_is_negative(r))
@@ -147,6 +151,10 @@ int
 BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
     BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_add(r, a, b))
 		return 0;
 	return BN_nnmod(r, r, m, ctx);
@@ -159,6 +167,10 @@ BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 int
 BN_mod_add_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_uadd(r, a, b))
 		return 0;
 	if (BN_ucmp(r, m) >= 0)
@@ -170,6 +182,10 @@ int
 BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
     BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_sub(r, a, b))
 		return 0;
 	return BN_nnmod(r, r, m, ctx);
@@ -182,6 +198,10 @@ BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 int
 BN_mod_sub_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (BN_ucmp(a, b) >= 0)
 		return BN_usub(r, a, b);
 	if (!BN_usub(r, b, a))
@@ -198,6 +218,11 @@ BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 
 	BN_CTX_start(ctx);
 
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		goto err;
+	}
+
 	rr = r;
 	if (rr == a || rr == b)
 		rr = BN_CTX_get(ctx);
@@ -231,6 +256,10 @@ BN_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 int
 BN_mod_lshift1(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_lshift1(r, a))
 		return 0;
 	return BN_nnmod(r, r, m, ctx);
@@ -243,6 +272,10 @@ BN_mod_lshift1(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 int
 BN_mod_lshift1_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *m)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_lshift1(r, a))
 		return 0;
 	if (BN_ucmp(r, m) >= 0)
@@ -258,6 +291,11 @@ BN_mod_lshift(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m, BN_CTX *ctx)
 
 	BN_CTX_start(ctx);
 
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		goto err;
+	}
+
 	if (!BN_nnmod(r, a, m, ctx))
 		goto err;
 
@@ -288,6 +326,11 @@ BN_mod_lshift_quick(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m)
 {
 	int max_shift;
 
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
+
 	if (!bn_copy(r, a))
 		return 0;
 
-- 
cgit v1.2.3-55-g6feb