From 1a1a85dd49b0d95ad37da975c7f03d879de1a8f1 Mon Sep 17 00:00:00 2001
From: jsing <>
Date: Tue, 7 Mar 2023 06:28:36 +0000
Subject: Limit bn_mul_mont() usage to sizes less than or equal to 8192 bits.

The assembly bn_mul_mont() implementations effectively use alloca() to
allocate space for computation (at up to 8x the input size), without
any limitation. This means that sufficiently large inputs lead to the
stack being blown. Prevent this by using the C based implementation
instead.

Thanks to Jiayi Lin <jlin139 at asu dot edu> for reporting this to us.

ok beck@ tb@
---
 src/lib/libcrypto/bn/bn_mont.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/lib/libcrypto/bn/bn_mont.c b/src/lib/libcrypto/bn/bn_mont.c
index e92ceae5f4..314d683782 100644
--- a/src/lib/libcrypto/bn/bn_mont.c
+++ b/src/lib/libcrypto/bn/bn_mont.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_mont.c,v 1.50 2023/03/07 06:19:44 jsing Exp $ */
+/* $OpenBSD: bn_mont.c,v 1.51 2023/03/07 06:28:36 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -435,6 +435,14 @@ bn_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
 	if (mctx->N.top <= 1 || a->top != mctx->N.top || b->top != mctx->N.top)
 		return bn_mod_mul_montgomery_simple(r, a, b, mctx, ctx);
 
+	/*
+	 * Legacy bn_mul_mont() performs stack based allocation, without
+	 * size limitation. Allowing a large size results in the stack
+	 * being blown.
+	 */
+	if (mctx->N.top > (8 * 1024 / sizeof(BN_ULONG)))
+		return bn_montgomery_multiply(r, a, b, mctx, ctx);
+
 	if (!bn_wexpand(r, mctx->N.top))
 		return 0;
 
-- 
cgit v1.2.3-55-g6feb