From 9a319239aa9791b8d59bd245ae1eb82cd3d46720 Mon Sep 17 00:00:00 2001 From: jsing <> Date: Tue, 7 Mar 2023 06:28:36 +0000 Subject: Limit bn_mul_mont() usage to sizes less than or equal to 8192 bits. The assembly bn_mul_mont() implementations effectively use alloca() to allocate space for computation (at up to 8x the input size), without any limitation. This means that sufficiently large inputs lead to the stack being blown. Prevent this by using the C based implementation instead. Thanks to Jiayi Lin for reporting this to us. ok beck@ tb@ --- src/lib/libcrypto/bn/bn_mont.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/lib/libcrypto') diff --git a/src/lib/libcrypto/bn/bn_mont.c b/src/lib/libcrypto/bn/bn_mont.c index e92ceae5f4..314d683782 100644 --- a/src/lib/libcrypto/bn/bn_mont.c +++ b/src/lib/libcrypto/bn/bn_mont.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bn_mont.c,v 1.50 2023/03/07 06:19:44 jsing Exp $ */ +/* $OpenBSD: bn_mont.c,v 1.51 2023/03/07 06:28:36 jsing Exp $ */ /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) * All rights reserved. * @@ -435,6 +435,14 @@ bn_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, if (mctx->N.top <= 1 || a->top != mctx->N.top || b->top != mctx->N.top) return bn_mod_mul_montgomery_simple(r, a, b, mctx, ctx); + /* + * Legacy bn_mul_mont() performs stack based allocation, without + * size limitation. Allowing a large size results in the stack + * being blown. + */ + if (mctx->N.top > (8 * 1024 / sizeof(BN_ULONG))) + return bn_montgomery_multiply(r, a, b, mctx, ctx); + if (!bn_wexpand(r, mctx->N.top)) return 0; -- cgit v1.2.3-55-g6feb