Avoid signed overflow in BN_MONT_CTX_set()

ri is an int, so the check relied on signed overflow (UB). It's not really
reachable, but shrug.

reported by smatch via jsg
ok beck jsing kenjiro

1 files changed, 3 insertions, 2 deletions

diff --git a/src/lib/libcrypto/bn/bn_mont.c b/src/lib/libcrypto/bn/bn_mont.c
index 950846fa5b..8280a8db27 100644
--- a/src/lib/libcrypto/bn/bn_mont.c
+++ b/src/lib/libcrypto/bn/bn_mont.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_mont.c,v 1.68 2025/05/25 05:12:05 jsing Exp $ */
+/* $OpenBSD: bn_mont.c,v 1.69 2025/08/03 10:33:46 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -116,6 +116,7 @@
  * sections 3.8 and 4.2 in http://security.ece.orst.edu/koc/papers/r01rsasw.pdf
  */
 
+#include <limits.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
@@ -214,7 +215,7 @@ BN_MONT_CTX_set(BN_MONT_CTX *mont, const BIGNUM *mod, BN_CTX *ctx)
                  goto err;
         mont->N.neg = 0;
         mont->ri = ((BN_num_bits(mod) + BN_BITS2 - 1) / BN_BITS2) * BN_BITS2;
-        if (mont->ri * 2 < mont->ri)
+        if (mont->ri > INT_MAX / 2)
                 goto err;
 
         /*