libcrypto: rsa gen: min. distance between p and q

This is required in NIST Special Publication 800-56B Revision 2
"Recommendation for Pair-Wise Key Establishment Using Integer
Factorization Cryptography":

 6 RSA Key Pairs
 6.2 Criteria for RSA Key Pairs for Key Establishment
 6.2.1 Definition of a Key Pair

	3. The prime factors p and q shall be generated using one of
	   the methods specified in Appendix B.3 of FIPS 186 such that:

		c. |p – q| > 2nBits/2−100

ok djm@, tb@

1 files changed, 19 insertions, 3 deletions

diff --git a/src/lib/libcrypto/rsa/rsa_gen.c b/src/lib/libcrypto/rsa/rsa_gen.c
index ebd0aeffd5..6a8bd08160 100644
--- a/src/lib/libcrypto/rsa/rsa_gen.c
+++ b/src/lib/libcrypto/rsa/rsa_gen.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_gen.c,v 1.31 2025/05/10 05:54:38 tb Exp $ */
+/* $OpenBSD: rsa_gen.c,v 1.32 2025/09/29 08:46:15 jan Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -84,6 +84,7 @@ rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
         BIGNUM pr0, d, p;
         int bitsp, bitsq, ok = -1, n = 0;
         BN_CTX *ctx = NULL;
+        BIGNUM *diff, *mindiff;
 
         ctx = BN_CTX_new();
         if (ctx == NULL)
@@ -97,10 +98,24 @@ rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
                 goto err;
         if ((r3 = BN_CTX_get(ctx)) == NULL)
                 goto err;
+        if ((diff = BN_CTX_get(ctx)) == NULL)
+                goto err;
+        if ((mindiff = BN_CTX_get(ctx)) == NULL)
+                goto err;
 
         bitsp = (bits + 1) / 2;
         bitsq = bits - bitsp;
 
+        /*
+         * To guarantee a minimum distance of 2^(bits/2 - 100) between p and q.
+         *
+         * NIST SP 800-56B, section 6.2.1, 3.c
+         */
+        if (bits < 200)
+                goto err;
+        if (!BN_set_bit(mindiff, bits/2 - 100))
+                goto err;
+
         /* We need the RSA components non-NULL */
         if (!rsa->n && ((rsa->n = BN_new()) == NULL))
                 goto err;
@@ -148,8 +163,9 @@ rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
                         if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL,
                             cb))
                                 goto err;
-                } while (BN_cmp(rsa->p, rsa->q) == 0 &&
-                    ++degenerate < 3);
+                        if (!BN_sub(diff, rsa->p, rsa->q))
+                                goto err;
+                } while (BN_ucmp(diff, mindiff) <= 0 && ++degenerate < 3);
                 if (degenerate == 3) {
                         ok = 0; /* we set our own err */
                         RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);