Stop supporting the long-retired X9.31 standard

This isolates the three API functions from the library so they can be
easily removed and any attempt to use RSA_X931_PADDING mode will now
result in an error.

ok jsing

1 files changed, 11 insertions, 19 deletions

diff --git a/src/lib/libcrypto/rsa/rsa_eay.c b/src/lib/libcrypto/rsa/rsa_eay.c
index b307a8bd88..e65319bda1 100644
--- a/src/lib/libcrypto/rsa/rsa_eay.c
+++ b/src/lib/libcrypto/rsa/rsa_eay.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_eay.c,v 1.58 2023/04/05 11:31:38 tb Exp $ */
+/* $OpenBSD: rsa_eay.c,v 1.59 2023/04/15 18:48:52 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -382,14 +382,11 @@ RSA_eay_private_encrypt(int flen, const unsigned char *from, unsigned char *to,
         case RSA_PKCS1_PADDING:
                 i = RSA_padding_add_PKCS1_type_1(buf, num, from, flen);
                 break;
-        case RSA_X931_PADDING:
-                i = RSA_padding_add_X931(buf, num, from, flen);
-                break;
         case RSA_NO_PADDING:
                 i = RSA_padding_add_none(buf, num, from, flen);
                 break;
         default:
-                RSAerror(RSA_R_UNKNOWN_PADDING_TYPE);
+                RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
                 goto err;
         }
         if (i <= 0)
@@ -449,14 +446,11 @@ RSA_eay_private_encrypt(int flen, const unsigned char *from, unsigned char *to,
                         goto err;
 
         if (padding == RSA_X931_PADDING) {
-                if (!BN_sub(f, rsa->n, ret))
-                        goto err;
-                if (BN_cmp(ret, f) > 0)
-                        res = f;
-                else
-                        res = ret;
-        } else
-                res = ret;
+                RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
+                goto err;
+        }
+
+        res = ret;
 
         /* put in leading 0 bytes if the number is less than the
          * length of the modulus */
@@ -667,9 +661,10 @@ RSA_eay_public_decrypt(int flen, const unsigned char *from, unsigned char *to,
             rsa->_method_mod_n))
                 goto err;
 
-        if (padding == RSA_X931_PADDING && (ret->d[0] & 0xf) != 12)
-                if (!BN_sub(ret, rsa->n, ret))
-                        goto err;
+        if (padding == RSA_X931_PADDING) {
+                RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
+                goto err;
+        }
 
         p = buf;
         i = BN_bn2bin(ret, p);
@@ -678,9 +673,6 @@ RSA_eay_public_decrypt(int flen, const unsigned char *from, unsigned char *to,
         case RSA_PKCS1_PADDING:
                 r = RSA_padding_check_PKCS1_type_1(to, num, buf, i, num);
                 break;
-        case RSA_X931_PADDING:
-                r = RSA_padding_check_X931(to, num, buf, i, num);
-                break;
         case RSA_NO_PADDING:
                 r = RSA_padding_check_none(to, num, buf, i, num);
                 break;