Fix two incorrect strtonum() conversions

The atoi() would also accept the magic negative values and old openssl
releases would expose these as arguments to -pkeyopt rsa_pss_saltlen:-1
in the openssl pkeyutl "app". While modern openssl switched to having
readable alternatives to these, the oseid component of opensc would use
the old syntax until yesterday.

Still, this is our bug and we need to keep accepting the magic values as
such, so do so. Everything below -3 will be rejected by the RSA_ctrl()
handler later.

Debugged by Doug Engert in https://github.com/OpenSC/OpenSC/issues/3317

ok jsing op

1 files changed, 13 insertions, 3 deletions

diff --git a/src/lib/libcrypto/rsa/rsa_pmeth.c b/src/lib/libcrypto/rsa/rsa_pmeth.c
index a1bdeb3b36..b4e0448ef2 100644
--- a/src/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/src/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_pmeth.c,v 1.41 2024/08/26 22:01:28 op Exp $ */
+/* $OpenBSD: rsa_pmeth.c,v 1.42 2025/01/17 08:50:07 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -668,7 +668,12 @@ pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
                 else if (!strcmp(value, "auto"))
                         saltlen = RSA_PSS_SALTLEN_AUTO;
                 else {
-                        saltlen = strtonum(value, 0, INT_MAX, &errstr);
+                        /*
+                         * Accept the special values -1, -2, -3 since that's
+                         * what atoi() historically did. Lower values are later
+                         * rejected in EVP_PKEY_CTRL_RSA_PSS_SALTLEN anyway.
+                         */
+                        saltlen = strtonum(value, -3, INT_MAX, &errstr);
                         if (errstr != NULL) {
                                 RSAerror(RSA_R_INVALID_PSS_SALTLEN);
                                 return -2;
@@ -718,7 +723,12 @@ pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
                 if (strcmp(type, "rsa_pss_keygen_saltlen") == 0) {
                         int saltlen;
 
-                        saltlen = strtonum(value, 0, INT_MAX, &errstr);
+                        /*
+                         * Accept the special values -1, -2, -3 since that's
+                         * what atoi() historically did. Lower values are later
+                         * rejected in EVP_PKEY_CTRL_RSA_PSS_SALTLEN anyway.
+                         */
+                        saltlen = strtonum(value, -3, INT_MAX, &errstr);
                         if (errstr != NULL) {
                                 RSAerror(RSA_R_INVALID_PSS_SALTLEN);
                                 return -2;