From aeaaf636a6726b17d0d27ea128a573bc90c4d04f Mon Sep 17 00:00:00 2001 From: tb <> Date: Sun, 5 Jan 2025 15:39:12 +0000 Subject: Stop requiring the RSA_FLAG_SIGN_VER You can set custom sign and verify handlers on an RSA method (wihch is used to create RSA private and public key handles). However, even if you set them explicitly with RSA_meth_set_{sign,verify}(3), these handlers aren't used for the sake of "backward compatibility" (with what?). In order to use them, you need to opt your objects into using the custom methods you set by setting the RSA_FLAG_SIGN_VER flag. OpenSSL 1.1 dropped this requirement and therefore nobody sets this flag anyore. Like most of the mechanically added accessors, almost nothing uses them, but, as found by kn, the yubco-piv-tool does. This resulted in a public key being passed to rsa_private_encrypt(), which of course doesn't end well. So follow OpenSSL 1.1 and drop this muppetry. This makes kn's problem with yubico-piv-tool go away. ok jsing kn --- src/lib/libcrypto/rsa/rsa_sign.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib/libcrypto/rsa/rsa_sign.c') diff --git a/src/lib/libcrypto/rsa/rsa_sign.c b/src/lib/libcrypto/rsa/rsa_sign.c index 5356768615..6edd20626d 100644 --- a/src/lib/libcrypto/rsa/rsa_sign.c +++ b/src/lib/libcrypto/rsa/rsa_sign.c @@ -1,4 +1,4 @@ -/* $OpenBSD: rsa_sign.c,v 1.36 2023/07/08 12:26:45 beck Exp $ */ +/* $OpenBSD: rsa_sign.c,v 1.37 2025/01/05 15:39:12 tb Exp $ */ /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) * All rights reserved. * @@ -130,7 +130,7 @@ RSA_sign(int type, const unsigned char *m, unsigned int m_len, unsigned char *tmps = NULL; int encrypt_len, encoded_len = 0, ret = 0; - if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign != NULL) + if (rsa->meth->rsa_sign != NULL) return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa); /* Compute the encoded digest. */ @@ -271,7 +271,7 @@ int RSA_verify(int dtype, const unsigned char *m, unsigned int m_len, const unsigned char *sigbuf, unsigned int siglen, RSA *rsa) { - if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_verify) + if (rsa->meth->rsa_verify != NULL) return rsa->meth->rsa_verify(dtype, m, m_len, sigbuf, siglen, rsa); -- cgit v1.2.3-55-g6feb