summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/rsa/rsa_pmeth.c
diff options
context:
space:
mode:
authormiod <>2014-07-09 08:20:08 +0000
committermiod <>2014-07-09 08:20:08 +0000
commit8cbe58f0d357b14b0ce292d336469d0554a567bc (patch)
tree07872a7ef59da8cea3b3b4a101fa3580e4d658c0 /src/lib/libcrypto/rsa/rsa_pmeth.c
parentbc1209e388500a20f5e75cab35d1b543ce0bbe74 (diff)
downloadopenbsd-8cbe58f0d357b14b0ce292d336469d0554a567bc.tar.gz
openbsd-8cbe58f0d357b14b0ce292d336469d0554a567bc.tar.bz2
openbsd-8cbe58f0d357b14b0ce292d336469d0554a567bc.zip
KNF
Diffstat (limited to 'src/lib/libcrypto/rsa/rsa_pmeth.c')
-rw-r--r--src/lib/libcrypto/rsa/rsa_pmeth.c426
1 files changed, 197 insertions, 229 deletions
diff --git a/src/lib/libcrypto/rsa/rsa_pmeth.c b/src/lib/libcrypto/rsa/rsa_pmeth.c
index 1f9d826014..4a662c2894 100644
--- a/src/lib/libcrypto/rsa/rsa_pmeth.c
+++ b/src/lib/libcrypto/rsa/rsa_pmeth.c
@@ -1,4 +1,4 @@
1/* $OpenBSD: rsa_pmeth.c,v 1.8 2014/06/12 20:40:57 deraadt Exp $ */ 1/* $OpenBSD: rsa_pmeth.c,v 1.9 2014/07/09 08:20:08 miod Exp $ */
2/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL 2/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3 * project 2006. 3 * project 2006.
4 */ 4 */
@@ -72,8 +72,7 @@
72 72
73/* RSA pkey context structure */ 73/* RSA pkey context structure */
74 74
75typedef struct 75typedef struct {
76 {
77 /* Key gen parameters */ 76 /* Key gen parameters */
78 int nbits; 77 int nbits;
79 BIGNUM *pub_exp; 78 BIGNUM *pub_exp;
@@ -89,11 +88,13 @@ typedef struct
89 int saltlen; 88 int saltlen;
90 /* Temp buffer */ 89 /* Temp buffer */
91 unsigned char *tbuf; 90 unsigned char *tbuf;
92 } RSA_PKEY_CTX; 91} RSA_PKEY_CTX;
93 92
94static int pkey_rsa_init(EVP_PKEY_CTX *ctx) 93static int
95 { 94pkey_rsa_init(EVP_PKEY_CTX *ctx)
95{
96 RSA_PKEY_CTX *rctx; 96 RSA_PKEY_CTX *rctx;
97
97 rctx = malloc(sizeof(RSA_PKEY_CTX)); 98 rctx = malloc(sizeof(RSA_PKEY_CTX));
98 if (!rctx) 99 if (!rctx)
99 return 0; 100 return 0;
@@ -111,424 +112,390 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
111 ctx->keygen_info_count = 2; 112 ctx->keygen_info_count = 2;
112 113
113 return 1; 114 return 1;
114 } 115}
115 116
116static int pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src) 117static int
117 { 118pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
119{
118 RSA_PKEY_CTX *dctx, *sctx; 120 RSA_PKEY_CTX *dctx, *sctx;
121
119 if (!pkey_rsa_init(dst)) 122 if (!pkey_rsa_init(dst))
120 return 0; 123 return 0;
121 sctx = src->data; 124 sctx = src->data;
122 dctx = dst->data; 125 dctx = dst->data;
123 dctx->nbits = sctx->nbits; 126 dctx->nbits = sctx->nbits;
124 if (sctx->pub_exp) 127 if (sctx->pub_exp) {
125 {
126 dctx->pub_exp = BN_dup(sctx->pub_exp); 128 dctx->pub_exp = BN_dup(sctx->pub_exp);
127 if (!dctx->pub_exp) 129 if (!dctx->pub_exp)
128 return 0; 130 return 0;
129 } 131 }
130 dctx->pad_mode = sctx->pad_mode; 132 dctx->pad_mode = sctx->pad_mode;
131 dctx->md = sctx->md; 133 dctx->md = sctx->md;
132 return 1; 134 return 1;
133 } 135}
134 136
135static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk) 137static int
136 { 138setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
139{
137 if (ctx->tbuf) 140 if (ctx->tbuf)
138 return 1; 141 return 1;
139 ctx->tbuf = malloc(EVP_PKEY_size(pk->pkey)); 142 ctx->tbuf = malloc(EVP_PKEY_size(pk->pkey));
140 if (!ctx->tbuf) 143 if (!ctx->tbuf)
141 return 0; 144 return 0;
142 return 1; 145 return 1;
143 } 146}
144 147
145static void pkey_rsa_cleanup(EVP_PKEY_CTX *ctx) 148static void
146 { 149pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
150{
147 RSA_PKEY_CTX *rctx = ctx->data; 151 RSA_PKEY_CTX *rctx = ctx->data;
148 if (rctx) 152
149 { 153 if (rctx) {
150 if (rctx->pub_exp) 154 if (rctx->pub_exp)
151 BN_free(rctx->pub_exp); 155 BN_free(rctx->pub_exp);
152 free(rctx->tbuf); 156 free(rctx->tbuf);
153 free(rctx); 157 free(rctx);
154 }
155 } 158 }
156static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen, 159}
157 const unsigned char *tbs, size_t tbslen) 160
158 { 161static int
162pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
163 const unsigned char *tbs, size_t tbslen)
164{
159 int ret; 165 int ret;
160 RSA_PKEY_CTX *rctx = ctx->data; 166 RSA_PKEY_CTX *rctx = ctx->data;
161 RSA *rsa = ctx->pkey->pkey.rsa; 167 RSA *rsa = ctx->pkey->pkey.rsa;
162 168
163 if (rctx->md) 169 if (rctx->md) {
164 { 170 if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
165 if (tbslen != (size_t)EVP_MD_size(rctx->md))
166 {
167 RSAerr(RSA_F_PKEY_RSA_SIGN, 171 RSAerr(RSA_F_PKEY_RSA_SIGN,
168 RSA_R_INVALID_DIGEST_LENGTH); 172 RSA_R_INVALID_DIGEST_LENGTH);
169 return -1; 173 return -1;
170 } 174 }
171 175
172 if (EVP_MD_type(rctx->md) == NID_mdc2) 176 if (EVP_MD_type(rctx->md) == NID_mdc2) {
173 {
174 unsigned int sltmp; 177 unsigned int sltmp;
178
175 if (rctx->pad_mode != RSA_PKCS1_PADDING) 179 if (rctx->pad_mode != RSA_PKCS1_PADDING)
176 return -1; 180 return -1;
177 ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2, 181 ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2, tbs, tbslen,
178 tbs, tbslen, sig, &sltmp, rsa); 182 sig, &sltmp, rsa);
179 183
180 if (ret <= 0) 184 if (ret <= 0)
181 return ret; 185 return ret;
182 ret = sltmp; 186 ret = sltmp;
183 } 187 } else if (rctx->pad_mode == RSA_X931_PADDING) {
184 else if (rctx->pad_mode == RSA_X931_PADDING)
185 {
186 if (!setup_tbuf(rctx, ctx)) 188 if (!setup_tbuf(rctx, ctx))
187 return -1; 189 return -1;
188 memcpy(rctx->tbuf, tbs, tbslen); 190 memcpy(rctx->tbuf, tbs, tbslen);
189 rctx->tbuf[tbslen] = 191 rctx->tbuf[tbslen] =
190 RSA_X931_hash_id(EVP_MD_type(rctx->md)); 192 RSA_X931_hash_id(EVP_MD_type(rctx->md));
191 ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, 193 ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
192 sig, rsa, RSA_X931_PADDING); 194 rsa, RSA_X931_PADDING);
193 } 195 } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
194 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
195 {
196 unsigned int sltmp; 196 unsigned int sltmp;
197 ret = RSA_sign(EVP_MD_type(rctx->md), 197
198 tbs, tbslen, sig, &sltmp, rsa); 198 ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
199 &sltmp, rsa);
199 if (ret <= 0) 200 if (ret <= 0)
200 return ret; 201 return ret;
201 ret = sltmp; 202 ret = sltmp;
202 } 203 } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
203 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
204 {
205 if (!setup_tbuf(rctx, ctx)) 204 if (!setup_tbuf(rctx, ctx))
206 return -1; 205 return -1;
207 if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, 206 if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
208 rctx->tbuf, tbs, 207 tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
209 rctx->md, rctx->mgf1md,
210 rctx->saltlen))
211 return -1; 208 return -1;
212 ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf, 209 ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
213 sig, rsa, RSA_NO_PADDING); 210 sig, rsa, RSA_NO_PADDING);
214 } 211 } else
215 else
216 return -1; 212 return -1;
217 } 213 } else
218 else
219 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa, 214 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
220 rctx->pad_mode); 215 rctx->pad_mode);
221 if (ret < 0) 216 if (ret < 0)
222 return ret; 217 return ret;
223 *siglen = ret; 218 *siglen = ret;
224 return 1; 219 return 1;
225 } 220}
226
227 221
228static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, 222static int
229 unsigned char *rout, size_t *routlen, 223pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, unsigned char *rout, size_t *routlen,
230 const unsigned char *sig, size_t siglen) 224 const unsigned char *sig, size_t siglen)
231 { 225{
232 int ret; 226 int ret;
233 RSA_PKEY_CTX *rctx = ctx->data; 227 RSA_PKEY_CTX *rctx = ctx->data;
234 228
235 if (rctx->md) 229 if (rctx->md) {
236 { 230 if (rctx->pad_mode == RSA_X931_PADDING) {
237 if (rctx->pad_mode == RSA_X931_PADDING)
238 {
239 if (!setup_tbuf(rctx, ctx)) 231 if (!setup_tbuf(rctx, ctx))
240 return -1; 232 return -1;
241 ret = RSA_public_decrypt(siglen, sig, 233 ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
242 rctx->tbuf, ctx->pkey->pkey.rsa, 234 ctx->pkey->pkey.rsa, RSA_X931_PADDING);
243 RSA_X931_PADDING);
244 if (ret < 1) 235 if (ret < 1)
245 return 0; 236 return 0;
246 ret--; 237 ret--;
247 if (rctx->tbuf[ret] != 238 if (rctx->tbuf[ret] !=
248 RSA_X931_hash_id(EVP_MD_type(rctx->md))) 239 RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
249 {
250 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER, 240 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
251 RSA_R_ALGORITHM_MISMATCH); 241 RSA_R_ALGORITHM_MISMATCH);
252 return 0; 242 return 0;
253 } 243 }
254 if (ret != EVP_MD_size(rctx->md)) 244 if (ret != EVP_MD_size(rctx->md)) {
255 {
256 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER, 245 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
257 RSA_R_INVALID_DIGEST_LENGTH); 246 RSA_R_INVALID_DIGEST_LENGTH);
258 return 0; 247 return 0;
259 } 248 }
260 if (rout) 249 if (rout)
261 memcpy(rout, rctx->tbuf, ret); 250 memcpy(rout, rctx->tbuf, ret);
262 } 251 } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
263 else if (rctx->pad_mode == RSA_PKCS1_PADDING)
264 {
265 size_t sltmp; 252 size_t sltmp;
266 ret = int_rsa_verify(EVP_MD_type(rctx->md), 253
267 NULL, 0, rout, &sltmp, 254 ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
268 sig, siglen, ctx->pkey->pkey.rsa); 255 rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
269 if (ret <= 0) 256 if (ret <= 0)
270 return 0; 257 return 0;
271 ret = sltmp; 258 ret = sltmp;
272 } 259 } else
273 else
274 return -1; 260 return -1;
275 } 261 } else
276 else
277 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa, 262 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
278 rctx->pad_mode); 263 rctx->pad_mode);
279 if (ret < 0) 264 if (ret < 0)
280 return ret; 265 return ret;
281 *routlen = ret; 266 *routlen = ret;
282 return 1; 267 return 1;
283 } 268}
284 269
285static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, 270static int
286 const unsigned char *sig, size_t siglen, 271pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
287 const unsigned char *tbs, size_t tbslen) 272 const unsigned char *tbs, size_t tbslen)
288 { 273{
289 RSA_PKEY_CTX *rctx = ctx->data; 274 RSA_PKEY_CTX *rctx = ctx->data;
290 RSA *rsa = ctx->pkey->pkey.rsa; 275 RSA *rsa = ctx->pkey->pkey.rsa;
291 size_t rslen; 276 size_t rslen;
292 if (rctx->md) 277
293 { 278 if (rctx->md) {
294 if (rctx->pad_mode == RSA_PKCS1_PADDING) 279 if (rctx->pad_mode == RSA_PKCS1_PADDING)
295 return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, 280 return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
296 sig, siglen, rsa); 281 sig, siglen, rsa);
297 if (rctx->pad_mode == RSA_X931_PADDING) 282 if (rctx->pad_mode == RSA_X931_PADDING) {
298 { 283 if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig,
299 if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, 284 siglen) <= 0)
300 sig, siglen) <= 0)
301 return 0; 285 return 0;
302 } 286 } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
303 else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING)
304 {
305 int ret; 287 int ret;
288
306 if (!setup_tbuf(rctx, ctx)) 289 if (!setup_tbuf(rctx, ctx))
307 return -1; 290 return -1;
308 ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, 291 ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
309 rsa, RSA_NO_PADDING); 292 rsa, RSA_NO_PADDING);
310 if (ret <= 0) 293 if (ret <= 0)
311 return 0; 294 return 0;
312 ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, 295 ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
313 rctx->md, rctx->mgf1md, 296 rctx->mgf1md, rctx->tbuf, rctx->saltlen);
314 rctx->tbuf, rctx->saltlen);
315 if (ret <= 0) 297 if (ret <= 0)
316 return 0; 298 return 0;
317 return 1; 299 return 1;
318 } 300 } else
319 else
320 return -1; 301 return -1;
321 } 302 } else {
322 else
323 {
324 if (!setup_tbuf(rctx, ctx)) 303 if (!setup_tbuf(rctx, ctx))
325 return -1; 304 return -1;
326 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, 305 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
327 rsa, rctx->pad_mode); 306 rctx->pad_mode);
328 if (rslen == 0) 307 if (rslen == 0)
329 return 0; 308 return 0;
330 } 309 }
331 310
332 if ((rslen != tbslen) || memcmp(tbs, rctx->tbuf, rslen)) 311 if (rslen != tbslen || memcmp(tbs, rctx->tbuf, rslen))
333 return 0; 312 return 0;
334 313
335 return 1; 314 return 1;
336 315}
337 }
338
339 316
340static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, 317static int
341 unsigned char *out, size_t *outlen, 318pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
342 const unsigned char *in, size_t inlen) 319 const unsigned char *in, size_t inlen)
343 { 320{
344 int ret; 321 int ret;
345 RSA_PKEY_CTX *rctx = ctx->data; 322 RSA_PKEY_CTX *rctx = ctx->data;
323
346 ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa, 324 ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
347 rctx->pad_mode); 325 rctx->pad_mode);
348 if (ret < 0) 326 if (ret < 0)
349 return ret; 327 return ret;
350 *outlen = ret; 328 *outlen = ret;
351 return 1; 329 return 1;
352 } 330}
353 331
354static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, 332static int
355 unsigned char *out, size_t *outlen, 333pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
356 const unsigned char *in, size_t inlen) 334 const unsigned char *in, size_t inlen)
357 { 335{
358 int ret; 336 int ret;
359 RSA_PKEY_CTX *rctx = ctx->data; 337 RSA_PKEY_CTX *rctx = ctx->data;
338
360 ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa, 339 ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
361 rctx->pad_mode); 340 rctx->pad_mode);
362 if (ret < 0) 341 if (ret < 0)
363 return ret; 342 return ret;
364 *outlen = ret; 343 *outlen = ret;
365 return 1; 344 return 1;
366 } 345}
367 346
368static int check_padding_md(const EVP_MD *md, int padding) 347static int
369 { 348check_padding_md(const EVP_MD *md, int padding)
349{
370 if (!md) 350 if (!md)
371 return 1; 351 return 1;
372 352
373 if (padding == RSA_NO_PADDING) 353 if (padding == RSA_NO_PADDING) {
374 {
375 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE); 354 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
376 return 0; 355 return 0;
377 } 356 }
378 357
379 if (padding == RSA_X931_PADDING) 358 if (padding == RSA_X931_PADDING) {
380 { 359 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
381 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1)
382 {
383 RSAerr(RSA_F_CHECK_PADDING_MD, 360 RSAerr(RSA_F_CHECK_PADDING_MD,
384 RSA_R_INVALID_X931_DIGEST); 361 RSA_R_INVALID_X931_DIGEST);
385 return 0; 362 return 0;
386 }
387 return 1;
388 } 363 }
364 return 1;
365 }
389 366
390 return 1; 367 return 1;
391 } 368}
392
393 369
394static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2) 370static int
395 { 371pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
372{
396 RSA_PKEY_CTX *rctx = ctx->data; 373 RSA_PKEY_CTX *rctx = ctx->data;
397 switch (type) 374 switch (type) {
398 { 375 case EVP_PKEY_CTRL_RSA_PADDING:
399 case EVP_PKEY_CTRL_RSA_PADDING: 376 if (p1 >= RSA_PKCS1_PADDING && p1 <= RSA_PKCS1_PSS_PADDING) {
400 if ((p1 >= RSA_PKCS1_PADDING) && (p1 <= RSA_PKCS1_PSS_PADDING))
401 {
402 if (!check_padding_md(rctx->md, p1)) 377 if (!check_padding_md(rctx->md, p1))
403 return 0; 378 return 0;
404 if (p1 == RSA_PKCS1_PSS_PADDING) 379 if (p1 == RSA_PKCS1_PSS_PADDING) {
405 {
406 if (!(ctx->operation & 380 if (!(ctx->operation &
407 (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY))) 381 (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
408 goto bad_pad; 382 goto bad_pad;
409 if (!rctx->md) 383 if (!rctx->md)
410 rctx->md = EVP_sha1(); 384 rctx->md = EVP_sha1();
411 } 385 }
412 if (p1 == RSA_PKCS1_OAEP_PADDING) 386 if (p1 == RSA_PKCS1_OAEP_PADDING) {
413 {
414 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT)) 387 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
415 goto bad_pad; 388 goto bad_pad;
416 if (!rctx->md) 389 if (!rctx->md)
417 rctx->md = EVP_sha1(); 390 rctx->md = EVP_sha1();
418 } 391 }
419 rctx->pad_mode = p1; 392 rctx->pad_mode = p1;
420 return 1; 393 return 1;
421 } 394 }
422 bad_pad: 395bad_pad:
423 RSAerr(RSA_F_PKEY_RSA_CTRL, 396 RSAerr(RSA_F_PKEY_RSA_CTRL,
424 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE); 397 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
425 return -2; 398 return -2;
426 399
427 case EVP_PKEY_CTRL_GET_RSA_PADDING: 400 case EVP_PKEY_CTRL_GET_RSA_PADDING:
428 *(int *)p2 = rctx->pad_mode; 401 *(int *)p2 = rctx->pad_mode;
429 return 1; 402 return 1;
430 403
431 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN: 404 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
432 case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN: 405 case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
433 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) 406 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
434 {
435 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN); 407 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
436 return -2; 408 return -2;
437 } 409 }
438 if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) 410 if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
439 *(int *)p2 = rctx->saltlen; 411 *(int *)p2 = rctx->saltlen;
440 else 412 else {
441 {
442 if (p1 < -2) 413 if (p1 < -2)
443 return -2; 414 return -2;
444 rctx->saltlen = p1; 415 rctx->saltlen = p1;
445 } 416 }
446 return 1; 417 return 1;
447 418
448 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS: 419 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
449 if (p1 < 256) 420 if (p1 < 256) {
450 {
451 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS); 421 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
452 return -2; 422 return -2;
453 } 423 }
454 rctx->nbits = p1; 424 rctx->nbits = p1;
455 return 1; 425 return 1;
456 426
457 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP: 427 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
458 if (!p2) 428 if (!p2)
459 return -2; 429 return -2;
460 rctx->pub_exp = p2; 430 rctx->pub_exp = p2;
461 return 1; 431 return 1;
462 432
463 case EVP_PKEY_CTRL_MD: 433 case EVP_PKEY_CTRL_MD:
464 if (!check_padding_md(p2, rctx->pad_mode)) 434 if (!check_padding_md(p2, rctx->pad_mode))
465 return 0; 435 return 0;
466 rctx->md = p2; 436 rctx->md = p2;
467 return 1; 437 return 1;
468 438
469 case EVP_PKEY_CTRL_RSA_MGF1_MD: 439 case EVP_PKEY_CTRL_RSA_MGF1_MD:
470 case EVP_PKEY_CTRL_GET_RSA_MGF1_MD: 440 case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
471 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) 441 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
472 {
473 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD); 442 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
474 return -2; 443 return -2;
475 } 444 }
476 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) 445 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
477 {
478 if (rctx->mgf1md) 446 if (rctx->mgf1md)
479 *(const EVP_MD **)p2 = rctx->mgf1md; 447 *(const EVP_MD **)p2 = rctx->mgf1md;
480 else 448 else
481 *(const EVP_MD **)p2 = rctx->md; 449 *(const EVP_MD **)p2 = rctx->md;
482 } 450 } else
483 else
484 rctx->mgf1md = p2; 451 rctx->mgf1md = p2;
485 return 1; 452 return 1;
486 453
487 case EVP_PKEY_CTRL_DIGESTINIT: 454 case EVP_PKEY_CTRL_DIGESTINIT:
488 case EVP_PKEY_CTRL_PKCS7_ENCRYPT: 455 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
489 case EVP_PKEY_CTRL_PKCS7_DECRYPT: 456 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
490 case EVP_PKEY_CTRL_PKCS7_SIGN: 457 case EVP_PKEY_CTRL_PKCS7_SIGN:
491 return 1; 458 return 1;
492#ifndef OPENSSL_NO_CMS 459#ifndef OPENSSL_NO_CMS
493 case EVP_PKEY_CTRL_CMS_DECRYPT: 460 case EVP_PKEY_CTRL_CMS_DECRYPT:
494 { 461 {
495 X509_ALGOR *alg = NULL; 462 X509_ALGOR *alg = NULL;
496 ASN1_OBJECT *encalg = NULL; 463 ASN1_OBJECT *encalg = NULL;
464
497 if (p2) 465 if (p2)
498 CMS_RecipientInfo_ktri_get0_algs(p2, NULL, NULL, &alg); 466 CMS_RecipientInfo_ktri_get0_algs(p2, NULL, NULL, &alg);
499 if (alg) 467 if (alg)
500 X509_ALGOR_get0(&encalg, NULL, NULL, alg); 468 X509_ALGOR_get0(&encalg, NULL, NULL, alg);
501 if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep) 469 if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep)
502 rctx->pad_mode = RSA_PKCS1_OAEP_PADDING; 470 rctx->pad_mode = RSA_PKCS1_OAEP_PADDING;
503 } 471 }
504 case EVP_PKEY_CTRL_CMS_ENCRYPT: 472 /* FALLTHROUGH */
505 case EVP_PKEY_CTRL_CMS_SIGN: 473
474 case EVP_PKEY_CTRL_CMS_ENCRYPT:
475 case EVP_PKEY_CTRL_CMS_SIGN:
506 return 1; 476 return 1;
507#endif 477#endif
508 case EVP_PKEY_CTRL_PEER_KEY: 478 case EVP_PKEY_CTRL_PEER_KEY:
509 RSAerr(RSA_F_PKEY_RSA_CTRL, 479 RSAerr(RSA_F_PKEY_RSA_CTRL,
510 RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); 480 RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
511 return -2; 481 return -2;
512 482
513 default: 483 default:
514 return -2; 484 return -2;
515
516 }
517 } 485 }
486}
518 487
519static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, 488static int
520 const char *type, const char *value) 489pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
521 { 490{
522 long lval; 491 long lval;
523 char *ep; 492 char *ep;
524 493
525 if (!value) 494 if (!value) {
526 {
527 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING); 495 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
528 return 0; 496 return 0;
529 } 497 }
530 if (!strcmp(type, "rsa_padding_mode")) 498 if (!strcmp(type, "rsa_padding_mode")) {
531 {
532 int pm; 499 int pm;
533 if (!strcmp(value, "pkcs1")) 500 if (!strcmp(value, "pkcs1"))
534 pm = RSA_PKCS1_PADDING; 501 pm = RSA_PKCS1_PADDING;
@@ -544,14 +511,13 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
544 pm = RSA_X931_PADDING; 511 pm = RSA_X931_PADDING;
545 else if (!strcmp(value, "pss")) 512 else if (!strcmp(value, "pss"))
546 pm = RSA_PKCS1_PSS_PADDING; 513 pm = RSA_PKCS1_PSS_PADDING;
547 else 514 else {
548 {
549 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, 515 RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
550 RSA_R_UNKNOWN_PADDING_TYPE); 516 RSA_R_UNKNOWN_PADDING_TYPE);
551 return -2; 517 return -2;
552 }
553 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
554 } 518 }
519 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
520 }
555 521
556 if (!strcmp(type, "rsa_pss_saltlen")) { 522 if (!strcmp(type, "rsa_pss_saltlen")) {
557 int saltlen; 523 int saltlen;
@@ -560,7 +526,8 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
560 lval = strtol(value, &ep, 10); 526 lval = strtol(value, &ep, 10);
561 if (value[0] == '\0' || *ep != '\0') 527 if (value[0] == '\0' || *ep != '\0')
562 goto not_a_number; 528 goto not_a_number;
563 if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) || 529 if ((errno == ERANGE &&
530 (lval == LONG_MAX || lval == LONG_MIN)) ||
564 (lval > INT_MAX || lval < INT_MIN)) 531 (lval > INT_MAX || lval < INT_MIN))
565 goto out_of_range; 532 goto out_of_range;
566 saltlen = lval; 533 saltlen = lval;
@@ -574,7 +541,8 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
574 lval = strtol(value, &ep, 10); 541 lval = strtol(value, &ep, 10);
575 if (value[0] == '\0' || *ep != '\0') 542 if (value[0] == '\0' || *ep != '\0')
576 goto not_a_number; 543 goto not_a_number;
577 if ((errno == ERANGE && (lval == LONG_MAX || lval == LONG_MIN)) || 544 if ((errno == ERANGE &&
545 (lval == LONG_MAX || lval == LONG_MIN)) ||
578 (lval > INT_MAX || lval < INT_MIN)) 546 (lval > INT_MAX || lval < INT_MIN))
579 goto out_of_range; 547 goto out_of_range;
580 nbits = lval; 548 nbits = lval;
@@ -584,6 +552,7 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
584 if (!strcmp(type, "rsa_keygen_pubexp")) { 552 if (!strcmp(type, "rsa_keygen_pubexp")) {
585 int ret; 553 int ret;
586 BIGNUM *pubexp = NULL; 554 BIGNUM *pubexp = NULL;
555
587 if (!BN_asc2bn(&pubexp, value)) 556 if (!BN_asc2bn(&pubexp, value))
588 return 0; 557 return 0;
589 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp); 558 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
@@ -597,27 +566,26 @@ out_of_range:
597 return -2; 566 return -2;
598} 567}
599 568
600static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) 569static int
601 { 570pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
571{
602 RSA *rsa = NULL; 572 RSA *rsa = NULL;
603 RSA_PKEY_CTX *rctx = ctx->data; 573 RSA_PKEY_CTX *rctx = ctx->data;
604 BN_GENCB *pcb, cb; 574 BN_GENCB *pcb, cb;
605 int ret; 575 int ret;
606 if (!rctx->pub_exp) 576
607 { 577 if (!rctx->pub_exp) {
608 rctx->pub_exp = BN_new(); 578 rctx->pub_exp = BN_new();
609 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4)) 579 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
610 return 0; 580 return 0;
611 } 581 }
612 rsa = RSA_new(); 582 rsa = RSA_new();
613 if (!rsa) 583 if (!rsa)
614 return 0; 584 return 0;
615 if (ctx->pkey_gencb) 585 if (ctx->pkey_gencb) {
616 {
617 pcb = &cb; 586 pcb = &cb;
618 evp_pkey_set_cb_translate(pcb, ctx); 587 evp_pkey_set_cb_translate(pcb, ctx);
619 } 588 } else
620 else
621 pcb = NULL; 589 pcb = NULL;
622 ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb); 590 ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
623 if (ret > 0) 591 if (ret > 0)
@@ -625,7 +593,7 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
625 else 593 else
626 RSA_free(rsa); 594 RSA_free(rsa);
627 return ret; 595 return ret;
628 } 596}
629 597
630const EVP_PKEY_METHOD rsa_pkey_meth = { 598const EVP_PKEY_METHOD rsa_pkey_meth = {
631 .pkey_id = EVP_PKEY_RSA, 599 .pkey_id = EVP_PKEY_RSA,
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-11-25 20:56:12 +0000