summaryrefslogtreecommitdiff
path: root/src/lib/libcrypto/rsa/rsa_pmeth.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/libcrypto/rsa/rsa_pmeth.c')
-rw-r--r--src/lib/libcrypto/rsa/rsa_pmeth.c627
1 files changed, 0 insertions, 627 deletions
diff --git a/src/lib/libcrypto/rsa/rsa_pmeth.c b/src/lib/libcrypto/rsa/rsa_pmeth.c
deleted file mode 100644
index 09166e105b..0000000000
--- a/src/lib/libcrypto/rsa/rsa_pmeth.c
+++ /dev/null
@@ -1,627 +0,0 @@
1/* $OpenBSD: rsa_pmeth.c,v 1.16 2015/02/11 04:05:14 beck Exp $ */
2/* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3 * project 2006.
4 */
5/* ====================================================================
6 * Copyright (c) 2006 The OpenSSL Project. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * 3. All advertising materials mentioning features or use of this
21 * software must display the following acknowledgment:
22 * "This product includes software developed by the OpenSSL Project
23 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24 *
25 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26 * endorse or promote products derived from this software without
27 * prior written permission. For written permission, please contact
28 * licensing@OpenSSL.org.
29 *
30 * 5. Products derived from this software may not be called "OpenSSL"
31 * nor may "OpenSSL" appear in their names without prior written
32 * permission of the OpenSSL Project.
33 *
34 * 6. Redistributions of any form whatsoever must retain the following
35 * acknowledgment:
36 * "This product includes software developed by the OpenSSL Project
37 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38 *
39 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
43 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50 * OF THE POSSIBILITY OF SUCH DAMAGE.
51 * ====================================================================
52 *
53 * This product includes cryptographic software written by Eric Young
54 * (eay@cryptsoft.com). This product includes software written by Tim
55 * Hudson (tjh@cryptsoft.com).
56 *
57 */
58
59#include <limits.h>
60#include <stdio.h>
61#include <string.h>
62
63#include <openssl/opensslconf.h>
64
65#include <openssl/asn1t.h>
66#include <openssl/bn.h>
67#include <openssl/err.h>
68#include <openssl/evp.h>
69#include <openssl/rsa.h>
70#include <openssl/x509.h>
71
72#ifndef OPENSSL_NO_CMS
73#include <openssl/cms.h>
74#endif
75
76#include "evp_locl.h"
77#include "rsa_locl.h"
78
79/* RSA pkey context structure */
80
81typedef struct {
82 /* Key gen parameters */
83 int nbits;
84 BIGNUM *pub_exp;
85 /* Keygen callback info */
86 int gentmp[2];
87 /* RSA padding mode */
88 int pad_mode;
89 /* message digest */
90 const EVP_MD *md;
91 /* message digest for MGF1 */
92 const EVP_MD *mgf1md;
93 /* PSS/OAEP salt length */
94 int saltlen;
95 /* Temp buffer */
96 unsigned char *tbuf;
97} RSA_PKEY_CTX;
98
99static int
100pkey_rsa_init(EVP_PKEY_CTX *ctx)
101{
102 RSA_PKEY_CTX *rctx;
103
104 rctx = malloc(sizeof(RSA_PKEY_CTX));
105 if (!rctx)
106 return 0;
107 rctx->nbits = 2048;
108 rctx->pub_exp = NULL;
109 rctx->pad_mode = RSA_PKCS1_PADDING;
110 rctx->md = NULL;
111 rctx->mgf1md = NULL;
112 rctx->tbuf = NULL;
113
114 rctx->saltlen = -2;
115
116 ctx->data = rctx;
117 ctx->keygen_info = rctx->gentmp;
118 ctx->keygen_info_count = 2;
119
120 return 1;
121}
122
123static int
124pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
125{
126 RSA_PKEY_CTX *dctx, *sctx;
127
128 if (!pkey_rsa_init(dst))
129 return 0;
130 sctx = src->data;
131 dctx = dst->data;
132 dctx->nbits = sctx->nbits;
133 if (sctx->pub_exp) {
134 dctx->pub_exp = BN_dup(sctx->pub_exp);
135 if (!dctx->pub_exp)
136 return 0;
137 }
138 dctx->pad_mode = sctx->pad_mode;
139 dctx->md = sctx->md;
140 return 1;
141}
142
143static int
144setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
145{
146 if (ctx->tbuf)
147 return 1;
148 ctx->tbuf = malloc(EVP_PKEY_size(pk->pkey));
149 if (!ctx->tbuf)
150 return 0;
151 return 1;
152}
153
154static void
155pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
156{
157 RSA_PKEY_CTX *rctx = ctx->data;
158
159 if (rctx) {
160 BN_free(rctx->pub_exp);
161 free(rctx->tbuf);
162 free(rctx);
163 }
164}
165
166static int
167pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
168 const unsigned char *tbs, size_t tbslen)
169{
170 int ret;
171 RSA_PKEY_CTX *rctx = ctx->data;
172 RSA *rsa = ctx->pkey->pkey.rsa;
173
174 if (rctx->md) {
175 if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
176 RSAerr(RSA_F_PKEY_RSA_SIGN,
177 RSA_R_INVALID_DIGEST_LENGTH);
178 return -1;
179 }
180
181 if (EVP_MD_type(rctx->md) == NID_mdc2) {
182 unsigned int sltmp;
183
184 if (rctx->pad_mode != RSA_PKCS1_PADDING)
185 return -1;
186 ret = RSA_sign_ASN1_OCTET_STRING(NID_mdc2, tbs, tbslen,
187 sig, &sltmp, rsa);
188
189 if (ret <= 0)
190 return ret;
191 ret = sltmp;
192 } else if (rctx->pad_mode == RSA_X931_PADDING) {
193 if (!setup_tbuf(rctx, ctx))
194 return -1;
195 memcpy(rctx->tbuf, tbs, tbslen);
196 rctx->tbuf[tbslen] =
197 RSA_X931_hash_id(EVP_MD_type(rctx->md));
198 ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
199 rsa, RSA_X931_PADDING);
200 } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
201 unsigned int sltmp;
202
203 ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
204 &sltmp, rsa);
205 if (ret <= 0)
206 return ret;
207 ret = sltmp;
208 } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
209 if (!setup_tbuf(rctx, ctx))
210 return -1;
211 if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
212 tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
213 return -1;
214 ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
215 sig, rsa, RSA_NO_PADDING);
216 } else
217 return -1;
218 } else
219 ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
220 rctx->pad_mode);
221 if (ret < 0)
222 return ret;
223 *siglen = ret;
224 return 1;
225}
226
227static int
228pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, unsigned char *rout, size_t *routlen,
229 const unsigned char *sig, size_t siglen)
230{
231 int ret;
232 RSA_PKEY_CTX *rctx = ctx->data;
233
234 if (rctx->md) {
235 if (rctx->pad_mode == RSA_X931_PADDING) {
236 if (!setup_tbuf(rctx, ctx))
237 return -1;
238 ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
239 ctx->pkey->pkey.rsa, RSA_X931_PADDING);
240 if (ret < 1)
241 return 0;
242 ret--;
243 if (rctx->tbuf[ret] !=
244 RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
245 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
246 RSA_R_ALGORITHM_MISMATCH);
247 return 0;
248 }
249 if (ret != EVP_MD_size(rctx->md)) {
250 RSAerr(RSA_F_PKEY_RSA_VERIFYRECOVER,
251 RSA_R_INVALID_DIGEST_LENGTH);
252 return 0;
253 }
254 if (rout)
255 memcpy(rout, rctx->tbuf, ret);
256 } else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
257 size_t sltmp;
258
259 ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
260 rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
261 if (ret <= 0)
262 return 0;
263 ret = sltmp;
264 } else
265 return -1;
266 } else
267 ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
268 rctx->pad_mode);
269 if (ret < 0)
270 return ret;
271 *routlen = ret;
272 return 1;
273}
274
275static int
276pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
277 const unsigned char *tbs, size_t tbslen)
278{
279 RSA_PKEY_CTX *rctx = ctx->data;
280 RSA *rsa = ctx->pkey->pkey.rsa;
281 size_t rslen;
282
283 if (rctx->md) {
284 if (rctx->pad_mode == RSA_PKCS1_PADDING)
285 return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
286 sig, siglen, rsa);
287 if (rctx->pad_mode == RSA_X931_PADDING) {
288 if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig,
289 siglen) <= 0)
290 return 0;
291 } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
292 int ret;
293
294 if (!setup_tbuf(rctx, ctx))
295 return -1;
296 ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
297 rsa, RSA_NO_PADDING);
298 if (ret <= 0)
299 return 0;
300 ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
301 rctx->mgf1md, rctx->tbuf, rctx->saltlen);
302 if (ret <= 0)
303 return 0;
304 return 1;
305 } else
306 return -1;
307 } else {
308 if (!setup_tbuf(rctx, ctx))
309 return -1;
310 rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
311 rctx->pad_mode);
312 if (rslen == 0)
313 return 0;
314 }
315
316 if (rslen != tbslen || memcmp(tbs, rctx->tbuf, rslen))
317 return 0;
318
319 return 1;
320}
321
322static int
323pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
324 const unsigned char *in, size_t inlen)
325{
326 int ret;
327 RSA_PKEY_CTX *rctx = ctx->data;
328
329 ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
330 rctx->pad_mode);
331 if (ret < 0)
332 return ret;
333 *outlen = ret;
334 return 1;
335}
336
337static int
338pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
339 const unsigned char *in, size_t inlen)
340{
341 int ret;
342 RSA_PKEY_CTX *rctx = ctx->data;
343
344 ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
345 rctx->pad_mode);
346 if (ret < 0)
347 return ret;
348 *outlen = ret;
349 return 1;
350}
351
352static int
353check_padding_md(const EVP_MD *md, int padding)
354{
355 if (!md)
356 return 1;
357
358 if (padding == RSA_NO_PADDING) {
359 RSAerr(RSA_F_CHECK_PADDING_MD, RSA_R_INVALID_PADDING_MODE);
360 return 0;
361 }
362
363 if (padding == RSA_X931_PADDING) {
364 if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
365 RSAerr(RSA_F_CHECK_PADDING_MD,
366 RSA_R_INVALID_X931_DIGEST);
367 return 0;
368 }
369 return 1;
370 }
371
372 return 1;
373}
374
375static int
376pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
377{
378 RSA_PKEY_CTX *rctx = ctx->data;
379
380 switch (type) {
381 case EVP_PKEY_CTRL_RSA_PADDING:
382 if (p1 >= RSA_PKCS1_PADDING && p1 <= RSA_PKCS1_PSS_PADDING) {
383 if (!check_padding_md(rctx->md, p1))
384 return 0;
385 if (p1 == RSA_PKCS1_PSS_PADDING) {
386 if (!(ctx->operation &
387 (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
388 goto bad_pad;
389 if (!rctx->md)
390 rctx->md = EVP_sha1();
391 }
392 if (p1 == RSA_PKCS1_OAEP_PADDING) {
393 if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
394 goto bad_pad;
395 if (!rctx->md)
396 rctx->md = EVP_sha1();
397 }
398 rctx->pad_mode = p1;
399 return 1;
400 }
401bad_pad:
402 RSAerr(RSA_F_PKEY_RSA_CTRL,
403 RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
404 return -2;
405
406 case EVP_PKEY_CTRL_GET_RSA_PADDING:
407 *(int *)p2 = rctx->pad_mode;
408 return 1;
409
410 case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
411 case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
412 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
413 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
414 return -2;
415 }
416 if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN)
417 *(int *)p2 = rctx->saltlen;
418 else {
419 if (p1 < -2)
420 return -2;
421 rctx->saltlen = p1;
422 }
423 return 1;
424
425 case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
426 if (p1 < 256) {
427 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_KEYBITS);
428 return -2;
429 }
430 rctx->nbits = p1;
431 return 1;
432
433 case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
434 if (!p2)
435 return -2;
436 rctx->pub_exp = p2;
437 return 1;
438
439 case EVP_PKEY_CTRL_MD:
440 if (!check_padding_md(p2, rctx->pad_mode))
441 return 0;
442 rctx->md = p2;
443 return 1;
444
445 case EVP_PKEY_CTRL_RSA_MGF1_MD:
446 case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
447 if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
448 RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_MGF1_MD);
449 return -2;
450 }
451 if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
452 if (rctx->mgf1md)
453 *(const EVP_MD **)p2 = rctx->mgf1md;
454 else
455 *(const EVP_MD **)p2 = rctx->md;
456 } else
457 rctx->mgf1md = p2;
458 return 1;
459
460 case EVP_PKEY_CTRL_DIGESTINIT:
461 case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
462 case EVP_PKEY_CTRL_PKCS7_DECRYPT:
463 case EVP_PKEY_CTRL_PKCS7_SIGN:
464 return 1;
465#ifndef OPENSSL_NO_CMS
466 case EVP_PKEY_CTRL_CMS_DECRYPT:
467 {
468 X509_ALGOR *alg = NULL;
469 ASN1_OBJECT *encalg = NULL;
470
471 if (p2)
472 CMS_RecipientInfo_ktri_get0_algs(p2, NULL,
473 NULL, &alg);
474 if (alg)
475 X509_ALGOR_get0(&encalg, NULL, NULL, alg);
476 if (encalg && OBJ_obj2nid(encalg) == NID_rsaesOaep)
477 rctx->pad_mode = RSA_PKCS1_OAEP_PADDING;
478 }
479 /* FALLTHROUGH */
480
481 case EVP_PKEY_CTRL_CMS_ENCRYPT:
482 case EVP_PKEY_CTRL_CMS_SIGN:
483 return 1;
484#endif
485 case EVP_PKEY_CTRL_PEER_KEY:
486 RSAerr(RSA_F_PKEY_RSA_CTRL,
487 RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
488 return -2;
489
490 default:
491 return -2;
492 }
493}
494
495static int
496pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
497{
498 long lval;
499 char *ep;
500
501 if (!value) {
502 RSAerr(RSA_F_PKEY_RSA_CTRL_STR, RSA_R_VALUE_MISSING);
503 return 0;
504 }
505 if (!strcmp(type, "rsa_padding_mode")) {
506 int pm;
507 if (!strcmp(value, "pkcs1"))
508 pm = RSA_PKCS1_PADDING;
509 else if (!strcmp(value, "sslv23"))
510 pm = RSA_SSLV23_PADDING;
511 else if (!strcmp(value, "none"))
512 pm = RSA_NO_PADDING;
513 else if (!strcmp(value, "oeap"))
514 pm = RSA_PKCS1_OAEP_PADDING;
515 else if (!strcmp(value, "oaep"))
516 pm = RSA_PKCS1_OAEP_PADDING;
517 else if (!strcmp(value, "x931"))
518 pm = RSA_X931_PADDING;
519 else if (!strcmp(value, "pss"))
520 pm = RSA_PKCS1_PSS_PADDING;
521 else {
522 RSAerr(RSA_F_PKEY_RSA_CTRL_STR,
523 RSA_R_UNKNOWN_PADDING_TYPE);
524 return -2;
525 }
526 return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
527 }
528
529 if (!strcmp(type, "rsa_pss_saltlen")) {
530 int saltlen;
531
532 errno = 0;
533 lval = strtol(value, &ep, 10);
534 if (value[0] == '\0' || *ep != '\0')
535 goto not_a_number;
536 if ((errno == ERANGE &&
537 (lval == LONG_MAX || lval == LONG_MIN)) ||
538 (lval > INT_MAX || lval < INT_MIN))
539 goto out_of_range;
540 saltlen = lval;
541 return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
542 }
543
544 if (!strcmp(type, "rsa_keygen_bits")) {
545 int nbits;
546
547 errno = 0;
548 lval = strtol(value, &ep, 10);
549 if (value[0] == '\0' || *ep != '\0')
550 goto not_a_number;
551 if ((errno == ERANGE &&
552 (lval == LONG_MAX || lval == LONG_MIN)) ||
553 (lval > INT_MAX || lval < INT_MIN))
554 goto out_of_range;
555 nbits = lval;
556 return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
557 }
558
559 if (!strcmp(type, "rsa_keygen_pubexp")) {
560 int ret;
561 BIGNUM *pubexp = NULL;
562
563 if (!BN_asc2bn(&pubexp, value))
564 return 0;
565 ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
566 if (ret <= 0)
567 BN_free(pubexp);
568 return ret;
569 }
570
571not_a_number:
572out_of_range:
573 return -2;
574}
575
576static int
577pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
578{
579 RSA *rsa = NULL;
580 RSA_PKEY_CTX *rctx = ctx->data;
581 BN_GENCB *pcb, cb;
582 int ret;
583
584 if (!rctx->pub_exp) {
585 rctx->pub_exp = BN_new();
586 if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4))
587 return 0;
588 }
589 rsa = RSA_new();
590 if (!rsa)
591 return 0;
592 if (ctx->pkey_gencb) {
593 pcb = &cb;
594 evp_pkey_set_cb_translate(pcb, ctx);
595 } else
596 pcb = NULL;
597 ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
598 if (ret > 0)
599 EVP_PKEY_assign_RSA(pkey, rsa);
600 else
601 RSA_free(rsa);
602 return ret;
603}
604
605const EVP_PKEY_METHOD rsa_pkey_meth = {
606 .pkey_id = EVP_PKEY_RSA,
607 .flags = EVP_PKEY_FLAG_AUTOARGLEN,
608
609 .init = pkey_rsa_init,
610 .copy = pkey_rsa_copy,
611 .cleanup = pkey_rsa_cleanup,
612
613 .keygen = pkey_rsa_keygen,
614
615 .sign = pkey_rsa_sign,
616
617 .verify = pkey_rsa_verify,
618
619 .verify_recover = pkey_rsa_verifyrecover,
620
621 .encrypt = pkey_rsa_encrypt,
622
623 .decrypt = pkey_rsa_decrypt,
624
625 .ctrl = pkey_rsa_ctrl,
626 .ctrl_str = pkey_rsa_ctrl_str
627};