summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorjsing <>2025-05-21 12:11:23 +0000
committerjsing <>2025-05-21 12:11:23 +0000
commitd60185c3dfd64fac68622c0241f71debf2d3e10b (patch)
tree3df95197dbad74c7fbc570a5c8b6383955927436 /src
parent1a3804b5e0099f4dabf04f04cfcd6566f3f31fb0 (diff)
downloadopenbsd-d60185c3dfd64fac68622c0241f71debf2d3e10b.tar.gz
openbsd-d60185c3dfd64fac68622c0241f71debf2d3e10b.tar.bz2
openbsd-d60185c3dfd64fac68622c0241f71debf2d3e10b.zip
Remove GHASH_CHUNK and size_t related code from GCM encrypt/decrypt.
This adds significant complexity to the code. On amd64 and aarch64 it results in a minimal slowdown for aligned inputs and a performance improvement for unaligned inputs. ok beck@ joshua@ tb@
Diffstat (limited to 'src')
-rw-r--r--src/lib/libcrypto/modes/gcm128.c221
1 files changed, 1 insertions, 220 deletions
diff --git a/src/lib/libcrypto/modes/gcm128.c b/src/lib/libcrypto/modes/gcm128.c
index 0d34be95c9..f2d216ab4a 100644
--- a/src/lib/libcrypto/modes/gcm128.c
+++ b/src/lib/libcrypto/modes/gcm128.c
@@ -1,4 +1,4 @@
1/* $OpenBSD: gcm128.c,v 1.43 2025/05/21 11:37:07 jsing Exp $ */ 1/* $OpenBSD: gcm128.c,v 1.44 2025/05/21 12:11:23 jsing Exp $ */
2/* ==================================================================== 2/* ====================================================================
3 * Copyright (c) 2010 The OpenSSL Project. All rights reserved. 3 * Copyright (c) 2010 The OpenSSL Project. All rights reserved.
4 * 4 *
@@ -236,13 +236,6 @@ gcm_ghash(GCM128_CONTEXT *ctx, const uint8_t *in, size_t len)
236} 236}
237#endif 237#endif
238 238
239/*
240 * GHASH_CHUNK is "stride parameter" missioned to mitigate cache
241 * trashing effect. In other words idea is to hash data while it's
242 * still in L1 cache after encryption pass...
243 */
244#define GHASH_CHUNK (3*1024)
245
246#if defined(GHASH_ASM) && \ 239#if defined(GHASH_ASM) && \
247 (defined(__i386) || defined(__i386__) || \ 240 (defined(__i386) || defined(__i386__) || \
248 defined(__x86_64) || defined(__x86_64__) || \ 241 defined(__x86_64) || defined(__x86_64__) || \
@@ -458,101 +451,7 @@ CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const unsigned char *in,
458 ctr = be32toh(ctx->Yi.d[3]); 451 ctr = be32toh(ctx->Yi.d[3]);
459 452
460 n = ctx->mres; 453 n = ctx->mres;
461 if (16 % sizeof(size_t) == 0)
462 do { /* always true actually */
463 if (n) {
464 while (n && len) {
465 ctx->Xi.c[n] ^= *(out++) = *(in++) ^
466 ctx->EKi.c[n];
467 --len;
468 n = (n + 1) % 16;
469 }
470 if (n == 0)
471 gcm_mul(ctx, ctx->Xi.u);
472 else {
473 ctx->mres = n;
474 return 0;
475 }
476 }
477#ifdef __STRICT_ALIGNMENT
478 if (((size_t)in|(size_t)out) % sizeof(size_t) != 0)
479 break;
480#endif
481#if defined(GHASH_CHUNK)
482 while (len >= GHASH_CHUNK) {
483 size_t j = GHASH_CHUNK;
484
485 while (j) {
486 size_t *out_t = (size_t *)out;
487 const size_t *in_t = (const size_t *)in;
488
489 (*block)(ctx->Yi.c, ctx->EKi.c, key);
490 ++ctr;
491 ctx->Yi.d[3] = htobe32(ctr);
492
493 for (i = 0; i < 16/sizeof(size_t); ++i)
494 out_t[i] = in_t[i] ^
495 ctx->EKi.t[i];
496 out += 16;
497 in += 16;
498 j -= 16;
499 }
500 gcm_ghash(ctx, out - GHASH_CHUNK, GHASH_CHUNK);
501 len -= GHASH_CHUNK;
502 }
503 if ((i = (len & (size_t)-16))) {
504 size_t j = i;
505
506 while (len >= 16) {
507 size_t *out_t = (size_t *)out;
508 const size_t *in_t = (const size_t *)in;
509
510 (*block)(ctx->Yi.c, ctx->EKi.c, key);
511 ++ctr;
512 ctx->Yi.d[3] = htobe32(ctr);
513
514 for (i = 0; i < 16/sizeof(size_t); ++i)
515 out_t[i] = in_t[i] ^
516 ctx->EKi.t[i];
517 out += 16;
518 in += 16;
519 len -= 16;
520 }
521 gcm_ghash(ctx, out - j, j);
522 }
523#else
524 while (len >= 16) {
525 size_t *out_t = (size_t *)out;
526 const size_t *in_t = (const size_t *)in;
527
528 (*block)(ctx->Yi.c, ctx->EKi.c, key);
529 ++ctr;
530 ctx->Yi.d[3] = htobe32(ctr);
531
532 for (i = 0; i < 16/sizeof(size_t); ++i)
533 ctx->Xi.t[i] ^=
534 out_t[i] = in_t[i] ^ ctx->EKi.t[i];
535 gcm_mul(ctx, ctx->Xi.u);
536 out += 16;
537 in += 16;
538 len -= 16;
539 }
540#endif
541 if (len) {
542 (*block)(ctx->Yi.c, ctx->EKi.c, key);
543 ++ctr;
544 ctx->Yi.d[3] = htobe32(ctr);
545
546 while (len--) {
547 ctx->Xi.c[n] ^= out[n] = in[n] ^
548 ctx->EKi.c[n];
549 ++n;
550 }
551 }
552 454
553 ctx->mres = n;
554 return 0;
555 } while (0);
556 for (i = 0; i < len; ++i) { 455 for (i = 0; i < len; ++i) {
557 if (n == 0) { 456 if (n == 0) {
558 (*block)(ctx->Yi.c, ctx->EKi.c, key); 457 (*block)(ctx->Yi.c, ctx->EKi.c, key);
@@ -594,103 +493,7 @@ CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const unsigned char *in,
594 ctr = be32toh(ctx->Yi.d[3]); 493 ctr = be32toh(ctx->Yi.d[3]);
595 494
596 n = ctx->mres; 495 n = ctx->mres;
597 if (16 % sizeof(size_t) == 0)
598 do { /* always true actually */
599 if (n) {
600 while (n && len) {
601 uint8_t c = *(in++);
602 *(out++) = c ^ ctx->EKi.c[n];
603 ctx->Xi.c[n] ^= c;
604 --len;
605 n = (n + 1) % 16;
606 }
607 if (n == 0)
608 gcm_mul(ctx, ctx->Xi.u);
609 else {
610 ctx->mres = n;
611 return 0;
612 }
613 }
614#ifdef __STRICT_ALIGNMENT
615 if (((size_t)in|(size_t)out) % sizeof(size_t) != 0)
616 break;
617#endif
618#if defined(GHASH_CHUNK)
619 while (len >= GHASH_CHUNK) {
620 size_t j = GHASH_CHUNK;
621
622 gcm_ghash(ctx, in, GHASH_CHUNK);
623 while (j) {
624 size_t *out_t = (size_t *)out;
625 const size_t *in_t = (const size_t *)in;
626
627 (*block)(ctx->Yi.c, ctx->EKi.c, key);
628 ++ctr;
629 ctx->Yi.d[3] = htobe32(ctr);
630
631 for (i = 0; i < 16/sizeof(size_t); ++i)
632 out_t[i] = in_t[i] ^
633 ctx->EKi.t[i];
634 out += 16;
635 in += 16;
636 j -= 16;
637 }
638 len -= GHASH_CHUNK;
639 }
640 if ((i = (len & (size_t)-16))) {
641 gcm_ghash(ctx, in, i);
642 while (len >= 16) {
643 size_t *out_t = (size_t *)out;
644 const size_t *in_t = (const size_t *)in;
645
646 (*block)(ctx->Yi.c, ctx->EKi.c, key);
647 ++ctr;
648 ctx->Yi.d[3] = htobe32(ctr);
649
650 for (i = 0; i < 16/sizeof(size_t); ++i)
651 out_t[i] = in_t[i] ^
652 ctx->EKi.t[i];
653 out += 16;
654 in += 16;
655 len -= 16;
656 }
657 }
658#else
659 while (len >= 16) {
660 size_t *out_t = (size_t *)out;
661 const size_t *in_t = (const size_t *)in;
662
663 (*block)(ctx->Yi.c, ctx->EKi.c, key);
664 ++ctr;
665 ctx->Yi.d[3] = htobe32(ctr);
666
667 for (i = 0; i < 16/sizeof(size_t); ++i) {
668 size_t c = in_t[i];
669 out_t[i] = c ^ ctx->EKi.t[i];
670 ctx->Xi.t[i] ^= c;
671 }
672 gcm_mul(ctx, ctx->Xi.u);
673 out += 16;
674 in += 16;
675 len -= 16;
676 }
677#endif
678 if (len) {
679 (*block)(ctx->Yi.c, ctx->EKi.c, key);
680 ++ctr;
681 ctx->Yi.d[3] = htobe32(ctr);
682
683 while (len--) {
684 uint8_t c = in[n];
685 ctx->Xi.c[n] ^= c;
686 out[n] = c ^ ctx->EKi.c[n];
687 ++n;
688 }
689 }
690 496
691 ctx->mres = n;
692 return 0;
693 } while (0);
694 for (i = 0; i < len; ++i) { 497 for (i = 0; i < len; ++i) {
695 uint8_t c; 498 uint8_t c;
696 if (n == 0) { 499 if (n == 0) {
@@ -747,17 +550,6 @@ CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const unsigned char *in,
747 return 0; 550 return 0;
748 } 551 }
749 } 552 }
750#if defined(GHASH_CHUNK)
751 while (len >= GHASH_CHUNK) {
752 (*stream)(in, out, GHASH_CHUNK/16, key, ctx->Yi.c);
753 ctr += GHASH_CHUNK/16;
754 ctx->Yi.d[3] = htobe32(ctr);
755 gcm_ghash(ctx, out, GHASH_CHUNK);
756 out += GHASH_CHUNK;
757 in += GHASH_CHUNK;
758 len -= GHASH_CHUNK;
759 }
760#endif
761 if ((i = (len & (size_t)-16))) { 553 if ((i = (len & (size_t)-16))) {
762 size_t j = i/16; 554 size_t j = i/16;
763 555
@@ -822,17 +614,6 @@ CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const unsigned char *in,
822 return 0; 614 return 0;
823 } 615 }
824 } 616 }
825#if defined(GHASH_CHUNK)
826 while (len >= GHASH_CHUNK) {
827 gcm_ghash(ctx, in, GHASH_CHUNK);
828 (*stream)(in, out, GHASH_CHUNK/16, key, ctx->Yi.c);
829 ctr += GHASH_CHUNK/16;
830 ctx->Yi.d[3] = htobe32(ctr);
831 out += GHASH_CHUNK;
832 in += GHASH_CHUNK;
833 len -= GHASH_CHUNK;
834 }
835#endif
836 if ((i = (len & (size_t)-16))) { 617 if ((i = (len & (size_t)-16))) {
837 size_t j = i/16; 618 size_t j = i/16;
838 619
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-10-04 01:15:25 +0000