Provide a SHA-256 assembly implementation for amd64 using SHA-NI.

This provides a SHA-256 assembly implementation for amd64, which uses
the Intel SHA Extensions (aka SHA New Instructions or SHA-NI). This
provides a 3-5x performance gain on some Intel CPUs and many AMD CPUs.

ok tb@

3 files changed, 220 insertions, 2 deletions

diff --git a/src/lib/libcrypto/arch/amd64/Makefile.inc b/src/lib/libcrypto/arch/amd64/Makefile.inc
index 9ba5634f87..fe22385633 100644
--- a/src/lib/libcrypto/arch/amd64/Makefile.inc
+++ b/src/lib/libcrypto/arch/amd64/Makefile.inc
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile.inc,v 1.32 2024/11/16 14:56:39 jsing Exp $
+# $OpenBSD: Makefile.inc,v 1.33 2024/11/16 15:31:36 jsing Exp $
 
 # amd64-specific libcrypto build rules
 
@@ -53,6 +53,7 @@ SSLASM+= sha sha1-x86_64
 CFLAGS+= -DSHA256_ASM
 SRCS+= sha256_amd64.c
 SRCS+= sha256_amd64_generic.S
+SRCS+= sha256_amd64_shani.S
 CFLAGS+= -DSHA512_ASM
 SRCS+= sha512_amd64.c
 SRCS+= sha512_amd64_generic.S
diff --git a/src/lib/libcrypto/sha/sha256_amd64.c b/src/lib/libcrypto/sha/sha256_amd64.c
index f7531b340f..6c5d3e897f 100644
--- a/src/lib/libcrypto/sha/sha256_amd64.c
+++ b/src/lib/libcrypto/sha/sha256_amd64.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sha256_amd64.c,v 1.1 2024/11/08 15:09:48 jsing Exp $ */
+/* $OpenBSD: sha256_amd64.c,v 1.2 2024/11/16 15:31:36 jsing Exp $ */
 /*
  * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
  *
@@ -17,10 +17,18 @@
 
 #include <openssl/sha.h>
 
+#include "crypto_arch.h"
+
 void sha256_block_generic(SHA256_CTX *ctx, const void *in, size_t num);
+void sha256_block_shani(SHA256_CTX *ctx, const void *in, size_t num);
 
 void
 sha256_block_data_order(SHA256_CTX *ctx, const void *in, size_t num)
 {
+        if ((crypto_cpu_caps_amd64 & CRYPTO_CPU_CAPS_AMD64_SHA) != 0) {
+                sha256_block_shani(ctx, in, num);
+                return;
+        }
+
         sha256_block_generic(ctx, in, num);
 }
diff --git a/src/lib/libcrypto/sha/sha256_amd64_shani.S b/src/lib/libcrypto/sha/sha256_amd64_shani.S
new file mode 100644
index 0000000000..df3a796b45
--- /dev/null
+++ b/src/lib/libcrypto/sha/sha256_amd64_shani.S
@@ -0,0 +1,209 @@
+/* $OpenBSD: sha256_amd64_shani.S,v 1.1 2024/11/16 15:31:36 jsing Exp $ */
+/*
+ * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifdef __CET__
+#include <cet.h>
+#else
+#define _CET_ENDBR
+#endif
+
+/*
+ * SHA-256 implementation using the Intel SHA extensions:
+ *
+ * https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sha-extensions.html
+ */
+
+#define ctx             %rdi
+#define in              %rsi
+#define num             %rdx
+
+#define end             %rbx
+
+#define k256            %rbp
+
+#define xmsg            %xmm0
+
+#define xhs0            %xmm1
+#define xhs1            %xmm2
+
+#define xabef           %xmm3
+#define xcdgh           %xmm4
+
+#define xmsgtmp0        %xmm6
+#define xmsgtmp1        %xmm7
+#define xmsgtmp2        %xmm8
+#define xmsgtmp3        %xmm9
+#define xmsgtmp4        %xmm10
+
+#define xshufmask       %xmm11
+
+#define xtmp0           %xmm12
+
+#define sha256_message_schedule_load(idx, m, xmsgtmp) \
+        movdqu  (idx*16)(m), xmsg;                                      \
+        pshufb  xshufmask, xmsg;                                        \
+        movdqa  xmsg, xmsgtmp;
+
+#define sha256_message_schedule_update(xmt0, xmt1, xmt2, xmt3) \
+        sha256msg1 xmt1, xmt0;                                          \
+        movdqa  xmt3, xmsgtmp4;                                         \
+        palignr $4, xmt2, xmsgtmp4;                                     \
+        paddd   xmsgtmp4, xmt0;                                         \
+        sha256msg2 xmt3, xmt0;
+
+#define sha256_shani_round(idx) \
+        paddd   (idx*16)(k256), xmsg;                                   \
+        sha256rnds2 xmsg, xhs0, xhs1;                                   \
+        pshufd  $0x0e, xmsg, xmsg;                                      \
+        sha256rnds2 xmsg, xhs1, xhs0;
+
+#define sha256_shani_round_load(idx, m, xmsgtmp) \
+        sha256_message_schedule_load(idx, m, xmsgtmp);                  \
+        sha256_shani_round(idx);
+
+#define sha256_shani_round_update(idx, xmt0, xmt1, xmt2, xmt3) \
+        sha256_message_schedule_update(xmt0, xmt1, xmt2, xmt3);         \
+        movdqa  xmt0, xmsg;                                             \
+        sha256_shani_round(idx);
+
+.text
+
+/*
+ * void sha256_block_shani(SHA256_CTX *ctx, const void *in, size_t num);
+ *
+ * Standard x86-64 ABI: rdi = ctx, rsi = in, rdx = num
+ */
+.align 16
+.globl  sha256_block_shani
+.type   sha256_block_shani,@function
+sha256_block_shani:
+        _CET_ENDBR
+
+        /* Save callee save registers. */
+        pushq   %rbx
+        pushq   %rbp
+
+        /* Compute end of message. */
+        shlq    $6, num
+        leaq    (in, num, 1), end
+
+        /* Address of SHA-256 constants. */
+        leaq    K256(%rip), k256
+
+        /* Load endian shuffle mask. */
+        movdqa  shufmask(%rip), xshufmask
+
+        /* Load current hash state from context. */
+        movdqu  (0*16)(ctx), xhs0       /* dcba */
+        movdqu  (1*16)(ctx), xhs1       /* hgfe */
+
+        /* Rearrange words to construct abef/cdgh. */
+        pshufd  $0xb1, xhs0, xhs0       /* cdab */
+        pshufd  $0x1b, xhs1, xhs1       /* efgh */
+        movdqa  xhs0, xtmp0
+        palignr $8, xhs1, xhs0          /* abef */
+        pblendw $0xf0, xtmp0, xhs1      /* cdgh */
+
+        jmp     .Lshani_block_loop
+
+.align 16
+.Lshani_block_loop:
+        /* Save state for accumulation. */
+        movdqa  xhs0, xabef
+        movdqa  xhs1, xcdgh
+
+        /* Rounds 0 through 15 (four rounds at a time). */
+        sha256_shani_round_load(0, in, xmsgtmp0)
+        sha256_shani_round_load(1, in, xmsgtmp1)
+        sha256_shani_round_load(2, in, xmsgtmp2)
+        sha256_shani_round_load(3, in, xmsgtmp3)
+
+        /* Rounds 16 through 63 (four rounds at a time). */
+        sha256_shani_round_update(4, xmsgtmp0, xmsgtmp1, xmsgtmp2, xmsgtmp3)
+        sha256_shani_round_update(5, xmsgtmp1, xmsgtmp2, xmsgtmp3, xmsgtmp0)
+        sha256_shani_round_update(6, xmsgtmp2, xmsgtmp3, xmsgtmp0, xmsgtmp1)
+        sha256_shani_round_update(7, xmsgtmp3, xmsgtmp0, xmsgtmp1, xmsgtmp2)
+
+        sha256_shani_round_update(8, xmsgtmp0, xmsgtmp1, xmsgtmp2, xmsgtmp3)
+        sha256_shani_round_update(9, xmsgtmp1, xmsgtmp2, xmsgtmp3, xmsgtmp0)
+        sha256_shani_round_update(10, xmsgtmp2, xmsgtmp3, xmsgtmp0, xmsgtmp1)
+        sha256_shani_round_update(11, xmsgtmp3, xmsgtmp0, xmsgtmp1, xmsgtmp2)
+
+        sha256_shani_round_update(12, xmsgtmp0, xmsgtmp1, xmsgtmp2, xmsgtmp3)
+        sha256_shani_round_update(13, xmsgtmp1, xmsgtmp2, xmsgtmp3, xmsgtmp0)
+        sha256_shani_round_update(14, xmsgtmp2, xmsgtmp3, xmsgtmp0, xmsgtmp1)
+        sha256_shani_round_update(15, xmsgtmp3, xmsgtmp0, xmsgtmp1, xmsgtmp2)
+
+        /* Accumulate hash state. */
+        paddd   xabef, xhs0
+        paddd   xcdgh, xhs1
+
+        addq    $64, in
+        cmpq    end, in
+        jb      .Lshani_block_loop
+
+        /* Rearrange words to construct dcba/hgfe. */
+        pshufd  $0x1b, xhs0, xhs0       /* feba */
+        pshufd  $0xb1, xhs1, xhs1       /* dchg */
+        movdqa  xhs0, xtmp0
+        pblendw $0xf0, xhs1, xhs0       /* dcba */
+        palignr $8, xtmp0, xhs1         /* hgfe */
+
+        /* Update stored hash context. */
+        movdqu  xhs0, (0*16)(ctx)
+        movdqu  xhs1, (1*16)(ctx)
+
+        /* Restore callee save registers. */
+        popq    %rbp
+        popq    %rbx
+
+        ret
+
+.rodata
+
+/*
+ * Shuffle mask - little endian to big endian word conversion.
+ */
+.align  16
+.type   shufmask,@object
+shufmask:
+.octa   0x0c0d0e0f08090a0b0405060700010203
+.size   shufmask,.-shufmask
+
+/*
+ * SHA-256 constants - see FIPS 180-4 section 4.2.2.
+ */
+.align  64
+.type   K256,@object
+K256:
+.long   0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
+.long   0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
+.long   0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
+.long   0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
+.long   0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
+.long   0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
+.long   0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
+.long   0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
+.long   0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
+.long   0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
+.long   0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
+.long   0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
+.long   0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
+.long   0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
+.long   0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
+.long   0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
+.size   K256,.-K256