Provide an accelerated SHA-512 assembly implementation for aarch64.

This provides a SHA-512 assembly implementation that makes use of the ARM
Cryptographic Extension (CE), which is found on many arm64 CPUs. This gives
a performance gain of up to 2.5x on an Apple M2 (dependent on block size).
If an aarch64 machine does not have SHA512 support, then we'll fall back to
using the existing C implementation.

ok kettenis@ tb@

4 files changed, 353 insertions, 2 deletions

diff --git a/src/lib/libcrypto/arch/aarch64/Makefile.inc b/src/lib/libcrypto/arch/aarch64/Makefile.inc
index 41198b069f..d93cb815ef 100644
--- a/src/lib/libcrypto/arch/aarch64/Makefile.inc
+++ b/src/lib/libcrypto/arch/aarch64/Makefile.inc
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile.inc,v 1.15 2025/03/07 14:21:22 jsing Exp $
+# $OpenBSD: Makefile.inc,v 1.16 2025/03/12 14:13:41 jsing Exp $
 
 # aarch64-specific libcrypto build rules
 
@@ -6,5 +6,7 @@ SRCS += crypto_cpu_caps.c
 
 SRCS += sha256_aarch64.c
 SRCS += sha256_aarch64_ce.S
+SRCS += sha512_aarch64.c
+SRCS += sha512_aarch64_ce.S
 
 AFLAGS+= -mmark-bti-property
diff --git a/src/lib/libcrypto/arch/aarch64/crypto_arch.h b/src/lib/libcrypto/arch/aarch64/crypto_arch.h
index adc91cd19f..35ecba9394 100644
--- a/src/lib/libcrypto/arch/aarch64/crypto_arch.h
+++ b/src/lib/libcrypto/arch/aarch64/crypto_arch.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: crypto_arch.h,v 1.3 2025/03/07 14:21:22 jsing Exp $ */
+/*      $OpenBSD: crypto_arch.h,v 1.4 2025/03/12 14:13:41 jsing Exp $ */
 /*
  * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
  *
@@ -34,7 +34,10 @@ extern uint64_t crypto_cpu_caps_aarch64;
 #define CRYPTO_CPU_CAPS_AARCH64_SHA3    (1ULL << 5)
 
 #ifndef OPENSSL_NO_ASM
+
 #define HAVE_SHA256_BLOCK_DATA_ORDER
+#define HAVE_SHA512_BLOCK_DATA_ORDER
+
 #endif
 
 #endif
diff --git a/src/lib/libcrypto/sha/sha512_aarch64.c b/src/lib/libcrypto/sha/sha512_aarch64.c
new file mode 100644
index 0000000000..3c997e3e89
--- /dev/null
+++ b/src/lib/libcrypto/sha/sha512_aarch64.c
@@ -0,0 +1,34 @@
+/* $OpenBSD: sha512_aarch64.c,v 1.1 2025/03/12 14:13:41 jsing Exp $ */
+/*
+ * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <openssl/sha.h>
+
+#include "crypto_arch.h"
+
+void sha512_block_ce(SHA512_CTX *ctx, const void *in, size_t num);
+void sha512_block_generic(SHA512_CTX *ctx, const void *in, size_t num);
+
+void
+sha512_block_data_order(SHA512_CTX *ctx, const void *in, size_t num)
+{
+        if ((crypto_cpu_caps_aarch64 & CRYPTO_CPU_CAPS_AARCH64_SHA512) != 0) {
+                sha512_block_ce(ctx, in, num);
+                return;
+        }
+
+        sha512_block_generic(ctx, in, num);
+}
diff --git a/src/lib/libcrypto/sha/sha512_aarch64_ce.S b/src/lib/libcrypto/sha/sha512_aarch64_ce.S
new file mode 100644
index 0000000000..89109a78ba
--- /dev/null
+++ b/src/lib/libcrypto/sha/sha512_aarch64_ce.S
@@ -0,0 +1,312 @@
+/* $OpenBSD: sha512_aarch64_ce.S,v 1.1 2025/03/12 14:13:41 jsing Exp $ */
+/*
+ * Copyright (c) 2023,2025 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * SHA-512 implementation using the ARM Cryptographic Extension (CE).
+ *
+ * The documentation for these is rather inadequate - each instruction is
+ * described in a mechanical sense, however their combined usage does not
+ * seem to be detailed anywhere.
+ *
+ * There are four instructions that enable hardware acceleration of SHA-512:
+ *
+ *  sha512h - hash update, part 1 (without a number to be inconsistent):
+ *    inputs <W1:W0 + K1:K0 + g:h>, <f:g>, <d:e>
+ *    output T1 for W0, T1 for W1
+ *
+ *  sha512h2 - hash update, part 2:
+ *    inputs <T1 for W0, T1 for W1>, <c:d>, <a:b>
+ *    output <T1 + T2 for W0, T1 + T2 for W1>
+ *
+ *  sha512su0 - message schedule update with sigma0 for two rounds:
+ *    inputs <W0:W1>, <W2:W3>
+ *    output W0 += sigma0(W1), W1 += sigma0(W2)
+ *
+ *  sha512su1 - message schedule update with sigma1 for two rounds:
+ *    inputs <W0:W1>, <W14:W15>, <W9:W10>
+ *    output W0 += sigma1(W14) + W9, W1 += sigma1(W15) + W10
+ */
+
+#define ctx             x0
+#define in              x1
+#define num             x2
+
+#define k512_base       x3
+#define k512            x4
+
+/* Note: the lower 64 bits of v8 through v15 are callee save. */
+
+#define hc0             v28
+#define hc1             v29
+#define hc2             v30
+#define hc3             v31
+
+#define hs0             v0
+#define hs1             v1
+#define hs2             v2
+#define hs3             v3
+#define hs4             v4
+#define hs5             v5
+#define hs6             v6
+#define hs7             v7
+
+#define w0              v10
+#define w1              v11
+#define w2              v12
+#define w3              v13
+#define w4              v14
+#define w5              v15
+#define w6              v16
+#define w7              v17
+
+#define k0              v20
+#define k1              v21
+#define k2              v22
+#define k3              v23
+#define k4              v24
+#define k5              v25
+#define k6              v26
+#define k7              v27
+
+#define tmp0            v8
+#define tmp1            v9
+#define tmp2            v18
+
+/*
+ * Update message schedule for m0 (W0:W1), using m1 (W2:W3), m4 (W8:W9),
+ * m5 (W10:W11) and m7 (W14:W15). The sha512su0 instruction computes the sigma0
+ * component of the message schedule update as m0 = sigma0(m1) + m0, while
+ * sha512su1 computes the sigma1 component as m0 = sigma1(m7) + W9:W10 + m0.
+ * Note that W9:W10 is split across two registers, hence this needs to be
+ * constructed before it is passed to sha512su1:
+ *
+ *   W0 = sigma1(W14) + W9 + sigma0(W1) + W0
+ */
+#define sha512_message_schedule_update(m0, m1, m4, m5, m7) \
+        sha512su0 m0.2d, m1.2d;                 /* W0 += sigma0(W1) */  \
+        ext     tmp2.16b, m4.16b, m5.16b, #8;   /* W9:W10 */            \
+        sha512su1 m0.2d, m7.2d, tmp2.2d;        /* W0 += sigma1(W14) + W9 */
+
+/*
+ * Compute two SHA-512 rounds by adding W0:W1 + K0:K1, then computing T1 for two
+ * rounds by swapping the double words, adding g:h and calling sha512h with this
+ * value (W1:W0 = W1:W0 + K1:K0 + g:h), f:g and d:e. The new e:f value is then
+ * computed by adding T1 + c:d (producing the next e:f values), before calling
+ * sha512h2 with T1, c:d and a:b, computing T1 + T2 for two rounds (producing
+ * the next a:b values):
+ *
+ *   T1 = h + Sigma1(e) + Ch(e, f, g) + Kt + Wt
+ *   T2 = Sigma0(a) + Maj(a, b, c)
+ * 
+ *   h = g
+ *   g = f
+ *   f = e
+ *   e = d + T1
+ *   d = c
+ *   c = b
+ *   b = a
+ *   a = T1 + T2
+ *
+ * The inputs are:
+ * 
+ *   h0 = a:b
+ *   h1 = c:d
+ *   h2 = e:f
+ *   h3 = g:h
+ *
+ * Producing the following outputs:
+ * 
+ *   h4 = next a:b
+ *   h5 = next e:f
+ *
+ * These values are then rotated by the caller to perform the next two rounds.
+ */
+#define sha512_round(h0, h1, h2, h3, h4, h5, w, k) \
+        add     h4.2d, w.2d, k.2d;                      /* W0:W1 += K0:K1 */    \
+        ext     h4.16b, h4.16b, h4.16b, #8;             /* W1:W0 (swap) */      \
+        add     h4.2d, h4.2d, h3.2d;                    /* W1:W0 += g:h */      \
+        ext     tmp0.16b, h2.16b, h3.16b, #8;           /* f:g */               \
+        ext     tmp1.16b, h1.16b, h2.16b, #8;           /* d:e */               \
+        sha512h h4, tmp0, tmp1.2d;                      /* T1 */                \
+        add     h5.2d, h1.2d, h4.2d;                    /* c:d + T1 */          \
+        sha512h2 h4, h1, h0.2d;                         /* T1 + T2 */
+
+#define sha512_round_update(h0, h1, h2, h3, h4, h5, m0, m1, m2, m3, m4, k) \
+        sha512_message_schedule_update(m0, m1, m2, m3, m4) \
+        sha512_round(h0, h1, h2, h3, h4, h5, m0, k)
+
+.arch   armv8-a+sha3
+
+.text
+
+/*
+ * void sha512_block_ce(SHA512_CTX *ctx, const void *in, size_t num);
+ *
+ * Standard ARM ABI: x0 = ctx, x1 = in, x2 = num
+ */
+.globl  sha512_block_ce
+sha512_block_ce:
+
+        /* Save low 64 bits of v8 through v15 to the stack. */
+        sub     sp, sp, #32
+        st4     {v8.d, v9.d, v10.d, v11.d}[0], [sp]
+        sub     sp, sp, #32
+        st4     {v12.d, v13.d, v14.d, v15.d}[0], [sp]
+
+        /* Address of SHA-512 constants. */
+        adrp    k512_base, K512
+        add     k512_base, k512_base, :lo12:K512
+
+        /*
+         * Load current hash state from context.
+         * hc0 = a:b, hc1 = c:d, hc2 = e:f, hc3 = g:h
+         */
+        ld1     {hc0.2d, hc1.2d, hc2.2d, hc3.2d}, [ctx]
+
+block_loop:
+        mov     k512, k512_base
+
+        /* Copy current hash state. */
+        mov     hs0.2d, hc0.2d
+        mov     hs1.2d, hc1.2d
+        mov     hs2.2d, hc2.2d
+        mov     hs3.2d, hc3.2d
+
+        /* Load and byte swap message schedule. */
+        ld1     {w0.16b, w1.16b, w2.16b, w3.16b}, [in], #64
+        rev64   w0.16b, w0.16b
+        rev64   w1.16b, w1.16b
+        rev64   w2.16b, w2.16b
+        rev64   w3.16b, w3.16b
+
+        ld1     {w4.2d, w5.2d, w6.2d, w7.2d}, [in], #64
+        rev64   w4.16b, w4.16b
+        rev64   w5.16b, w5.16b
+        rev64   w6.16b, w6.16b
+        rev64   w7.16b, w7.16b
+
+        /* Rounds 0 through 15 (two rounds at a time). */
+        ld1     {k0.2d, k1.2d, k2.2d, k3.2d}, [k512], #64
+        ld1     {k4.2d, k5.2d, k6.2d, k7.2d}, [k512], #64
+
+        sha512_round(hs0, hs1, hs2, hs3, hs4, hs5, w0, k0)
+        sha512_round(hs4, hs0, hs5, hs2, hs6, hs7, w1, k1)
+        sha512_round(hs6, hs4, hs7, hs5, hs1, hs3, w2, k2)
+        sha512_round(hs1, hs6, hs3, hs7, hs0, hs2, w3, k3)
+        sha512_round(hs0, hs1, hs2, hs3, hs4, hs5, w4, k4)
+        sha512_round(hs4, hs0, hs5, hs2, hs6, hs7, w5, k5)
+        sha512_round(hs6, hs4, hs7, hs5, hs1, hs3, w6, k6)
+        sha512_round(hs1, hs6, hs3, hs7, hs0, hs2, w7, k7)
+
+        /* Rounds 16 through 31 (two rounds at a time). */
+        ld1     {k0.2d, k1.2d, k2.2d, k3.2d}, [k512], #64
+        ld1     {k4.2d, k5.2d, k6.2d, k7.2d}, [k512], #64
+
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w0, w1, w4, w5, w7, k0)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w1, w2, w5, w6, w0, k1)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w2, w3, w6, w7, w1, k2)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w3, w4, w7, w0, w2, k3)
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w4, w5, w0, w1, w3, k4)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w5, w6, w1, w2, w4, k5)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w6, w7, w2, w3, w5, k6)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w7, w0, w3, w4, w6, k7)
+
+        /* Rounds 32 through 47 (two rounds at a time). */
+        ld1     {k0.2d, k1.2d, k2.2d, k3.2d}, [k512], #64
+        ld1     {k4.2d, k5.2d, k6.2d, k7.2d}, [k512], #64
+
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w0, w1, w4, w5, w7, k0)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w1, w2, w5, w6, w0, k1)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w2, w3, w6, w7, w1, k2)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w3, w4, w7, w0, w2, k3)
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w4, w5, w0, w1, w3, k4)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w5, w6, w1, w2, w4, k5)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w6, w7, w2, w3, w5, k6)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w7, w0, w3, w4, w6, k7)
+
+        /* Rounds 48 through 63 (two rounds at a time). */
+        ld1     {k0.2d, k1.2d, k2.2d, k3.2d}, [k512], #64
+        ld1     {k4.2d, k5.2d, k6.2d, k7.2d}, [k512], #64
+
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w0, w1, w4, w5, w7, k0)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w1, w2, w5, w6, w0, k1)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w2, w3, w6, w7, w1, k2)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w3, w4, w7, w0, w2, k3)
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w4, w5, w0, w1, w3, k4)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w5, w6, w1, w2, w4, k5)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w6, w7, w2, w3, w5, k6)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w7, w0, w3, w4, w6, k7)
+
+        /* Rounds 64 through 79 (two rounds at a time). */
+        ld1     {k0.2d, k1.2d, k2.2d, k3.2d}, [k512], #64
+        ld1     {k4.2d, k5.2d, k6.2d, k7.2d}, [k512], #64
+
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w0, w1, w4, w5, w7, k0)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w1, w2, w5, w6, w0, k1)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w2, w3, w6, w7, w1, k2)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w3, w4, w7, w0, w2, k3)
+        sha512_round_update(hs0, hs1, hs2, hs3, hs4, hs5, w4, w5, w0, w1, w3, k4)
+        sha512_round_update(hs4, hs0, hs5, hs2, hs6, hs7, w5, w6, w1, w2, w4, k5)
+        sha512_round_update(hs6, hs4, hs7, hs5, hs1, hs3, w6, w7, w2, w3, w5, k6)
+        sha512_round_update(hs1, hs6, hs3, hs7, hs0, hs2, w7, w0, w3, w4, w6, k7)
+
+        /* Add intermediate state to hash state. */
+        add     hc0.2d, hc0.2d, hs0.2d
+        add     hc1.2d, hc1.2d, hs1.2d
+        add     hc2.2d, hc2.2d, hs2.2d
+        add     hc3.2d, hc3.2d, hs3.2d
+
+        sub     num, num, #1
+        cbnz    num, block_loop
+
+        /* Store hash state to context. */
+        st1     {hc0.2d, hc1.2d, hc2.2d, hc3.2d}, [ctx]
+
+        /* Restore low 64 bits of v8 through v15 from the stack. */
+        ld4     {v12.d, v13.d, v14.d, v15.d}[0], [sp], #32
+        ld4     {v8.d, v9.d, v10.d, v11.d}[0], [sp], #32
+
+        ret
+
+/*
+ * SHA-512 constants - see FIPS 180-4 section 4.2.3.
+ */
+.rodata
+.align  4
+.type   K512,@object
+K512:
+.quad   0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc
+.quad   0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118
+.quad   0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2
+.quad   0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694
+.quad   0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65
+.quad   0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5
+.quad   0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4
+.quad   0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70
+.quad   0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df
+.quad   0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b
+.quad   0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30
+.quad   0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8
+.quad   0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8
+.quad   0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3
+.quad   0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec
+.quad   0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b
+.quad   0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178
+.quad   0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b
+.quad   0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c
+.quad   0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817
+.size   K512,.-K512