Move AES-NI from EVP to AES for CTR mode.

The mode implementation for CTR has two variants - one takes the block
function, while the other takes a "ctr32" function. The latter is expected
to handle the lower 32 bits of the IV/counter, but is not expected to
handle overflow. The AES-NI implementation for CTR currently uses the
second variant.

Provide aes_ctr32_encrypt_internal() as a function that can be replaced on
a machine dependent basis, along with an aes_ctr32_encrypt_generic()
function that provides the default implementation and can be used as a
fallback. Wire up the AES-NI version for amd64 and i386, change
AES_ctr128_encrypt() to use CRYPTO_ctr128_encrypt_ctr32() (which calls
aes_ctr32_encrypt_internal()) and remove the various AES-NI specific
EVP_CIPHER methods for CTR.

Callers of AES_ctr128_encrypt() will now use AES-NI, if available.

ok tb@

6 files changed, 95 insertions, 76 deletions

diff --git a/src/lib/libcrypto/aes/aes.c b/src/lib/libcrypto/aes/aes.c
index e630c3f81a..e9dbe975e3 100644
--- a/src/lib/libcrypto/aes/aes.c
+++ b/src/lib/libcrypto/aes/aes.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: aes.c,v 1.9 2025/06/03 08:42:15 kenjiro Exp $ */
+/* $OpenBSD: aes.c,v 1.10 2025/06/27 17:10:45 jsing Exp $ */
 /* ====================================================================
  * Copyright (c) 2002-2006 The OpenSSL Project.  All rights reserved.
  *
@@ -56,6 +56,7 @@
 #include <openssl/modes.h>
 
 #include "crypto_arch.h"
+#include "crypto_internal.h"
 
 static const unsigned char aes_wrap_default_iv[] = {
         0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6,
@@ -171,12 +172,58 @@ AES_cfb8_encrypt(const unsigned char *in, unsigned char *out, size_t length,
 LCRYPTO_ALIAS(AES_cfb8_encrypt);
 
 void
+aes_ctr32_encrypt_generic(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE])
+{
+        uint8_t iv[AES_BLOCK_SIZE], buf[AES_BLOCK_SIZE];
+        uint32_t ctr;
+        int i;
+
+        memcpy(iv, ivec, sizeof(iv));
+
+        ctr = crypto_load_be32toh(&iv[12]);
+
+        while (blocks > 0) {
+                crypto_store_htobe32(&iv[12], ctr);
+                aes_encrypt_internal(iv, buf, key);
+                ctr++;
+
+                for (i = 0; i < AES_BLOCK_SIZE; i++)
+                        out[i] = in[i] ^ buf[i];
+
+                in += 16;
+                out += 16;
+                blocks--;
+        }
+}
+
+#ifdef HAVE_AES_CTR32_ENCRYPT_INTERNAL
+void aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]);
+
+#else
+static inline void
+aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE])
+{
+        aes_ctr32_encrypt_generic(in, out, blocks, key, ivec);
+}
+#endif
+
+void
+aes_ctr32_encrypt_ctr128f(const unsigned char *in, unsigned char *out, size_t blocks,
+    const void *key, const unsigned char ivec[AES_BLOCK_SIZE])
+{
+        aes_ctr32_encrypt_internal(in, out, blocks, key, ivec);
+}
+
+void
 AES_ctr128_encrypt(const unsigned char *in, unsigned char *out,
     size_t length, const AES_KEY *key, unsigned char ivec[AES_BLOCK_SIZE],
     unsigned char ecount_buf[AES_BLOCK_SIZE], unsigned int *num)
 {
-        CRYPTO_ctr128_encrypt(in, out, length, key, ivec, ecount_buf, num,
-            aes_encrypt_block128);
+        CRYPTO_ctr128_encrypt_ctr32(in, out, length, key, ivec, ecount_buf,
+            num, aes_ctr32_encrypt_ctr128f);
 }
 LCRYPTO_ALIAS(AES_ctr128_encrypt);
 
diff --git a/src/lib/libcrypto/aes/aes_amd64.c b/src/lib/libcrypto/aes/aes_amd64.c
index 302d1ac91d..456409d186 100644
--- a/src/lib/libcrypto/aes/aes_amd64.c
+++ b/src/lib/libcrypto/aes/aes_amd64.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: aes_amd64.c,v 1.1 2025/06/15 15:11:50 jsing Exp $ */
+/* $OpenBSD: aes_amd64.c,v 1.2 2025/06/27 17:10:45 jsing Exp $ */
 /*
  * Copyright (c) 2025 Joel Sing <jsing@openbsd.org>
  *
@@ -32,6 +32,9 @@ void aes_decrypt_generic(const unsigned char *in, unsigned char *out,
 void aes_cbc_encrypt_generic(const unsigned char *in, unsigned char *out,
     size_t len, const AES_KEY *key, unsigned char *ivec, const int enc);
 
+void aes_ctr32_encrypt_generic(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]);
+
 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
     AES_KEY *key);
 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
@@ -45,6 +48,9 @@ void aesni_decrypt(const unsigned char *in, unsigned char *out,
 void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out,
     size_t len, const AES_KEY *key, unsigned char *ivec, const int enc);
 
+void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+    size_t blocks, const void *key, const unsigned char *ivec);
+
 int
 aes_set_encrypt_key_internal(const unsigned char *userKey, const int bits,
     AES_KEY *key)
@@ -100,3 +106,15 @@ aes_cbc_encrypt_internal(const unsigned char *in, unsigned char *out,
 
         aes_cbc_encrypt_generic(in, out, len, key, ivec, enc);
 }
+
+void
+aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE])
+{
+        if ((crypto_cpu_caps_amd64 & CRYPTO_CPU_CAPS_AMD64_AES) != 0) {
+                aesni_ctr32_encrypt_blocks(in, out, blocks, key, ivec);
+                return;
+        }
+
+        aes_ctr32_encrypt_generic(in, out, blocks, key, ivec);
+}
diff --git a/src/lib/libcrypto/aes/aes_i386.c b/src/lib/libcrypto/aes/aes_i386.c
index 0b5c89af70..2da02a8d35 100644
--- a/src/lib/libcrypto/aes/aes_i386.c
+++ b/src/lib/libcrypto/aes/aes_i386.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: aes_i386.c,v 1.1 2025/06/15 15:11:50 jsing Exp $ */
+/* $OpenBSD: aes_i386.c,v 1.2 2025/06/27 17:10:45 jsing Exp $ */
 /*
  * Copyright (c) 2025 Joel Sing <jsing@openbsd.org>
  *
@@ -32,6 +32,9 @@ void aes_decrypt_generic(const unsigned char *in, unsigned char *out,
 void aes_cbc_encrypt_generic(const unsigned char *in, unsigned char *out,
     size_t len, const AES_KEY *key, unsigned char *ivec, const int enc);
 
+void aes_ctr32_encrypt_generic(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]);
+
 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
     AES_KEY *key);
 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
@@ -45,6 +48,9 @@ void aesni_decrypt(const unsigned char *in, unsigned char *out,
 void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out,
     size_t len, const AES_KEY *key, unsigned char *ivec, const int enc);
 
+void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
+    size_t blocks, const void *key, const unsigned char *ivec);
+
 int
 aes_set_encrypt_key_internal(const unsigned char *userKey, const int bits,
     AES_KEY *key)
@@ -100,3 +106,15 @@ aes_cbc_encrypt_internal(const unsigned char *in, unsigned char *out,
 
         aes_cbc_encrypt_generic(in, out, len, key, ivec, enc);
 }
+
+void
+aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out,
+    size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE])
+{
+        if ((crypto_cpu_caps_i386 & CRYPTO_CPU_CAPS_I386_AES) != 0) {
+                aesni_ctr32_encrypt_blocks(in, out, blocks, key, ivec);
+                return;
+        }
+
+        aes_ctr32_encrypt_generic(in, out, blocks, key, ivec);
+}
diff --git a/src/lib/libcrypto/arch/amd64/crypto_arch.h b/src/lib/libcrypto/arch/amd64/crypto_arch.h
index da1a22e003..9eb99229d1 100644
--- a/src/lib/libcrypto/arch/amd64/crypto_arch.h
+++ b/src/lib/libcrypto/arch/amd64/crypto_arch.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: crypto_arch.h,v 1.6 2025/06/15 14:16:11 jsing Exp $ */
+/*      $OpenBSD: crypto_arch.h,v 1.7 2025/06/27 17:10:45 jsing Exp $ */
 /*
  * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
  *
@@ -32,11 +32,12 @@ extern uint64_t crypto_cpu_caps_amd64;
 
 #ifndef OPENSSL_NO_ASM
 
-#define HAVE_AES_CBC_ENCRYPT_INTERNAL
 #define HAVE_AES_SET_ENCRYPT_KEY_INTERNAL
 #define HAVE_AES_SET_DECRYPT_KEY_INTERNAL
 #define HAVE_AES_ENCRYPT_INTERNAL
 #define HAVE_AES_DECRYPT_INTERNAL
+#define HAVE_AES_CBC_ENCRYPT_INTERNAL
+#define HAVE_AES_CTR32_ENCRYPT_INTERNAL
 
 #define HAVE_RC4_INTERNAL
 #define HAVE_RC4_SET_KEY_INTERNAL
diff --git a/src/lib/libcrypto/arch/i386/crypto_arch.h b/src/lib/libcrypto/arch/i386/crypto_arch.h
index a693bd20d8..e03e6cd765 100644
--- a/src/lib/libcrypto/arch/i386/crypto_arch.h
+++ b/src/lib/libcrypto/arch/i386/crypto_arch.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: crypto_arch.h,v 1.5 2025/06/15 14:18:31 jsing Exp $ */
+/*      $OpenBSD: crypto_arch.h,v 1.6 2025/06/27 17:10:45 jsing Exp $ */
 /*
  * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
  *
@@ -31,11 +31,12 @@ extern uint64_t crypto_cpu_caps_i386;
 
 #ifndef OPENSSL_NO_ASM
 
-#define HAVE_AES_CBC_ENCRYPT_INTERNAL
 #define HAVE_AES_SET_ENCRYPT_KEY_INTERNAL
 #define HAVE_AES_SET_DECRYPT_KEY_INTERNAL
 #define HAVE_AES_ENCRYPT_INTERNAL
 #define HAVE_AES_DECRYPT_INTERNAL
+#define HAVE_AES_CBC_ENCRYPT_INTERNAL
+#define HAVE_AES_CTR32_ENCRYPT_INTERNAL
 
 #define HAVE_RC4_INTERNAL
 #define HAVE_RC4_SET_KEY_INTERNAL
diff --git a/src/lib/libcrypto/evp/e_aes.c b/src/lib/libcrypto/evp/e_aes.c
index 34f5513300..b00eb048e8 100644
--- a/src/lib/libcrypto/evp/e_aes.c
+++ b/src/lib/libcrypto/evp/e_aes.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_aes.c,v 1.75 2025/06/16 14:50:30 jsing Exp $ */
+/* $OpenBSD: e_aes.c,v 1.76 2025/06/27 17:10:45 jsing Exp $ */
 /* ====================================================================
  * Copyright (c) 2001-2011 The OpenSSL Project.  All rights reserved.
  *
@@ -160,21 +160,6 @@ void aesni_ccm64_decrypt_blocks (const unsigned char *in, unsigned char *out,
     unsigned char cmac[16]);
 
 static int
-aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
-    const unsigned char *in, size_t len)
-{
-        EVP_AES_KEY *eak = ctx->cipher_data;
-        unsigned int num = ctx->num;
-
-        CRYPTO_ctr128_encrypt_ctr32(in, out, len, &eak->ks, ctx->iv, ctx->buf,
-            &num, aesni_ctr32_encrypt_blocks);
-
-        ctx->num = (size_t)num;
-
-        return 1;
-}
-
-static int
 aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t len)
 {
@@ -562,19 +547,6 @@ EVP_aes_128_cfb8(void)
 }
 LCRYPTO_ALIAS(EVP_aes_128_cfb8);
 
-#ifdef AESNI_CAPABLE
-static const EVP_CIPHER aesni_128_ctr = {
-        .nid = NID_aes_128_ctr,
-        .block_size = 1,
-        .key_len = 16,
-        .iv_len = 16,
-        .flags = EVP_CIPH_CTR_MODE,
-        .init = aes_init_key,
-        .do_cipher = aesni_ctr_cipher,
-        .ctx_size = sizeof(EVP_AES_KEY),
-};
-#endif
-
 static const EVP_CIPHER aes_128_ctr = {
         .nid = NID_aes_128_ctr,
         .block_size = 1,
@@ -589,11 +561,7 @@ static const EVP_CIPHER aes_128_ctr = {
 const EVP_CIPHER *
 EVP_aes_128_ctr(void)
 {
-#ifdef AESNI_CAPABLE
-        return AESNI_CAPABLE ? &aesni_128_ctr : &aes_128_ctr;
-#else
         return &aes_128_ctr;
-#endif
 }
 LCRYPTO_ALIAS(EVP_aes_128_ctr);
 
@@ -722,19 +690,6 @@ EVP_aes_192_cfb8(void)
 }
 LCRYPTO_ALIAS(EVP_aes_192_cfb8);
 
-#ifdef AESNI_CAPABLE
-static const EVP_CIPHER aesni_192_ctr = {
-        .nid = NID_aes_192_ctr,
-        .block_size = 1,
-        .key_len = 24,
-        .iv_len = 16,
-        .flags = EVP_CIPH_CTR_MODE,
-        .init = aes_init_key,
-        .do_cipher = aesni_ctr_cipher,
-        .ctx_size = sizeof(EVP_AES_KEY),
-};
-#endif
-
 static const EVP_CIPHER aes_192_ctr = {
         .nid = NID_aes_192_ctr,
         .block_size = 1,
@@ -749,11 +704,7 @@ static const EVP_CIPHER aes_192_ctr = {
 const EVP_CIPHER *
 EVP_aes_192_ctr(void)
 {
-#ifdef AESNI_CAPABLE
-        return AESNI_CAPABLE ? &aesni_192_ctr : &aes_192_ctr;
-#else
         return &aes_192_ctr;
-#endif
 }
 LCRYPTO_ALIAS(EVP_aes_192_ctr);
 
@@ -882,19 +833,6 @@ EVP_aes_256_cfb8(void)
 }
 LCRYPTO_ALIAS(EVP_aes_256_cfb8);
 
-#ifdef AESNI_CAPABLE
-static const EVP_CIPHER aesni_256_ctr = {
-        .nid = NID_aes_256_ctr,
-        .block_size = 1,
-        .key_len = 32,
-        .iv_len = 16,
-        .flags = EVP_CIPH_CTR_MODE,
-        .init = aes_init_key,
-        .do_cipher = aesni_ctr_cipher,
-        .ctx_size = sizeof(EVP_AES_KEY),
-};
-#endif
-
 static const EVP_CIPHER aes_256_ctr = {
         .nid = NID_aes_256_ctr,
         .block_size = 1,
@@ -909,11 +847,7 @@ static const EVP_CIPHER aes_256_ctr = {
 const EVP_CIPHER *
 EVP_aes_256_ctr(void)
 {
-#ifdef AESNI_CAPABLE
-        return AESNI_CAPABLE ? &aesni_256_ctr : &aes_256_ctr;
-#else
         return &aes_256_ctr;
-#endif
 }
 LCRYPTO_ALIAS(EVP_aes_256_ctr);