From abb03e21a8d0fc7f97a871f5aee5a8084176540f Mon Sep 17 00:00:00 2001 From: jsing <> Date: Fri, 27 Jun 2025 17:10:45 +0000 Subject: Move AES-NI from EVP to AES for CTR mode. The mode implementation for CTR has two variants - one takes the block function, while the other takes a "ctr32" function. The latter is expected to handle the lower 32 bits of the IV/counter, but is not expected to handle overflow. The AES-NI implementation for CTR currently uses the second variant. Provide aes_ctr32_encrypt_internal() as a function that can be replaced on a machine dependent basis, along with an aes_ctr32_encrypt_generic() function that provides the default implementation and can be used as a fallback. Wire up the AES-NI version for amd64 and i386, change AES_ctr128_encrypt() to use CRYPTO_ctr128_encrypt_ctr32() (which calls aes_ctr32_encrypt_internal()) and remove the various AES-NI specific EVP_CIPHER methods for CTR. Callers of AES_ctr128_encrypt() will now use AES-NI, if available. ok tb@ --- src/lib/libcrypto/aes/aes.c | 53 +++++++++++++++++++++-- src/lib/libcrypto/aes/aes_amd64.c | 20 ++++++++- src/lib/libcrypto/aes/aes_i386.c | 20 ++++++++- src/lib/libcrypto/arch/amd64/crypto_arch.h | 5 ++- src/lib/libcrypto/arch/i386/crypto_arch.h | 5 ++- src/lib/libcrypto/evp/e_aes.c | 68 +----------------------------- 6 files changed, 95 insertions(+), 76 deletions(-) (limited to 'src/lib') diff --git a/src/lib/libcrypto/aes/aes.c b/src/lib/libcrypto/aes/aes.c index e630c3f81a..e9dbe975e3 100644 --- a/src/lib/libcrypto/aes/aes.c +++ b/src/lib/libcrypto/aes/aes.c @@ -1,4 +1,4 @@ -/* $OpenBSD: aes.c,v 1.9 2025/06/03 08:42:15 kenjiro Exp $ */ +/* $OpenBSD: aes.c,v 1.10 2025/06/27 17:10:45 jsing Exp $ */ /* ==================================================================== * Copyright (c) 2002-2006 The OpenSSL Project. All rights reserved. * @@ -56,6 +56,7 @@ #include #include "crypto_arch.h" +#include "crypto_internal.h" static const unsigned char aes_wrap_default_iv[] = { 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, @@ -170,13 +171,59 @@ AES_cfb8_encrypt(const unsigned char *in, unsigned char *out, size_t length, } LCRYPTO_ALIAS(AES_cfb8_encrypt); +void +aes_ctr32_encrypt_generic(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]) +{ + uint8_t iv[AES_BLOCK_SIZE], buf[AES_BLOCK_SIZE]; + uint32_t ctr; + int i; + + memcpy(iv, ivec, sizeof(iv)); + + ctr = crypto_load_be32toh(&iv[12]); + + while (blocks > 0) { + crypto_store_htobe32(&iv[12], ctr); + aes_encrypt_internal(iv, buf, key); + ctr++; + + for (i = 0; i < AES_BLOCK_SIZE; i++) + out[i] = in[i] ^ buf[i]; + + in += 16; + out += 16; + blocks--; + } +} + +#ifdef HAVE_AES_CTR32_ENCRYPT_INTERNAL +void aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]); + +#else +static inline void +aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]) +{ + aes_ctr32_encrypt_generic(in, out, blocks, key, ivec); +} +#endif + +void +aes_ctr32_encrypt_ctr128f(const unsigned char *in, unsigned char *out, size_t blocks, + const void *key, const unsigned char ivec[AES_BLOCK_SIZE]) +{ + aes_ctr32_encrypt_internal(in, out, blocks, key, ivec); +} + void AES_ctr128_encrypt(const unsigned char *in, unsigned char *out, size_t length, const AES_KEY *key, unsigned char ivec[AES_BLOCK_SIZE], unsigned char ecount_buf[AES_BLOCK_SIZE], unsigned int *num) { - CRYPTO_ctr128_encrypt(in, out, length, key, ivec, ecount_buf, num, - aes_encrypt_block128); + CRYPTO_ctr128_encrypt_ctr32(in, out, length, key, ivec, ecount_buf, + num, aes_ctr32_encrypt_ctr128f); } LCRYPTO_ALIAS(AES_ctr128_encrypt); diff --git a/src/lib/libcrypto/aes/aes_amd64.c b/src/lib/libcrypto/aes/aes_amd64.c index 302d1ac91d..456409d186 100644 --- a/src/lib/libcrypto/aes/aes_amd64.c +++ b/src/lib/libcrypto/aes/aes_amd64.c @@ -1,4 +1,4 @@ -/* $OpenBSD: aes_amd64.c,v 1.1 2025/06/15 15:11:50 jsing Exp $ */ +/* $OpenBSD: aes_amd64.c,v 1.2 2025/06/27 17:10:45 jsing Exp $ */ /* * Copyright (c) 2025 Joel Sing * @@ -32,6 +32,9 @@ void aes_decrypt_generic(const unsigned char *in, unsigned char *out, void aes_cbc_encrypt_generic(const unsigned char *in, unsigned char *out, size_t len, const AES_KEY *key, unsigned char *ivec, const int enc); +void aes_ctr32_encrypt_generic(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]); + int aesni_set_encrypt_key(const unsigned char *userKey, int bits, AES_KEY *key); int aesni_set_decrypt_key(const unsigned char *userKey, int bits, @@ -45,6 +48,9 @@ void aesni_decrypt(const unsigned char *in, unsigned char *out, void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out, size_t len, const AES_KEY *key, unsigned char *ivec, const int enc); +void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t blocks, const void *key, const unsigned char *ivec); + int aes_set_encrypt_key_internal(const unsigned char *userKey, const int bits, AES_KEY *key) @@ -100,3 +106,15 @@ aes_cbc_encrypt_internal(const unsigned char *in, unsigned char *out, aes_cbc_encrypt_generic(in, out, len, key, ivec, enc); } + +void +aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]) +{ + if ((crypto_cpu_caps_amd64 & CRYPTO_CPU_CAPS_AMD64_AES) != 0) { + aesni_ctr32_encrypt_blocks(in, out, blocks, key, ivec); + return; + } + + aes_ctr32_encrypt_generic(in, out, blocks, key, ivec); +} diff --git a/src/lib/libcrypto/aes/aes_i386.c b/src/lib/libcrypto/aes/aes_i386.c index 0b5c89af70..2da02a8d35 100644 --- a/src/lib/libcrypto/aes/aes_i386.c +++ b/src/lib/libcrypto/aes/aes_i386.c @@ -1,4 +1,4 @@ -/* $OpenBSD: aes_i386.c,v 1.1 2025/06/15 15:11:50 jsing Exp $ */ +/* $OpenBSD: aes_i386.c,v 1.2 2025/06/27 17:10:45 jsing Exp $ */ /* * Copyright (c) 2025 Joel Sing * @@ -32,6 +32,9 @@ void aes_decrypt_generic(const unsigned char *in, unsigned char *out, void aes_cbc_encrypt_generic(const unsigned char *in, unsigned char *out, size_t len, const AES_KEY *key, unsigned char *ivec, const int enc); +void aes_ctr32_encrypt_generic(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]); + int aesni_set_encrypt_key(const unsigned char *userKey, int bits, AES_KEY *key); int aesni_set_decrypt_key(const unsigned char *userKey, int bits, @@ -45,6 +48,9 @@ void aesni_decrypt(const unsigned char *in, unsigned char *out, void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out, size_t len, const AES_KEY *key, unsigned char *ivec, const int enc); +void aesni_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out, + size_t blocks, const void *key, const unsigned char *ivec); + int aes_set_encrypt_key_internal(const unsigned char *userKey, const int bits, AES_KEY *key) @@ -100,3 +106,15 @@ aes_cbc_encrypt_internal(const unsigned char *in, unsigned char *out, aes_cbc_encrypt_generic(in, out, len, key, ivec, enc); } + +void +aes_ctr32_encrypt_internal(const unsigned char *in, unsigned char *out, + size_t blocks, const AES_KEY *key, const unsigned char ivec[AES_BLOCK_SIZE]) +{ + if ((crypto_cpu_caps_i386 & CRYPTO_CPU_CAPS_I386_AES) != 0) { + aesni_ctr32_encrypt_blocks(in, out, blocks, key, ivec); + return; + } + + aes_ctr32_encrypt_generic(in, out, blocks, key, ivec); +} diff --git a/src/lib/libcrypto/arch/amd64/crypto_arch.h b/src/lib/libcrypto/arch/amd64/crypto_arch.h index da1a22e003..9eb99229d1 100644 --- a/src/lib/libcrypto/arch/amd64/crypto_arch.h +++ b/src/lib/libcrypto/arch/amd64/crypto_arch.h @@ -1,4 +1,4 @@ -/* $OpenBSD: crypto_arch.h,v 1.6 2025/06/15 14:16:11 jsing Exp $ */ +/* $OpenBSD: crypto_arch.h,v 1.7 2025/06/27 17:10:45 jsing Exp $ */ /* * Copyright (c) 2024 Joel Sing * @@ -32,11 +32,12 @@ extern uint64_t crypto_cpu_caps_amd64; #ifndef OPENSSL_NO_ASM -#define HAVE_AES_CBC_ENCRYPT_INTERNAL #define HAVE_AES_SET_ENCRYPT_KEY_INTERNAL #define HAVE_AES_SET_DECRYPT_KEY_INTERNAL #define HAVE_AES_ENCRYPT_INTERNAL #define HAVE_AES_DECRYPT_INTERNAL +#define HAVE_AES_CBC_ENCRYPT_INTERNAL +#define HAVE_AES_CTR32_ENCRYPT_INTERNAL #define HAVE_RC4_INTERNAL #define HAVE_RC4_SET_KEY_INTERNAL diff --git a/src/lib/libcrypto/arch/i386/crypto_arch.h b/src/lib/libcrypto/arch/i386/crypto_arch.h index a693bd20d8..e03e6cd765 100644 --- a/src/lib/libcrypto/arch/i386/crypto_arch.h +++ b/src/lib/libcrypto/arch/i386/crypto_arch.h @@ -1,4 +1,4 @@ -/* $OpenBSD: crypto_arch.h,v 1.5 2025/06/15 14:18:31 jsing Exp $ */ +/* $OpenBSD: crypto_arch.h,v 1.6 2025/06/27 17:10:45 jsing Exp $ */ /* * Copyright (c) 2024 Joel Sing * @@ -31,11 +31,12 @@ extern uint64_t crypto_cpu_caps_i386; #ifndef OPENSSL_NO_ASM -#define HAVE_AES_CBC_ENCRYPT_INTERNAL #define HAVE_AES_SET_ENCRYPT_KEY_INTERNAL #define HAVE_AES_SET_DECRYPT_KEY_INTERNAL #define HAVE_AES_ENCRYPT_INTERNAL #define HAVE_AES_DECRYPT_INTERNAL +#define HAVE_AES_CBC_ENCRYPT_INTERNAL +#define HAVE_AES_CTR32_ENCRYPT_INTERNAL #define HAVE_RC4_INTERNAL #define HAVE_RC4_SET_KEY_INTERNAL diff --git a/src/lib/libcrypto/evp/e_aes.c b/src/lib/libcrypto/evp/e_aes.c index 34f5513300..b00eb048e8 100644 --- a/src/lib/libcrypto/evp/e_aes.c +++ b/src/lib/libcrypto/evp/e_aes.c @@ -1,4 +1,4 @@ -/* $OpenBSD: e_aes.c,v 1.75 2025/06/16 14:50:30 jsing Exp $ */ +/* $OpenBSD: e_aes.c,v 1.76 2025/06/27 17:10:45 jsing Exp $ */ /* ==================================================================== * Copyright (c) 2001-2011 The OpenSSL Project. All rights reserved. * @@ -159,21 +159,6 @@ void aesni_ccm64_decrypt_blocks (const unsigned char *in, unsigned char *out, size_t blocks, const void *key, const unsigned char ivec[16], unsigned char cmac[16]); -static int -aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, - const unsigned char *in, size_t len) -{ - EVP_AES_KEY *eak = ctx->cipher_data; - unsigned int num = ctx->num; - - CRYPTO_ctr128_encrypt_ctr32(in, out, len, &eak->ks, ctx->iv, ctx->buf, - &num, aesni_ctr32_encrypt_blocks); - - ctx->num = (size_t)num; - - return 1; -} - static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t len) @@ -562,19 +547,6 @@ EVP_aes_128_cfb8(void) } LCRYPTO_ALIAS(EVP_aes_128_cfb8); -#ifdef AESNI_CAPABLE -static const EVP_CIPHER aesni_128_ctr = { - .nid = NID_aes_128_ctr, - .block_size = 1, - .key_len = 16, - .iv_len = 16, - .flags = EVP_CIPH_CTR_MODE, - .init = aes_init_key, - .do_cipher = aesni_ctr_cipher, - .ctx_size = sizeof(EVP_AES_KEY), -}; -#endif - static const EVP_CIPHER aes_128_ctr = { .nid = NID_aes_128_ctr, .block_size = 1, @@ -589,11 +561,7 @@ static const EVP_CIPHER aes_128_ctr = { const EVP_CIPHER * EVP_aes_128_ctr(void) { -#ifdef AESNI_CAPABLE - return AESNI_CAPABLE ? &aesni_128_ctr : &aes_128_ctr; -#else return &aes_128_ctr; -#endif } LCRYPTO_ALIAS(EVP_aes_128_ctr); @@ -722,19 +690,6 @@ EVP_aes_192_cfb8(void) } LCRYPTO_ALIAS(EVP_aes_192_cfb8); -#ifdef AESNI_CAPABLE -static const EVP_CIPHER aesni_192_ctr = { - .nid = NID_aes_192_ctr, - .block_size = 1, - .key_len = 24, - .iv_len = 16, - .flags = EVP_CIPH_CTR_MODE, - .init = aes_init_key, - .do_cipher = aesni_ctr_cipher, - .ctx_size = sizeof(EVP_AES_KEY), -}; -#endif - static const EVP_CIPHER aes_192_ctr = { .nid = NID_aes_192_ctr, .block_size = 1, @@ -749,11 +704,7 @@ static const EVP_CIPHER aes_192_ctr = { const EVP_CIPHER * EVP_aes_192_ctr(void) { -#ifdef AESNI_CAPABLE - return AESNI_CAPABLE ? &aesni_192_ctr : &aes_192_ctr; -#else return &aes_192_ctr; -#endif } LCRYPTO_ALIAS(EVP_aes_192_ctr); @@ -882,19 +833,6 @@ EVP_aes_256_cfb8(void) } LCRYPTO_ALIAS(EVP_aes_256_cfb8); -#ifdef AESNI_CAPABLE -static const EVP_CIPHER aesni_256_ctr = { - .nid = NID_aes_256_ctr, - .block_size = 1, - .key_len = 32, - .iv_len = 16, - .flags = EVP_CIPH_CTR_MODE, - .init = aes_init_key, - .do_cipher = aesni_ctr_cipher, - .ctx_size = sizeof(EVP_AES_KEY), -}; -#endif - static const EVP_CIPHER aes_256_ctr = { .nid = NID_aes_256_ctr, .block_size = 1, @@ -909,11 +847,7 @@ static const EVP_CIPHER aes_256_ctr = { const EVP_CIPHER * EVP_aes_256_ctr(void) { -#ifdef AESNI_CAPABLE - return AESNI_CAPABLE ? &aesni_256_ctr : &aes_256_ctr; -#else return &aes_256_ctr; -#endif } LCRYPTO_ALIAS(EVP_aes_256_ctr); -- cgit v1.2.3-55-g6feb