Validate AES_set_{encrypt,decrypt}_key() inputs at API boundary.

Every aes_set_{encrypt,decrypt}_key_internal() implementation is currently
required to check the inputs and return appropriate error codes. Pull the
input validation up to the API boundary, setting key->rounds at the same
time. Additionally, call aes_set_encrypt_key_internal() directly from
aes_set_decrypt_key_internal(), rather than going back through the public
API.

ok tb@

2 files changed, 28 insertions, 18 deletions

diff --git a/src/lib/libcrypto/aes/aes.c b/src/lib/libcrypto/aes/aes.c
index 693badcd66..f9b2cfd9dd 100644
--- a/src/lib/libcrypto/aes/aes.c
+++ b/src/lib/libcrypto/aes/aes.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: aes.c,v 1.14 2025/07/22 09:13:49 jsing Exp $ */
+/* $OpenBSD: aes.c,v 1.15 2025/09/08 12:46:38 jsing Exp $ */
 /* ====================================================================
  * Copyright (c) 2002-2006 The OpenSSL Project.  All rights reserved.
  *
@@ -72,9 +72,27 @@ void aes_encrypt_internal(const unsigned char *in, unsigned char *out,
 void aes_decrypt_internal(const unsigned char *in, unsigned char *out,
     const AES_KEY *key);
 
+static int
+aes_rounds_for_key_length(int bits)
+{
+        if (bits == 128)
+                return 10;
+        if (bits == 192)
+                return 12;
+        if (bits == 256)
+                return 14;
+
+        return 0;
+}
+
 int
 AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key)
 {
+        if (userKey == NULL || key == NULL)
+                return -1;
+        if ((key->rounds = aes_rounds_for_key_length(bits)) <= 0)
+                return -2;
+
         return aes_set_encrypt_key_internal(userKey, bits, key);
 }
 LCRYPTO_ALIAS(AES_set_encrypt_key);
@@ -82,6 +100,11 @@ LCRYPTO_ALIAS(AES_set_encrypt_key);
 int
 AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key)
 {
+        if (userKey == NULL || key == NULL)
+                return -1;
+        if ((key->rounds = aes_rounds_for_key_length(bits)) <= 0)
+                return -2;
+
         return aes_set_decrypt_key_internal(userKey, bits, key);
 }
 LCRYPTO_ALIAS(AES_set_decrypt_key);
diff --git a/src/lib/libcrypto/aes/aes_core.c b/src/lib/libcrypto/aes/aes_core.c
index 8eccb998d3..2311547100 100644
--- a/src/lib/libcrypto/aes/aes_core.c
+++ b/src/lib/libcrypto/aes/aes_core.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: aes_core.c,v 1.27 2025/04/21 12:23:09 jsing Exp $ */
+/* $OpenBSD: aes_core.c,v 1.28 2025/09/08 12:46:38 jsing Exp $ */
 /**
  * rijndael-alg-fst.c
  *
@@ -645,20 +645,8 @@ aes_set_encrypt_key_internal(const unsigned char *userKey, const int bits,
         int i = 0;
         uint32_t temp;
 
-        if (!userKey || !key)
-                return -1;
-        if (bits != 128 && bits != 192 && bits != 256)
-                return -2;
-
         rk = key->rd_key;
 
-        if (bits == 128)
-                key->rounds = 10;
-        else if (bits == 192)
-                key->rounds = 12;
-        else
-                key->rounds = 14;
-
         rk[0] = crypto_load_be32toh(&userKey[0 * 4]);
         rk[1] = crypto_load_be32toh(&userKey[1 * 4]);
         rk[2] = crypto_load_be32toh(&userKey[2 * 4]);
@@ -746,13 +734,12 @@ aes_set_decrypt_key_internal(const unsigned char *userKey, const int bits,
     AES_KEY *key)
 {
         uint32_t *rk;
-        int i, j, status;
         uint32_t temp;
+        int i, j, ret;
 
         /* first, start with an encryption schedule */
-        status = AES_set_encrypt_key(userKey, bits, key);
-        if (status < 0)
-                return status;
+        if ((ret = aes_set_encrypt_key_internal(userKey, bits, key)) < 0)
+                return ret;
 
         rk = key->rd_key;