Add bounds checks for various EVP cipher implementations.

The EVP cipher API uses size_t, however a number of the underlying
implementations use long in their API. This means that an input with
size > LONG_MAX will go negative.

Found by Coverity, hiding under a large pile of macros.

ok tb@

6 files changed, 98 insertions, 7 deletions

diff --git a/src/lib/libcrypto/evp/e_bf.c b/src/lib/libcrypto/evp/e_bf.c
index ab6dc4f7de..4122f701da 100644
--- a/src/lib/libcrypto/evp/e_bf.c
+++ b/src/lib/libcrypto/evp/e_bf.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_bf.c,v 1.11 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_bf.c,v 1.12 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -85,6 +86,9 @@ bf_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 static int
 bf_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 BF_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_BF_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
                 inl -= EVP_MAXCHUNK;
@@ -103,6 +107,9 @@ bf_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in
 {
         size_t chunk = EVP_MAXCHUNK;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         if (inl < chunk)
                 chunk = inl;
 
@@ -123,6 +130,9 @@ bf_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 {
         size_t i, bl;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -139,6 +149,9 @@ bf_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 static int
 bf_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 BF_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_BF_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
                 inl -= EVP_MAXCHUNK;
diff --git a/src/lib/libcrypto/evp/e_cast.c b/src/lib/libcrypto/evp/e_cast.c
index d6f1b1d1a0..e654962c75 100644
--- a/src/lib/libcrypto/evp/e_cast.c
+++ b/src/lib/libcrypto/evp/e_cast.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_cast.c,v 1.10 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_cast.c,v 1.11 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -85,6 +86,9 @@ cast_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 static int
 cast5_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 CAST_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
                 inl -= EVP_MAXCHUNK;
@@ -103,6 +107,9 @@ cast5_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char
 {
         size_t chunk = EVP_MAXCHUNK;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         if (inl < chunk)
                 chunk = inl;
 
@@ -123,6 +130,9 @@ cast5_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 {
         size_t i, bl;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -139,6 +149,9 @@ cast5_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 static int
 cast5_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 CAST_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
                 inl -= EVP_MAXCHUNK;
diff --git a/src/lib/libcrypto/evp/e_des.c b/src/lib/libcrypto/evp/e_des.c
index bf037591be..9205128cf4 100644
--- a/src/lib/libcrypto/evp/e_des.c
+++ b/src/lib/libcrypto/evp/e_des.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_des.c,v 1.17 2022/09/04 13:17:18 jsing Exp $ */
+/* $OpenBSD: e_des.c,v 1.18 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -98,6 +99,9 @@ des_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
         size_t i, bl;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -108,6 +112,7 @@ des_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
         for (i = 0; i <= inl; i += bl)
                 DES_ecb_encrypt((DES_cblock *)(in + i), (DES_cblock *)(out + i),
                     ctx->cipher_data, ctx->encrypt);
+
         return 1;
 }
 
@@ -115,6 +120,9 @@ static int
 des_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data,
                     (DES_cblock *)ctx->iv, &ctx->num);
@@ -132,6 +140,9 @@ static int
 des_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_ncbc_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data,
                     (DES_cblock *)ctx->iv, ctx->encrypt);
@@ -149,6 +160,9 @@ static int
 des_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_cfb64_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data,
                     (DES_cblock *)ctx->iv, &ctx->num, ctx->encrypt);
@@ -171,6 +185,9 @@ des_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
         size_t n, chunk = EVP_MAXCHUNK/8;
         unsigned char c[1], d[1];
 
+        if (inl > LONG_MAX)
+                return 0;
+
         if (inl < chunk)
                 chunk = inl;
 
@@ -197,6 +214,9 @@ static int
 des_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_cfb_encrypt(in, out, 8, (long)EVP_MAXCHUNK,
                     ctx->cipher_data, (DES_cblock *)ctx->iv, ctx->encrypt);
diff --git a/src/lib/libcrypto/evp/e_des3.c b/src/lib/libcrypto/evp/e_des3.c
index e9d7f56809..1171a53b74 100644
--- a/src/lib/libcrypto/evp/e_des3.c
+++ b/src/lib/libcrypto/evp/e_des3.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_des3.c,v 1.23 2022/09/04 13:17:18 jsing Exp $ */
+/* $OpenBSD: e_des3.c,v 1.24 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -129,6 +130,9 @@ des_ede_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
         size_t i, bl;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -146,6 +150,9 @@ static int
 des_ede_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_ede3_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK,
                     &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
@@ -166,6 +173,9 @@ static int
 des_ede_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_ede3_cbc_encrypt(in, out, (long)EVP_MAXCHUNK,
                     &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
@@ -185,6 +195,9 @@ static int
 des_ede_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_ede3_cfb64_encrypt(in, out, (long)EVP_MAXCHUNK,
                     &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
@@ -208,6 +221,10 @@ des_ede3_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
         size_t n;
         unsigned char c[1], d[1];
+
+        if (inl > LONG_MAX)
+                return 0;
+
         if (!(ctx->flags & EVP_CIPH_FLAG_LENGTH_BITS))
                 inl *= 8;
 
@@ -227,6 +244,9 @@ static int
 des_ede3_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 DES_ede3_cfb_encrypt(in, out, 8, (long)EVP_MAXCHUNK,
                     &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
diff --git a/src/lib/libcrypto/evp/e_idea.c b/src/lib/libcrypto/evp/e_idea.c
index c25f031871..c7f2b30a44 100644
--- a/src/lib/libcrypto/evp/e_idea.c
+++ b/src/lib/libcrypto/evp/e_idea.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_idea.c,v 1.14 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_idea.c,v 1.15 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -102,6 +103,9 @@ idea_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
         size_t i, bl;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -121,6 +125,9 @@ typedef struct {
 static int
 idea_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 idea_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_IDEA_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
                 inl -= EVP_MAXCHUNK;
@@ -137,6 +144,9 @@ idea_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in
 static int
 idea_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 idea_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_IDEA_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
                 inl -= EVP_MAXCHUNK;
@@ -155,6 +165,9 @@ idea_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *
 {
         size_t chunk = EVP_MAXCHUNK;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         if (inl < chunk)
                 chunk = inl;
 
@@ -170,7 +183,6 @@ idea_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *
         return 1;
 }
 
-
 static const EVP_CIPHER idea_cbc = {
         .nid = NID_idea_cbc,
         .block_size = 8,
diff --git a/src/lib/libcrypto/evp/e_rc2.c b/src/lib/libcrypto/evp/e_rc2.c
index 6567e75b0c..72e582d5e0 100644
--- a/src/lib/libcrypto/evp/e_rc2.c
+++ b/src/lib/libcrypto/evp/e_rc2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_rc2.c,v 1.16 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_rc2.c,v 1.17 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -87,6 +88,9 @@ typedef struct {
 static int
 rc2_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 RC2_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
                 inl -= EVP_MAXCHUNK;
@@ -105,6 +109,9 @@ rc2_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 {
         size_t chunk = EVP_MAXCHUNK;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         if (inl < chunk)
                 chunk = inl;
 
@@ -125,6 +132,9 @@ rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 {
         size_t i, bl;
 
+        if (inl > LONG_MAX)
+                return 0;
+
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -141,6 +151,9 @@ rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 static int
 rc2_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+        if (inl > LONG_MAX)
+                return 0;
+
         while (inl >= EVP_MAXCHUNK) {
                 RC2_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
                 inl -= EVP_MAXCHUNK;