Use LONG_MAX as the limit for ciphers with long based APIs.

These ciphers have long based APIs, while EVP has a size_t based API. The
intent of these loops is to handle sizes that are bigger than LONG_MAX.
Rather than using the rather crazy EVP_MAXCHUNK construct, use LONG_MAX
rounded down to a large block size, ensuring that it is a block size
multiple. Revert the recently added overflow checks now that this is
handled more appropriately.

ok tb@

1 files changed, 16 insertions, 24 deletions

diff --git a/src/lib/libcrypto/evp/e_cast.c b/src/lib/libcrypto/evp/e_cast.c
index f5654d9f3e..702c26e0c3 100644
--- a/src/lib/libcrypto/evp/e_cast.c
+++ b/src/lib/libcrypto/evp/e_cast.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_cast.c,v 1.12 2022/09/10 17:39:47 jsing Exp $ */
+/* $OpenBSD: e_cast.c,v 1.13 2022/09/15 07:04:19 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -86,14 +86,13 @@ cast_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 static int
 cast5_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
-        if (inl > LONG_MAX)
-                return 0;
-
-        while (inl >= EVP_MAXCHUNK) {
-                CAST_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
-                inl -= EVP_MAXCHUNK;
-                in += EVP_MAXCHUNK;
-                out += EVP_MAXCHUNK;
+        size_t chunk = LONG_MAX & ~0xff;
+
+        while (inl >= chunk) {
+                CAST_cbc_encrypt(in, out, (long)chunk, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
+                inl -= chunk;
+                in += chunk;
+                out += chunk;
         }
 
         if (inl)
@@ -105,10 +104,7 @@ cast5_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 static int
 cast5_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
-        size_t chunk = EVP_MAXCHUNK;
-
-        if (inl > LONG_MAX)
-                return 0;
+        size_t chunk = LONG_MAX & ~0xff;
 
         if (inl < chunk)
                 chunk = inl;
@@ -130,9 +126,6 @@ cast5_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 {
         size_t i, bl;
 
-        if (inl > LONG_MAX)
-                return 0;
-
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -149,14 +142,13 @@ cast5_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 static int
 cast5_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
-        if (inl > LONG_MAX)
-                return 0;
-
-        while (inl >= EVP_MAXCHUNK) {
-                CAST_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
-                inl -= EVP_MAXCHUNK;
-                in += EVP_MAXCHUNK;
-                out += EVP_MAXCHUNK;
+        size_t chunk = LONG_MAX & ~0xff;
+
+        while (inl >= chunk) {
+                CAST_ofb64_encrypt(in, out, (long)chunk, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
+                inl -= chunk;
+                in += chunk;
+                out += chunk;
         }
 
         if (inl)