Use LONG_MAX as the limit for ciphers with long based APIs.

These ciphers have long based APIs, while EVP has a size_t based API. The
intent of these loops is to handle sizes that are bigger than LONG_MAX.
Rather than using the rather crazy EVP_MAXCHUNK construct, use LONG_MAX
rounded down to a large block size, ensuring that it is a block size
multiple. Revert the recently added overflow checks now that this is
handled more appropriately.

ok tb@

1 files changed, 14 insertions, 22 deletions

diff --git a/src/lib/libcrypto/evp/e_rc2.c b/src/lib/libcrypto/evp/e_rc2.c
index 4f92365e7e..1af17a7c41 100644
--- a/src/lib/libcrypto/evp/e_rc2.c
+++ b/src/lib/libcrypto/evp/e_rc2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_rc2.c,v 1.18 2022/09/10 17:39:47 jsing Exp $ */
+/* $OpenBSD: e_rc2.c,v 1.19 2022/09/15 07:04:19 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -88,14 +88,13 @@ typedef struct {
 static int
 rc2_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
-        if (inl > LONG_MAX)
-                return 0;
+        size_t chunk = LONG_MAX & ~0xff;
 
-        while (inl >= EVP_MAXCHUNK) {
-                RC2_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
-                inl -= EVP_MAXCHUNK;
-                in += EVP_MAXCHUNK;
-                out += EVP_MAXCHUNK;
+        while (inl >= chunk) {
+                RC2_cbc_encrypt(in, out, (long)chunk, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
+                inl -= chunk;
+                in += chunk;
+                out += chunk;
         }
 
         if (inl)
@@ -107,10 +106,7 @@ rc2_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 static int
 rc2_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
-        size_t chunk = EVP_MAXCHUNK;
-
-        if (inl > LONG_MAX)
-                return 0;
+        size_t chunk = LONG_MAX & ~0xff;
 
         if (inl < chunk)
                 chunk = inl;
@@ -132,9 +128,6 @@ rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 {
         size_t i, bl;
 
-        if (inl > LONG_MAX)
-                return 0;
-
         bl = ctx->cipher->block_size;
 
         if (inl < bl)
@@ -151,14 +144,13 @@ rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 static int
 rc2_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
-        if (inl > LONG_MAX)
-                return 0;
+        size_t chunk = LONG_MAX & ~0xff;
 
-        while (inl >= EVP_MAXCHUNK) {
-                RC2_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
-                inl -= EVP_MAXCHUNK;
-                in += EVP_MAXCHUNK;
-                out += EVP_MAXCHUNK;
+        while (inl >= chunk) {
+                RC2_ofb64_encrypt(in, out, (long)chunk, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
+                inl -= chunk;
+                in += chunk;
+                out += chunk;
         }
 
         if (inl)