fix for integer overflow in encode and encrypt update functions.

additionally, in EncodeUpdate, if the amount written would overflow,
return 0 instead to prevent bugs in the caller.
CVE-2016-2105 and CVE-2016-2106 from openssl.

4 files changed, 22 insertions, 12 deletions

diff --git a/src/lib/libcrypto/evp/encode.c b/src/lib/libcrypto/evp/encode.c
index 637870cef6..1097a7c903 100644
--- a/src/lib/libcrypto/evp/encode.c
+++ b/src/lib/libcrypto/evp/encode.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: encode.c,v 1.23 2016/05/04 14:53:29 tedu Exp $ */
+/* $OpenBSD: encode.c,v 1.24 2016/05/04 15:05:13 tedu Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,13 +125,13 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
         int i, j;
-        unsigned int total = 0;
+        size_t total = 0;
 
         *outl = 0;
         if (inl == 0)
                 return;
         OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
-        if ((ctx->num + inl) < ctx->length) {
+        if (ctx->length - ctx->num > inl) {
                 memcpy(&(ctx->enc_data[ctx->num]), in, inl);
                 ctx->num += inl;
                 return;
@@ -148,7 +148,7 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                 *out = '\0';
                 total = j + 1;
         }
-        while (inl >= ctx->length) {
+        while (inl >= ctx->length && total <= INT_MAX) {
                 j = EVP_EncodeBlock(out, in, ctx->length);
                 in += ctx->length;
                 inl -= ctx->length;
@@ -157,6 +157,11 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                 *out = '\0';
                 total += j + 1;
         }
+        if (total > INT_MAX) {
+                /* Too much output data! */
+                *outl = 0;
+                return;
+        }
         if (inl != 0)
                 memcpy(&(ctx->enc_data[0]), in, inl);
         ctx->num = inl;
diff --git a/src/lib/libcrypto/evp/evp_enc.c b/src/lib/libcrypto/evp/evp_enc.c
index a4e369f622..556908fd10 100644
--- a/src/lib/libcrypto/evp/evp_enc.c
+++ b/src/lib/libcrypto/evp/evp_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_enc.c,v 1.29 2016/05/04 14:53:29 tedu Exp $ */
+/* $OpenBSD: evp_enc.c,v 1.30 2016/05/04 15:05:13 tedu Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -334,7 +334,7 @@ EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                 return 0;
         }
         if (i != 0) {
-                if (i + inl < bl) {
+                if (bl - i > inl) {
                         memcpy(&(ctx->buf[i]), in, inl);
                         ctx->buf_len += inl;
                         *outl = 0;
diff --git a/src/lib/libssl/src/crypto/evp/encode.c b/src/lib/libssl/src/crypto/evp/encode.c
index 637870cef6..1097a7c903 100644
--- a/src/lib/libssl/src/crypto/evp/encode.c
+++ b/src/lib/libssl/src/crypto/evp/encode.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: encode.c,v 1.23 2016/05/04 14:53:29 tedu Exp $ */
+/* $OpenBSD: encode.c,v 1.24 2016/05/04 15:05:13 tedu Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,13 +125,13 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
         int i, j;
-        unsigned int total = 0;
+        size_t total = 0;
 
         *outl = 0;
         if (inl == 0)
                 return;
         OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
-        if ((ctx->num + inl) < ctx->length) {
+        if (ctx->length - ctx->num > inl) {
                 memcpy(&(ctx->enc_data[ctx->num]), in, inl);
                 ctx->num += inl;
                 return;
@@ -148,7 +148,7 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                 *out = '\0';
                 total = j + 1;
         }
-        while (inl >= ctx->length) {
+        while (inl >= ctx->length && total <= INT_MAX) {
                 j = EVP_EncodeBlock(out, in, ctx->length);
                 in += ctx->length;
                 inl -= ctx->length;
@@ -157,6 +157,11 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                 *out = '\0';
                 total += j + 1;
         }
+        if (total > INT_MAX) {
+                /* Too much output data! */
+                *outl = 0;
+                return;
+        }
         if (inl != 0)
                 memcpy(&(ctx->enc_data[0]), in, inl);
         ctx->num = inl;
diff --git a/src/lib/libssl/src/crypto/evp/evp_enc.c b/src/lib/libssl/src/crypto/evp/evp_enc.c
index a4e369f622..556908fd10 100644
--- a/src/lib/libssl/src/crypto/evp/evp_enc.c
+++ b/src/lib/libssl/src/crypto/evp/evp_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_enc.c,v 1.29 2016/05/04 14:53:29 tedu Exp $ */
+/* $OpenBSD: evp_enc.c,v 1.30 2016/05/04 15:05:13 tedu Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -334,7 +334,7 @@ EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                 return 0;
         }
         if (i != 0) {
-                if (i + inl < bl) {
+                if (bl - i > inl) {
                         memcpy(&(ctx->buf[i]), in, inl);
                         ctx->buf_len += inl;
                         *outl = 0;