cherrypick fix for CVE-2012-2110: libcrypto ASN.1 parsing heap overflow

ok miod@ deraadt@

1 files changed, 17 insertions, 0 deletions

diff --git a/src/lib/libcrypto/buffer/buffer.c b/src/lib/libcrypto/buffer/buffer.c
index 620ea8d536..bc803ab6c8 100644
--- a/src/lib/libcrypto/buffer/buffer.c
+++ b/src/lib/libcrypto/buffer/buffer.c
@@ -60,6 +60,11 @@
 #include "cryptlib.h"
 #include <openssl/buffer.h>
 
+/* LIMIT_BEFORE_EXPANSION is the maximum n such that (n+3)/3*4 < 2**31. That
+ * function is applied in several functions in this file and this limit ensures
+ * that the result fits in an int. */
+#define LIMIT_BEFORE_EXPANSION 0x5ffffffc
+
 BUF_MEM *BUF_MEM_new(void)
         {
         BUF_MEM *ret;
@@ -105,6 +110,12 @@ int BUF_MEM_grow(BUF_MEM *str, size_t len)
                 str->length=len;
                 return(len);
                 }
+        /* This limit is sufficient to ensure (len+3)/3*4 < 2**31 */
+        if (len > LIMIT_BEFORE_EXPANSION)
+                {
+                BUFerr(BUF_F_BUF_MEM_GROW,ERR_R_MALLOC_FAILURE);
+                return 0;
+                }
         n=(len+3)/3*4;
         if (str->data == NULL)
                 ret=OPENSSL_malloc(n);
@@ -142,6 +153,12 @@ int BUF_MEM_grow_clean(BUF_MEM *str, size_t len)
                 str->length=len;
                 return(len);
                 }
+        /* This limit is sufficient to ensure (len+3)/3*4 < 2**31 */
+        if (len > LIMIT_BEFORE_EXPANSION)
+                {
+                BUFerr(BUF_F_BUF_MEM_GROW,ERR_R_MALLOC_FAILURE);
+                return 0;
+                }
         n=(len+3)/3*4;
         if (str->data == NULL)
                 ret=OPENSSL_malloc(n);