Replace assert() and OPENSSL_assert() calls with proper error return paths.

Careful review, feedback & ok doug@ jsing@

8 files changed, 87 insertions, 30 deletions

diff --git a/src/lib/libcrypto/evp/digest.c b/src/lib/libcrypto/evp/digest.c
index 4a18aff657..c9fb60d49b 100644
--- a/src/lib/libcrypto/evp/digest.c
+++ b/src/lib/libcrypto/evp/digest.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: digest.c,v 1.24 2014/11/09 19:12:18 miod Exp $ */
+/* $OpenBSD: digest.c,v 1.25 2015/02/10 09:52:35 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -249,7 +249,10 @@ EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *size)
 {
         int ret;
 
-        OPENSSL_assert(ctx->digest->md_size <= EVP_MAX_MD_SIZE);
+        if ((size_t)ctx->digest->md_size > EVP_MAX_MD_SIZE) {
+                EVPerr(EVP_F_EVP_DIGESTFINAL_EX, EVP_R_TOO_LARGE);
+                return 0;
+        }
         ret = ctx->digest->final(ctx, md);
         if (size != NULL)
                 *size = ctx->digest->md_size;
diff --git a/src/lib/libcrypto/evp/e_rc2.c b/src/lib/libcrypto/evp/e_rc2.c
index 456a22eeeb..9052195ac2 100644
--- a/src/lib/libcrypto/evp/e_rc2.c
+++ b/src/lib/libcrypto/evp/e_rc2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_rc2.c,v 1.10 2014/07/11 08:44:48 jsing Exp $ */
+/* $OpenBSD: e_rc2.c,v 1.11 2015/02/10 09:52:35 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -187,7 +187,11 @@ rc2_get_asn1_type_and_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 
         if (type != NULL) {
                 l = EVP_CIPHER_CTX_iv_length(c);
-                OPENSSL_assert(l <= sizeof(iv));
+                if (l > sizeof(iv)) {
+                        EVPerr(EVP_F_RC2_GET_ASN1_TYPE_AND_IV,
+                            EVP_R_IV_TOO_LARGE);
+                        return -1;
+                }
                 i = ASN1_TYPE_get_int_octetstring(type, &num, iv, l);
                 if (i != (int)l)
                         return (-1);
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index dd4d2245e6..6de762a4ff 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.42 2015/02/08 22:22:13 miod Exp $ */
+/* $OpenBSD: evp.h,v 1.43 2015/02/10 09:52:35 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1353,13 +1353,19 @@ void ERR_load_EVP_strings(void);
 #define EVP_F_EVP_AEAD_CTX_INIT                          180
 #define EVP_F_EVP_AEAD_CTX_OPEN                          190
 #define EVP_F_EVP_AEAD_CTX_SEAL                          191
+#define EVP_F_EVP_BYTESTOKEY                             200
 #define EVP_F_EVP_CIPHERINIT_EX                          123
 #define EVP_F_EVP_CIPHER_CTX_COPY                        163
 #define EVP_F_EVP_CIPHER_CTX_CTRL                        124
 #define EVP_F_EVP_CIPHER_CTX_SET_KEY_LENGTH              122
+#define EVP_F_EVP_CIPHER_GET_ASN1_IV                     201
+#define EVP_F_EVP_CIPHER_SET_ASN1_IV                     202
 #define EVP_F_EVP_DECRYPTFINAL_EX                        101
+#define EVP_F_EVP_DECRYPTUPDATE                          199
+#define EVP_F_EVP_DIGESTFINAL_EX                         196
 #define EVP_F_EVP_DIGESTINIT_EX                          128
 #define EVP_F_EVP_ENCRYPTFINAL_EX                        127
+#define EVP_F_EVP_ENCRYPTUPDATE                          198
 #define EVP_F_EVP_MD_CTX_COPY_EX                         110
 #define EVP_F_EVP_MD_CTX_CTRL                            195
 #define EVP_F_EVP_MD_SIZE                                162
@@ -1415,6 +1421,7 @@ void ERR_load_EVP_strings(void);
 #define EVP_F_PKCS5_V2_PBKDF2_KEYIVGEN                   164
 #define EVP_F_PKCS8_SET_BROKEN                           112
 #define EVP_F_PKEY_SET_TYPE                              158
+#define EVP_F_RC2_GET_ASN1_TYPE_AND_IV                   197
 #define EVP_F_RC2_MAGIC_TO_METH                          109
 #define EVP_F_RC5_CTRL                                   125
 
diff --git a/src/lib/libcrypto/evp/evp_enc.c b/src/lib/libcrypto/evp/evp_enc.c
index 49ceacefad..42ccfceec9 100644
--- a/src/lib/libcrypto/evp/evp_enc.c
+++ b/src/lib/libcrypto/evp/evp_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_enc.c,v 1.25 2014/10/22 13:02:04 jsing Exp $ */
+/* $OpenBSD: evp_enc.c,v 1.26 2015/02/10 09:52:35 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -140,10 +140,6 @@ EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
                         const EVP_CIPHER *c =
                             ENGINE_get_cipher(impl, cipher->nid);
                         if (!c) {
-                                /* One positive side-effect of US's export
-                                 * control history, is that we should at least
-                                 * be able to avoid using US mispellings of
-                                 * "initialisation"? */
                                 EVPerr(EVP_F_EVP_CIPHERINIT_EX,
                                     EVP_R_INITIALIZATION_ERROR);
                                 return 0;
@@ -186,9 +182,12 @@ EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
 skip_to_init:
 #endif
         /* we assume block size is a power of 2 in *cryptUpdate */
-        OPENSSL_assert(ctx->cipher->block_size == 1 ||
-            ctx->cipher->block_size == 8 ||
-            ctx->cipher->block_size == 16);
+        if (ctx->cipher->block_size != 1 &&
+            ctx->cipher->block_size != 8 &&
+            ctx->cipher->block_size != 16) {
+                EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_BAD_BLOCK_LENGTH);
+                return 0;
+        }
 
         if (!(EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_CUSTOM_IV)) {
                 switch (EVP_CIPHER_CTX_mode(ctx)) {
@@ -205,8 +204,12 @@ skip_to_init:
 
                 case EVP_CIPH_CBC_MODE:
 
-                        OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) <=
-                            (int)sizeof(ctx->iv));
+                        if ((size_t)EVP_CIPHER_CTX_iv_length(ctx) >
+                            sizeof(ctx->iv)) {
+                                EVPerr(EVP_F_EVP_CIPHERINIT_EX,
+                                    EVP_R_IV_TOO_LARGE);
+                                return 0;
+                        }
                         if (iv)
                                 memcpy(ctx->oiv, iv,
                                     EVP_CIPHER_CTX_iv_length(ctx));
@@ -325,7 +328,11 @@ EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
         }
         i = ctx->buf_len;
         bl = ctx->cipher->block_size;
-        OPENSSL_assert(bl <= (int)sizeof(ctx->buf));
+        if ((size_t)bl > sizeof(ctx->buf)) {
+                EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_BAD_BLOCK_LENGTH);
+                *outl = 0;
+                return 0;
+        }
         if (i != 0) {
                 if (i + inl < bl) {
                         memcpy(&(ctx->buf[i]), in, inl);
@@ -383,7 +390,10 @@ EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
         }
 
         b = ctx->cipher->block_size;
-        OPENSSL_assert(b <= sizeof ctx->buf);
+        if (b > sizeof ctx->buf) {
+                EVPerr(EVP_F_EVP_ENCRYPTFINAL_EX, EVP_R_BAD_BLOCK_LENGTH);
+                return 0;
+        }
         if (b == 1) {
                 *outl = 0;
                 return 1;
@@ -437,7 +447,10 @@ EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                 return EVP_EncryptUpdate(ctx, out, outl, in, inl);
 
         b = ctx->cipher->block_size;
-        OPENSSL_assert(b <= sizeof ctx->final);
+        if (b > sizeof ctx->final) {
+                EVPerr(EVP_F_EVP_DECRYPTUPDATE, EVP_R_BAD_BLOCK_LENGTH);
+                return 0;
+        }
 
         if (ctx->final_used) {
                 memcpy(out, ctx->final, b);
@@ -506,7 +519,11 @@ EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
                             EVP_R_WRONG_FINAL_BLOCK_LENGTH);
                         return (0);
                 }
-                OPENSSL_assert(b <= sizeof ctx->final);
+                if (b > sizeof ctx->final) {
+                        EVPerr(EVP_F_EVP_DECRYPTFINAL_EX,
+                            EVP_R_BAD_BLOCK_LENGTH);
+                        return 0;
+                }
                 n = ctx->final[b - 1];
                 if (n == 0 || n > (int)b) {
                         EVPerr(EVP_F_EVP_DECRYPTFINAL_EX, EVP_R_BAD_DECRYPT);
diff --git a/src/lib/libcrypto/evp/evp_key.c b/src/lib/libcrypto/evp/evp_key.c
index 1493ca9103..4718ab6175 100644
--- a/src/lib/libcrypto/evp/evp_key.c
+++ b/src/lib/libcrypto/evp/evp_key.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_key.c,v 1.20 2014/08/06 04:28:21 guenther Exp $ */
+/* $OpenBSD: evp_key.c,v 1.21 2015/02/10 09:52:35 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -59,6 +59,7 @@
 #include <stdio.h>
 #include <string.h>
 
+#include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/objects.h>
 #include <openssl/ui.h>
@@ -129,10 +130,18 @@ EVP_BytesToKey(const EVP_CIPHER *type, const EVP_MD *md,
         int niv, nkey, addmd = 0;
         unsigned int mds = 0, i;
         int rv = 0;
+
         nkey = type->key_len;
         niv = type->iv_len;
-        OPENSSL_assert(nkey <= EVP_MAX_KEY_LENGTH);
-        OPENSSL_assert(niv <= EVP_MAX_IV_LENGTH);
+
+        if ((size_t)nkey > EVP_MAX_KEY_LENGTH) {
+                EVPerr(EVP_F_EVP_BYTESTOKEY, EVP_R_BAD_KEY_LENGTH);
+                return 0;
+        }
+        if ((size_t)niv > EVP_MAX_IV_LENGTH) {
+                EVPerr(EVP_F_EVP_BYTESTOKEY, EVP_R_IV_TOO_LARGE);
+                return 0;
+        }
 
         if (data == NULL)
                 return (nkey);
diff --git a/src/lib/libcrypto/evp/evp_lib.c b/src/lib/libcrypto/evp/evp_lib.c
index 310252d0e8..491c8d6f67 100644
--- a/src/lib/libcrypto/evp/evp_lib.c
+++ b/src/lib/libcrypto/evp/evp_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_lib.c,v 1.13 2014/07/11 08:44:48 jsing Exp $ */
+/* $OpenBSD: evp_lib.c,v 1.14 2015/02/10 09:52:35 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -99,7 +99,11 @@ EVP_CIPHER_get_asn1_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 
         if (type != NULL) {
                 l = EVP_CIPHER_CTX_iv_length(c);
-                OPENSSL_assert(l <= sizeof(c->iv));
+                if (l > sizeof(c->iv)) {
+                        EVPerr(EVP_F_EVP_CIPHER_GET_ASN1_IV,
+                             EVP_R_IV_TOO_LARGE);
+                        return 0;
+                }
                 i = ASN1_TYPE_get_octetstring(type, c->oiv, l);
                 if (i != (int)l)
                         return (-1);
@@ -117,7 +121,11 @@ EVP_CIPHER_set_asn1_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 
         if (type != NULL) {
                 j = EVP_CIPHER_CTX_iv_length(c);
-                OPENSSL_assert(j <= sizeof(c->iv));
+                if (j > sizeof(c->iv)) {
+                        EVPerr(EVP_F_EVP_CIPHER_SET_ASN1_IV,
+                             EVP_R_IV_TOO_LARGE);
+                        return 0;
+                }
                 i = ASN1_TYPE_set_octetstring(type, c->oiv, j);
         }
         return (i);
diff --git a/src/lib/libcrypto/evp/p5_crpt.c b/src/lib/libcrypto/evp/p5_crpt.c
index 3b1419b545..112a69114c 100644
--- a/src/lib/libcrypto/evp/p5_crpt.c
+++ b/src/lib/libcrypto/evp/p5_crpt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: p5_crpt.c,v 1.14 2014/07/13 12:46:44 miod Exp $ */
+/* $OpenBSD: p5_crpt.c,v 1.15 2015/02/10 09:52:35 miod Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 1999.
  */
@@ -134,9 +134,15 @@ PKCS5_PBE_keyivgen(EVP_CIPHER_CTX *cctx, const char *pass, int passlen,
                 if (!EVP_DigestFinal_ex (&ctx, md_tmp, NULL))
                         goto err;
         }
-        OPENSSL_assert(EVP_CIPHER_key_length(cipher) <= (int)sizeof(md_tmp));
+        if ((size_t)EVP_CIPHER_key_length(cipher) > sizeof(md_tmp)) {
+                EVPerr(EVP_F_PKCS5_PBE_KEYIVGEN, EVP_R_BAD_KEY_LENGTH);
+                goto err;
+        }
         memcpy(key, md_tmp, EVP_CIPHER_key_length(cipher));
-        OPENSSL_assert(EVP_CIPHER_iv_length(cipher) <= 16);
+        if ((size_t)EVP_CIPHER_iv_length(cipher) > 16) {
+                EVPerr(EVP_F_PKCS5_PBE_KEYIVGEN, EVP_R_IV_TOO_LARGE);
+                goto err;
+        }
         memcpy(iv, md_tmp + (16 - EVP_CIPHER_iv_length(cipher)),
             EVP_CIPHER_iv_length(cipher));
         if (!EVP_CipherInit_ex(cctx, cipher, NULL, key, iv, en_de))
diff --git a/src/lib/libcrypto/evp/p5_crpt2.c b/src/lib/libcrypto/evp/p5_crpt2.c
index 61eadec804..c9eef8f49a 100644
--- a/src/lib/libcrypto/evp/p5_crpt2.c
+++ b/src/lib/libcrypto/evp/p5_crpt2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: p5_crpt2.c,v 1.17 2014/07/11 08:44:48 jsing Exp $ */
+/* $OpenBSD: p5_crpt2.c,v 1.18 2015/02/10 09:52:35 miod Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 1999.
  */
@@ -255,7 +255,10 @@ PKCS5_v2_PBKDF2_keyivgen(EVP_CIPHER_CTX *ctx, const char *pass, int passlen,
                 goto err;
         }
         keylen = EVP_CIPHER_CTX_key_length(ctx);
-        OPENSSL_assert(keylen <= sizeof key);
+        if (keylen > sizeof key) {
+                EVPerr(EVP_F_PKCS5_V2_PBKDF2_KEYIVGEN, EVP_R_BAD_KEY_LENGTH);
+                goto err;
+        }
 
         /* Decode parameter */