Return an int in BIO_set_cipher() to be able to report errors.

tested in a bulk by sthen
ok jsing

2 files changed, 22 insertions, 13 deletions

diff --git a/src/lib/libcrypto/evp/bio_enc.c b/src/lib/libcrypto/evp/bio_enc.c
index 712222a7cd..7b55998952 100644
--- a/src/lib/libcrypto/evp/bio_enc.c
+++ b/src/lib/libcrypto/evp/bio_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bio_enc.c,v 1.21 2018/05/02 15:51:41 tb Exp $ */
+/* $OpenBSD: bio_enc.c,v 1.22 2018/08/24 19:30:24 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -404,23 +404,32 @@ EVP_CIPHER_ctx *c;
         }
 */
 
-void
+int
 BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k,
     const unsigned char *i, int e)
 {
         BIO_ENC_CTX *ctx;
+        long (*cb)(BIO *, int, const char *, int, long, long);
 
         if (b == NULL)
-                return;
+                return 0;
 
-        if ((b->callback != NULL) &&
-            (b->callback(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 0L) <= 0))
-                return;
+        if ((ctx = BIO_get_data(b)) == NULL)
+                return 0;
 
-        b->init = 1;
-        ctx = (BIO_ENC_CTX *)b->ptr;
-        EVP_CipherInit_ex(&(ctx->cipher), c, NULL, k, i, e);
+        if ((cb = BIO_get_callback(b)) != NULL) {
+                if (cb(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 0L)
+                    <= 0)
+                        return 0;
+        }
 
-        if (b->callback != NULL)
-                b->callback(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 1L);
+        BIO_set_init(b, 1);
+
+        if (!EVP_CipherInit_ex(&(ctx->cipher), c, NULL, k, i, e))
+                return 0;
+
+        if (cb != NULL)
+                return cb(b, BIO_CB_CTRL, (const char *)c, BIO_CTRL_SET, e, 1L);
+
+        return 1;
 }
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index c557ba9f39..c6ec0e35d8 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.64 2018/05/30 15:40:50 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.65 2018/08/24 19:30:24 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -655,7 +655,7 @@ int EVP_CIPHER_CTX_rand_key(EVP_CIPHER_CTX *ctx, unsigned char *key);
 const BIO_METHOD *BIO_f_md(void);
 const BIO_METHOD *BIO_f_base64(void);
 const BIO_METHOD *BIO_f_cipher(void);
-void BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k,
+int BIO_set_cipher(BIO *b, const EVP_CIPHER *c, const unsigned char *k,
     const unsigned char *i, int enc);
 #endif