Convert EVP_EncodeUpdate() to return an int to allow for error

checking. Matches our documented behavior.

Based on OpenSSL commit c5ebfcab713a82a1d46a51c8c2668c419425b387

tested in a bulk by sthen
ok jsing

2 files changed, 10 insertions, 8 deletions

diff --git a/src/lib/libcrypto/evp/encode.c b/src/lib/libcrypto/evp/encode.c
index 1097a7c903..07cfd7f2bc 100644
--- a/src/lib/libcrypto/evp/encode.c
+++ b/src/lib/libcrypto/evp/encode.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: encode.c,v 1.24 2016/05/04 15:05:13 tedu Exp $ */
+/* $OpenBSD: encode.c,v 1.25 2018/08/24 19:45:11 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -120,7 +120,7 @@ EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
         ctx->line_num = 0;
 }
 
-void
+int
 EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
@@ -128,13 +128,13 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
         size_t total = 0;
 
         *outl = 0;
-        if (inl == 0)
-                return;
+        if (inl <= 0)
+                return 0;
         OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
         if (ctx->length - ctx->num > inl) {
                 memcpy(&(ctx->enc_data[ctx->num]), in, inl);
                 ctx->num += inl;
-                return;
+                return 1;
         }
         if (ctx->num != 0) {
                 i = ctx->length - ctx->num;
@@ -160,12 +160,14 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
         if (total > INT_MAX) {
                 /* Too much output data! */
                 *outl = 0;
-                return;
+                return 0;
         }
         if (inl != 0)
                 memcpy(&(ctx->enc_data[0]), in, inl);
         ctx->num = inl;
         *outl = total;
+
+        return 1;
 }
 
 void
diff --git a/src/lib/libcrypto/evp/evp.h b/src/lib/libcrypto/evp/evp.h
index b607be7ae5..0ec0431bc7 100644
--- a/src/lib/libcrypto/evp/evp.h
+++ b/src/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.66 2018/08/24 19:36:52 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.67 2018/08/24 19:45:11 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -631,7 +631,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 int EVP_SealFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl);
 
 void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl);
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl);
 int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int n);