From 1e04f96479c885fa94175f42f348872cbdd3c9d4 Mon Sep 17 00:00:00 2001
From: jsing <>
Date: Mon, 26 May 2014 13:01:58 +0000
Subject: Implement an improved version of the EVP AEAD API. The
 EVP_AEAD_CTX_{open,seal} functions previously returned an ssize_t that was
 overloaded to indicate success/failure, along with the number of bytes
 written as output. This change adds an explicit *out_len argument which is
 used to return the number of output bytes and the return value is now an int
 that is purely used to identify success or failure.

This change effectively rides the last libcrypto crank (although I do not
expect there to be many users of the EVP AEAD API currently).

Thanks to Adam Langley for providing the improved code that this diff is
based on.

ok miod@
---
 src/lib/libcrypto/evp/evp_aead.c | 50 +++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 29 deletions(-)

(limited to 'src/lib/libcrypto/evp/evp_aead.c')

diff --git a/src/lib/libcrypto/evp/evp_aead.c b/src/lib/libcrypto/evp/evp_aead.c
index c8ba1df54a..427bf05467 100644
--- a/src/lib/libcrypto/evp/evp_aead.c
+++ b/src/lib/libcrypto/evp/evp_aead.c
@@ -126,67 +126,59 @@ check_alias(const unsigned char *in, size_t in_len, const unsigned char *out)
 	return 0;
 }
 
-ssize_t
-EVP_AEAD_CTX_seal(const EVP_AEAD_CTX *ctx, unsigned char *out,
+int
+EVP_AEAD_CTX_seal(const EVP_AEAD_CTX *ctx, unsigned char *out, size_t *out_len,
     size_t max_out_len, const unsigned char *nonce, size_t nonce_len,
     const unsigned char *in, size_t in_len, const unsigned char *ad,
     size_t ad_len)
 {
 	size_t possible_out_len = in_len + ctx->aead->overhead;
-	ssize_t r;
 
-	if (possible_out_len < in_len /* overflow */ ||
-	    possible_out_len > SSIZE_MAX /* return value cannot be
-					    represented */) {
-		EVPerr(EVP_F_EVP_AEAD_CTX_SEAL, EVP_R_TOO_LARGE);
+	/* Overflow. */
+	if (possible_out_len < in_len) {
+		EVPerr(EVP_F_AEAD_CTX_SEAL, EVP_R_TOO_LARGE);
 		goto error;
 	}
 
 	if (!check_alias(in, in_len, out)) {
-		EVPerr(EVP_F_EVP_AEAD_CTX_SEAL, EVP_R_OUTPUT_ALIASES_INPUT);
+		EVPerr(EVP_F_AEAD_CTX_SEAL, EVP_R_OUTPUT_ALIASES_INPUT);
 		goto error;
 	}
 
-	r = ctx->aead->seal(ctx, out, max_out_len, nonce, nonce_len,
-	    in, in_len, ad, ad_len);
-	if (r >= 0)
-		return r;
+	if (ctx->aead->seal(ctx, out, out_len, max_out_len, nonce, nonce_len,
+	    in, in_len, ad, ad_len)) {
+		return 1;
+	}
 
 error:
 	/* In the event of an error, clear the output buffer so that a caller
 	 * that doesn't check the return value doesn't send raw data. */
 	memset(out, 0, max_out_len);
-	return -1;
+	*out_len = 0;
+	return 0;
 }
 
-ssize_t
-EVP_AEAD_CTX_open(const EVP_AEAD_CTX *ctx, unsigned char *out,
+int
+EVP_AEAD_CTX_open(const EVP_AEAD_CTX *ctx, unsigned char *out, size_t *out_len,
     size_t max_out_len, const unsigned char *nonce, size_t nonce_len,
     const unsigned char *in, size_t in_len, const unsigned char *ad,
     size_t ad_len)
 {
-	ssize_t r;
-
-	if (in_len > SSIZE_MAX) {
-		EVPerr(EVP_F_EVP_AEAD_CTX_OPEN, EVP_R_TOO_LARGE);
-		goto error;  /* may not be able to represent return value. */
-	}
-
 	if (!check_alias(in, in_len, out)) {
-		EVPerr(EVP_F_EVP_AEAD_CTX_OPEN, EVP_R_OUTPUT_ALIASES_INPUT);
+		EVPerr(EVP_F_AEAD_CTX_OPEN, EVP_R_OUTPUT_ALIASES_INPUT);
 		goto error;
 	}
 
-	r = ctx->aead->open(ctx, out, max_out_len, nonce, nonce_len,
-	    in, in_len, ad, ad_len);
-
-	if (r >= 0)
-		return r;
+	if (ctx->aead->open(ctx, out, out_len, max_out_len, nonce, nonce_len,
+	    in, in_len, ad, ad_len)) {
+		return 1;
+	}
 
 error:
 	/* In the event of an error, clear the output buffer so that a caller
 	 * that doesn't check the return value doesn't try and process bad
 	 * data. */
 	memset(out, 0, max_out_len);
-	return -1;
+	*out_len = 0;
+	return 0;
 }
-- 
cgit v1.2.3-55-g6feb