Fix return values of ecx methods

It is hard to get your return values right if you choose them to be a
random subset of {-2, ..., 3}. The item_verify() and the digestverify()
methods don't return 0 on error, but -1. Here 0 means "failed to verify",
obviously.

ok jsing

1 files changed, 5 insertions, 5 deletions

diff --git a/src/lib/libcrypto/ec/ecx_methods.c b/src/lib/libcrypto/ec/ecx_methods.c
index 8510d1a471..cc757d31b4 100644
--- a/src/lib/libcrypto/ec/ecx_methods.c
+++ b/src/lib/libcrypto/ec/ecx_methods.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ecx_methods.c,v 1.5 2023/03/15 06:34:07 tb Exp $ */
+/*      $OpenBSD: ecx_methods.c,v 1.6 2023/07/02 15:02:52 tb Exp $ */
 /*
  * Copyright (c) 2022 Joel Sing <jsing@openbsd.org>
  *
@@ -683,11 +683,11 @@ ecx_item_verify(EVP_MD_CTX *md_ctx, const ASN1_ITEM *it, void *asn,
 
         if (nid != NID_ED25519 || param_type != V_ASN1_UNDEF) {
                 ECerror(EC_R_INVALID_ENCODING);
-                return 0;
+                return -1;
         }
 
         if (!EVP_DigestVerifyInit(md_ctx, NULL, NULL, NULL, pkey))
-                return 0;
+                return -1;
 
         return 2;
 }
@@ -757,9 +757,9 @@ pkey_ecx_digestverify(EVP_MD_CTX *md_ctx, const unsigned char *sig,
         ecx_key = pkey_ctx->pkey->pkey.ecx;
 
         if (ecx_key == NULL || ecx_key->pub_key == NULL)
-                return 0;
+                return -1;
         if (sig_len != ecx_sig_size(pkey_ctx->pkey))
-                return 0;
+                return -1;
 
         return ED25519_verify(message, message_len, sig, ecx_key->pub_key);
 }