Add TLS_ERROR_INVALID_ARGUMENT error code to libtls

This is an initial pass, defining the error code and using it for
"too long"/length-related errors.

ok beck jsing

3 files changed, 15 insertions, 12 deletions

diff --git a/src/lib/libtls/tls.c b/src/lib/libtls/tls.c
index a8b03f0d4a..c2f7f3722c 100644
--- a/src/lib/libtls/tls.c
+++ b/src/lib/libtls/tls.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls.c,v 1.102 2024/03/26 08:54:48 joshua Exp $ */
+/* $OpenBSD: tls.c,v 1.103 2024/03/27 07:35:30 joshua Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -359,9 +359,9 @@ tls_keypair_to_pkey(struct tls *ctx, struct tls_keypair *keypair, EVP_PKEY **pke
                 return (0);
 
         if (len > INT_MAX) {
-                tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+                tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
                     ctx->config->use_fake_private_key ?
-                    "cert too long" : "key too long");
+                    "certificate too long" : "key too long");
                 goto err;
         }
 
@@ -491,7 +491,7 @@ tls_configure_ssl_keypair(struct tls *ctx, SSL_CTX *ssl_ctx,
 
         if (keypair->cert_mem != NULL) {
                 if (keypair->cert_len > INT_MAX) {
-                        tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+                        tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
                             "certificate too long");
                         goto err;
                 }
@@ -647,7 +647,8 @@ tls_configure_ssl_verify(struct tls *ctx, SSL_CTX *ssl_ctx, int verify)
 
         if (ca_mem != NULL) {
                 if (ca_len > INT_MAX) {
-                        tls_set_errorx(ctx, TLS_ERROR_UNKNOWN, "ca too long");
+                        tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
+                            "ca too long");
                         goto err;
                 }
                 if (SSL_CTX_load_verify_mem(ssl_ctx, ca_mem, ca_len) != 1) {
@@ -664,7 +665,8 @@ tls_configure_ssl_verify(struct tls *ctx, SSL_CTX *ssl_ctx, int verify)
 
         if (crl_mem != NULL) {
                 if (crl_len > INT_MAX) {
-                        tls_set_errorx(ctx, TLS_ERROR_UNKNOWN, "crl too long");
+                        tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
+                            "crl too long");
                         goto err;
                 }
                 if ((bio = BIO_new_mem_buf(crl_mem, crl_len)) == NULL) {
@@ -865,7 +867,7 @@ tls_read(struct tls *ctx, void *buf, size_t buflen)
         }
 
         if (buflen > INT_MAX) {
-                tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+                tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
                     "buflen too long");
                 goto out;
         }
@@ -897,7 +899,7 @@ tls_write(struct tls *ctx, const void *buf, size_t buflen)
         }
 
         if (buflen > INT_MAX) {
-                tls_set_errorx(ctx, TLS_ERROR_UNKNOWN,
+                tls_set_errorx(ctx, TLS_ERROR_INVALID_ARGUMENT,
                     "buflen too long");
                 goto out;
         }
diff --git a/src/lib/libtls/tls.h b/src/lib/libtls/tls.h
index b69c4af58c..67804d7cd8 100644
--- a/src/lib/libtls/tls.h
+++ b/src/lib/libtls/tls.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls.h,v 1.65 2024/03/26 08:54:48 joshua Exp $ */
+/* $OpenBSD: tls.h,v 1.66 2024/03/27 07:35:30 joshua Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -81,6 +81,7 @@ extern "C" {
 #define TLS_ERROR_UNKNOWN                       0x0000
 #define TLS_ERROR_OUT_OF_MEMORY                 0x1000
 #define TLS_ERROR_INVALID_CONTEXT               0x2000
+#define TLS_ERROR_INVALID_ARGUMENT              0x2001
 #endif
 
 struct tls;
diff --git a/src/lib/libtls/tls_config.c b/src/lib/libtls/tls_config.c
index 449071641b..645562e838 100644
--- a/src/lib/libtls/tls_config.c
+++ b/src/lib/libtls/tls_config.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls_config.c,v 1.68 2024/03/26 06:24:52 joshua Exp $ */
+/* $OpenBSD: tls_config.c,v 1.69 2024/03/27 07:35:30 joshua Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -321,12 +321,12 @@ tls_config_parse_alpn(struct tls_config *config, const char *alpn,
         q = s;
         while ((p = strsep(&q, ",")) != NULL) {
                 if ((len = strlen(p)) == 0) {
-                        tls_config_set_errorx(config, TLS_ERROR_UNKNOWN,
+                        tls_config_set_errorx(config, TLS_ERROR_INVALID_ARGUMENT,
                             "alpn protocol with zero length");
                         goto err;
                 }
                 if (len > 255) {
-                        tls_config_set_errorx(config, TLS_ERROR_UNKNOWN,
+                        tls_config_set_errorx(config, TLS_ERROR_INVALID_ARGUMENT,
                             "alpn protocol too long");
                         goto err;
                 }