libtls: add missing length checks before BIO_new_mem_buf()

Like all proper libcrypto APIs, BIO_new_mem_buf() takes an int as a length
argument. Check the size_t passed in to be at most INT_MAX to avoid issues
with truncation and overflow like it's done everywhere else. After release
this should probably be clamped down further since legitimate files (certs
and keys) are nowhere near this large.

Prompted by a diff by Michael Forney

ok jsing

2 files changed, 17 insertions, 2 deletions

diff --git a/src/lib/libtls/tls_keypair.c b/src/lib/libtls/tls_keypair.c
index ffda91df8e..594b9af438 100644
--- a/src/lib/libtls/tls_keypair.c
+++ b/src/lib/libtls/tls_keypair.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls_keypair.c,v 1.9 2024/03/26 06:24:52 joshua Exp $ */
+/* $OpenBSD: tls_keypair.c,v 1.10 2026/04/16 05:16:48 tb Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -148,6 +148,11 @@ tls_keypair_load_cert(struct tls_keypair *keypair, struct tls_error *error,
                     "keypair has no certificate");
                 goto err;
         }
+        if (keypair->cert_len > INT_MAX) {
+                tls_error_setx(error, TLS_ERROR_INVALID_ARGUMENT,
+                    "certificate too long");
+                goto err;
+        }
         if ((cert_bio = BIO_new_mem_buf(keypair->cert_mem,
             keypair->cert_len)) == NULL) {
                 tls_error_set(error, TLS_ERROR_UNKNOWN,
diff --git a/src/lib/libtls/tls_signer.c b/src/lib/libtls/tls_signer.c
index 2573803ec1..ad80296830 100644
--- a/src/lib/libtls/tls_signer.c
+++ b/src/lib/libtls/tls_signer.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls_signer.c,v 1.13 2024/06/11 16:35:24 op Exp $ */
+/* $OpenBSD: tls_signer.c,v 1.14 2026/04/16 05:16:48 tb Exp $ */
 /*
  * Copyright (c) 2021 Eric Faurot <eric@openbsd.org>
  *
@@ -99,6 +99,11 @@ tls_signer_add_keypair_mem(struct tls_signer *signer, const uint8_t *cert,
         char *hash = NULL;
 
         /* Compute certificate hash */
+        if (cert_len > INT_MAX) {
+                tls_error_setx(&signer->error, TLS_ERROR_INVALID_ARGUMENT,
+                    "certificate too long");
+                goto err;
+        }
         if ((bio = BIO_new_mem_buf(cert, cert_len)) == NULL) {
                 tls_error_setx(&signer->error, TLS_ERROR_UNKNOWN,
                     "failed to create certificate bio");
@@ -124,6 +129,11 @@ tls_signer_add_keypair_mem(struct tls_signer *signer, const uint8_t *cert,
         bio = NULL;
 
         /* Read private key */
+        if (key_len > INT_MAX) {
+                tls_error_setx(&signer->error, TLS_ERROR_INVALID_ARGUMENT,
+                    "private key too long");
+                goto err;
+        }
         if ((bio = BIO_new_mem_buf(key, key_len)) == NULL) {
                 tls_error_setx(&signer->error, TLS_ERROR_UNKNOWN,
                     "failed to create key bio");