Provide our own signature padding defines.

Rather than leaking libcrypto defines through the tls_sign_cb and
tls_signer_sign() interfaces, provide and use our own TLS_PADDING_*
defines.

ok inoguchi@ tb@

2 files changed, 38 insertions, 6 deletions

diff --git a/src/lib/libtls/tls.h b/src/lib/libtls/tls.h
index 91166bf9a7..91218b729d 100644
--- a/src/lib/libtls/tls.h
+++ b/src/lib/libtls/tls.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls.h,v 1.60 2022/02/01 17:13:10 jsing Exp $ */
+/* $OpenBSD: tls.h,v 1.61 2022/02/01 17:18:38 jsing Exp $ */
 /*
  * Copyright (c) 2014 Joel Sing <jsing@openbsd.org>
  *
@@ -72,6 +72,10 @@ extern "C" {
 #define TLS_MAX_SESSION_ID_LENGTH               32
 #define TLS_TICKET_KEY_SIZE                     48
 
+#define TLS_PADDING_NONE                        0
+#define TLS_PADDING_RSA_PKCS1                   1
+#define TLS_PADDING_RSA_X9_31                   2
+
 struct tls;
 struct tls_config;
 
diff --git a/src/lib/libtls/tls_signer.c b/src/lib/libtls/tls_signer.c
index d6429762e9..1f11096792 100644
--- a/src/lib/libtls/tls_signer.c
+++ b/src/lib/libtls/tls_signer.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tls_signer.c,v 1.3 2022/02/01 17:13:10 jsing Exp $ */
+/* $OpenBSD: tls_signer.c,v 1.4 2022/02/01 17:18:38 jsing Exp $ */
 /*
  * Copyright (c) 2021 Eric Faurot <eric@openbsd.org>
  *
@@ -183,12 +183,24 @@ tls_sign_rsa(struct tls_signer *signer, struct tls_signer_key *skey,
     const uint8_t *input, size_t input_len, int padding_type,
     uint8_t **out_signature, size_t *out_signature_len)
 {
-        int rsa_size, signature_len;
+        int rsa_padding, rsa_size, signature_len;
         char *signature = NULL;
 
         *out_signature = NULL;
         *out_signature_len = 0;
 
+        if (padding_type == TLS_PADDING_NONE) {
+                rsa_padding = RSA_NO_PADDING;
+        } else if (padding_type == TLS_PADDING_RSA_PKCS1) {
+                rsa_padding = RSA_PKCS1_PADDING;
+        } else if (padding_type == TLS_PADDING_RSA_X9_31) {
+                rsa_padding = RSA_X931_PADDING;
+        } else {
+                tls_error_setx(&signer->error, "invalid RSA padding type (%d)",
+                    padding_type);
+                return (-1);
+        }
+
         if (input_len > INT_MAX) {
                 tls_error_setx(&signer->error, "input too large");
                 return (-1);
@@ -204,7 +216,7 @@ tls_sign_rsa(struct tls_signer *signer, struct tls_signer_key *skey,
         }
 
         if ((signature_len = RSA_private_encrypt((int)input_len, input,
-            signature, skey->rsa, padding_type)) <= 0) {
+            signature, skey->rsa, rsa_padding)) <= 0) {
                 /* XXX - include further details from libcrypto. */
                 tls_error_setx(&signer->error, "RSA signing failed");
                 free(signature);
@@ -228,6 +240,11 @@ tls_sign_ecdsa(struct tls_signer *signer, struct tls_signer_key *skey,
         *out_signature = NULL;
         *out_signature_len = 0;
 
+        if (padding_type != TLS_PADDING_NONE) {
+                tls_error_setx(&signer->error, "invalid ECDSA padding");
+                return (-1);
+        }
+
         if (input_len > INT_MAX) {
                 tls_error_setx(&signer->error, "digest too large");
                 return (-1);
@@ -296,6 +313,7 @@ tls_rsa_priv_enc(int from_len, const unsigned char *from, unsigned char *to,
         uint8_t *signature = NULL;
         size_t signature_len = 0;
         const char *pubkey_hash;
+        int padding_type;
 
         /*
          * This function is called via RSA_private_encrypt() and has to conform
@@ -309,11 +327,21 @@ tls_rsa_priv_enc(int from_len, const unsigned char *from, unsigned char *to,
         if (pubkey_hash == NULL || config == NULL)
                 goto err;
 
+        if (rsa_padding == RSA_NO_PADDING) {
+                padding_type = TLS_PADDING_NONE;
+        } else if (rsa_padding == RSA_PKCS1_PADDING) {
+                padding_type = TLS_PADDING_RSA_PKCS1;
+        } else if (rsa_padding == RSA_X931_PADDING) {
+                padding_type = TLS_PADDING_RSA_X9_31;
+        } else {
+                goto err;
+        }
+
         if (from_len < 0)
                 goto err;
 
         if (config->sign_cb(config->sign_cb_arg, pubkey_hash, from, from_len,
-            rsa_padding, &signature, &signature_len) == -1)
+            padding_type, &signature, &signature_len) == -1)
                 goto err;
 
         if (signature_len > INT_MAX || (int)signature_len > RSA_size(rsa))
@@ -378,7 +406,7 @@ tls_ecdsa_do_sign(const unsigned char *dgst, int dgst_len, const BIGNUM *inv,
                 goto err;
 
         if (config->sign_cb(config->sign_cb_arg, pubkey_hash, dgst, dgst_len,
-            0, &signature, &signature_len) == -1)
+            TLS_PADDING_NONE, &signature, &signature_len) == -1)
                 goto err;
 
         p = signature;