Add initial regress for RSA padding.

2 files changed, 328 insertions, 2 deletions

diff --git a/src/regress/lib/libcrypto/rsa/Makefile b/src/regress/lib/libcrypto/rsa/Makefile
index 53065e6408..494d40d867 100644
--- a/src/regress/lib/libcrypto/rsa/Makefile
+++ b/src/regress/lib/libcrypto/rsa/Makefile
@@ -1,6 +1,7 @@
-#       $OpenBSD: Makefile,v 1.3 2024/03/30 00:34:40 jsing Exp $
+#       $OpenBSD: Makefile,v 1.4 2024/03/30 00:36:14 jsing Exp $
 
-PROG=           rsa_test
+PROGS=          rsa_test \
+                rsa_padding_test
 LDADD=          -lcrypto
 DPADD=          ${LIBCRYPTO}
 WARNINGS=       Yes
diff --git a/src/regress/lib/libcrypto/rsa/rsa_padding_test.c b/src/regress/lib/libcrypto/rsa/rsa_padding_test.c
new file mode 100644
index 0000000000..64357bf405
--- /dev/null
+++ b/src/regress/lib/libcrypto/rsa/rsa_padding_test.c
@@ -0,0 +1,325 @@
+/*      $OpenBSD: rsa_padding_test.c,v 1.1 2024/03/30 00:36:14 jsing Exp $      */
+/*
+ * Copyright (c) 2024 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+
+#if 0
+static void
+hexdump(const unsigned char *buf, size_t len)
+{
+        size_t i;
+
+        for (i = 1; i <= len; i++)
+                fprintf(stderr, " 0x%02hhx,%s", buf[i - 1], i % 8 ? "" : "\n");
+
+        fprintf(stderr, "\n");
+}
+#endif
+
+struct pkcs1_test {
+        uint8_t in[128];
+        size_t in_len;
+        int want;
+        int want_error;
+};
+
+static const struct pkcs1_test pkcs1_type1_tests[] = {
+        {
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
+                        0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                        0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                },
+                .in_len = 32,
+                .want = 19,
+        },
+        {
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00,
+                },
+                .in_len = 11,
+                .want = 0,
+        },
+        {
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = 1,
+        },
+        {
+                /* Insufficient padding bytes (< 8). */
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0x00, 0xff, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BAD_PAD_BYTE_COUNT,
+        },
+        {
+                /* Incorrect padding type (0x00). */
+                .in = {
+                        0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
+        },
+        {
+                /* Incorrect padding type (0x02). */
+                .in = {
+                        0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BLOCK_TYPE_IS_NOT_01,
+        },
+        {
+                /* Non-padding byte before end of padding marker. */
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xfe, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BAD_FIXED_HEADER_DECRYPT,
+        },
+        {
+                /* No end of padding marker. */
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                },
+                .in_len = 32,
+                .want = -1,
+                .want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
+        },
+};
+
+#define N_PKCS1_TYPE1_TESTS \
+        (sizeof(pkcs1_type1_tests) / sizeof(pkcs1_type1_tests[0]))
+
+static int
+test_pkcs1_type1(void)
+{
+        const struct pkcs1_test *pt;
+        uint8_t buf[32], in[19], out[512];
+        int pad_len;
+        long err;
+        size_t i;
+        int failed = 1;
+
+        memset(in, 0x6f, sizeof(in));
+
+        if (!RSA_padding_add_PKCS1_type_1(buf, sizeof(buf), in, sizeof(in))) {
+                fprintf(stderr, "FAIL: failed to add PKCS1 type 1 padding\n");
+                goto failed;
+        }
+
+        pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
+            buf + 1, sizeof(buf) - 1, sizeof(buf));
+        if (pad_len != sizeof(in)) {
+                fprintf(stderr, "FAIL: failed to check PKCS1 type 1 padding\n");
+                ERR_print_errors_fp(stderr);
+                goto failed;
+        }
+
+        for (i = 0; i < N_PKCS1_TYPE1_TESTS; i++) {
+                pt = &pkcs1_type1_tests[i];
+
+                ERR_clear_error();
+
+                pad_len = RSA_padding_check_PKCS1_type_1(out, sizeof(out) - 1,
+                    pt->in + 1, pt->in_len - 1, pt->in_len);
+
+                if (pad_len != pt->want) {
+                        fprintf(stderr, "FAIL: test %zu - failed to check "
+                            "PKCS1 type 1 padding (%d != %d)\n", i, pad_len,
+                            pt->want);
+                        ERR_print_errors_fp(stderr);
+                        goto failed;
+                }
+
+                err = ERR_peek_error();
+                if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
+                        fprintf(stderr, "FAIL: test %zu - PKCS1 type 1 padding "
+                            "check failed with error reason %i, want %i\n",
+                            i, ERR_GET_REASON(err), pt->want_error);
+                        ERR_print_errors_fp(stderr);
+                        goto failed;
+                }
+        }
+
+        failed = 0;
+
+ failed:
+        return failed;
+}
+
+static const struct pkcs1_test pkcs1_type2_tests[] = {
+        {
+                .in = {
+                        0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0xff, 0xff, 0x00, 0x6f, 0x6f, 0x6f,
+                        0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                        0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                },
+                .in_len = 32,
+                .want = 19,
+        },
+        {
+                .in = {
+                        0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00,
+                },
+                .in_len = 11,
+                .want = 0,
+        },
+        {
+                .in = {
+                        0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = 1,
+        },
+        {
+                /* Insufficient padding bytes (< 8). */
+                .in = {
+                        0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0x00, 0xff, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BAD_PAD_BYTE_COUNT,
+        },
+        {
+                /* Incorrect padding type (0x00). */
+                .in = {
+                        0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
+        },
+        {
+                /* Incorrect padding type (0x01). */
+                .in = {
+                        0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0x00, 0xff,
+                },
+                .in_len = 12,
+                .want = -1,
+                .want_error = RSA_R_BLOCK_TYPE_IS_NOT_02,
+        },
+        {
+                /* No end of padding marker. */
+                .in = {
+                        0x00, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+                        0xff, 0xff, 0xff, 0xff, 0xff, 0x6f, 0x6f, 0x6f,
+                        0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                        0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f, 0x6f,
+                },
+                .in_len = 32,
+                .want = -1,
+                .want_error = RSA_R_NULL_BEFORE_BLOCK_MISSING,
+        },
+};
+
+#define N_PKCS1_TYPE2_TESTS \
+        (sizeof(pkcs1_type2_tests) / sizeof(pkcs1_type2_tests[0]))
+
+static int
+test_pkcs1_type2(void)
+{
+        const struct pkcs1_test *pt;
+        uint8_t buf[32], in[19], out[32];
+        int pad_len;
+        long err;
+        size_t i;
+        int failed = 1;
+
+        memset(in, 0x6f, sizeof(in));
+
+        if (!RSA_padding_add_PKCS1_type_2(buf, sizeof(buf), in, sizeof(in))) {
+                fprintf(stderr, "FAIL: failed to add PKCS1 type 2 padding\n");
+                goto failed;
+        }
+
+        pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
+            buf + 1, sizeof(buf) - 1, sizeof(out));
+        if (pad_len != sizeof(in)) {
+                fprintf(stderr, "FAIL: failed to check PKCS1 type 2 padding\n");
+                ERR_print_errors_fp(stderr);
+                goto failed;
+        }
+
+        for (i = 0; i < N_PKCS1_TYPE2_TESTS; i++) {
+                pt = &pkcs1_type2_tests[i];
+
+                ERR_clear_error();
+
+                pad_len = RSA_padding_check_PKCS1_type_2(out, sizeof(out) - 1,
+                    pt->in + 1, pt->in_len - 1, pt->in_len);
+
+                if (pad_len != pt->want) {
+                        fprintf(stderr, "FAIL: test %zu - failed to check "
+                            "PKCS1 type 2 padding (%d != %d)\n", i, pad_len,
+                            pt->want);
+                        ERR_print_errors_fp(stderr);
+                        goto failed;
+                }
+
+                err = ERR_peek_error();
+                if (pt->want == -1 && ERR_GET_REASON(err) != pt->want_error) {
+                        fprintf(stderr, "FAIL: test %zu - PKCS1 type 2 padding "
+                            "check failed with error reason %i, want %i\n",
+                            i, ERR_GET_REASON(err), pt->want_error);
+                        ERR_print_errors_fp(stderr);
+                        goto failed;
+                }
+        }
+
+        failed = 0;
+
+ failed:
+        return failed;
+}
+
+int
+main(int argc, char **argv)
+{
+        int failed = 0;
+
+        failed |= test_pkcs1_type1();
+        failed |= test_pkcs1_type2();
+
+        return failed;
+}