Add initial regress test framework

Add a test framework for use in LibreSSL regression tests. This test
framework aims to be as lightweight and as simple to use as possible.

The design is mostly inspired by Go's test system, and aims to be a
drop-in utility in most existing regress tests.

ok jsing tb beck

3 files changed, 408 insertions, 0 deletions

diff --git a/src/regress/lib/libcrypto/test/test.c b/src/regress/lib/libcrypto/test/test.c
new file mode 100644
index 0000000000..b48711919d
--- /dev/null
+++ b/src/regress/lib/libcrypto/test/test.c
@@ -0,0 +1,225 @@
+/*      $OpenBSD: test.c,v 1.1 2025/05/21 08:57:13 joshua Exp $ */
+/*
+ * Copyright (c) 2025 Joshua Sing <joshua@joshuasing.dev>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <err.h>
+#include <libgen.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "test.h"
+
+struct test {
+        struct test *parent;
+        char *name;
+        FILE *out;
+        int skipped;
+        int failed;
+};
+
+static struct test *
+test_new(struct test *pt, const char *name)
+{
+        struct test *t;
+
+        if ((t = calloc(1, sizeof(*t))) == NULL)
+                err(1, "calloc");
+
+        if (name != NULL) {
+                if ((t->name = strdup(name)) == NULL)
+                        err(1, "strdup");
+        }
+
+        t->out = pt->out;
+        t->parent = pt;
+
+        return t;
+}
+
+struct test *
+test_init(void)
+{
+        struct test *t;
+        char *tmp_file;
+        int out_fd;
+        char *v;
+
+        t = test_new(NULL, NULL);
+        t->out = stderr;
+
+        if (((v = getenv("TEST_VERBOSE")) != NULL) && strcmp(v, "0") != 0)
+                return t;
+
+        /* Create a temporary file for logging in non-verbose mode */
+        if ((tmp_file = strdup("/tmp/libressl-test.XXXXXXXX")) == NULL)
+                err(1, "strdup");
+        if ((out_fd = mkstemp(tmp_file)) == -1)
+                err(1, "mkstemp");
+
+        unlink(tmp_file);
+        if ((t->out = fdopen(out_fd, "w+")) == NULL)
+                err(1, "fdopen");
+
+        return t;
+}
+
+static void
+test_cleanup(struct test *t)
+{
+        free(t->name);
+        free(t);
+}
+
+int
+test_result(struct test *t)
+{
+        int failed = t->failed;
+
+        if (t->parent == NULL && t->out != stderr)
+                fclose(t->out);
+
+        test_cleanup(t);
+
+        return failed;
+}
+
+void
+test_fail(struct test *t)
+{
+        t->failed = 1;
+
+        /* Also fail parent. */
+        if (t->parent != NULL)
+                test_fail(t->parent);
+}
+
+static void
+test_vprintf(struct test *t, const char *fmt, va_list ap)
+{
+        if (vfprintf(t->out, fmt, ap) == -1)
+                err(1, "vfprintf");
+}
+
+void
+test_printf(struct test *t, const char *fmt, ...)
+{
+        va_list ap;
+
+        va_start(ap, fmt);
+        test_vprintf(t, fmt, ap);
+        va_end(ap);
+}
+
+static void
+test_vlogf_internal(struct test *t, const char *label, const char *func,
+    const char *file, int line, const char *fmt, va_list ap)
+{
+        char *msg = NULL;
+        char *l = ": ";
+        const char *filename;
+
+        if (label == NULL) {
+                label = "";
+                l = "";
+        }
+
+        if (vasprintf(&msg, fmt, ap) == -1)
+                err(1, "vasprintf");
+
+        if ((filename = strrchr(file, '/')) != NULL)
+                filename++;
+        else
+                filename = file;
+
+        test_printf(t, "%s [%s:%d]%s%s: %s\n",
+            func, filename, line, l, label, msg);
+
+        free(msg);
+}
+
+void
+test_logf_internal(struct test *t, const char *label, const char *func,
+    const char *file, int line, const char *fmt, ...)
+{
+        va_list ap;
+
+        va_start(ap, fmt);
+        test_vlogf_internal(t, label, func, file, line, fmt, ap);
+        va_end(ap);
+}
+
+void
+test_skip(struct test *t, const char *reason)
+{
+        t->skipped = 1;
+        test_printf(t, "%s\n", reason);
+}
+
+void
+test_skipf(struct test *t, const char *fmt, ...)
+{
+        va_list ap;
+
+        t->skipped = 1;
+
+        va_start(ap, fmt);
+        test_vprintf(t, fmt, ap);
+        if (fputc('\n', t->out) == EOF)
+                err(1, "fputc");
+        va_end(ap);
+}
+
+void
+test_run(struct test *pt, const char *name, test_run_func *fn, const void *arg)
+{
+        struct test *t = test_new(pt, name);
+        char *status = "PASS";
+        char buf[1024];
+        size_t buflen;
+        int ferr;
+
+        /* Run test */
+        test_printf(t, "=== RUN   %s\n", t->name);
+        fn(t, arg);
+
+        if (t->skipped)
+                status = "SKIP";
+        if (t->failed)
+                status = "FAIL";
+
+        test_printf(t, "--- %s: %s\n\n", status, t->name);
+
+        /* Print result of test */
+        if (t->failed && t->out != stderr) {
+                /* Copy logs to stderr */
+                rewind(t->out);
+                while ((buflen = fread(buf, 1, sizeof(buf), t->out)) > 0)
+                        fwrite(buf, 1, buflen, stderr);
+                if ((ferr = ferror(t->out)) != 0)
+                        errx(1, "ferror: %d", ferr);
+        }
+
+        if (t->out != NULL && t->out != stderr)  {
+                /* Reset output file */
+                rewind(t->out);
+                ftruncate(fileno(t->out), 0);
+        }
+
+        test_cleanup(t);
+}
diff --git a/src/regress/lib/libcrypto/test/test.h b/src/regress/lib/libcrypto/test/test.h
new file mode 100644
index 0000000000..e6cfec80ea
--- /dev/null
+++ b/src/regress/lib/libcrypto/test/test.h
@@ -0,0 +1,132 @@
+/*      $OpenBSD: test.h,v 1.1 2025/05/21 08:57:13 joshua Exp $ */
+/*
+ * Copyright (c) 2025 Joshua Sing <joshua@joshuasing.dev>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef HEADER_TEST_H
+#define HEADER_TEST_H
+
+#include <stdint.h>
+
+struct test;
+
+/*
+ * test_init creates a new root test struct.
+ *
+ * Additional tests may be run under the root test struct by calling test_run.
+ *
+ * If the TEST_VERBOSE environment variable is set and not equal to "0", then
+ * verbose mode will be enabled and all test logs will be written to stderr.
+ */
+struct test *test_init(void);
+
+/*
+ * test_result cleans up after all tests have completed and returns an
+ * appropriate exit code indicating the result of the tests.
+ */
+int test_result(struct test *_t);
+
+/*
+ * test_run_func is an individual test function. It is passed the test struct
+ * and an arbitrary argument which may be passed when test_run is called.
+ */
+typedef void (test_run_func)(struct test *_t, const void *_arg);
+
+/*
+ * test_fail marks the test and its parents as failed.
+ */
+void test_fail(struct test *_t);
+
+/*
+ * test_printf prints a test log message. When in verbose mode, the log message
+ * will be written to stderr, otherwise it will be buffered and only written to
+ * stderr if the test fails.
+ *
+ * This printf will write directly, without any additional formatting.
+ */
+void test_printf(struct test *_t, const char *_fmt, ...);
+    __attribute__((__format__ (printf, 2, 3)))
+    __attribute__((__nonnull__ (2)));
+
+/*
+ * test_logf_internal prints a test log message. When in verbose mode, the
+ * log message will be written to stderr, otherwise it will be buffered and
+ * only written to stderr if the test fails.
+ *
+ * label is an optional label indicating the severity of the log.
+ * func, file and line are used to show where the log comes from and are
+ * automatically set in the test log macros.
+ *
+ * This function should never be called directly.
+ */
+void test_logf_internal(struct test *_t, const char *_label, const char *_func,
+    const char *_file, int _line, const char *_fmt, ...);
+    __attribute__((__format__ (printf, 6, 7)))
+    __attribute__((__nonnull__ (6)));
+
+/*
+ * test_logf prints an informational log message. When in verbose mode, the log
+ * will be written to stderr, otherwise it will be buffered and only written to
+ * stderr if the test fails.
+ */
+#define test_logf(t, fmt, ...) \
+    test_logf_internal(t, NULL, __func__, __FILE__, __LINE__, fmt, ##__VA_ARGS__)
+
+/*
+ * test_errorf prints an error message. It will also cause the test to fail.
+ * If the test cannot proceed, it is recommended to return or goto a cleanup
+ * label.
+ *
+ * Tests should not fail-fast if continuing will provide more detailed
+ * information about what is broken.
+ */
+#define test_errorf(t, fmt, ...) \
+    test_logf_internal(t, "ERROR", __func__, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \
+    test_fail(t)
+
+/*
+ * test_skip marks the test as skipped. Once called, the test should return.
+ */
+void test_skip(struct test *_t, const char *_reason);
+
+/*
+ * test_skipf marks the test as skipped with a formatted reason. Once called,
+ * the test should return.
+ */
+void test_skipf(struct test *_t, const char *_fmt, ...);
+    __attribute__((__format__ (printf, 2, 3)))
+    __attribute__((__nonnull__ (2)));
+
+/*
+ * test_run runs a test function. It will create a new test struct with the
+ * given test as the parent. An argument may be provided to pass data to the
+ * test function, otherwise NULL should be passed.
+ *
+ * Each test should have a unique and informational name.
+ */
+void test_run(struct test *_t, const char *_name, test_run_func *_fn, const void *_arg);
+
+/*
+ * test_hexdump prints the given data as hexadecimal.
+ */
+void test_hexdump(struct test *_t, const unsigned char *_buf, size_t _len);
+
+/*
+ * test_hexdiff prints the given data as hexadecimal. If a second comparison
+ * buffer is not NULL, any differing bytes will be marked with an astrix.
+ */
+void test_hexdiff(struct test *_t, const uint8_t *_buf, size_t _len, const uint8_t *_compare);
+
+#endif /* HEADER_TEST_H */
diff --git a/src/regress/lib/libcrypto/test/test_util.c b/src/regress/lib/libcrypto/test/test_util.c
new file mode 100644
index 0000000000..6ecb574788
--- /dev/null
+++ b/src/regress/lib/libcrypto/test/test_util.c
@@ -0,0 +1,51 @@
+/*      $OpenBSD: test_util.c,v 1.1 2025/05/21 08:57:13 joshua Exp $ */
+/*
+ * Copyright (c) 2017 Joel Sing <jsing@openbsd.org>
+ * Copyright (c) 2024 Theo Buehler <tb@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+
+#include "test.h"
+
+void
+test_hexdump(struct test *t, const unsigned char *buf, size_t len)
+{
+        size_t i;
+
+        for (i = 1; i <= len; i++)
+                test_printf(t, " 0x%02x,%s", buf[i - 1], i % 8 ? "" : "\n");
+
+        if ((len % 8) != 0)
+                test_printf(t, "\n");
+}
+
+void
+test_hexdiff(struct test *t, const uint8_t *buf, size_t len, const uint8_t *compare)
+{
+        const char *mark = "", *newline;
+        size_t i;
+
+        for (i = 1; i <= len; i++) {
+                if (compare != NULL)
+                        mark = (buf[i - 1] != compare[i - 1]) ? "*" : " ";
+                newline = i % 8 ? "" : "\n";
+                test_printf(t, " %s0x%02x,%s", mark, buf[i - 1], newline);
+        }
+
+        if ((len % 8) != 0)
+                test_printf(t, "\n");
+}