rework tests Makefile.am

There are so many test exceptions that need handling that it is easier
to simply edit it directly rather than doing autogeneration anymore.

This also puts biotest and pidwraptest behind a new --enable-extratests
option, so they are easy to run but are not enabled by default.

5 files changed, 289 insertions, 89 deletions

diff --git a/.gitignore b/.gitignore
index 6c70806..471ca3a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -92,7 +92,6 @@ INSTALL
 /stamp-h2
 
 include/openssl/Makefile.am
-tests/Makefile.am
 
 crypto/VERSION
 ssl/VERSION
diff --git a/configure.ac b/configure.ac
index 6a50c1b..af4b26e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -287,6 +287,10 @@ AC_ARG_ENABLE([asm],
         AS_HELP_STRING([--disable-asm], [Disable assembly]))
 AM_CONDITIONAL([OPENSSL_NO_ASM], [test "x$enable_asm" = "xno"])
 
+AC_ARG_ENABLE([extratests],
+        AS_HELP_STRING([--enable-extratests], [Enable extra tests that may be unreliable on some platforms]))
+AM_CONDITIONAL([ENABLE_EXTRATESTS], [test "x$enable_extratests" = xyes])
+
 old_cflags=$CFLAGS
 CFLAGS="$old_cflags -I$srcdir/include"
 AC_TRY_COMPILE([#include "$srcdir/crypto/modes/modes_lcl.h"],
diff --git a/tests/Makefile.am b/tests/Makefile.am
new file mode 100644
index 0000000..2ed7a44
--- /dev/null
+++ b/tests/Makefile.am
@@ -0,0 +1,282 @@
+include $(top_srcdir)/Makefile.am.common
+
+AM_CPPFLAGS += -I $(top_srcdir)/crypto/modes
+AM_CPPFLAGS += -I $(top_srcdir)/crypto/asn1
+AM_CPPFLAGS += -I $(top_srcdir)/ssl
+
+LDADD = $(PLATFORM_LDADD) $(PROG_LDADD)
+LDADD += $(top_builddir)/ssl/libssl.la
+LDADD += $(top_builddir)/crypto/libcrypto.la
+
+TESTS =
+check_PROGRAMS =
+EXTRA_DIST =
+DISTCLEANFILES = pidwraptest.txt
+
+# aeadtest
+TESTS += aeadtest.sh
+check_PROGRAMS += aeadtest
+aeadtest_SOURCES = aeadtest.c
+EXTRA_DIST += aeadtest.sh
+EXTRA_DIST += aeadtests.txt
+
+# aes_wrap
+TESTS += aes_wrap
+check_PROGRAMS += aes_wrap
+aes_wrap_SOURCES = aes_wrap.c
+
+# arc4randomforktest
+# Windows/mingw does not have fork, but Cygwin does.
+if !HOST_WIN
+TESTS += arc4randomforktest.sh
+check_PROGRAMS += arc4randomforktest
+arc4randomforktest_SOURCES = arc4randomforktest.c
+endif
+EXTRA_DIST += arc4randomforktest.sh
+
+# asn1test
+TESTS += asn1test
+check_PROGRAMS += asn1test
+asn1test_SOURCES = asn1test.c
+
+# base64test
+TESTS += base64test
+check_PROGRAMS += base64test
+base64test_SOURCES = base64test.c
+
+# bftest
+TESTS += bftest
+check_PROGRAMS += bftest
+bftest_SOURCES = bftest.c
+
+# biotest
+# the BIO tests rely on resolver results that are OS and environment-specific
+if ENABLE_EXTRATESTS
+TESTS += biotest
+check_PROGRAMS += biotest
+biotest_SOURCES = biotest.c
+endif
+
+# bntest
+TESTS += bntest
+check_PROGRAMS += bntest
+bntest_SOURCES = bntest.c
+
+# bytestringtest
+TESTS += bytestringtest
+check_PROGRAMS += bytestringtest
+bytestringtest_SOURCES = bytestringtest.c
+
+# casttest
+TESTS += casttest
+check_PROGRAMS += casttest
+casttest_SOURCES = casttest.c
+
+# chachatest
+TESTS += chachatest
+check_PROGRAMS += chachatest
+chachatest_SOURCES = chachatest.c
+
+# cipherstest
+TESTS += cipherstest
+check_PROGRAMS += cipherstest
+cipherstest_SOURCES = cipherstest.c
+
+# cts128test
+TESTS += cts128test
+check_PROGRAMS += cts128test
+cts128test_SOURCES = cts128test.c
+
+# destest
+TESTS += destest
+check_PROGRAMS += destest
+destest_SOURCES = destest.c
+
+# dhtest
+TESTS += dhtest
+check_PROGRAMS += dhtest
+dhtest_SOURCES = dhtest.c
+
+# dsatest
+TESTS += dsatest
+check_PROGRAMS += dsatest
+dsatest_SOURCES = dsatest.c
+
+# ecdhtest
+TESTS += ecdhtest
+check_PROGRAMS += ecdhtest
+ecdhtest_SOURCES = ecdhtest.c
+
+# ecdsatest
+TESTS += ecdsatest
+check_PROGRAMS += ecdsatest
+ecdsatest_SOURCES = ecdsatest.c
+
+# ectest
+TESTS += ectest
+check_PROGRAMS += ectest
+ectest_SOURCES = ectest.c
+
+# enginetest
+TESTS += enginetest
+check_PROGRAMS += enginetest
+enginetest_SOURCES = enginetest.c
+
+# evptest
+TESTS += evptest.sh
+check_PROGRAMS += evptest
+evptest_SOURCES = evptest.c
+EXTRA_DIST += evptest.sh
+EXTRA_DIST += evptests.txt
+
+# explicit_bzero
+# explicit_bzero relies on SA_ONSTACK, which is unavailable on Windows
+if !HOST_WIN
+if !HOST_CYGWIN
+TESTS += explicit_bzero
+check_PROGRAMS += explicit_bzero
+explicit_bzero_SOURCES = explicit_bzero.c
+if !HAVE_MEMMEM
+explicit_bzero_SOURCES += memmem.c
+endif
+endif
+endif
+
+# exptest
+TESTS += exptest
+check_PROGRAMS += exptest
+exptest_SOURCES = exptest.c
+
+# gcm128test
+TESTS += gcm128test
+check_PROGRAMS += gcm128test
+gcm128test_SOURCES = gcm128test.c
+
+# gost2814789t
+TESTS += gost2814789t
+check_PROGRAMS += gost2814789t
+gost2814789t_SOURCES = gost2814789t.c
+
+# hmactest
+TESTS += hmactest
+check_PROGRAMS += hmactest
+hmactest_SOURCES = hmactest.c
+
+# ideatest
+TESTS += ideatest
+check_PROGRAMS += ideatest
+ideatest_SOURCES = ideatest.c
+
+# igetest
+TESTS += igetest
+check_PROGRAMS += igetest
+igetest_SOURCES = igetest.c
+
+# md4test
+TESTS += md4test
+check_PROGRAMS += md4test
+md4test_SOURCES = md4test.c
+
+# md5test
+TESTS += md5test
+check_PROGRAMS += md5test
+md5test_SOURCES = md5test.c
+
+# mdc2test
+TESTS += mdc2test
+check_PROGRAMS += mdc2test
+mdc2test_SOURCES = mdc2test.c
+
+# mont
+TESTS += mont
+check_PROGRAMS += mont
+mont_SOURCES = mont.c
+
+# pbkdf2
+TESTS += pbkdf2
+check_PROGRAMS += pbkdf2
+pbkdf2_SOURCES = pbkdf2.c
+
+# pidwraptest
+# pidwraptest relies on an OS-specific way to give out pids and is generally
+# awkward on systems with slow fork
+if ENABLE_EXTRATESTS
+TESTS += pidwraptest
+check_PROGRAMS += pidwraptest
+pidwraptest_SOURCES = pidwraptest.c
+endif
+
+# pkcs7test
+TESTS += pkcs7test
+check_PROGRAMS += pkcs7test
+pkcs7test_SOURCES = pkcs7test.c
+
+# poly1305test
+TESTS += poly1305test
+check_PROGRAMS += poly1305test
+poly1305test_SOURCES = poly1305test.c
+
+# pq_test
+TESTS += pq_test.sh
+check_PROGRAMS += pq_test
+pq_test_SOURCES = pq_test.c
+EXTRA_DIST += pq_test.sh
+EXTRA_DIST += pq_expected.txt
+
+# randtest
+TESTS += randtest
+check_PROGRAMS += randtest
+randtest_SOURCES = randtest.c
+
+# rc2test
+TESTS += rc2test
+check_PROGRAMS += rc2test
+rc2test_SOURCES = rc2test.c
+
+# rc4test
+TESTS += rc4test
+check_PROGRAMS += rc4test
+rc4test_SOURCES = rc4test.c
+
+# rmdtest
+TESTS += rmdtest
+check_PROGRAMS += rmdtest
+rmdtest_SOURCES = rmdtest.c
+
+# sha1test
+TESTS += sha1test
+check_PROGRAMS += sha1test
+sha1test_SOURCES = sha1test.c
+
+# sha256test
+TESTS += sha256test
+check_PROGRAMS += sha256test
+sha256test_SOURCES = sha256test.c
+
+# sha512test
+TESTS += sha512test
+check_PROGRAMS += sha512test
+sha512test_SOURCES = sha512test.c
+
+# shatest
+TESTS += shatest
+check_PROGRAMS += shatest
+shatest_SOURCES = shatest.c
+
+# ssltest
+TESTS += ssltest.sh
+check_PROGRAMS += ssltest
+ssltest_SOURCES = ssltest.c
+EXTRA_DIST += ssltest.sh
+EXTRA_DIST += testssl ca.pem server.pem
+
+# timingsafe
+TESTS += timingsafe
+check_PROGRAMS += timingsafe
+timingsafe_SOURCES = timingsafe.c
+
+# utf8test
+TESTS += utf8test
+check_PROGRAMS += utf8test
+utf8test_SOURCES = utf8test.c
+
diff --git a/tests/Makefile.am.tpl b/tests/Makefile.am.tpl
deleted file mode 100644
index 26334af..0000000
--- a/tests/Makefile.am.tpl
+++ /dev/null
@@ -1,15 +0,0 @@
-include $(top_srcdir)/Makefile.am.common
-
-AM_CPPFLAGS += -I $(top_srcdir)/crypto/modes
-AM_CPPFLAGS += -I $(top_srcdir)/crypto/asn1
-AM_CPPFLAGS += -I $(top_srcdir)/ssl
-
-LDADD = $(PLATFORM_LDADD) $(PROG_LDADD)
-LDADD += $(top_builddir)/ssl/libssl.la
-LDADD += $(top_builddir)/crypto/libcrypto.la
-
-TESTS =
-check_PROGRAMS =
-EXTRA_DIST =
-DISTCLEANFILES = pidwraptest.txt
-
diff --git a/update.sh b/update.sh
index 435c888..7222266 100755
--- a/update.sh
+++ b/update.sh
@@ -208,82 +208,12 @@ $CP $libssl_regress/ssl/testssl tests
 for i in `find $libssl_regress -name '*.c'`; do
          $CP "$i" tests
 done
+
 $CP $libssl_regress/certs/ca.pem tests
 $CP $libssl_regress/certs/server.pem tests
-
-# setup test drivers
-# do not directly run all test programs
-test_drivers=(
-        aeadtest
-        evptest
-        pq_test
-        ssltest
-        arc4randomforktest
-        pidwraptest
-)
-tests_posix_only=(
-        arc4randomforktest
-        explicit_bzero
-        pidwraptest
-)
-# the BIO tests rely on resolver results that are OS and environment-specific
-# explicit_bzero relies on SA_ONSTACK, not available on all systems
-# pidwraptest relies on an OS-specific way to give out pids and is generally
-# awkward on systems with slow fork
-tests_disabled=(
-        biotest
-        explicit_bzero
-        pidwraptest
-)
-$CP $libc_src/string/memmem.c tests/
-(cd tests
-        $CP Makefile.am.tpl Makefile.am
-
-        for i in `ls -1 *.c|sort|grep -v memmem.c`; do
-                TEST=`echo $i|sed -e "s/\.c//"`
-                if ! [[ ${tests_disabled[*]} =~ "$TEST" ]]; then
-                        if [[ ${tests_posix_only[*]} =~ "$TEST" ]]; then
-                                echo "if !HOST_WIN" >> Makefile.am
-                        fi
-                        if ! [[ ${test_drivers[*]} =~ "$TEST" ]]; then
-                                echo "TESTS += $TEST" >> Makefile.am
-                        fi
-                        echo "check_PROGRAMS += $TEST" >> Makefile.am
-                        echo "${TEST}_SOURCES = $i" >> Makefile.am
-                        if [[ ${TEST} = "explicit_bzero" ]]; then
-                                echo "if !HAVE_MEMMEM" >> Makefile.am
-                                echo "explicit_bzero_SOURCES += memmem.c" >> Makefile.am
-                                echo "endif" >> Makefile.am
-                        fi
-                        if [[ ${tests_posix_only[*]} =~ "$TEST" ]]; then
-                                echo "endif" >> Makefile.am
-                        fi
-                fi
-        done
-)
-$CP $libcrypto_regress/evp/evptests.txt tests
-$CP $libcrypto_regress/aead/aeadtests.txt tests
-$CP $libcrypto_regress/pqueue/expected.txt tests/pq_expected.txt
 chmod 755 tests/testssl
-for i in "${test_drivers[@]}"; do
-        if [ -e tests/${i}.sh ]; then
-                if ! [[ ${tests_disabled[*]} =~ "$i" ]]; then
-                        if [[ ${tests_posix_only[*]} =~ "$i" ]]; then
-                                echo "if !HOST_WIN" >> tests/Makefile.am
-                        fi
-                        echo "TESTS += ${i}.sh" >> tests/Makefile.am
-                        if [[ ${tests_posix_only[*]} =~ "$i" ]]; then
-                                echo "endif" >> tests/Makefile.am
-                        fi
-                        echo "EXTRA_DIST += ${i}.sh" >> tests/Makefile.am
-                fi
-        fi
-done
-echo "EXTRA_DIST += aeadtests.txt" >> tests/Makefile.am
-echo "EXTRA_DIST += evptests.txt" >> tests/Makefile.am
-echo "EXTRA_DIST += pq_expected.txt" >> tests/Makefile.am
-echo "EXTRA_DIST += testssl ca.pem server.pem" >> tests/Makefile.am
 
+# add headers
 (cd include/openssl
         $CP Makefile.am.tpl Makefile.am
         for i in `ls -1 *.h|sort`; do
@@ -291,8 +221,8 @@ echo "EXTRA_DIST += testssl ca.pem server.pem" >> tests/Makefile.am
         done
 )
 
-echo "copying manpages"
 # copy manpages
+echo "copying manpages"
 (cd man
         $CP Makefile.am.tpl Makefile.am