refresh the test infrastructure a bit.

3 files changed, 93 insertions, 90 deletions

diff --git a/src/regress/lib/libc/asr/Makefile b/src/regress/lib/libc/asr/Makefile
index 76891ec176..12e59536e0 100644
--- a/src/regress/lib/libc/asr/Makefile
+++ b/src/regress/lib/libc/asr/Makefile
@@ -1,53 +1,52 @@
-#       $OpenBSD: Makefile,v 1.3 2013/04/01 10:18:28 eric Exp $
+#       $OpenBSD: Makefile,v 1.4 2017/03/10 17:23:48 eric Exp $
 
 #
-# Note on building and running the regress tests:
+# Notes on building and running the regression tests
 #
-# The regress suite builds two sets of static executables: one linked
-# against the current libc, and one linked against a libc with the resolver
-# replaced. The idea is to compare the output of all programs in both cases.
+# The regress suite builds two sets of static executables: the former linked
+# against the system libc, and the latter against the libc found in /usr/obj
+# The idea is to compare the output of all programs with internal changes in
+# the libc (more specifically the resolver).  They will be run in a chrooted
+# environment to test various /etc configurations without touching the local
+# machine config files.
 #
-# So before building the regression test programs, you need to have two
-# static libc: the system libc.a in /usr/lib/ which is assumed to be built
-# with asr, and a pre-asr libc.a in the "./lib/" directory (you can build
-# one by commenting out the ".include "${LIBCSRCDIR}/asr/Makefile.inc"
-# line from the /usr/src/lib/libc/Makefile.in and rebuild the libc).
-# 
-# When done, run "make && make install". This builds and installs the two sets
-# of programs in REGRESSDIR. They will run chrooted there to allow testing with
-# various /etc environment without messing up the local machine config files.
+# First, build a libc that needs testing, then:
 #
-# When you are sure everything is in place, run "make regress". This will
-# create two files: $REGRESSDIR/output.log with the output of all tests,
-# and $REGRESSDIR/regress.log with the diffs for those that "failed".
+# $ make
+# $ doas make install
+# $ doas make regress
 #
-# Note that the "install" and "regress" targets need to be run as root.
+# Tests output goes into a $REGRESSDIR/tmp.XXXXXXXXXX directory, and a digest
+# is displayed at the end.
 #
 
-LIBCDIRSTD?=    ${.CURDIR}/lib
-LIBCDIRASR?=    /usr/lib
-
-REGRESSDIR?=    /tmp/regress
+REGRESSDIR?=    /tmp/regress-asr
 REGRESS?=       regress.sh
 
+A?=     .a
+B?=     .b
+
+DIRA?=  /usr/lib
+DIRB?=  /usr/obj/lib/libc
+
+
 all: build
 
 build:
-        cd bin && EXT=.asr LDFLAGS=-L${LIBCDIRASR} make
-        cd bin && EXT=.std LDFLAGS=-L${LIBCDIRSTD} make
+        cd bin && EXT=${A} LDFLAGS=-L${DIRA} make
+        cd bin && EXT=${B} LDFLAGS=-L${DIRB} make
 
 clean:
-        cd bin && EXT=.std make clean
-        cd bin && EXT=.asr make clean
+        cd bin && EXT=${A} make clean
+        cd bin && EXT=${B} make clean
 
 install:
-        mkdir -p ${REGRESSDIR}/etc
         mkdir -p ${REGRESSDIR}/bin
-        cd bin && BINDIR=${REGRESSDIR}/bin EXT=.std make install
-        cd bin && BINDIR=${REGRESSDIR}/bin EXT=.asr make install
+        cd bin && BINDIR=${REGRESSDIR}/bin EXT=${A} make install
+        cd bin && BINDIR=${REGRESSDIR}/bin EXT=${B} make install
 
 uninstall:
         rm -rf ${REGRESSDIR}
 
 regress:
-        RUNDIR=${REGRESSDIR} sh ${.CURDIR}/${REGRESS}
+        RUNDIR=${REGRESSDIR} A=${A} B=${B} sh ${.CURDIR}/${REGRESS}
diff --git a/src/regress/lib/libc/asr/regress.sh b/src/regress/lib/libc/asr/regress.sh
index 3388f0dd2c..a0950cefde 100644
--- a/src/regress/lib/libc/asr/regress.sh
+++ b/src/regress/lib/libc/asr/regress.sh
@@ -1,9 +1,7 @@
-#       $OpenBSD: regress.sh,v 1.6 2013/04/01 10:17:45 eric Exp $
+#       $OpenBSD: regress.sh,v 1.7 2017/03/10 17:23:48 eric Exp $
 
 . regress.subr
 
-EFLAG=
-
 test_res_mkquery()
 {
         for i in $@; do
diff --git a/src/regress/lib/libc/asr/regress.subr b/src/regress/lib/libc/asr/regress.subr
index 3e8cc0ebd5..43b7280158 100644
--- a/src/regress/lib/libc/asr/regress.subr
+++ b/src/regress/lib/libc/asr/regress.subr
@@ -1,13 +1,5 @@
 #!/bin/sh
-#       $OpenBSD: regress.subr,v 1.2 2013/03/28 09:36:03 eric Exp $
-
-TOTAL=0
-FAIL=0
-OK=0
-
-EXT0=.std
-EXT1=.asr
-EFLAG=-ee
+#       $OpenBSD: regress.subr,v 1.3 2017/03/10 17:23:48 eric Exp $
 
 set -e
 
@@ -19,70 +11,75 @@ fail()
 
 regress()
 {
-        local out;
-        local _cmd=$1;
-        local _bin0=/bin/$_cmd$EXT0
-        local _bin1=/bin/$_cmd$EXT1
-        shift;
-
-        TOTAL=$((TOTAL+1))
-
-        # XXX with user "bin"
-        test -x $_RUNDIR$_bin0 || fail $_RUNDIR$_bin0 not executable
-        test -x $_RUNDIR$_bin1 || fail $_RUNDIR$_bin1 not executable
-
-        out=/tmp/asr_regress
-
-        echo -n $_cmd $EFLAG $@ "."
+        local cmd="$1"
+        local bin0="/bin/${cmd}${A}"
+        local bin1="/bin/${cmd}${B}"
+        local out="${OUTDIR}/test-${COUNT}.out"
+        local out0="${OUTDIR}/test-${COUNT}${A}.out"
+        local out1="${OUTDIR}/test-${COUNT}${B}.out"
+        local err0="${OUTDIR}/test-${COUNT}${A}.err"
+        local err1="${OUTDIR}/test-${COUNT}${B}.err"
+        local outdiff="${OUTDIR}/test-${COUNT}${A}${B}.diff"
+        shift
+
+        test -x "${ROOTDIR}${bin0}" || fail "${ROOTDIR}${bin0}" not executable
+        test -x "${ROOTDIR}${bin1}" || fail "${ROOTDIR}${bin1}" not executable
+
+        echo -n "${cmd} $@ ."
 
         set +e
-        chroot -u bin "$_RUNDIR" $_bin0 $EFLAG $@ > $out.0
+        chroot -u bin "${ROOTDIR}" ${bin0} $@ > ${out0}
         echo -n .
-        chroot -u bin "$_RUNDIR" $_bin1 $EFLAG $@ > $out.1
-        echo -n ". "
+        chroot -u bin "${ROOTDIR}" ${bin1} $@ > ${out1}
+        echo -n "."
 
-        diff -u $out.0 $out.1 > $out.diff
+        diff -u ${out0} ${out1} > ${outdiff}
         set -e
-        if test -s $out.diff; then
+        if test -s ${outdiff}; then
                 FAIL=$((FAIL+1))
                 echo fail
-                echo "*** FAIL (env=$REGRESSENV)" $_cmd $EFLAG $@ >> $REG
-                tail -n +3 $out.diff >> $REG
-                echo >> $REG
+                echo "### FAIL [#${COUNT} env=${REGRESSENV}] ${cmd} $@" >> $ERR
+                cat ${outdiff} >> $ERR
+                echo >> $ERR
         else
+                rm ${outdiff}
+                rm ${out1}
+                mv ${out0} ${out}
                 OK=$((OK+1))
                 echo ok
-                echo "OK (env=$REGRESSENV)" $_cmd $EFLAG $@ >> $OUT
-                cat $out.0 >> $OUT
+                echo "### OK [#${COUNT} env=$REGRESSENV] ${cmd} $@" >> $OUT
+                cat ${out} >> $OUT
                 echo >> $OUT
         fi
-        rm $out.diff $out.0 $out.1
+
+        COUNT=$((COUNT+1))
 }
 
 regress_setenv()
 {
         local _name="$1"
 
-        echo "===> using env $_name"
+        echo "==> using env $_name"
 
-        cp /etc/hosts $_RUNDIR/etc/
-        cp /etc/resolv.conf $_RUNDIR/etc/
-        cp /etc/protocols $_RUNDIR/etc/
-        cp /etc/networks $_RUNDIR/etc/
+        mkdir -p "${ROOTDIR}/etc/"
+        cp /etc/hosts "${ROOTDIR}/etc/"
+        cp /etc/resolv.conf "${ROOTDIR}/etc/"
+        cp /etc/protocols "${ROOTDIR}/etc/"
+        cp /etc/networks "${ROOTDIR}/etc/"
 
         case $_name in
         empty)
-                rm -f $_RUNDIR/etc/*
+                rm -f "${ROOTDIR}/etc/*"
                 ;;
         local)
                 ;;
         file)
-                grep -v lookup /etc/resolv.conf > $_RUNDIR/etc/resolv.conf
-                echo "lookup file" >> $_RUNDIR/etc/resolv.conf
+                grep -v lookup /etc/resolv.conf > "${ROOTDIR}/etc/resolv.conf"
+                echo "lookup file" >> "${ROOTDIR}/etc/resolv.conf"
                 ;;
         bind)
-                grep -v lookup /etc/resolv.conf > $_RUNDIR/etc/resolv.conf
-                echo "lookup bind" >> $_RUNDIR/etc/resolv.conf
+                grep -v lookup /etc/resolv.conf > "${ROOTDIR}/etc/resolv.conf"
+                echo "lookup bind" >> "${ROOTDIR}/etc/resolv.conf"
                 ;;
         *)
                 fail unknown env $_name
@@ -93,23 +90,32 @@ regress_setenv()
 
 regress_digest()
 {
-        echo
-        cat $REG
-        echo "===>" run=$TOTAL fail=$FAIL
-}
+        echo "===> done"
+        test -f $ERR && (echo; cat $ERR; echo "===> errfile=$ERR")
+        echo "===> logfile=$OUT"
+        echo "===> run=$COUNT ok=$OK fail=$FAIL error=$ERROR"
 
+}
 
 # needed for chroot
 test "$(id -u)" -ne 0 && fail need root privileges to run this script
 
 # we really really want to avoid erasing /etc later
-test "$RUNDIR" || fail RUNDIR is not set
-_RUNDIR=$(readlink -fn ${RUNDIR})
-test "$_RUNDIR" == / && fail RUNDIR is root dir: $RUNDIR
+test "${RUNDIR}" || fail RUNDIR is not set
+
+ROOTDIR=$(readlink -fn ${RUNDIR})
+test "${ROOTDIR}" == "/" && fail RUNDIR is root dir: ${RUNDIR}
+
+COUNT=0
+OK=0
+FAIL=0
+ERROR=0
+A=${A:=.a}
+B=${B:=.b}
 
-OUT=$_RUNDIR/output.log
-REG=$_RUNDIR/regress.log
-ETC=$_RUNDIR/etc
+OUTDIR=$(mktemp -d -p $ROOTDIR)
+chmod a+rx ${OUTDIR}
+echo "===> ${OUTDIR}"
 
-echo -n > $REG
-echo -n > $OUT
+OUT=$OUTDIR/regress.log
+ERR=$OUTDIR/regress.err