Some new testcases, some improvements to the tester.sh script.

1 files changed, 76 insertions, 26 deletions

diff --git a/tests/tester.sh b/tests/tester.sh
index bb49609c3..a4fa38d34 100755
--- a/tests/tester.sh
+++ b/tests/tester.sh
@@ -3,6 +3,8 @@
 # tester.sh - reads testcases from file and tests busybox applets vs GNU
 # counterparts
 #
+# This should be run from within the tests/ directory. Before you run it, you
+# should compile up a busybox that has all applets and all features turned on.
 
 # set up defaults (can be changed with cmd-line options)
 BUSYBOX=../busybox
@@ -12,32 +14,38 @@ BB_OUT=bb.out
 GNU_OUT=gnu.out
 SETUP=""
 CLEANUP=""
+KEEPTMPFILES="no"
+DEBUG=2
 
-# internal-use vars
-fail_only=0
 
-
-while getopts 'p:t:l:b:g:s:c:f' opt
+#while getopts 'p:t:l:b:g:s:c:kd:' opt
+while getopts 'p:t:l:s:c:kd:' opt
 do
         case $opt in
                 p) BUSYBOX=$OPTARG; ;;
                 t) TESTCASES=$OPTARG; ;;
                 l) LOGFILE=$OPTARG; ;;
-                b) BB_OUT=$OPTARG; ;;
-                g) GNU_OUT=$OPTARG; ;;
+#               b) BB_OUT=$OPTARG; ;;
+#               g) GNU_OUT=$OPTARG; ;;
                 s) SETUP=$OPTARG; ;;
                 c) CLEANUP=$OPTARG; ;;
-                f) fail_only=1; ;;
+                k) KEEPTMPFILES="yes"; ;;
+                d) DEBUG=$OPTARG; ;;
                 *)
                         echo "usage: $0 [-ptlbgsc]"
-                        echo "  -p PATH  path to busybox executable"
-                        echo "  -t FILE  run testcases in FILE"
-                        echo "  -l FILE  log test results in FILE"
-                        echo "  -b FILE  store temporary busybox output in FILE"
-                        echo "  -g FILE  store temporary GNU output in FILE"
+                        echo "  -p PATH  path to busybox executable (default=$BUSYBOX)"
+                        echo "  -t FILE  run testcases in FILE (default=$TESTCASES)"
+                        echo "  -l FILE  log test results in FILE (default=$LOGFILE)"
+#                       echo "  -b FILE  store temporary busybox output in FILE"
+#                       echo "  -g FILE  store temporary GNU output in FILE"
                         echo "  -s FILE  (setup) run commands in FILE before testcases"
                         echo "  -c FILE  (cleanup) run commands in FILE after testcases"
-                        echo "  -f       display only testcases that fail"
+                        echo "  -k       keep temporary output files (don't delete them)"
+                        echo "  -d NUM   set level of debugging output"
+                        echo "           0 = no output"
+                        echo "           1 = output failures / whoops lines only"
+                        echo "           2 = (default) output setup / cleanup msgs and testcase lines"
+                        echo "           3+= other debug noise (internal stuff)"
                         exit 1
                         ;;
         esac
@@ -45,16 +53,44 @@ done
 #shift `expr $OPTIND - 1`
 
 
+# maybe print some debug output
+if [ $DEBUG -ge 3 ]
+then
+        echo "BUSYBOX=$BUSYBOX"
+        echo "TESTCASES=$TESTCASES"
+        echo "LOGFILE=$LOGFILE"
+        echo "BB_OUT=$BB_OUT"
+        echo "GNU_OUT=$GNU_OUT"
+        echo "SETUP=$SETUP"
+        echo "CLEANUP=$CLEANUP"
+        echo "DEBUG=$DEBUG"
+fi
+
+
+# do sanity checks
+if [ ! -e $BUSYBOX ]
+then
+        echo "Busybox executable: $BUSYBOX not found!"
+        exit 1
+fi
+
+if [ ! -e $TESTCASES ]
+then
+        echo "Testcases file: $TESTCASES not found!"
+        exit 1
+fi
+
+
 # do normal setup
 [ -e $LOGFILE ] && rm $LOGFILE
 unalias -a      # gets rid of aliases that might create different output
 
+
 # do extra setup (if any)
 if [ ! -z $SETUP ] 
 then
-        echo "running setup commands in $SETUP"
-        sh $SETUP
-        # XXX: Would 'eval' or 'source' work better instead of 'sh'?
+        [ $DEBUG -ge 2 ] && echo "running setup commands in $SETUP"
+        source $SETUP
 fi
 
 
@@ -67,21 +103,35 @@ do
         then
                 if [ `echo "$line" | cut -c1` != "#" ]
                 then
-                        [ $fail_only -eq 0 ] && echo "testing: $line" | tee -a $LOGFILE
 
                         # test if the applet was compiled into busybox
-                        applet=`echo $line | cut -d' ' -f1`
+                        # (this only tests the applet at the beginning of the line)
+                        #applet=`echo $line | cut -d' ' -f1`
+                        applet=`echo $line | sed 's/\(^[^ ;]*\)[ ;].*/\1/'`
                         $BUSYBOX 2>&1 | grep -qw $applet
                         if [ $? -eq 1 ]
                         then
                                 echo "WHOOPS: $applet not compiled into busybox" | tee -a $LOGFILE
                         else
-                                $BUSYBOX $line > $BB_OUT
-                                $line > $GNU_OUT
+
+                                # execute line using gnu / system programs
+                                [ $DEBUG -ge 2 ] && echo "testing: $line" | tee -a $LOGFILE
+                                sh -c "$line" > $GNU_OUT
+
+                                # change line to include "busybox" before every statement
+                                line="$BUSYBOX $line"
+                                line=${line//;/; $BUSYBOX }
+                                line=${line//|/| $BUSYBOX }
+
+                                # execute line using busybox programs
+                                [ $DEBUG -ge 2 ] && echo "testing: $line" | tee -a $LOGFILE
+                                sh -c "$line" > $BB_OUT
+
+                                # see if they match
                                 diff -q $BB_OUT $GNU_OUT > /dev/null
                                 if [ $? -eq 1 ]
                                 then
-                                        echo "FAILED: $line" | tee -a $LOGFILE
+                                        [ $DEBUG -ge 1 ] && echo "FAILED: $line" | tee -a $LOGFILE
                                         diff -u $BB_OUT $GNU_OUT >> $LOGFILE 
                                 fi
                         fi
@@ -89,16 +139,16 @@ do
         fi
 done
 
-echo "Finished. Results are in $LOGFILE"
+[ $DEBUG -gt 0 ] && echo "Finished. Results are in $LOGFILE"
 
 
 # do normal cleanup
-rm -f $BB_OUT $GNU_OUT
+[ $KEEPTMPFILES == "no" ] && rm -f $BB_OUT $GNU_OUT
+
 
 # do extra cleanup (if any)
 if [ ! -z $CLEANUP ] 
 then
-        echo "running cleanup commands in $CLEANUP"
-        sh $CLEANUP
-        # XXX: Would 'eval' or 'source' work better instead of 'sh'?
+        [ $DEBUG -ge 2 ] && echo "running cleanup commands in $CLEANUP"
+        source $CLEANUP
 fi