From 396a3565cf44c68b380d4e5f7f6ce67513e58a19 Mon Sep 17 00:00:00 2001
From: Remzi Arpaci-Dusseau <remzi.arpacidusseau@gmail.com>
Date: Sat, 16 Feb 2019 08:06:40 -0500
Subject: [PATCH] some bug fixes and enhancements to tester

---
 tester/run-tests.sh | 91 ++++++++++++++++++++++++++++++++-------------
 1 file changed, 65 insertions(+), 26 deletions(-)

diff --git a/tester/run-tests.sh b/tester/run-tests.sh
index 2f0f92c..70cbefc 100755
--- a/tester/run-tests.sh
+++ b/tester/run-tests.sh
@@ -2,46 +2,81 @@
 
 # run_test testnumber
 run_test () {
-    testfile=tests/$1.run
-    eval $(cat $testfile) > tests-out/$1.out 2> tests-out/$1.err
-    return 0
+    local testnum=$1
+    local testfile=tests/$testnum.run
+    eval $(cat $testfile) > tests-out/$testnum.out 2> tests-out/$testnum.err
 }
 
-# check_test testnumber out/err
-check_test () {
-    outdiff=$(diff tests/$1.$2 tests-out/$1.$2)
-    outerr=$?
-    if (( $outerr != 0 )); then
-	builtin echo -e "\e[31mtest $testnum: standard $2 incorrect\e[0m"
-	echo "should be:"
-	cat tests/$1.$2
-	echo "is:"
-	cat tests-out/$1.$2
+print_error_message () {
+    local testnum=$1
+    local contrunning=$2
+    local filetype=$3
+    builtin echo -e "\e[31mtest $testnum: standard $filetype incorrect\e[0m"
+    echo "  what results should be found in file: tests/$testnum.$filetype"
+    echo "  what results produced by your program: tests-out/$testnum.$filetype"
+    echo "  compare the two using diff, cmp, or related tools to debug"
+    if (( $contrunning == 0 )); then
 	exit 1
     fi
-    return 0
 }
 
-# run_and_check testnumber printerror
+# check_test testnumber contrunning out/err
+check_test () {
+    local testnum=$1
+    local contrunning=$2
+    local filetype=$3
+
+    # option to use cmp instead?
+    returnval=$(diff tests/$testnum.$filetype tests-out/$testnum.$filetype)
+    if (( $? != 0 )); then
+	echo -n 1
+    else
+	echo -n 0
+    fi
+}
+
+# run_and_check testnumber contrunning verbose printerror
 #   testnumber: the test to run and check
 #   printerrer: if 1, print an error if test does not exist
 run_and_check () {
-    if [[ ! -f tests/$1.run ]]; then
-	if (( $2 == 1 )); then
-	    echo "test $1 does not exist" >&2; exit 1
+    local testnum=$1
+    local contrunning=$2
+    local verbose=$3
+    local failmode=$4
+
+    if [[ ! -f tests/$testnum.run ]]; then
+	if (( $failmode == 1 )); then
+	    echo "test $testnum does not exist" >&2; exit 1
 	fi
 	exit 0
     fi
-    run_test $1
-    check_test $1 out
-    check_test $1 err
-    builtin echo -e "\e[32mtest $1: passed\e[0m"
-    return 0
+    if (( $verbose == 1 )); then
+	echo "running test $testnum"
+	cat tests/$testnum.desc
+    fi
+    run_test $testnum
+    outcheck=$(check_test $testnum $contrunning out)
+    errcheck=$(check_test $testnum $contrunning err)
+    # echo "results: outcheck:$outcheck errcheck:$errcheck"
+    if (( $outcheck == 0 )) && (( $errcheck == 0 )); then
+	builtin echo -e "\e[32mtest $testnum: passed\e[0m"
+    else
+	if (( $outcheck == 1 )); then
+	    print_error_message $testnum $contrunning out
+	fi
+	if (( $errcheck == 1 )); then
+	    print_error_message $testnum $contrunning err
+	fi
+    fi
 }
 
 # usage: call when args not parsed, or when help needed
 usage () {
     echo "usage: run-tests.sh [-h] [-v] [-t test]"
+    echo "  -h                help message"
+    echo "  -v                verbose"
+    echo "  -t n              run only test n"
+    echo "  -c                continue even after failure"
     return 0
 }
 
@@ -49,9 +84,10 @@ usage () {
 # main program
 #
 verbose=0
+contrunning=0
 specific=""
 
-args=`getopt hvt: $*`
+args=`getopt hvct: $*`
 if [[ $? != 0 ]]; then
     usage; exit 1
 fi
@@ -66,6 +102,9 @@ for i; do
     -v)
         verbose=1
         shift;;
+    -c)
+        contrunning=1
+        shift;;
     -t)
         specific=$2
 	shift
@@ -87,14 +126,14 @@ fi
 
 # run just one test
 if [[ $specific != "" ]]; then
-    run_and_check $specific 1
+    run_and_check $specific $contrunning $verbose 1
     exit 0
 fi
 
 # run all tests
 (( testnum = 1 ))
 while true; do
-    run_and_check $testnum 0
+    run_and_check $testnum $contrunning $verbose 0
     (( testnum = $testnum + 1 ))
 done