test: aggregate-results updates

notmuch-test will now call aggregate-results.sh with file list
that it compiles based on the test ran, and aggregate-results
will report failure is any of the test files are missing.

With this notmuch-test no longer has to exit in non-parallel
run if some test fail to write its report file -- so it works
as parallel tests in this sense.

Changed test_done() in test-lib.sh write report file in one write(2),
so there is (even) less chance it being partially written. Also,
now it writes 'total' last and aggregate-results.sh expects this
line to exist in all report files for reporting to be successful.

Added 'set -eu' to notmuch-test and modified code to work with
these settings. That makes it harder to get mistakes slipped
into committed code.

diff --git a/test/aggregate-results.sh b/test/aggregate-results.sh
index 75400e6e3e3f6f14dbecff9902df84ec5aaf26f9..6845fcf0b2ae06b55e8a94e926c1f30484e3f14d 100755 (executable)
--- a/test/aggregate-results.sh
+++ b/test/aggregate-results.sh
@@ -8,9 +8,16 @@ failed=0
 broken=0
 total=0
 all_skipped=0
+rep_failed=0
 
 for file
 do
+       if [ ! -f "$file" ]; then
+               echo "'$file' does not exist!"
+               rep_failed=$((rep_failed + 1))
+               continue
+       fi
+       has_total=0
        while read type value
        do
                case $type in
@@ -24,18 +31,23 @@ do
                        broken=$((broken + value)) ;;
                total)
                        total=$((total + value))
+                       has_total=1
                        if [ "$value" -eq 0 ]; then
                                all_skipped=$((all_skipped + 1))
                        fi
                esac
        done <"$file"
+       if [ "$has_total" -eq 0 ]; then
+               echo "'$file' lacks 'total ...'; results may be inconsistent."
+               failed=$((failed + 1))
+       fi
 done
 
 pluralize_s () { [ "$1" -eq 1 ] && s='' || s='s'; }
 
 echo "Notmuch test suite complete."
 
-if [ "$fixed" -eq 0 ] && [ "$failed" -eq 0 ]; then
+if [ "$fixed" -eq 0 ] && [ "$failed" -eq 0 ] && [ "$rep_failed" -eq 0 ]; then
        pluralize_s "$total"
        printf "All $total test$s "
        if [ "$broken" -eq 0 ]; then
@@ -70,10 +82,16 @@ if [ "$all_skipped" -ne 0 ]; then
        echo "All tests in $all_skipped file$s skipped."
 fi
 
+if [ "$rep_failed" -ne 0 ]; then
+       pluralize_s "$rep_failed"
+       echo "$rep_failed test$s failed to report results."
+fi
+
 # Note that we currently do not consider skipped tests as failing the
 # build.
 
-if [ "$success" -gt 0 ] && [ "$fixed" -eq 0 ] && [ "$failed" -eq 0 ]
+if [ "$success" -gt 0 ] && [ "$fixed" -eq 0 ] &&
+       [ "$failed" -eq 0 ] && [ "$rep_failed" -eq 0 ]
 then
        exit 0
 else

diff --git a/test/notmuch-test b/test/notmuch-test
index cbd33f9374805c96d0bf03312e0cde7255f394b4..ce142f7cd73094f880f72d870d913b7613b7a9b4 100755 (executable)
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -18,12 +18,14 @@ fi
 # Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set.
 . $(dirname "$0")/export-dirs.sh || exit 1
 
+set -eu
+
 TESTS=
-for test in $NOTMUCH_TESTS; do
+for test in ${NOTMUCH_TESTS-}; do
     TESTS="$TESTS $NOTMUCH_SRCDIR/test/$test"
 done
 
-if [[ -z "$TESTS" ]]; then
+if [ -z "$TESTS" ]; then
     TESTS="$NOTMUCH_SRCDIR/test/T[0-9][0-9][0-9]-*.sh"
 fi
 
@@ -44,43 +46,46 @@ else
     TEST_TIMEOUT_CMD=""
 fi
 
-trap 'e=$?; kill $!; exit $e' HUP INT TERM
-
 META_FAILURE=
+RES=0
 # Run the tests
-if test -z "$NOTMUCH_TEST_SERIALIZE" && command -v parallel >/dev/null ; then
+if test -z "${NOTMUCH_TEST_SERIALIZE-}" && command -v parallel >/dev/null ; then
     test -t 1 && export COLORS_WITHOUT_TTY=t || :
     if parallel --version 2>&1 | grep -q GNU ; then
         echo "INFO: running tests with GNU parallel"
-        printf '%s\n' $TESTS | $TEST_TIMEOUT_CMD parallel
+        printf '%s\n' $TESTS | $TEST_TIMEOUT_CMD parallel || RES=$?
     else
         echo "INFO: running tests with moreutils parallel"
-        $TEST_TIMEOUT_CMD parallel -- $TESTS
+        $TEST_TIMEOUT_CMD parallel -- $TESTS || RES=$?
     fi
-    RES=$?
-    if [[ $RES != 0 ]]; then
+    if [ $RES != 0 ]; then
         META_FAILURE="parallel test suite returned error code $RES"
     fi
 else
+    trap 'e=$?; trap - 0; kill ${!-}; exit $e' 0 HUP INT TERM
     for test in $TESTS; do
         $TEST_TIMEOUT_CMD $test "$@" &
-        wait $!
-        # If the test failed without producing results, then it aborted,
-        # so we should abort, too.
-        RES=$?
-        testname=$(basename $test .sh)
-        if [[ $RES != 0 && ! -e "$NOTMUCH_BUILDDIR/test/test-results/$testname" ]]; then
-            META_FAILURE="Aborting on $testname (returned $RES)"
-            break
-        fi
+        wait $! && ev=0 || ev=$?
+        test $ev = 0 || RES=$ev
     done
+    trap - 0 HUP INT TERM
+    if [ $RES != 0 ]; then
+        META_FAILURE="some tests failed; first failed returned error code $RES"
+    fi
 fi
-trap - HUP INT TERM
 
 # Report results
+RESULT_FILES=
+for file in $TESTS
+do
+    file=${file##*/} # drop leading path components
+    file=${file%.sh} # drop trailing '.sh'
+    RESULT_FILES="$RESULT_FILES $NOTMUCH_BUILDDIR/test/test-results/$file"
+done
+
 echo
-$NOTMUCH_SRCDIR/test/aggregate-results.sh $NOTMUCH_BUILDDIR/test/test-results/*
-ev=$?
+$NOTMUCH_SRCDIR/test/aggregate-results.sh $RESULT_FILES && ev=0 || ev=$?
+
 if [ -n "$META_FAILURE" ]; then
     printf 'ERROR: %s\n' "$META_FAILURE"
     if [ $ev = 0 ]; then

diff --git a/test/test-lib.sh b/test/test-lib.sh
index 0bca76df217afbb062ac2157eefd3206b5cb196e..23f7c8f80aaf741cf81a8f2af249cfc05675b94b 100644 (file)
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -858,15 +858,16 @@ test_when_finished () {
 test_done () {
        GIT_EXIT_OK=t
        test_results_dir="$TEST_DIRECTORY/test-results"
-       mkdir -p "$test_results_dir"
+       test -d "$test_results_dir" || mkdir "$test_results_dir"
        test_results_path="$test_results_dir/$this_test"
 
-       echo "total $test_count" >> $test_results_path
-       echo "success $test_success" >> $test_results_path
-       echo "fixed $test_fixed" >> $test_results_path
-       echo "broken $test_broken" >> $test_results_path
-       echo "failed $test_failure" >> $test_results_path
-       echo "" >> $test_results_path
+       printf %s\\n \
+               "success $test_success" \
+               "fixed $test_fixed" \
+               "broken $test_broken" \
+               "failed $test_failure" \
+               "total $test_count" \
+           > $test_results_path
 
        [ -n "$EMACS_SERVER" ] && test_emacs '(kill-emacs)'