]> git.notmuchmail.org Git - notmuch/blobdiff - test/test-lib.sh
test: renamed test scripts to format T\d\d\d-name.sh
[notmuch] / test / test-lib.sh
index 6ce3b3189f25c354b471c42db1730ab8fe172e89..487075e5e7e4cf8c13226e3ca2426f3c27904bc9 100644 (file)
@@ -22,6 +22,13 @@ if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
     exit 1
 fi
 
+# Make sure echo builtin does not expand backslash-escape sequences by default.
+shopt -u xpg_echo
+
+this_test=${0##*/}
+this_test=${this_test%.sh}
+this_test=${this_test#T[0-9][0-9][0-9]-}
+
 # if --tee was passed, write the output not only to the terminal, but
 # additionally to the file test-results/$BASENAME.out, too.
 case "$GIT_TEST_TEE_STARTED, $* " in
@@ -30,7 +37,7 @@ done,*)
        ;;
 *' --tee '*|*' --va'*)
        mkdir -p test-results
-       BASE=test-results/$(basename "$0" .sh)
+       BASE=test-results/$this_test
        (GIT_TEST_TEE_STARTED=done ${SHELL-sh} "$0" "$@" 2>&1;
         echo $? > $BASE.exit) | tee $BASE.out
        test "$(cat $BASE.exit)" = 0
@@ -41,6 +48,10 @@ esac
 # Keep the original TERM for say_color and test_emacs
 ORIGINAL_TERM=$TERM
 
+# dtach(1) provides more capable terminal environment to anything
+# that requires more than dumb terminal...
+[ x"${TERM:-dumb}" = xdumb ] && DTACH_TERM=vt100 || DTACH_TERM=$TERM
+
 # For repeatability, reset the environment to known value.
 LANG=C
 LC_ALL=C
@@ -180,7 +191,18 @@ then
        exit 0
 fi
 
-echo $(basename "$0"): "Testing ${test_description}"
+test_description_printed=
+print_test_description ()
+{
+       test -z "$test_description_printed" || return 0
+       echo
+       echo $this_test: "Testing ${test_description}"
+       test_description_printed=1
+}
+if [ -z "$NOTMUCH_TEST_QUIET" ]
+then
+       print_test_description
+fi
 
 exec 5>&1
 
@@ -190,22 +212,39 @@ test_fixed=0
 test_broken=0
 test_success=0
 
-die () {
+_die_common () {
        code=$?
+       trap - EXIT
+       set +ex
        rm -rf "$TEST_TMPDIR"
+}
+
+die () {
+       _die_common
        if test -n "$GIT_EXIT_OK"
        then
                exit $code
        else
-               echo >&5 "FATAL: Unexpected exit with code $code"
+               exec >&5
+               say_color error '%-6s' FATAL
+               echo " $test_subtest_name"
+               echo
+               echo "Unexpected exit while executing $0. Exit code $code."
                exit 1
        fi
 }
 
+die_signal () {
+       _die_common
+       echo >&5 "FATAL: $0: interrupted by signal" $((code - 128))
+       exit $code
+}
+
 GIT_EXIT_OK=
 # Note: TEST_TMPDIR *NOT* exported!
 TEST_TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-test-$$.XXXXXX")
 trap 'die' EXIT
+trap 'die_signal' HUP INT TERM
 
 test_decode_color () {
        sed     -e 's/.\[1m/<WHITE>/g' \
@@ -332,7 +371,11 @@ generate_message ()
     fi
 
     if [ -z "${template[date]}" ]; then
-       template[date]="Fri, 05 Jan 2001 15:43:57 +0000"
+       # we use decreasing timestamps here for historical reasons;
+       # the existing test suite when we converted to unique timestamps just
+       # happened to have signicantly fewer failures with that choice.
+       template[date]=$(TZ=UTC printf "%(%a, %d %b %Y %T %z)T\n" \
+                       $((978709437 - gen_msg_cnt)))
     fi
 
     additional_headers=""
@@ -421,9 +464,9 @@ emacs_deliver_message ()
 
     test_emacs \
        "(let ((message-send-mail-function 'message-smtpmail-send-it)
+               (mail-host-address \"example.com\")
               (smtpmail-smtp-server \"localhost\")
               (smtpmail-smtp-service \"25025\"))
-          (notmuch-hello)
           (notmuch-mua-mail)
           (message-goto-to)
           (insert \"test_suite@notmuchmail.org\nDate: 01 Jan 2000 12:00:00 -0000\")
@@ -441,6 +484,36 @@ emacs_deliver_message ()
     notmuch new >/dev/null
 }
 
+# Pretend to deliver a message with emacs. Really save it to a file
+# and add it to the database
+#
+# Uses emacs to generate and deliver a message to the mail store.
+# Accepts arbitrary extra emacs/elisp functions to modify the message
+# before sending, which is useful to doing things like attaching files
+# to the message and encrypting/signing.
+emacs_fcc_message ()
+{
+    local subject="$1"
+    local body="$2"
+    shift 2
+    # before we can send a message, we have to prepare the FCC maildir
+    mkdir -p "$MAIL_DIR"/sent/{cur,new,tmp}
+
+    test_emacs \
+       "(let ((message-send-mail-function (lambda () t))
+               (mail-host-address \"example.com\"))
+          (notmuch-mua-mail)
+          (message-goto-to)
+          (insert \"test_suite@notmuchmail.org\nDate: 01 Jan 2000 12:00:00 -0000\")
+          (message-goto-subject)
+          (insert \"${subject}\")
+          (message-goto-body)
+          (insert \"${body}\")
+          $@
+          (message-send-and-exit))" || return 1
+    notmuch new >/dev/null
+}
+
 # Generate a corpus of email and add it to the database.
 #
 # This corpus is fixed, (it happens to be 50 messages from early in
@@ -494,12 +567,12 @@ test_expect_equal ()
        if ! test_skip "$test_subtest_name"
        then
                if [ "$output" = "$expected" ]; then
-                       test_ok_ "$test_subtest_name"
+                       test_ok_
                else
                        testname=$this_test.$test_count
                        echo "$expected" > $testname.expected
                        echo "$output" > $testname.output
-                       test_failure_ "$test_subtest_name" "$(diff -u $testname.expected $testname.output)"
+                       test_failure_ "$(diff -u $testname.expected $testname.output)"
                fi
     fi
 }
@@ -520,12 +593,12 @@ test_expect_equal_file ()
        if ! test_skip "$test_subtest_name"
        then
                if diff -q "$file1" "$file2" >/dev/null ; then
-                       test_ok_ "$test_subtest_name"
+                       test_ok_
                else
                        testname=$this_test.$test_count
                        cp "$file1" "$testname.$basename1"
                        cp "$file2" "$testname.$basename2"
-                       test_failure_ "$test_subtest_name" "$(diff -u "$testname.$basename1" "$testname.$basename2")"
+                       test_failure_ "$(diff -u "$testname.$basename1" "$testname.$basename2")"
                fi
     fi
 }
@@ -563,9 +636,9 @@ test_emacs_expect_t () {
                result=$(cat OUTPUT)
                if [ "$result" = t ]
                then
-                       test_ok_ "$test_subtest_name"
+                       test_ok_
                else
-                       test_failure_ "$test_subtest_name" "${result}"
+                       test_failure_ "${result}"
                fi
        else
                # Restore state after the (non) test.
@@ -593,16 +666,36 @@ notmuch_show_sanitize_all ()
 {
     sed \
        -e 's| filename:.*| filename:XXXXX|' \
-       -e 's| id:[^ ]* | id:XXXXX |'
+       -e 's| id:[^ ]* | id:XXXXX |' | \
+       notmuch_date_sanitize
 }
 
 notmuch_json_show_sanitize ()
 {
     sed \
        -e 's|"id": "[^"]*",|"id": "XXXXX",|g' \
-       -e 's|"filename": "/[^"]*",|"filename": "YYYYY",|g'
+       -e 's|"Date": "Fri, 05 Jan 2001 [^"]*0000"|"Date": "GENERATED_DATE"|g' \
+       -e 's|"filename": "/[^"]*",|"filename": "YYYYY",|g' \
+       -e 's|"timestamp": 97.......|"timestamp": 42|g'
 }
 
+notmuch_emacs_error_sanitize ()
+{
+    local command=$1
+    shift
+    for file in "$@"; do
+       echo "=== $file ==="
+       cat "$file"
+    done | sed  \
+       -e 's/^\[.*\]$/[XXX]/' \
+       -e "s|^\(command: \)\{0,1\}/.*/$command|\1YYY/$command|"
+}
+
+notmuch_date_sanitize ()
+{
+    sed \
+       -e 's/^Date: Fri, 05 Jan 2001 .*0000/Date: GENERATED_DATE/'
+}
 # End of notmuch helper functions
 
 # Use test_set_prereq to tell that a particular prerequisite is available.
@@ -666,12 +759,15 @@ test_require_external_prereq () {
 
 test_ok_ () {
        if test "$test_subtest_known_broken_" = "t"; then
-               test_known_broken_ok_ "$@"
+               test_known_broken_ok_
                return
        fi
        test_success=$(($test_success + 1))
+       if test -n "$NOTMUCH_TEST_QUIET"; then
+               return 0
+       fi
        say_color pass "%-6s" "PASS"
-       echo " $@"
+       echo " $test_subtest_name"
 }
 
 test_failure_ () {
@@ -680,7 +776,8 @@ test_failure_ () {
                return
        fi
        test_failure=$(($test_failure + 1))
-       test_failure_message_ "FAIL" "$@"
+       print_test_description
+       test_failure_message_ "FAIL" "$test_subtest_name" "$@"
        test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
        return 1
 }
@@ -697,13 +794,13 @@ test_known_broken_ok_ () {
        test_reset_state_
        test_fixed=$(($test_fixed+1))
        say_color pass "%-6s" "FIXED"
-       echo " $@"
+       echo " $test_subtest_name"
 }
 
 test_known_broken_failure_ () {
        test_reset_state_
        test_broken=$(($test_broken+1))
-       test_failure_message_ "BROKEN" "$@"
+       test_failure_message_ "BROKEN" "$test_subtest_name" "$@"
        return 1
 }
 
@@ -771,6 +868,7 @@ test_expect_success () {
        test "$#" = 3 && { prereq=$1; shift; } || prereq=
        test "$#" = 2 ||
        error "bug in the test script: not 2 or 3 parameters to test-expect-success"
+       test_subtest_name="$1"
        test_reset_state_
        if ! test_skip "$@"
        then
@@ -780,9 +878,9 @@ test_expect_success () {
                test_check_missing_external_prereqs_ "$@" ||
                if [ "$run_ret" = 0 -a "$eval_ret" = 0 ]
                then
-                       test_ok_ "$1"
+                       test_ok_
                else
-                       test_failure_ "$@"
+                       test_failure_ "$2"
                fi
        fi
 }
@@ -791,6 +889,7 @@ test_expect_code () {
        test "$#" = 4 && { prereq=$1; shift; } || prereq=
        test "$#" = 3 ||
        error "bug in the test script: not 3 or 4 parameters to test-expect-code"
+       test_subtest_name="$2"
        test_reset_state_
        if ! test_skip "$@"
        then
@@ -800,9 +899,9 @@ test_expect_code () {
                test_check_missing_external_prereqs_ "$@" ||
                if [ "$run_ret" = 0 -a "$eval_ret" = "$1" ]
                then
-                       test_ok_ "$2"
+                       test_ok_
                else
-                       test_failure_ "$@"
+                       test_failure_ "exit code $eval_ret, expected $1" "$3"
                fi
        fi
 }
@@ -819,10 +918,10 @@ test_external () {
        test "$#" = 4 && { prereq=$1; shift; } || prereq=
        test "$#" = 3 ||
        error >&5 "bug in the test script: not 3 or 4 parameters to test_external"
-       descr="$1"
+       test_subtest_name="$1"
        shift
        test_reset_state_
-       if ! test_skip "$descr" "$@"
+       if ! test_skip "$test_subtest_name" "$@"
        then
                # Announce the script to reduce confusion about the
                # test output that follows.
@@ -833,9 +932,9 @@ test_external () {
                "$@" 2>&4
                if [ "$?" = 0 ]
                then
-                       test_ok_ "$descr"
+                       test_ok_
                else
-                       test_failure_ "$descr" "$@"
+                       test_failure_ "$@"
                fi
        fi
 }
@@ -849,11 +948,11 @@ test_external_without_stderr () {
        stderr="$tmp/git-external-stderr.$$.tmp"
        test_external "$@" 4> "$stderr"
        [ -f "$stderr" ] || error "Internal error: $stderr disappeared."
-       descr="no stderr: $1"
+       test_subtest_name="no stderr: $1"
        shift
        if [ ! -s "$stderr" ]; then
                rm "$stderr"
-               test_ok_ "$descr"
+               test_ok_
        else
                if [ "$verbose" = t ]; then
                        output=`echo; echo Stderr is:; cat "$stderr"`
@@ -862,7 +961,7 @@ test_external_without_stderr () {
                fi
                # rm first in case test_failure exits.
                rm "$stderr"
-               test_failure_ "$descr" "$@" "$output"
+               test_failure_ "$@" "$output"
        fi
 }
 
@@ -929,7 +1028,7 @@ test_done () {
        GIT_EXIT_OK=t
        test_results_dir="$TEST_DIRECTORY/test-results"
        mkdir -p "$test_results_dir"
-       test_results_path="$test_results_dir/${0%.sh}"
+       test_results_path="$test_results_dir/$this_test"
 
        echo "total $test_count" >> $test_results_path
        echo "success $test_success" >> $test_results_path
@@ -938,8 +1037,6 @@ test_done () {
        echo "failed $test_failure" >> $test_results_path
        echo "" >> $test_results_path
 
-       echo
-
        [ -n "$EMACS_SERVER" ] && test_emacs '(kill-emacs)'
 
        if [ "$test_failure" = "0" ]; then
@@ -988,7 +1085,7 @@ test_emacs () {
        test -z "$missing_dependencies" || return
 
        if [ -z "$EMACS_SERVER" ]; then
-               emacs_tests="$(basename $0).el"
+               emacs_tests="${this_test}.el"
                if [ -f "$TEST_DIRECTORY/$emacs_tests" ]; then
                        load_emacs_tests="--eval '(load \"$emacs_tests\")'"
                else
@@ -996,9 +1093,10 @@ test_emacs () {
                fi
                server_name="notmuch-test-suite-$$"
                # start a detached session with an emacs server
-               # user's TERM is given to dtach which assumes a minimally
+               # user's TERM (or 'vt100' in case user's TERM is unset, empty
+               # or 'dumb') is given to dtach which assumes a minimally
                # VT100-compatible terminal -- and emacs inherits that
-               TERM=$ORIGINAL_TERM dtach -n "$TEST_TMPDIR/emacs-dtach-socket.$$" \
+               TERM=$DTACH_TERM dtach -n "$TEST_TMPDIR/emacs-dtach-socket.$$" \
                        sh -c "stty rows 24 cols 80; exec '$TMP_DIRECTORY/run_emacs' \
                                --no-window-system \
                                $load_emacs_tests \
@@ -1101,7 +1199,6 @@ else
        exec 4>test.output 3>&4
 fi
 
-this_test=${0##*/}
 for skp in $NOTMUCH_SKIP_TESTS
 do
        to_skip=