X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=test%2Ftest-lib.sh;h=db3b6aa198c7319bafb13d7b319f768169877d33;hp=948752613dda3e2cf5711b0b477176651c592d60;hb=23d86773b9e1b8111921af94f7f14ea0867eaee2;hpb=21326a1e6b23f0dc98d13c93cd5023e148fd1a5d diff --git a/test/test-lib.sh b/test/test-lib.sh index 94875261..db3b6aa1 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -22,6 +22,13 @@ if [ ${BASH_VERSINFO[0]} -lt 4 ]; then exit 1 fi +# Make sure echo builtin does not expand backslash-escape sequences by default. +shopt -u xpg_echo + +this_test=${0##*/} +this_test=${this_test%.sh} +this_test_bare=${this_test#T[0-9][0-9][0-9]-} + # if --tee was passed, write the output not only to the terminal, but # additionally to the file test-results/$BASENAME.out, too. case "$GIT_TEST_TEE_STARTED, $* " in @@ -30,7 +37,7 @@ done,*) ;; *' --tee '*|*' --va'*) mkdir -p test-results - BASE=test-results/$(basename "$0" .sh) + BASE=test-results/$this_test (GIT_TEST_TEE_STARTED=done ${SHELL-sh} "$0" "$@" 2>&1; echo $? > $BASE.exit) | tee $BASE.out test "$(cat $BASE.exit)" = 0 @@ -38,9 +45,19 @@ done,*) ;; esac +# Save STDOUT to fd 6 and STDERR to fd 7. +exec 6>&1 7>&2 +# Make xtrace debugging (when used) use redirected STDERR, with verbose lead: +BASH_XTRACEFD=7 +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + # Keep the original TERM for say_color and test_emacs ORIGINAL_TERM=$TERM +# dtach(1) provides more capable terminal environment to anything +# that requires more than dumb terminal... +[ x"${TERM:-dumb}" = xdumb ] && DTACH_TERM=vt100 || DTACH_TERM=$TERM + # For repeatability, reset the environment to known value. LANG=C LC_ALL=C @@ -56,6 +73,8 @@ if [[ ( -n "$TEST_EMACS" && -z "$TEST_EMACSCLIENT" ) || \ fi TEST_EMACS=${TEST_EMACS:-${EMACS:-emacs}} TEST_EMACSCLIENT=${TEST_EMACSCLIENT:-emacsclient} +TEST_CC=${TEST_CC:-cc} +TEST_CFLAGS=${TEST_CFLAGS:-"-g -O0"} # Protect ourselves from common misconfiguration to export # CDPATH into the environment @@ -180,9 +199,18 @@ then exit 0 fi -echo $(basename "$0"): "Testing ${test_description}" - -exec 5>&1 +test_description_printed= +print_test_description () +{ + test -z "$test_description_printed" || return 0 + echo + echo $this_test: "Testing ${test_description}" + test_description_printed=1 +} +if [ -z "$NOTMUCH_TEST_QUIET" ] +then + print_test_description +fi test_failure=0 test_count=0 @@ -190,22 +218,39 @@ test_fixed=0 test_broken=0 test_success=0 -die () { +_die_common () { code=$? + trap - EXIT + set +ex rm -rf "$TEST_TMPDIR" +} + +die () { + _die_common if test -n "$GIT_EXIT_OK" then exit $code else - echo >&5 "FATAL: Unexpected exit with code $code" + exec >&6 + say_color error '%-6s' FATAL + echo " $test_subtest_name" + echo + echo "Unexpected exit while executing $0. Exit code $code." exit 1 fi } +die_signal () { + _die_common + echo >&6 "FATAL: $0: interrupted by signal" $((code - 128)) + exit $code +} + GIT_EXIT_OK= # Note: TEST_TMPDIR *NOT* exported! TEST_TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-test-$$.XXXXXX") trap 'die' EXIT +trap 'die_signal' HUP INT TERM test_decode_color () { sed -e 's/.\[1m//g' \ @@ -329,10 +374,20 @@ generate_message () else template[subject]="Test message #${gen_msg_cnt}" fi + elif [ "${template[subject]}" = "@FORCE_EMPTY" ]; then + template[subject]="" fi if [ -z "${template[date]}" ]; then - template[date]="Fri, 05 Jan 2001 15:43:57 +0000" + # we use decreasing timestamps here for historical reasons; + # the existing test suite when we converted to unique timestamps just + # happened to have signicantly fewer failures with that choice. + local date_secs=$((978709437 - gen_msg_cnt)) + # printf %(..)T is bash 4.2+ feature. use perl fallback if needed... + TZ=UTC printf -v template[date] "%(%a, %d %b %Y %T %z)T" $date_secs 2>/dev/null || + template[date]=`perl -le 'use POSIX "strftime"; + @time = gmtime '"$date_secs"'; + print strftime "%a, %d %b %Y %T +0000", @time'` fi additional_headers="" @@ -421,9 +476,9 @@ emacs_deliver_message () test_emacs \ "(let ((message-send-mail-function 'message-smtpmail-send-it) + (mail-host-address \"example.com\") (smtpmail-smtp-server \"localhost\") (smtpmail-smtp-service \"25025\")) - (notmuch-hello) (notmuch-mua-mail) (message-goto-to) (insert \"test_suite@notmuchmail.org\nDate: 01 Jan 2000 12:00:00 -0000\") @@ -441,6 +496,36 @@ emacs_deliver_message () notmuch new >/dev/null } +# Pretend to deliver a message with emacs. Really save it to a file +# and add it to the database +# +# Uses emacs to generate and deliver a message to the mail store. +# Accepts arbitrary extra emacs/elisp functions to modify the message +# before sending, which is useful to doing things like attaching files +# to the message and encrypting/signing. +emacs_fcc_message () +{ + local subject="$1" + local body="$2" + shift 2 + # before we can send a message, we have to prepare the FCC maildir + mkdir -p "$MAIL_DIR"/sent/{cur,new,tmp} + + test_emacs \ + "(let ((message-send-mail-function (lambda () t)) + (mail-host-address \"example.com\")) + (notmuch-mua-mail) + (message-goto-to) + (insert \"test_suite@notmuchmail.org\nDate: 01 Jan 2000 12:00:00 -0000\") + (message-goto-subject) + (insert \"${subject}\") + (message-goto-body) + (insert \"${body}\") + $@ + (message-send-and-exit))" || return 1 + notmuch new >/dev/null +} + # Generate a corpus of email and add it to the database. # # This corpus is fixed, (it happens to be 50 messages from early in @@ -467,11 +552,10 @@ test_begin_subtest () fi test_subtest_name="$1" test_reset_state_ - # Remember stdout and stderr file descriptors and redirect test - # output to the previously prepared file descriptors 3 and 4 (see - # below) + # Redirect test output to the previously prepared file descriptors + # 3 and 4 (see below) if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi - exec 6>&1 7>&2 >&3 2>&4 + exec >&3 2>&4 inside_subtest=t } @@ -494,12 +578,12 @@ test_expect_equal () if ! test_skip "$test_subtest_name" then if [ "$output" = "$expected" ]; then - test_ok_ "$test_subtest_name" + test_ok_ else testname=$this_test.$test_count echo "$expected" > $testname.expected echo "$output" > $testname.output - test_failure_ "$test_subtest_name" "$(diff -u $testname.expected $testname.output)" + test_failure_ "$(diff -u $testname.expected $testname.output)" fi fi } @@ -520,12 +604,12 @@ test_expect_equal_file () if ! test_skip "$test_subtest_name" then if diff -q "$file1" "$file2" >/dev/null ; then - test_ok_ "$test_subtest_name" + test_ok_ else testname=$this_test.$test_count cp "$file1" "$testname.$basename1" cp "$file2" "$testname.$basename2" - test_failure_ "$test_subtest_name" "$(diff -u "$testname.$basename1" "$testname.$basename2")" + test_failure_ "$(diff -u "$testname.$basename1" "$testname.$basename2")" fi fi } @@ -537,14 +621,20 @@ test_expect_equal_json () { # The test suite forces LC_ALL=C, but this causes Python 3 to # decode stdin as ASCII. We need to read JSON in UTF-8, so # override Python's stdio encoding defaults. - output=$(echo "$1" | PYTHONIOENCODING=utf-8 python -mjson.tool \ + output=$(echo "$1" | PYTHONIOENCODING=utf-8 $NOTMUCH_PYTHON -mjson.tool \ || echo "$1") - expected=$(echo "$2" | PYTHONIOENCODING=utf-8 python -mjson.tool \ + expected=$(echo "$2" | PYTHONIOENCODING=utf-8 $NOTMUCH_PYTHON -mjson.tool \ || echo "$2") shift 2 test_expect_equal "$output" "$expected" "$@" } +# Sort the top-level list of JSON data from stdin. +test_sort_json () { + PYTHONIOENCODING=utf-8 python -c \ + "import sys, json; json.dump(sorted(json.load(sys.stdin)),sys.stdout)" +} + test_emacs_expect_t () { test "$#" = 2 && { prereq=$1; shift; } || prereq= test "$#" = 1 || @@ -563,9 +653,9 @@ test_emacs_expect_t () { result=$(cat OUTPUT) if [ "$result" = t ] then - test_ok_ "$test_subtest_name" + test_ok_ else - test_failure_ "$test_subtest_name" "${result}" + test_failure_ "${result}" fi else # Restore state after the (non) test. @@ -581,7 +671,12 @@ NOTMUCH_NEW () notmuch_search_sanitize () { - sed -r -e 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/' + perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/' +} + +notmuch_search_files_sanitize() +{ + sed -e "s,$MAIL_DIR,MAIL_DIR," } NOTMUCH_SHOW_FILENAME_SQUELCH='s,filename:.*/mail,filename:/XXX/mail,' @@ -593,16 +688,37 @@ notmuch_show_sanitize_all () { sed \ -e 's| filename:.*| filename:XXXXX|' \ - -e 's| id:[^ ]* | id:XXXXX |' + -e 's| id:[^ ]* | id:XXXXX |' | \ + notmuch_date_sanitize } notmuch_json_show_sanitize () { sed \ -e 's|"id": "[^"]*",|"id": "XXXXX",|g' \ - -e 's|"filename": "[^"]*",|"filename": "YYYYY",|g' + -e 's|"Date": "Fri, 05 Jan 2001 [^"]*0000"|"Date": "GENERATED_DATE"|g' \ + -e 's|"filename": "signature.asc",||g' \ + -e 's|"filename": "/[^"]*",|"filename": "YYYYY",|g' \ + -e 's|"timestamp": 97.......|"timestamp": 42|g' } +notmuch_emacs_error_sanitize () +{ + local command=$1 + shift + for file in "$@"; do + echo "=== $file ===" + cat "$file" + done | sed \ + -e 's/^\[.*\]$/[XXX]/' \ + -e "s|^\(command: \)\{0,1\}/.*/$command|\1YYY/$command|" +} + +notmuch_date_sanitize () +{ + sed \ + -e 's/^Date: Fri, 05 Jan 2001 .*0000/Date: GENERATED_DATE/' +} # End of notmuch helper functions # Use test_set_prereq to tell that a particular prerequisite is available. @@ -666,12 +782,15 @@ test_require_external_prereq () { test_ok_ () { if test "$test_subtest_known_broken_" = "t"; then - test_known_broken_ok_ "$@" + test_known_broken_ok_ return fi test_success=$(($test_success + 1)) + if test -n "$NOTMUCH_TEST_QUIET"; then + return 0 + fi say_color pass "%-6s" "PASS" - echo " $@" + echo " $test_subtest_name" } test_failure_ () { @@ -680,7 +799,8 @@ test_failure_ () { return fi test_failure=$(($test_failure + 1)) - test_failure_message_ "FAIL" "$@" + print_test_description + test_failure_message_ "FAIL" "$test_subtest_name" "$@" test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; } return 1 } @@ -697,13 +817,13 @@ test_known_broken_ok_ () { test_reset_state_ test_fixed=$(($test_fixed+1)) say_color pass "%-6s" "FIXED" - echo " $@" + echo " $test_subtest_name" } test_known_broken_failure_ () { test_reset_state_ test_broken=$(($test_broken+1)) - test_failure_message_ "BROKEN" "$@" + test_failure_message_ "BROKEN" "$test_subtest_name" "$@" return 1 } @@ -728,6 +848,12 @@ test_skip () { case $this_test.$test_count in $skp) to_skip=t + break + esac + case $this_test_bare.$test_count in + $skp) + to_skip=t + break esac done if test -z "$to_skip" && test -n "$prereq" && @@ -771,6 +897,7 @@ test_expect_success () { test "$#" = 3 && { prereq=$1; shift; } || prereq= test "$#" = 2 || error "bug in the test script: not 2 or 3 parameters to test-expect-success" + test_subtest_name="$1" test_reset_state_ if ! test_skip "$@" then @@ -780,9 +907,9 @@ test_expect_success () { test_check_missing_external_prereqs_ "$@" || if [ "$run_ret" = 0 -a "$eval_ret" = 0 ] then - test_ok_ "$1" + test_ok_ else - test_failure_ "$@" + test_failure_ "$2" fi fi } @@ -791,6 +918,7 @@ test_expect_code () { test "$#" = 4 && { prereq=$1; shift; } || prereq= test "$#" = 3 || error "bug in the test script: not 3 or 4 parameters to test-expect-code" + test_subtest_name="$2" test_reset_state_ if ! test_skip "$@" then @@ -800,9 +928,9 @@ test_expect_code () { test_check_missing_external_prereqs_ "$@" || if [ "$run_ret" = 0 -a "$eval_ret" = "$1" ] then - test_ok_ "$2" + test_ok_ else - test_failure_ "$@" + test_failure_ "exit code $eval_ret, expected $1" "$3" fi fi } @@ -818,11 +946,11 @@ test_expect_code () { test_external () { test "$#" = 4 && { prereq=$1; shift; } || prereq= test "$#" = 3 || - error >&5 "bug in the test script: not 3 or 4 parameters to test_external" - descr="$1" + error >&6 "bug in the test script: not 3 or 4 parameters to test_external" + test_subtest_name="$1" shift test_reset_state_ - if ! test_skip "$descr" "$@" + if ! test_skip "$test_subtest_name" "$@" then # Announce the script to reduce confusion about the # test output that follows. @@ -833,9 +961,9 @@ test_external () { "$@" 2>&4 if [ "$?" = 0 ] then - test_ok_ "$descr" + test_ok_ else - test_failure_ "$descr" "$@" + test_failure_ "$@" fi fi } @@ -849,11 +977,11 @@ test_external_without_stderr () { stderr="$tmp/git-external-stderr.$$.tmp" test_external "$@" 4> "$stderr" [ -f "$stderr" ] || error "Internal error: $stderr disappeared." - descr="no stderr: $1" + test_subtest_name="no stderr: $1" shift if [ ! -s "$stderr" ]; then rm "$stderr" - test_ok_ "$descr" + test_ok_ else if [ "$verbose" = t ]; then output=`echo; echo Stderr is:; cat "$stderr"` @@ -862,7 +990,7 @@ test_external_without_stderr () { fi # rm first in case test_failure exits. rm "$stderr" - test_failure_ "$descr" "$@" "$output" + test_failure_ "$@" "$output" fi } @@ -929,7 +1057,7 @@ test_done () { GIT_EXIT_OK=t test_results_dir="$TEST_DIRECTORY/test-results" mkdir -p "$test_results_dir" - test_results_path="$test_results_dir/${0%.sh}" + test_results_path="$test_results_dir/$this_test" echo "total $test_count" >> $test_results_path echo "success $test_success" >> $test_results_path @@ -938,8 +1066,6 @@ test_done () { echo "failed $test_failure" >> $test_results_path echo "" >> $test_results_path - echo - [ -n "$EMACS_SERVER" ] && test_emacs '(kill-emacs)' if [ "$test_failure" = "0" ]; then @@ -963,15 +1089,14 @@ export NOTMUCH_CONFIG=$NOTMUCH_CONFIG # Here's what we are using here: # -# --no-init-file Don't load users ~/.emacs -# -# --no-site-file Don't load the site-wide startup stuff +# --quick Use minimal customization. This implies --no-init-file, +# --no-site-file and (emacs 24) --no-site-lisp # # --directory Ensure that the local elisp sources are found # # --load Force loading of notmuch.el and test-lib.el -exec ${TEST_EMACS} --no-init-file --no-site-file \ +exec ${TEST_EMACS} --quick \ --directory "$TEST_DIRECTORY/../emacs" --load notmuch.el \ --directory "$TEST_DIRECTORY" --load test-lib.el \ "\$@" @@ -988,7 +1113,7 @@ test_emacs () { test -z "$missing_dependencies" || return if [ -z "$EMACS_SERVER" ]; then - emacs_tests="$(basename $0).el" + emacs_tests="${this_test_bare}.el" if [ -f "$TEST_DIRECTORY/$emacs_tests" ]; then load_emacs_tests="--eval '(load \"$emacs_tests\")'" else @@ -996,9 +1121,10 @@ test_emacs () { fi server_name="notmuch-test-suite-$$" # start a detached session with an emacs server - # user's TERM is given to dtach which assumes a minimally + # user's TERM (or 'vt100' in case user's TERM is unset, empty + # or 'dumb') is given to dtach which assumes a minimally # VT100-compatible terminal -- and emacs inherits that - TERM=$ORIGINAL_TERM dtach -n "$TEST_TMPDIR/emacs-dtach-socket.$$" \ + TERM=$DTACH_TERM dtach -n "$TEST_TMPDIR/emacs-dtach-socket.$$" \ sh -c "stty rows 24 cols 80; exec '$TMP_DIRECTORY/run_emacs' \ --no-window-system \ $load_emacs_tests \ @@ -1020,23 +1146,35 @@ test_emacs () { rm -f OUTPUT touch OUTPUT - ${TEST_EMACSCLIENT} --socket-name="$EMACS_SERVER" --eval "(progn $@)" + ${TEST_EMACSCLIENT} --socket-name="$EMACS_SERVER" --eval "(notmuch-test-progn $@)" } test_python() { export LD_LIBRARY_PATH=$TEST_DIRECTORY/../lib export PYTHONPATH=$TEST_DIRECTORY/../bindings/python - # Some distros (e.g. Arch Linux) ship Python 2.* as /usr/bin/python2, - # most others as /usr/bin/python. So first try python2, and fallback to - # python if python2 doesn't exist. - cmd=python2 - [[ ${test_missing_external_prereq_[python2]} == t ]] && cmd=python - (echo "import sys; _orig_stdout=sys.stdout; sys.stdout=open('OUTPUT', 'w')"; cat) \ - | $cmd - + | $NOTMUCH_PYTHON - } +test_ruby() { + export LD_LIBRARY_PATH=$TEST_DIRECTORY/../lib + MAIL_DIR=$MAIL_DIR ruby -I $TEST_DIRECTORY/../bindings/ruby> OUTPUT +} + +test_C () { + exec_file="test${test_count}" + test_file="${exec_file}.c" + cat > ${test_file} + export LD_LIBRARY_PATH=${TEST_DIRECTORY}/../lib + ${TEST_CC} ${TEST_CFLAGS} -I${TEST_DIRECTORY}/../lib -o ${exec_file} ${test_file} -L${TEST_DIRECTORY}/../lib/ -lnotmuch -ltalloc + echo "== stdout ==" > OUTPUT.stdout + echo "== stderr ==" > OUTPUT.stderr + ./${exec_file} "$@" 1>>OUTPUT.stdout 2>>OUTPUT.stderr + sed "s,${PWD},CWD,g" OUTPUT.stdout OUTPUT.stderr > OUTPUT +} + + # Creates a script that counts how much time it is executed and calls # notmuch. $notmuch_counter_command is set to the path to the # generated script. Use notmuch_counter_value() function to get the @@ -1092,7 +1230,7 @@ emacs_generate_script # Use -P to resolve symlinks in our working directory so that the cwd # in subprocesses like git equals our $PWD (for pathname comparisons). -cd -P "$test" || error "Cannot setup test environment" +cd -P "$test" || error "Cannot set up test environment" if test "$verbose" = "t" then @@ -1101,7 +1239,6 @@ else exec 4>test.output 3>&4 fi -this_test=${0##*/} for skp in $NOTMUCH_SKIP_TESTS do to_skip= @@ -1110,6 +1247,12 @@ do case "$this_test" in $skp) to_skip=t + break + esac + case "$this_test_bare" in + $skp) + to_skip=t + break esac done case "$to_skip" in @@ -1176,5 +1319,4 @@ test_declare_external_prereq emacs test_declare_external_prereq ${TEST_EMACSCLIENT} test_declare_external_prereq gdb test_declare_external_prereq gpg -test_declare_external_prereq python -test_declare_external_prereq python2 +test_declare_external_prereq ${NOTMUCH_PYTHON}