]> git.notmuchmail.org Git - notmuch/blobdiff - test/test-lib.sh
test: renamed test scripts to format T\d\d\d-name.sh
[notmuch] / test / test-lib.sh
index 076f92951673ef42e7e0ed56840fc7843bf78bd4..487075e5e7e4cf8c13226e3ca2426f3c27904bc9 100644 (file)
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 #
 # Copyright (c) 2005 Junio C Hamano
 #
@@ -23,6 +22,13 @@ if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
     exit 1
 fi
 
+# Make sure echo builtin does not expand backslash-escape sequences by default.
+shopt -u xpg_echo
+
+this_test=${0##*/}
+this_test=${this_test%.sh}
+this_test=${this_test#T[0-9][0-9][0-9]-}
+
 # if --tee was passed, write the output not only to the terminal, but
 # additionally to the file test-results/$BASENAME.out, too.
 case "$GIT_TEST_TEE_STARTED, $* " in
@@ -31,7 +37,7 @@ done,*)
        ;;
 *' --tee '*|*' --va'*)
        mkdir -p test-results
-       BASE=test-results/$(basename "$0" .sh)
+       BASE=test-results/$this_test
        (GIT_TEST_TEE_STARTED=done ${SHELL-sh} "$0" "$@" 2>&1;
         echo $? > $BASE.exit) | tee $BASE.out
        test "$(cat $BASE.exit)" = 0
@@ -42,6 +48,10 @@ esac
 # Keep the original TERM for say_color and test_emacs
 ORIGINAL_TERM=$TERM
 
+# dtach(1) provides more capable terminal environment to anything
+# that requires more than dumb terminal...
+[ x"${TERM:-dumb}" = xdumb ] && DTACH_TERM=vt100 || DTACH_TERM=$TERM
+
 # For repeatability, reset the environment to known value.
 LANG=C
 LC_ALL=C
@@ -50,6 +60,13 @@ TZ=UTC
 TERM=dumb
 export LANG LC_ALL PAGER TERM TZ
 GIT_TEST_CMP=${GIT_TEST_CMP:-diff -u}
+if [[ ( -n "$TEST_EMACS" && -z "$TEST_EMACSCLIENT" ) || \
+      ( -z "$TEST_EMACS" && -n "$TEST_EMACSCLIENT" ) ]]; then
+    echo "error: must specify both or neither of TEST_EMACS and TEST_EMACSCLIENT" >&2
+    exit 1
+fi
+TEST_EMACS=${TEST_EMACS:-${EMACS:-emacs}}
+TEST_EMACSCLIENT=${TEST_EMACSCLIENT:-emacsclient}
 
 # Protect ourselves from common misconfiguration to export
 # CDPATH into the environment
@@ -116,6 +133,16 @@ do
        esac
 done
 
+if test -n "$debug"; then
+    print_subtest () {
+       printf " %-4s" "[$((test_count - 1))]"
+    }
+else
+    print_subtest () {
+       true
+    }
+fi
+
 if test -n "$color"; then
        say_color () {
                (
@@ -130,8 +157,9 @@ if test -n "$color"; then
                esac
                shift
                printf " "
-                printf "$@"
+               printf "$@"
                tput sgr0
+               print_subtest
                )
        }
 else
@@ -139,7 +167,8 @@ else
                test -z "$1" && test -n "$quiet" && return
                shift
                printf " "
-                printf "$@"
+               printf "$@"
+               print_subtest
        }
 fi
 
@@ -162,7 +191,18 @@ then
        exit 0
 fi
 
-echo $(basename "$0"): "Testing ${test_description}"
+test_description_printed=
+print_test_description ()
+{
+       test -z "$test_description_printed" || return 0
+       echo
+       echo $this_test: "Testing ${test_description}"
+       test_description_printed=1
+}
+if [ -z "$NOTMUCH_TEST_QUIET" ]
+then
+       print_test_description
+fi
 
 exec 5>&1
 
@@ -172,22 +212,39 @@ test_fixed=0
 test_broken=0
 test_success=0
 
-die () {
+_die_common () {
        code=$?
+       trap - EXIT
+       set +ex
        rm -rf "$TEST_TMPDIR"
+}
+
+die () {
+       _die_common
        if test -n "$GIT_EXIT_OK"
        then
                exit $code
        else
-               echo >&5 "FATAL: Unexpected exit with code $code"
+               exec >&5
+               say_color error '%-6s' FATAL
+               echo " $test_subtest_name"
+               echo
+               echo "Unexpected exit while executing $0. Exit code $code."
                exit 1
        fi
 }
 
+die_signal () {
+       _die_common
+       echo >&5 "FATAL: $0: interrupted by signal" $((code - 128))
+       exit $code
+}
+
 GIT_EXIT_OK=
 # Note: TEST_TMPDIR *NOT* exported!
 TEST_TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-test-$$.XXXXXX")
 trap 'die' EXIT
+trap 'die_signal' HUP INT TERM
 
 test_decode_color () {
        sed     -e 's/.\[1m/<WHITE>/g' \
@@ -237,7 +294,7 @@ remove_cr () {
 #      Store the message in file 'name'. The default is to store it
 #      in 'msg-<count>', where <count> is three-digit number of the
 #      message.
-#      
+#
 #  [body]=text
 #
 #      Text to use as the body of the email message
@@ -306,11 +363,19 @@ generate_message ()
     fi
 
     if [ -z "${template[subject]}" ]; then
-       template[subject]="Test message #${gen_msg_cnt}"
+       if [ -n "$test_subtest_name" ]; then
+           template[subject]="$test_subtest_name"
+       else
+           template[subject]="Test message #${gen_msg_cnt}"
+       fi
     fi
 
     if [ -z "${template[date]}" ]; then
-       template[date]="Tue, 05 Jan 2001 15:43:57 -0000"
+       # we use decreasing timestamps here for historical reasons;
+       # the existing test suite when we converted to unique timestamps just
+       # happened to have signicantly fewer failures with that choice.
+       template[date]=$(TZ=UTC printf "%(%a, %d %b %Y %T %z)T\n" \
+                       $((978709437 - gen_msg_cnt)))
     fi
 
     additional_headers=""
@@ -334,6 +399,11 @@ ${additional_headers}"
 ${additional_headers}"
     fi
 
+    if [ ! -z "${template[bcc]}" ]; then
+       additional_headers="Bcc: ${template[bcc]}
+${additional_headers}"
+    fi
+
     if [ ! -z "${template[references]}" ]; then
        additional_headers="References: ${template[references]}
 ${additional_headers}"
@@ -344,6 +414,11 @@ ${additional_headers}"
 ${additional_headers}"
     fi
 
+    if [ ! -z "${template[content-transfer-encoding]}" ]; then
+       additional_headers="Content-Transfer-Encoding: ${template[content-transfer-encoding]}
+${additional_headers}"
+    fi
+
     # Note that in the way we're setting it above and using it below,
     # `additional_headers' will also serve as the header / body separator
     # (empty line in between).
@@ -382,13 +457,16 @@ emacs_deliver_message ()
     shift 2
     # before we can send a message, we have to prepare the FCC maildir
     mkdir -p "$MAIL_DIR"/sent/{cur,new,tmp}
-    $TEST_DIRECTORY/smtp-dummy sent_message &
-    smtp_dummy_pid=$!
+    # eval'ing smtp-dummy --background will set smtp_dummy_pid
+    smtp_dummy_pid=
+    eval `$TEST_DIRECTORY/smtp-dummy --background sent_message`
+    test -n "$smtp_dummy_pid" || return 1
+
     test_emacs \
        "(let ((message-send-mail-function 'message-smtpmail-send-it)
+               (mail-host-address \"example.com\")
               (smtpmail-smtp-server \"localhost\")
               (smtpmail-smtp-service \"25025\"))
-          (notmuch-hello)
           (notmuch-mua-mail)
           (message-goto-to)
           (insert \"test_suite@notmuchmail.org\nDate: 01 Jan 2000 12:00:00 -0000\")
@@ -398,9 +476,41 @@ emacs_deliver_message ()
           (insert \"${body}\")
           $@
           (message-send-and-exit))"
-    # opportunistically quit smtp-dummy in case above fails.
-    { echo QUIT > /dev/tcp/localhost/25025; } 2>/dev/null
-    wait ${smtp_dummy_pid}
+
+    # In case message was sent properly, client waits for confirmation
+    # before exiting and resuming control here; therefore making sure
+    # that server exits by sending (KILL) signal to it is safe.
+    kill -9 $smtp_dummy_pid
+    notmuch new >/dev/null
+}
+
+# Pretend to deliver a message with emacs. Really save it to a file
+# and add it to the database
+#
+# Uses emacs to generate and deliver a message to the mail store.
+# Accepts arbitrary extra emacs/elisp functions to modify the message
+# before sending, which is useful to doing things like attaching files
+# to the message and encrypting/signing.
+emacs_fcc_message ()
+{
+    local subject="$1"
+    local body="$2"
+    shift 2
+    # before we can send a message, we have to prepare the FCC maildir
+    mkdir -p "$MAIL_DIR"/sent/{cur,new,tmp}
+
+    test_emacs \
+       "(let ((message-send-mail-function (lambda () t))
+               (mail-host-address \"example.com\"))
+          (notmuch-mua-mail)
+          (message-goto-to)
+          (insert \"test_suite@notmuchmail.org\nDate: 01 Jan 2000 12:00:00 -0000\")
+          (message-goto-subject)
+          (insert \"${subject}\")
+          (message-goto-body)
+          (insert \"${body}\")
+          $@
+          (message-send-and-exit))" || return 1
     notmuch new >/dev/null
 }
 
@@ -457,12 +567,12 @@ test_expect_equal ()
        if ! test_skip "$test_subtest_name"
        then
                if [ "$output" = "$expected" ]; then
-                       test_ok_ "$test_subtest_name"
+                       test_ok_
                else
                        testname=$this_test.$test_count
                        echo "$expected" > $testname.expected
                        echo "$output" > $testname.output
-                       test_failure_ "$test_subtest_name" "$(diff -u $testname.expected $testname.output)"
+                       test_failure_ "$(diff -u $testname.expected $testname.output)"
                fi
     fi
 }
@@ -476,29 +586,75 @@ test_expect_equal_file ()
        test "$#" = 2 ||
        error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
 
-       output="$1"
-       expected="$2"
+       file1="$1"
+       basename1=`basename "$file1"`
+       file2="$2"
+       basename2=`basename "$file2"`
        if ! test_skip "$test_subtest_name"
        then
-               if diff -q "$expected" "$output" >/dev/null ; then
-                       test_ok_ "$test_subtest_name"
+               if diff -q "$file1" "$file2" >/dev/null ; then
+                       test_ok_
                else
                        testname=$this_test.$test_count
-                       cp "$output" $testname.output
-                       cp "$expected" $testname.expected
-                       test_failure_ "$test_subtest_name" "$(diff -u $testname.expected $testname.output)"
+                       cp "$file1" "$testname.$basename1"
+                       cp "$file2" "$testname.$basename2"
+                       test_failure_ "$(diff -u "$testname.$basename1" "$testname.$basename2")"
                fi
     fi
 }
 
+# Like test_expect_equal, but arguments are JSON expressions to be
+# canonicalized before diff'ing.  If an argument cannot be parsed, it
+# is used unchanged so that there's something to diff against.
+test_expect_equal_json () {
+    # The test suite forces LC_ALL=C, but this causes Python 3 to
+    # decode stdin as ASCII.  We need to read JSON in UTF-8, so
+    # override Python's stdio encoding defaults.
+    output=$(echo "$1" | PYTHONIOENCODING=utf-8 python -mjson.tool \
+        || echo "$1")
+    expected=$(echo "$2" | PYTHONIOENCODING=utf-8 python -mjson.tool \
+        || echo "$2")
+    shift 2
+    test_expect_equal "$output" "$expected" "$@"
+}
+
+test_emacs_expect_t () {
+       test "$#" = 2 && { prereq=$1; shift; } || prereq=
+       test "$#" = 1 ||
+       error "bug in the test script: not 1 or 2 parameters to test_emacs_expect_t"
+
+       # Run the test.
+       if ! test_skip "$test_subtest_name"
+       then
+               test_emacs "(notmuch-test-run $1)" >/dev/null
+
+               # Restore state after the test.
+               exec 1>&6 2>&7          # Restore stdout and stderr
+               inside_subtest=
+
+               # Report success/failure.
+               result=$(cat OUTPUT)
+               if [ "$result" = t ]
+               then
+                       test_ok_
+               else
+                       test_failure_ "${result}"
+               fi
+       else
+               # Restore state after the (non) test.
+               exec 1>&6 2>&7          # Restore stdout and stderr
+               inside_subtest=
+       fi
+}
+
 NOTMUCH_NEW ()
 {
-    notmuch new | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'
+    notmuch new "${@}" | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'
 }
 
 notmuch_search_sanitize ()
 {
-    sed -r -e 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'
+    perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'
 }
 
 NOTMUCH_SHOW_FILENAME_SQUELCH='s,filename:.*/mail,filename:/XXX/mail,'
@@ -510,17 +666,36 @@ notmuch_show_sanitize_all ()
 {
     sed \
        -e 's| filename:.*| filename:XXXXX|' \
-       -e 's| id:[^ ]* | id:XXXXX |'
+       -e 's| id:[^ ]* | id:XXXXX |' | \
+       notmuch_date_sanitize
 }
 
 notmuch_json_show_sanitize ()
 {
-    sed -e 's|, |,\n |g' | \
-       sed \
-       -e 's|"id": "[^"]*",|"id": "XXXXX",|' \
-       -e 's|"filename": "[^"]*",|"filename": "YYYYY",|'
+    sed \
+       -e 's|"id": "[^"]*",|"id": "XXXXX",|g' \
+       -e 's|"Date": "Fri, 05 Jan 2001 [^"]*0000"|"Date": "GENERATED_DATE"|g' \
+       -e 's|"filename": "/[^"]*",|"filename": "YYYYY",|g' \
+       -e 's|"timestamp": 97.......|"timestamp": 42|g'
 }
 
+notmuch_emacs_error_sanitize ()
+{
+    local command=$1
+    shift
+    for file in "$@"; do
+       echo "=== $file ==="
+       cat "$file"
+    done | sed  \
+       -e 's/^\[.*\]$/[XXX]/' \
+       -e "s|^\(command: \)\{0,1\}/.*/$command|\1YYY/$command|"
+}
+
+notmuch_date_sanitize ()
+{
+    sed \
+       -e 's/^Date: Fri, 05 Jan 2001 .*0000/Date: GENERATED_DATE/'
+}
 # End of notmuch helper functions
 
 # Use test_set_prereq to tell that a particular prerequisite is available.
@@ -548,17 +723,51 @@ test_have_prereq () {
        esac
 }
 
+declare -A test_missing_external_prereq_
+declare -A test_subtest_missing_external_prereq_
+
+# declare prerequisite for the given external binary
+test_declare_external_prereq () {
+       binary="$1"
+       test "$#" = 2 && name=$2 || name="$binary(1)"
+
+       if ! hash $binary 2>/dev/null; then
+               test_missing_external_prereq_["${binary}"]=t
+               eval "
+$binary () {
+       test_subtest_missing_external_prereq_[\"${name}\"]=t
+       false
+}"
+       fi
+}
+
+# Explicitly require external prerequisite.  Useful when binary is
+# called indirectly (e.g. from emacs).
+# Returns success if dependency is available, failure otherwise.
+test_require_external_prereq () {
+       binary="$1"
+       if [[ ${test_missing_external_prereq_["${binary}"]} == t ]]; then
+               # dependency is missing, call the replacement function to note it
+               eval "$binary"
+       else
+               true
+       fi
+}
+
 # You are not expected to call test_ok_ and test_failure_ directly, use
 # the text_expect_* functions instead.
 
 test_ok_ () {
        if test "$test_subtest_known_broken_" = "t"; then
-               test_known_broken_ok_ "$@"
+               test_known_broken_ok_
                return
        fi
        test_success=$(($test_success + 1))
+       if test -n "$NOTMUCH_TEST_QUIET"; then
+               return 0
+       fi
        say_color pass "%-6s" "PASS"
-       echo " $@"
+       echo " $test_subtest_name"
 }
 
 test_failure_ () {
@@ -567,7 +776,8 @@ test_failure_ () {
                return
        fi
        test_failure=$(($test_failure + 1))
-       test_failure_message_ "FAIL" "$@"
+       print_test_description
+       test_failure_message_ "FAIL" "$test_subtest_name" "$@"
        test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
        return 1
 }
@@ -584,13 +794,13 @@ test_known_broken_ok_ () {
        test_reset_state_
        test_fixed=$(($test_fixed+1))
        say_color pass "%-6s" "FIXED"
-       echo " $@"
+       echo " $test_subtest_name"
 }
 
 test_known_broken_failure_ () {
        test_reset_state_
        test_broken=$(($test_broken+1))
-       test_failure_message_ "BROKEN" "$@"
+       test_failure_message_ "BROKEN" "$test_subtest_name" "$@"
        return 1
 }
 
@@ -624,18 +834,32 @@ test_skip () {
        fi
        case "$to_skip" in
        t)
-               test_reset_state_
-               say_color skip >&3 "skipping test: $@"
-               say_color skip "%-6s" "SKIP"
-               echo " $1"
-               : true
+               test_report_skip_ "$@"
                ;;
        *)
-               false
+               test_check_missing_external_prereqs_ "$@"
                ;;
        esac
 }
 
+test_check_missing_external_prereqs_ () {
+       if [[ ${#test_subtest_missing_external_prereq_[@]} != 0 ]]; then
+               say_color skip >&1 "missing prerequisites: "
+               echo ${!test_subtest_missing_external_prereq_[@]} >&1
+               test_report_skip_ "$@"
+       else
+               false
+       fi
+}
+
+test_report_skip_ () {
+       test_reset_state_
+       say_color skip >&3 "skipping test:"
+       echo " $@" >&3
+       say_color skip "%-6s" "SKIP"
+       echo " $1"
+}
+
 test_subtest_known_broken () {
        test_subtest_known_broken_=t
 }
@@ -644,15 +868,19 @@ test_expect_success () {
        test "$#" = 3 && { prereq=$1; shift; } || prereq=
        test "$#" = 2 ||
        error "bug in the test script: not 2 or 3 parameters to test-expect-success"
+       test_subtest_name="$1"
        test_reset_state_
        if ! test_skip "$@"
        then
                test_run_ "$2"
-               if [ "$?" = 0 -a "$eval_ret" = 0 ]
+               run_ret="$?"
+               # test_run_ may update missing external prerequisites
+               test_check_missing_external_prereqs_ "$@" ||
+               if [ "$run_ret" = 0 -a "$eval_ret" = 0 ]
                then
-                       test_ok_ "$1"
+                       test_ok_
                else
-                       test_failure_ "$@"
+                       test_failure_ "$2"
                fi
        fi
 }
@@ -661,15 +889,19 @@ test_expect_code () {
        test "$#" = 4 && { prereq=$1; shift; } || prereq=
        test "$#" = 3 ||
        error "bug in the test script: not 3 or 4 parameters to test-expect-code"
+       test_subtest_name="$2"
        test_reset_state_
        if ! test_skip "$@"
        then
                test_run_ "$3"
-               if [ "$?" = 0 -a "$eval_ret" = "$1" ]
+               run_ret="$?"
+               # test_run_ may update missing external prerequisites,
+               test_check_missing_external_prereqs_ "$@" ||
+               if [ "$run_ret" = 0 -a "$eval_ret" = "$1" ]
                then
-                       test_ok_ "$2"
+                       test_ok_
                else
-                       test_failure_ "$@"
+                       test_failure_ "exit code $eval_ret, expected $1" "$3"
                fi
        fi
 }
@@ -686,10 +918,10 @@ test_external () {
        test "$#" = 4 && { prereq=$1; shift; } || prereq=
        test "$#" = 3 ||
        error >&5 "bug in the test script: not 3 or 4 parameters to test_external"
-       descr="$1"
+       test_subtest_name="$1"
        shift
        test_reset_state_
-       if ! test_skip "$descr" "$@"
+       if ! test_skip "$test_subtest_name" "$@"
        then
                # Announce the script to reduce confusion about the
                # test output that follows.
@@ -700,9 +932,9 @@ test_external () {
                "$@" 2>&4
                if [ "$?" = 0 ]
                then
-                       test_ok_ "$descr"
+                       test_ok_
                else
-                       test_failure_ "$descr" "$@"
+                       test_failure_ "$@"
                fi
        fi
 }
@@ -716,11 +948,11 @@ test_external_without_stderr () {
        stderr="$tmp/git-external-stderr.$$.tmp"
        test_external "$@" 4> "$stderr"
        [ -f "$stderr" ] || error "Internal error: $stderr disappeared."
-       descr="no stderr: $1"
+       test_subtest_name="no stderr: $1"
        shift
        if [ ! -s "$stderr" ]; then
                rm "$stderr"
-               test_ok_ "$descr"
+               test_ok_
        else
                if [ "$verbose" = t ]; then
                        output=`echo; echo Stderr is:; cat "$stderr"`
@@ -729,7 +961,7 @@ test_external_without_stderr () {
                fi
                # rm first in case test_failure exits.
                rm "$stderr"
-               test_failure_ "$descr" "$@" "$output"
+               test_failure_ "$@" "$output"
        fi
 }
 
@@ -796,7 +1028,7 @@ test_done () {
        GIT_EXIT_OK=t
        test_results_dir="$TEST_DIRECTORY/test-results"
        mkdir -p "$test_results_dir"
-       test_results_path="$test_results_dir/${0%.sh}-$$"
+       test_results_path="$test_results_dir/$this_test"
 
        echo "total $test_count" >> $test_results_path
        echo "success $test_success" >> $test_results_path
@@ -805,12 +1037,10 @@ test_done () {
        echo "failed $test_failure" >> $test_results_path
        echo "" >> $test_results_path
 
-       echo
-
        [ -n "$EMACS_SERVER" ] && test_emacs '(kill-emacs)'
 
        if [ "$test_failure" = "0" ]; then
-           if [ "$test_broken" = "0" ]; then       
+           if [ "$test_broken" = "0" ]; then
                rm -rf "$remove_tmp"
            fi
            exit 0
@@ -822,7 +1052,7 @@ test_done () {
 emacs_generate_script () {
        # Construct a little test script here for the benefit of the user,
        # (who can easily run "run_emacs" to get the same emacs environment
-       # for investigating any failures).    
+       # for investigating any failures).
        cat <<EOF >"$TMP_DIRECTORY/run_emacs"
 #!/bin/sh
 export PATH=$PATH
@@ -838,7 +1068,7 @@ export NOTMUCH_CONFIG=$NOTMUCH_CONFIG
 #
 # --load               Force loading of notmuch.el and test-lib.el
 
-exec emacs --no-init-file --no-site-file \
+exec ${TEST_EMACS} --no-init-file --no-site-file \
        --directory "$TEST_DIRECTORY/../emacs" --load notmuch.el \
        --directory "$TEST_DIRECTORY" --load test-lib.el \
        "\$@"
@@ -847,155 +1077,113 @@ EOF
 }
 
 test_emacs () {
+       # test dependencies beforehand to avoid the waiting loop below
+       missing_dependencies=
+       test_require_external_prereq dtach || missing_dependencies=1
+       test_require_external_prereq emacs || missing_dependencies=1
+       test_require_external_prereq ${TEST_EMACSCLIENT} || missing_dependencies=1
+       test -z "$missing_dependencies" || return
+
        if [ -z "$EMACS_SERVER" ]; then
+               emacs_tests="${this_test}.el"
+               if [ -f "$TEST_DIRECTORY/$emacs_tests" ]; then
+                       load_emacs_tests="--eval '(load \"$emacs_tests\")'"
+               else
+                       load_emacs_tests=
+               fi
                server_name="notmuch-test-suite-$$"
                # start a detached session with an emacs server
-               # user's TERM is given to dtach which assumes a minimally
+               # user's TERM (or 'vt100' in case user's TERM is unset, empty
+               # or 'dumb') is given to dtach which assumes a minimally
                # VT100-compatible terminal -- and emacs inherits that
-               TERM=$ORIGINAL_TERM dtach -n "$TEST_TMPDIR/emacs-dtach-socket.$$" \
+               TERM=$DTACH_TERM dtach -n "$TEST_TMPDIR/emacs-dtach-socket.$$" \
                        sh -c "stty rows 24 cols 80; exec '$TMP_DIRECTORY/run_emacs' \
                                --no-window-system \
+                               $load_emacs_tests \
                                --eval '(setq server-name \"$server_name\")' \
                                --eval '(server-start)' \
                                --eval '(orphan-watchdog $$)'" || return
                EMACS_SERVER="$server_name"
                # wait until the emacs server is up
-               until test_emacs '()' 2>/dev/null; do
+               until test_emacs '()' >/dev/null 2>/dev/null; do
                        sleep 1
                done
        fi
 
-       emacsclient --socket-name="$EMACS_SERVER" --eval "(progn $@)"
-}
+       # Clear test-output output file.  Most Emacs tests end with a
+       # call to (test-output).  If the test code fails with an
+       # exception before this call, the output file won't get
+       # updated.  Since we don't want to compare against an output
+       # file from another test, so start out with an empty file.
+       rm -f OUTPUT
+       touch OUTPUT
 
-test_reset_state_ () {
-       test_subtest_known_broken_=
+       ${TEST_EMACSCLIENT} --socket-name="$EMACS_SERVER" --eval "(progn $@)"
 }
 
+test_python() {
+       export LD_LIBRARY_PATH=$TEST_DIRECTORY/../lib
+       export PYTHONPATH=$TEST_DIRECTORY/../bindings/python
 
-find_notmuch_path ()
-{
-    dir="$1"
+       # Some distros (e.g. Arch Linux) ship Python 2.* as /usr/bin/python2,
+       # most others as /usr/bin/python. So first try python2, and fallback to
+       # python if python2 doesn't exist.
+       cmd=python2
+       [[ ${test_missing_external_prereq_[python2]} == t ]] && cmd=python
 
-    while [ -n "$dir" ]; do
-       bin="$dir/notmuch"
-       if [ -x "$bin" ]; then
-           echo "$dir"
-           return
-       fi
-       dir="$(dirname "$dir")"
-       if [ "$dir" = "/" ]; then
-           break
-       fi
-    done
+       (echo "import sys; _orig_stdout=sys.stdout; sys.stdout=open('OUTPUT', 'w')"; cat) \
+               | $cmd -
 }
 
-# Test the binaries we have just built.  The tests are kept in
-# test/ subdirectory and are run in 'trash directory' subdirectory.
-TEST_DIRECTORY=$(pwd)
-if test -n "$valgrind"
-then
-       make_symlink () {
-               test -h "$2" &&
-               test "$1" = "$(readlink "$2")" || {
-                       # be super paranoid
-                       if mkdir "$2".lock
-                       then
-                               rm -f "$2" &&
-                               ln -s "$1" "$2" &&
-                               rm -r "$2".lock
-                       else
-                               while test -d "$2".lock
-                               do
-                                       say "Waiting for lock on $2."
-                                       sleep 1
-                               done
-                       fi
-               }
-       }
+# Creates a script that counts how much time it is executed and calls
+# notmuch.  $notmuch_counter_command is set to the path to the
+# generated script.  Use notmuch_counter_value() function to get the
+# current counter value.
+notmuch_counter_reset () {
+       notmuch_counter_command="$TMP_DIRECTORY/notmuch_counter"
+       if [ ! -x "$notmuch_counter_command" ]; then
+               notmuch_counter_state_path="$TMP_DIRECTORY/notmuch_counter.state"
+               cat >"$notmuch_counter_command" <<EOF || return
+#!/bin/sh
 
-       make_valgrind_symlink () {
-               # handle only executables
-               test -x "$1" || return
+read count < "$notmuch_counter_state_path"
+echo \$((count + 1)) > "$notmuch_counter_state_path"
 
-               base=$(basename "$1")
-               symlink_target=$TEST_DIRECTORY/../$base
-               # do not override scripts
-               if test -x "$symlink_target" &&
-                   test ! -d "$symlink_target" &&
-                   test "#!" != "$(head -c 2 < "$symlink_target")"
-               then
-                       symlink_target=$TEST_DIRECTORY/valgrind.sh
-               fi
-               case "$base" in
-               *.sh|*.perl)
-                       symlink_target=$TEST_DIRECTORY/unprocessed-script
-               esac
-               # create the link, or replace it if it is out of date
-               make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
-       }
+exec notmuch "\$@"
+EOF
+               chmod +x "$notmuch_counter_command" || return
+       fi
 
-       # override notmuch executable in TEST_DIRECTORY/..
-       GIT_VALGRIND=$TEST_DIRECTORY/valgrind
-       mkdir -p "$GIT_VALGRIND"/bin
-       make_valgrind_symlink $TEST_DIRECTORY/../notmuch
-       OLDIFS=$IFS
-       IFS=:
-       for path in $PATH
-       do
-               ls "$path"/notmuch 2> /dev/null |
-               while read file
-               do
-                       make_valgrind_symlink "$file"
-               done
-       done
-       IFS=$OLDIFS
-       PATH=$GIT_VALGRIND/bin:$PATH
-       GIT_EXEC_PATH=$GIT_VALGRIND/bin
-       export GIT_VALGRIND
-else # normal case
-       notmuch_path=`find_notmuch_path "$TEST_DIRECTORY"`
-       test -n "$notmuch_path" && PATH="$notmuch_path:$PATH"
-fi
-export PATH
-
-# Test repository
-test="tmp.$(basename "$0" .sh)"
-test -n "$root" && test="$root/$test"
-case "$test" in
-/*) TMP_DIRECTORY="$test" ;;
- *) TMP_DIRECTORY="$TEST_DIRECTORY/$test" ;;
-esac
-test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
-rm -fr "$test" || {
-       GIT_EXIT_OK=t
-       echo >&5 "FATAL: Cannot prepare test area"
-       exit 1
+       echo 0 > "$notmuch_counter_state_path"
 }
 
-# A temporary home directory is needed by at least:
-# - emacs/"Sending a message via (fake) SMTP"
-# - emacs/"Reply within emacs"
-# - crypto/emacs_deliver_message
-export HOME="${TMP_DIRECTORY}/home"
-mkdir -p "${HOME}"
+# Returns the current notmuch counter value.
+notmuch_counter_value () {
+       if [ -r "$notmuch_counter_state_path" ]; then
+               read count < "$notmuch_counter_state_path"
+       else
+               count=0
+       fi
+       echo $count
+}
 
-MAIL_DIR="${TMP_DIRECTORY}/mail"
-export GNUPGHOME="${TMP_DIRECTORY}/gnupg"
-export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config"
+test_reset_state_ () {
+       test -z "$test_init_done_" && test_init_
 
-mkdir -p "${test}"
-mkdir -p "${MAIL_DIR}"
+       test_subtest_known_broken_=
+       test_subtest_missing_external_prereq_=()
+}
 
-cat <<EOF >"${NOTMUCH_CONFIG}"
-[database]
-path=${MAIL_DIR}
+# called once before the first subtest
+test_init_ () {
+       test_init_done_=t
 
-[user]
-name=Notmuch Test Suite
-primary_email=test_suite@notmuchmail.org
-other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
-EOF
+       # skip all tests if there were external prerequisites missing during init
+       test_check_missing_external_prereqs_ "all tests in $this_test" && test_done
+}
+
+
+. ./test-lib-common.sh
 
 emacs_generate_script
 
@@ -1011,7 +1199,6 @@ else
        exec 4>test.output 3>&4
 fi
 
-this_test=${0##*/}
 for skp in $NOTMUCH_SKIP_TESTS
 do
        to_skip=
@@ -1079,3 +1266,12 @@ test -z "$NO_PYTHON" && test_set_prereq PYTHON
 # test whether the filesystem supports symbolic links
 ln -s x y 2>/dev/null && test -h y 2>/dev/null && test_set_prereq SYMLINKS
 rm -f y
+
+# declare prerequisites for external binaries used in tests
+test_declare_external_prereq dtach
+test_declare_external_prereq emacs
+test_declare_external_prereq ${TEST_EMACSCLIENT}
+test_declare_external_prereq gdb
+test_declare_external_prereq gpg
+test_declare_external_prereq python
+test_declare_external_prereq python2