test: require test_begin_subtest before test_expect_code

Unify the subtests by requiring test_begin_subtest before
test_expect_code. (Similar change for test_expect_success has already
been done.)

This increases clarity in the test scripts by having a separate line
for the start of the subtest with the heading, and makes it possible
to simplify the test infrastructure by making all subtests similar.

diff --git a/test/README b/test/README
index 7acdb4b81f4d89b769af41e271759448e0183f03..dcd05237a0623c859af0ae67e033994137dd0d54 100644 (file)
--- a/test/README
+++ b/test/README
@@ -200,6 +200,11 @@ library for your script to use.
    <script>.  If it yields success, test is considered
    successful.
 
    <script>.  If it yields success, test is considered
    successful.
 

+ test_expect_code <code> <script>
+
+   This takes two strings as parameter, and evaluates the <script>.
+   If it yields <code> exit status, test is considered successful.
+

  test_subtest_known_broken
 
    Mark the current test as broken.  Such tests are expected to fail.
  test_subtest_known_broken
 
    Mark the current test as broken.  Such tests are expected to fail.

diff --git a/test/T000-basic.sh b/test/T000-basic.sh
index 003ab95f5681eb0a9dc625bc2664ba861458bfda..36a7ca4c5bdfb78ed28430ab6c471123780b85f9 100755 (executable)
--- a/test/T000-basic.sh
+++ b/test/T000-basic.sh
@@ -30,11 +30,9 @@ test_begin_subtest 'tests clean up after themselves'
 clean=no
 test_expect_success 'test_when_finished clean=yes'
 
 clean=no
 test_expect_success 'test_when_finished clean=yes'
 

+test_begin_subtest 'tests clean up even after a failure'

 cleaner=no
 cleaner=no

-test_expect_code 1 'tests clean up even after a failure' '
-    test_when_finished cleaner=yes &&
-    (exit 1)
-'
+test_expect_code 1 'test_when_finished cleaner=yes && (exit 1)'

 
 if test $clean$cleaner != yesyes
 then
 
 if test $clean$cleaner != yesyes
 then


@@ -42,9 +40,8 @@ then
        exit 1
 fi
 
        exit 1
 fi
 

-test_expect_code 2 'failure to clean up causes the test to fail' '
-    test_when_finished "(exit 2)"
-'
+test_begin_subtest 'failure to clean up causes the test to fail'
+test_expect_code 2 'test_when_finished "(exit 2)"'

 
 EXPECTED=$TEST_DIRECTORY/test.expected-output
 suppress_diff_date() {
 
 EXPECTED=$TEST_DIRECTORY/test.expected-output
 suppress_diff_date() {

diff --git a/test/T020-compact.sh b/test/T020-compact.sh
index 1fecd071e77e07faaf6348beb4df82f82fc7dd30..a3d7380e81c8c12f1a685ca1f4e61cac292ea63c 100755 (executable)
--- a/test/T020-compact.sh
+++ b/test/T020-compact.sh
@@ -16,7 +16,8 @@ if [ $NOTMUCH_HAVE_XAPIAN_COMPACT -eq 0 ]; then
     test_expect_equal "$output" "notmuch was compiled against a xapian version lacking compaction support.
 Compaction failed: Unsupported operation"
 
     test_expect_equal "$output" "notmuch was compiled against a xapian version lacking compaction support.
 Compaction failed: Unsupported operation"
 

-    test_expect_code 1 "Compact unsupported: status code" "notmuch compact"
+    test_begin_subtest "Compact unsupported: status code"
+    test_expect_code 1 "notmuch compact"

 
     test_done
 fi
 
     test_done
 fi

diff --git a/test/T050-new.sh b/test/T050-new.sh
index 9115de820770181bd6ff8f8c9b655d047e9fd8c9..ffa303efaceca99cdc7c0d6cbf75a83f6184bb1d 100755 (executable)
--- a/test/T050-new.sh
+++ b/test/T050-new.sh
@@ -286,8 +286,8 @@ notmuch config set new.tags "-foo;bar"
 output=$(NOTMUCH_NEW --debug 2>&1)
 test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
 
 output=$(NOTMUCH_NEW --debug 2>&1)
 test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
 

-test_expect_code 1 "Invalid tags set exit code" \
-    "NOTMUCH_NEW --debug 2>&1"
+test_begin_subtest "Invalid tags set exit code"
+test_expect_code 1 "NOTMUCH_NEW --debug 2>&1"

 
 notmuch config set new.tags $OLDCONFIG
 
 
 notmuch config set new.tags $OLDCONFIG
 

diff --git a/test/T070-insert.sh b/test/T070-insert.sh
index 9120debabf8c3f6ffe07c6d2b74a5cc17f746698..48f212ee09283f45468a405c55544223c31daf0f 100755 (executable)
--- a/test/T070-insert.sh
+++ b/test/T070-insert.sh
@@ -20,13 +20,13 @@ gen_insert_msg() {
        "[body]=\"insert-message\""
 }
 
        "[body]=\"insert-message\""
 }
 

-test_expect_code 1 "Insert zero-length file" \
-    "notmuch insert < /dev/null"
+test_begin_subtest "Insert zero-length file"
+test_expect_code 1 "notmuch insert < /dev/null"

 
 # This test is a proxy for other errors that may occur while trying to
 # add a message to the notmuch database, e.g. database locked.
 
 # This test is a proxy for other errors that may occur while trying to
 # add a message to the notmuch database, e.g. database locked.

-test_expect_code 1 "Insert non-message" \
-    "echo bad_message | notmuch insert"
+test_begin_subtest "Insert non-message"
+test_expect_code 1 "echo bad_message | notmuch insert"

 
 test_begin_subtest "Database empty so far"
 test_expect_equal "0" "`notmuch count --output=messages '*'`"
 
 test_begin_subtest "Database empty so far"
 test_expect_equal "0" "`notmuch count --output=messages '*'`"


@@ -138,9 +138,9 @@ notmuch insert --folder=Drafts +draft -unread < "$gen_msg_filename"
 output=$(notmuch search --output=messages path:Drafts/cur tag:draft NOT tag:unread)
 test_expect_equal "$output" "id:$gen_msg_id"
 
 output=$(notmuch search --output=messages path:Drafts/cur tag:draft NOT tag:unread)
 test_expect_equal "$output" "id:$gen_msg_id"
 

+test_begin_subtest "Insert message into non-existent folder"

 gen_insert_msg
 gen_insert_msg

-test_expect_code 1 "Insert message into non-existent folder" \
-    "notmuch insert --folder=nonesuch < $gen_msg_filename"
+test_expect_code 1 "notmuch insert --folder=nonesuch < $gen_msg_filename"

 
 test_begin_subtest "Insert message, create folder"
 gen_insert_msg
 
 test_begin_subtest "Insert message, create folder"
 gen_insert_msg


@@ -162,9 +162,9 @@ notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
 output=$(notmuch count path:F/G/H/I/J/new tag:folder)
 test_expect_equal "$output" "2"
 
 output=$(notmuch count path:F/G/H/I/J/new tag:folder)
 test_expect_equal "$output" "2"
 

+test_begin_subtest "Insert message, create invalid subfolder"

 gen_insert_msg
 gen_insert_msg

-test_expect_code 1 "Insert message, create invalid subfolder" \
-    "notmuch insert --folder=../G --create-folder $gen_msg_filename"
+test_expect_code 1 "notmuch insert --folder=../G --create-folder $gen_msg_filename"

 
 OLDCONFIG=$(notmuch config get new.tags)
 
 
 OLDCONFIG=$(notmuch config get new.tags)
 


@@ -180,8 +180,8 @@ gen_insert_msg
 output=$(notmuch insert $gen_msg_filename 2>&1)
 test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
 
 output=$(notmuch insert $gen_msg_filename 2>&1)
 test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
 

-test_expect_code 1 "Invalid tags set exit code" \
-    "notmuch insert $gen_msg_filename 2>&1"
+test_begin_subtest "Invalid tags set exit code"
+test_expect_code 1 "notmuch insert $gen_msg_filename 2>&1"

 
 notmuch config set new.tags $OLDCONFIG
 
 
 notmuch config set new.tags $OLDCONFIG
 


@@ -205,22 +205,28 @@ done
 gen_insert_msg
 
 for code in  FILE_NOT_EMAIL READ_ONLY_DATABASE UPGRADE_REQUIRED PATH_ERROR; do
 gen_insert_msg
 
 for code in  FILE_NOT_EMAIL READ_ONLY_DATABASE UPGRADE_REQUIRED PATH_ERROR; do

-    test_expect_code 1 "EXIT_FAILURE when add_message returns $code" \
+    test_begin_subtest "EXIT_FAILURE when add_message returns $code"
+    test_expect_code 1 \

          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"
          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"

-    test_expect_code 0 "success exit with --keep when add_message returns $code" \
+
+    test_begin_subtest "success exit with --keep when add_message returns $code"
+    test_expect_code 0 \

          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert --keep < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"
 done
 
 for code in OUT_OF_MEMORY XAPIAN_EXCEPTION ; do
          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert --keep < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"
 done
 
 for code in OUT_OF_MEMORY XAPIAN_EXCEPTION ; do

-    test_expect_code 75 "EX_TEMPFAIL when add_message returns $code" \
+    test_begin_subtest "EX_TEMPFAIL when add_message returns $code"
+    test_expect_code 75 \

          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"
          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"

-    test_expect_code 0 "success exit with --keep when add_message returns $code" \
+
+    test_begin_subtest "success exit with --keep when add_message returns $code"
+    test_expect_code 0 \

          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert --keep < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"
          "${TEST_GDB} --batch-silent --return-child-result \
             -ex 'set args insert --keep < $gen_msg_filename' \
             -x index-file-$code.gdb notmuch"

diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh
index 61d1311608a755c5815bf545ff46170036339fa5..0d0a3b874526dccd24166066be3767fbe29efea5 100755 (executable)
--- a/test/T150-tagging.sh
+++ b/test/T150-tagging.sh
@@ -19,8 +19,11 @@ test_expect_equal "$output" "\
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag3 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)"
 
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; One (inbox tag3 unread)
 thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Two (inbox tag3 unread)"
 

-test_expect_code 1 "No tag operations" 'notmuch tag One'
-test_expect_code 1 "No query" 'notmuch tag +tag2'
+test_begin_subtest "No tag operations"
+test_expect_code 1 'notmuch tag One'
+
+test_begin_subtest "No query"
+test_expect_code 1 'notmuch tag +tag2'

 
 test_begin_subtest "Redundant tagging"
 notmuch tag +tag1 -tag3 One
 
 test_begin_subtest "Redundant tagging"
 notmuch tag +tag1 -tag3 One


@@ -282,9 +285,11 @@ notmuch dump --format=batch-tag| \
 
 test_expect_equal_file EXPECTED OUTPUT
 
 
 test_expect_equal_file EXPECTED OUTPUT
 

-test_expect_code 1 "Empty tag names" 'notmuch tag + One'
+test_begin_subtest "Empty tag names"
+test_expect_code 1 'notmuch tag + One'

 
 

-test_expect_code 1 "Tag name beginning with -" 'notmuch tag +- One'
+test_begin_subtest "Tag name beginning with -"
+test_expect_code 1 'notmuch tag +- One'

 
 test_begin_subtest "Xapian exception: read only files"
 chmod u-w  ${MAIL_DIR}/.notmuch/xapian/*.${db_ending}
 
 test_begin_subtest "Xapian exception: read only files"
 chmod u-w  ${MAIL_DIR}/.notmuch/xapian/*.${db_ending}

diff --git a/test/T160-json.sh b/test/T160-json.sh
index 5a954e3baf0d477e26bc570985c768fc7c20a10c..ac51895e826d260601260d6aae1a55bb90138fa9 100755 (executable)
--- a/test/T160-json.sh
+++ b/test/T160-json.sh
@@ -64,11 +64,11 @@ test_expect_equal_json "$output" "[{\"thread\": \"XXX\",
  \"tags\": [\"inbox\",
  \"unread\"]}]"
 
  \"tags\": [\"inbox\",
  \"unread\"]}]"
 

-test_expect_code 20 "Format version: too low" \
-    "notmuch search --format-version=0 \\*"
+test_begin_subtest "Format version: too low"
+test_expect_code 20 "notmuch search --format-version=0 \\*"

 
 

-test_expect_code 21 "Format version: too high" \
-    "notmuch search --format-version=999 \\*"
+test_begin_subtest "Format version: too high"
+test_expect_code 21 "notmuch search --format-version=999 \\*"

 
 test_begin_subtest "Show message: multiple filenames"
 add_message "[id]=message-id@example.com [filename]=copy1"
 
 test_begin_subtest "Show message: multiple filenames"
 add_message "[id]=message-id@example.com [filename]=copy1"

diff --git a/test/T400-hooks.sh b/test/T400-hooks.sh
index ae571ba282d9b1b33973aa1b44def8b1ed0e6e90..7917a82f0154ef1d8355e62d7ae30ebb796e3fa8 100755 (executable)
--- a/test/T400-hooks.sh
+++ b/test/T400-hooks.sh
@@ -70,7 +70,8 @@ output=`notmuch new 2>&1`
 test_expect_equal "$output" "Error: pre-new hook failed with status 13"
 
 # depends on the previous subtest leaving broken hook behind
 test_expect_equal "$output" "Error: pre-new hook failed with status 13"
 
 # depends on the previous subtest leaving broken hook behind

-test_expect_code 1 "pre-new non-zero exit status (notmuch status)" "notmuch new"
+test_begin_subtest "pre-new non-zero exit status (notmuch status)"
+test_expect_code 1 "notmuch new"

 
 # depends on the previous subtests leaving 1 new message behind
 test_begin_subtest "pre-new non-zero exit status aborts new"
 
 # depends on the previous subtests leaving 1 new message behind
 test_begin_subtest "pre-new non-zero exit status aborts new"


@@ -89,7 +90,8 @@ echo "Error: post-new hook failed with status 13" >> expected
 test_expect_equal_file expected output
 
 # depends on the previous subtest leaving broken hook behind
 test_expect_equal_file expected output
 
 # depends on the previous subtest leaving broken hook behind

-test_expect_code 1 "post-new non-zero exit status (notmuch status)" "notmuch new"
+test_begin_subtest "post-new non-zero exit status (notmuch status)"
+test_expect_code 1 "notmuch new"

 
 test_begin_subtest "post-insert hook does not affect insert status"
 rm_hooks
 
 test_begin_subtest "post-insert hook does not affect insert status"
 rm_hooks


@@ -97,7 +99,7 @@ generate_message
 create_failing_hook "post-insert"
 test_expect_success "notmuch insert < \"$gen_msg_filename\" > /dev/null"
 
 create_failing_hook "post-insert"
 test_expect_success "notmuch insert < \"$gen_msg_filename\" > /dev/null"
 

-# test_begin_subtest "hook without executable permissions"
+test_begin_subtest "hook without executable permissions"

 rm_hooks
 mkdir -p ${HOOK_DIR}
 cat <<EOF >"${HOOK_DIR}/pre-new"
 rm_hooks
 mkdir -p ${HOOK_DIR}
 cat <<EOF >"${HOOK_DIR}/pre-new"


@@ -105,15 +107,15 @@ cat <<EOF >"${HOOK_DIR}/pre-new"
 echo foo
 EOF
 output=`notmuch new 2>&1`
 echo foo
 EOF
 output=`notmuch new 2>&1`

-test_expect_code 1 "hook without executable permissions" "notmuch new"
+test_expect_code 1 "notmuch new"

 
 

-# test_begin_subtest "hook execution failure"
+test_begin_subtest "hook execution failure"

 rm_hooks
 mkdir -p ${HOOK_DIR}
 cat <<EOF >"${HOOK_DIR}/pre-new"
 no hashbang, execl fails
 EOF
 chmod +x "${HOOK_DIR}/pre-new"
 rm_hooks
 mkdir -p ${HOOK_DIR}
 cat <<EOF >"${HOOK_DIR}/pre-new"
 no hashbang, execl fails
 EOF
 chmod +x "${HOOK_DIR}/pre-new"

-test_expect_code 1 "hook execution failure" "notmuch new"
+test_expect_code 1 "notmuch new"

 
 test_done
 
 test_done

diff --git a/test/T570-revision-tracking.sh b/test/T570-revision-tracking.sh
index a63e691cd3d4b931659f9cd33d99947ee9370d07..76ad227944a0636b53168c8c0afd401dc26ef767 100755 (executable)
--- a/test/T570-revision-tracking.sh
+++ b/test/T570-revision-tracking.sh
@@ -55,23 +55,23 @@ test_expect_success "notmuch search --uuid=$(cat UUID) '*'"
 test_begin_subtest "uuid works as global option"
 test_expect_success "notmuch --uuid=$(cat UUID) search '*'"
 
 test_begin_subtest "uuid works as global option"
 test_expect_success "notmuch --uuid=$(cat UUID) search '*'"
 

-test_expect_code 1 'uuid works as global option II' \
-                   "notmuch --uuid=this-is-no-uuid search '*'"
+test_begin_subtest "uuid works as global option II"
+test_expect_code 1 "notmuch --uuid=this-is-no-uuid search '*'"

 
 

-test_expect_code 1 'search fails with incorrect uuid' \
-                "notmuch search --uuid=this-is-no-uuid '*'"
+test_begin_subtest "search fails with incorrect uuid"
+test_expect_code 1 "notmuch search --uuid=this-is-no-uuid '*'"

 
 test_begin_subtest "show succeeds with correct uuid"
 test_expect_success "notmuch show --uuid=$(cat UUID) '*'"
 
 
 test_begin_subtest "show succeeds with correct uuid"
 test_expect_success "notmuch show --uuid=$(cat UUID) '*'"
 

-test_expect_code 1 'show fails with incorrect uuid' \
-                "notmuch show --uuid=this-is-no-uuid '*'"
+test_begin_subtest "show fails with incorrect uuid"
+test_expect_code 1 "notmuch show --uuid=this-is-no-uuid '*'"

 
 test_begin_subtest "tag succeeds with correct uuid"
 test_expect_success "notmuch tag --uuid=$(cat UUID) +test '*'"
 
 
 test_begin_subtest "tag succeeds with correct uuid"
 test_expect_success "notmuch tag --uuid=$(cat UUID) +test '*'"
 

-test_expect_code 1 'tag fails with incorrect uuid' \
-                "notmuch tag --uuid=this-is-no-uuid '*' +test2"
+test_begin_subtest "tag fails with incorrect uuid"
+test_expect_code 1 "notmuch tag --uuid=this-is-no-uuid '*' +test2"

 
 test_begin_subtest 'lastmod:0.. matches everything'
 total=$(notmuch count '*')
 
 test_begin_subtest 'lastmod:0.. matches everything'
 total=$(notmuch count '*')

diff --git a/test/T600-named-queries.sh b/test/T600-named-queries.sh
index 450da9b0c25ae0d37dd79cd7b77d0fdb738d8e65..495b7699a0a781beee246fb6070bdb67e3ae584d 100755 (executable)
--- a/test/T600-named-queries.sh
+++ b/test/T600-named-queries.sh
@@ -4,8 +4,8 @@ test_description='named queries'
 
 QUERYSTR="date:2009-11-18..2009-11-18 and tag:unread"
 
 
 QUERYSTR="date:2009-11-18..2009-11-18 and tag:unread"
 

-test_expect_code 1 "error adding named query before initializing DB" \
-                "notmuch config set query.test \"$QUERYSTR\""
+test_begin_subtest "error adding named query before initializing DB"
+test_expect_code 1 "notmuch config set query.test \"$QUERYSTR\""

 
 add_email_corpus
 
 
 add_email_corpus
 

diff --git a/test/test-lib.sh b/test/test-lib.sh
index b68290c3f02fb2be28a296da9b7f89e5c00c9595..32bd3f1278a446b230dab2c11cca4c01eab6d3be 100644 (file)
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -940,13 +940,17 @@ test_expect_success () {
 }
 
 test_expect_code () {
 }
 
 test_expect_code () {

-       test "$#" = 3 ||
-       error "bug in the test script: not 3 parameters to test_expect_code"
-       test_subtest_name="$2"
-       test_reset_state_
-       if ! test_skip "$@"
+       exec 1>&6 2>&7          # Restore stdout and stderr
+       if [ -z "$inside_subtest" ]; then
+               error "bug in the test script: test_expect_code without test_begin_subtest"
+       fi
+       inside_subtest=
+       test "$#" = 2 ||
+       error "bug in the test script: not 2 parameters to test_expect_code"
+
+       if ! test_skip "$test_subtest_name"

        then
        then

-               test_run_ "$3"
+               test_run_ "$2"

                run_ret="$?"
                # test_run_ may update missing external prerequisites,
                test_check_missing_external_prereqs_ "$@" ||
                run_ret="$?"
                # test_run_ may update missing external prerequisites,
                test_check_missing_external_prereqs_ "$@" ||


@@ -954,7 +958,7 @@ test_expect_code () {
                then
                        test_ok_
                else
                then
                        test_ok_
                else

-                       test_failure_ "exit code $eval_ret, expected $1" "$3"
+                       test_failure_ "exit code $eval_ret, expected $1" "$2"

                fi
        fi
 }
                fi
        fi
 }