git.notmuchmail.org Git - notmuch/blob - test/T240-dump-restore.sh

   1 #!/usr/bin/env bash
   2 test_description="\"notmuch dump\" and \"notmuch restore\""
   3 . $(dirname "$0")/test-lib.sh || exit 1
   4
   5 NOTMUCH_NEW > /dev/null
   6 test_begin_subtest "dump header"
   7 cat <<EOF > EXPECTED
   8 #notmuch-dump batch-tag:3 config,properties,tags
   9 EOF
  10 notmuch dump > OUTPUT
  11 test_expect_equal_file EXPECTED OUTPUT
  12 add_email_corpus
  13
  14 test_begin_subtest "Dumping all tags"
  15 test_expect_success 'generate_message && notmuch new && notmuch dump > dump.expected'
  16
  17 # The use of from:cworth is rather arbitrary: it matches some of the
  18 # email corpus' messages, but not all of them.
  19
  20 test_begin_subtest "Dumping all tags II"
  21 test_expect_success \
  22   'notmuch tag +ABC +DEF -- from:cworth &&
  23   notmuch dump > dump-ABC_DEF.expected &&
  24   ! cmp dump.expected dump-ABC_DEF.expected'
  25
  26 test_begin_subtest "Clearing all tags"
  27 test_expect_success \
  28   'sed -e "s/(\([^(]*\))$/()/" < dump.expected > clear.expected &&
  29   notmuch restore --input=clear.expected &&
  30   notmuch dump > clear.actual &&
  31   test_cmp clear.expected clear.actual'
  32
  33 test_begin_subtest "Clearing all tags"
  34 test_expect_success \
  35   'notmuch tag +ABC +DEF -- from:cworth &&
  36   notmuch restore --accumulate < dump.expected &&
  37   notmuch dump > dump.actual &&
  38   test_cmp dump-ABC_DEF.expected dump.actual'
  39
  40 test_begin_subtest "Restoring original tags"
  41 test_expect_success \
  42   'notmuch restore --input=dump.expected &&
  43   notmuch dump > dump.actual &&
  44   test_cmp dump.expected dump.actual'
  45
  46 test_begin_subtest "Restore with nothing to do"
  47 test_expect_success \
  48   'notmuch restore < dump.expected &&
  49   notmuch dump > dump.actual &&
  50   test_cmp dump.expected dump.actual'
  51
  52 test_begin_subtest "Accumulate with existing tags"
  53 test_expect_success \
  54   'notmuch restore --accumulate --input=dump.expected &&
  55   notmuch dump > dump.actual &&
  56   test_cmp dump.expected dump.actual'
  57
  58 test_begin_subtest "Accumulate with no tags"
  59 test_expect_success \
  60   'notmuch restore --accumulate < clear.expected &&
  61   notmuch dump > dump.actual &&
  62   test_cmp dump.expected dump.actual'
  63
  64 test_begin_subtest "Accumulate with new tags"
  65 test_expect_success \
  66   'notmuch restore --input=dump.expected &&
  67   notmuch restore --accumulate --input=dump-ABC_DEF.expected &&
  68   notmuch dump > OUTPUT.$test_count &&
  69   notmuch restore --input=dump.expected &&
  70   test_cmp dump-ABC_DEF.expected OUTPUT.$test_count'
  71
  72 # notmuch restore currently only considers the first argument.
  73 test_begin_subtest "Invalid restore invocation"
  74 test_expect_success \
  75   'test_must_fail notmuch restore --input=dump.expected another_one'
  76
  77 test_begin_subtest "dump --output=outfile"
  78 notmuch dump --output=dump-outfile.actual
  79 test_expect_equal_file dump.expected dump-outfile.actual
  80
  81 test_begin_subtest "dump --output=outfile --"
  82 notmuch dump --output=dump-1-arg-dash.actual --
  83 test_expect_equal_file dump.expected dump-1-arg-dash.actual
  84
  85 # gzipped output
  86
  87 test_begin_subtest "dump --gzip"
  88 notmuch dump --gzip > dump-gzip.gz
  89 gunzip dump-gzip.gz
  90 test_expect_equal_file dump.expected dump-gzip
  91
  92 test_begin_subtest "dump --gzip --output=outfile"
  93 notmuch dump --gzip --output=dump-gzip-outfile.gz
  94 gunzip dump-gzip-outfile.gz
  95 test_expect_equal_file dump.expected dump-gzip-outfile
  96
  97 test_begin_subtest "restoring gzipped stdin"
  98 notmuch dump --gzip --output=backup.gz
  99 notmuch tag +new_tag '*'
 100 notmuch restore < backup.gz
 101 notmuch dump --output=dump.actual
 102 test_expect_equal_file dump.expected dump.actual
 103
 104 test_begin_subtest "restoring gzipped file"
 105 notmuch dump --gzip --output=backup.gz
 106 notmuch tag +new_tag '*'
 107 notmuch restore --input=backup.gz
 108 notmuch dump --output=dump.actual
 109 test_expect_equal_file dump.expected dump.actual
 110
 111 # Note, we assume all messages from cworth have a message-id
 112 # containing cworth.org
 113
 114 { head -1 dump.expected ; grep 'cworth[.]org' dump.expected; } > dump-cworth.expected
 115
 116 test_begin_subtest "dump -- from:cworth"
 117 notmuch dump -- from:cworth > dump-dash-cworth.actual
 118 test_expect_equal_file dump-cworth.expected dump-dash-cworth.actual
 119
 120
 121 if [ "${NOTMUCH_HAVE_SFSEXP-0}" = "1" ]; then
 122
 123     test_begin_subtest "dump --query=sexp -- '(from cworth)'"
 124     notmuch dump --query=sexp -- '(from cworth)' > dump-dash-cworth.actual2
 125     test_expect_equal_file_nonempty dump-cworth.expected dump-dash-cworth.actual2
 126
 127     test_begin_subtest "dump --query=sexp --output=outfile '(from cworth)'"
 128     notmuch dump --output=dump-outfile-cworth.actual2 --query=sexp '(from cworth)'
 129     test_expect_equal_file dump-cworth.expected dump-outfile-cworth.actual2
 130
 131 fi
 132
 133 test_begin_subtest "dump --output=outfile from:cworth"
 134 notmuch dump --output=dump-outfile-cworth.actual from:cworth
 135 test_expect_equal_file dump-cworth.expected dump-outfile-cworth.actual
 136
 137 test_begin_subtest "dump --output=outfile -- from:cworth"
 138 notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
 139 test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
 140
 141 test_begin_subtest "Check for a safe set of message-ids"
 142 test_subtest_broken_for_installed
 143 notmuch search --output=messages from:cworth | sed s/^id:// > EXPECTED
 144 notmuch search --output=messages from:cworth | sed s/^id:// |\
 145         $TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT
 146 test_expect_equal_file EXPECTED OUTPUT
 147
 148 test_begin_subtest "format=batch-tag, dump sanity check."
 149 NOTMUCH_DUMP_TAGS --format=sup from:cworth | cut -f1 -d' ' | \
 150     sort > EXPECTED.$test_count
 151 NOTMUCH_DUMP_TAGS --format=batch-tag from:cworth | sed 's/^.*-- id://' | \
 152     sort > OUTPUT.$test_count
 153 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 154
 155 test_begin_subtest "format=batch-tag, missing newline"
 156 printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN
 157 notmuch restore --accumulate < IN
 158 NOTMUCH_DUMP_TAGS id:20091117232137.GA7669@griffis1.net > OUT
 159 cat <<EOF > EXPECTED
 160 +a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net
 161 EOF
 162 test_expect_equal_file EXPECTED OUT
 163
 164 test_begin_subtest "format=batch-tag, # round-trip"
 165 notmuch dump --format=sup | sort > EXPECTED.$test_count
 166 notmuch dump --format=batch-tag > DUMPFILE
 167 notmuch restore --format=batch-tag < DUMPFILE
 168 notmuch dump --format=sup | sort > OUTPUT.$test_count
 169 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 170
 171 test_begin_subtest "format=batch-tag, # blank lines and comments"
 172 notmuch dump --format=batch-tag| sort > EXPECTED.$test_count
 173 notmuch restore <<EOF
 174 # this line is a comment; the next has only white space
 175
 176
 177 # the previous line is empty
 178 EOF
 179 notmuch dump --format=batch-tag | sort > OUTPUT.$test_count
 180 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 181
 182 test_begin_subtest "format=batch-tag, # reverse-round-trip empty tag"
 183 cat <<EOF >EXPECTED.$test_count
 184 + -- id:20091117232137.GA7669@griffis1.net
 185 EOF
 186 notmuch restore --format=batch-tag < EXPECTED.$test_count
 187 NOTMUCH_DUMP_TAGS --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count
 188 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 189
 190 tag1='comic_swear=$&^%$^%\\//-+$^%$'
 191 enc1=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag1")
 192
 193 tag2=$(printf 'this\n tag\t has\n spaces')
 194 enc2=$($TEST_DIRECTORY/hex-xcode --direction=encode "$tag2")
 195
 196 enc3='%c3%91%c3%a5%c3%b0%c3%a3%c3%a5%c3%a9-%c3%8f%c3%8a'
 197 tag3=$($TEST_DIRECTORY/hex-xcode --direction=decode $enc3)
 198
 199 notmuch dump --format=batch-tag > BACKUP
 200
 201 notmuch tag +"$tag1" +"$tag2" +"$tag3" -inbox -unread "*"
 202
 203 # initial segment of file used for several tests below.
 204 cat <<EOF > comments-and-blanks
 205 # this is a comment
 206
 207 # next line has leading whitespace
 208
 209
 210 EOF
 211
 212 test_begin_subtest 'restoring empty file is not an error'
 213 notmuch restore < /dev/null 2>OUTPUT.$test_count
 214 cp /dev/null EXPECTED
 215 test_expect_equal_file EXPECTED OUTPUT.$test_count
 216
 217 test_begin_subtest 'file of comments and blank lines is not an error'
 218 notmuch restore --input=comments-and-blanks
 219 ret_val=$?
 220 test_expect_equal "$ret_val" "0"
 221
 222 cp comments-and-blanks leading-comments-blanks-batch-tag
 223 echo "+some_tag -- id:yun1vjwegii.fsf@aiko.keithp.com" \
 224     >> leading-comments-blanks-batch-tag
 225
 226 test_begin_subtest 'detect format=batch-tag with leading comments and blanks'
 227 notmuch restore --input=leading-comments-blanks-batch-tag
 228 notmuch search --output=tags id:yun1vjwegii.fsf@aiko.keithp.com > OUTPUT.$test_count
 229 echo "some_tag" > EXPECTED
 230 test_expect_equal_file EXPECTED OUTPUT.$test_count
 231
 232 cp comments-and-blanks leading-comments-blanks-sup
 233 echo "yun1vjwegii.fsf@aiko.keithp.com (another_tag)" \
 234     >> leading-comments-blanks-sup
 235
 236 test_begin_subtest 'detect format=sup with leading comments and blanks'
 237 notmuch restore --input=leading-comments-blanks-sup
 238 notmuch search --output=tags id:yun1vjwegii.fsf@aiko.keithp.com > OUTPUT.$test_count
 239 echo "another_tag" > EXPECTED
 240 test_expect_equal_file EXPECTED OUTPUT.$test_count
 241
 242 test_begin_subtest 'format=batch-tag, round trip with strange tags'
 243 notmuch dump --format=batch-tag > EXPECTED.$test_count
 244 notmuch dump --format=batch-tag > DUMPFILE
 245 notmuch restore --format=batch-tag < DUMPFILE
 246 notmuch dump --format=batch-tag > OUTPUT.$test_count
 247 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 248
 249 test_begin_subtest 'format=batch-tag, checking encoded output'
 250 test_subtest_broken_for_installed
 251 NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\
 252          awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count
 253 NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth > OUTPUT.$test_count
 254 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 255
 256 test_begin_subtest 'restoring sane tags'
 257 notmuch restore --format=batch-tag < BACKUP
 258 notmuch dump --format=batch-tag > OUTPUT.$test_count
 259 test_expect_equal_file BACKUP OUTPUT.$test_count
 260
 261 test_begin_subtest 'format=batch-tag, restore=auto'
 262 notmuch dump --format=batch-tag > EXPECTED.$test_count
 263 notmuch tag -inbox -unread "*"
 264 notmuch restore --format=auto < EXPECTED.$test_count
 265 notmuch dump --format=batch-tag > OUTPUT.$test_count
 266 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 267
 268 test_begin_subtest 'format=sup, restore=auto'
 269 notmuch dump --format=sup > EXPECTED.$test_count
 270 notmuch tag -inbox -unread "*"
 271 notmuch restore --format=auto < EXPECTED.$test_count
 272 notmuch dump --format=sup > OUTPUT.$test_count
 273 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 274
 275 test_begin_subtest 'format=batch-tag, restore=default'
 276 notmuch dump --format=batch-tag > EXPECTED.$test_count
 277 notmuch tag -inbox -unread "*"
 278 notmuch restore < EXPECTED.$test_count
 279 notmuch dump --format=batch-tag > OUTPUT.$test_count
 280 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 281
 282 test_begin_subtest 'format=sup, restore=default'
 283 notmuch dump --format=sup > EXPECTED.$test_count
 284 notmuch tag -inbox -unread "*"
 285 notmuch restore < EXPECTED.$test_count
 286 notmuch dump --format=sup > OUTPUT.$test_count
 287 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 288
 289 test_begin_subtest 'restore: checking error messages'
 290 notmuch restore <<EOF 2>OUTPUT
 291 # the next line has a space
 292
 293 a
 294 +0
 295 +a +b
 296 # trailing whitespace
 297 +a +b
 298 +c +d --
 299 # this is a harmless comment, do not yell about it.
 300
 301 # the previous line was blank; also no yelling please
 302 +%zz -- id:whatever
 303 +e +f id:"
 304 +e +f tag:abc
 305 # the next non-comment line should report an an empty tag error for
 306 # batch tagging, but not for restore
 307 + +e -- id:20091117232137.GA7669@griffis1.net
 308 # valid id, but warning about missing message
 309 +e id:missing_message_id
 310 # exercise parser
 311 +e -- id:some)stuff
 312 +e -- id:some stuff
 313 +e -- id:some"stuff
 314 +e -- id:"a_message_id_with""_a_quote"
 315 +e -- id:"a message id with spaces"
 316 +e --  id:an_id_with_leading_and_trailing_ws \
 317
 318 EOF
 319
 320 cat <<EOF > EXPECTED
 321 Warning: cannot parse query: a (skipping)
 322 Warning: no query string [+0]
 323 Warning: no query string [+a +b]
 324 Warning: missing query string [+a +b ]
 325 Warning: no query string after -- [+c +d --]
 326 Warning: hex decoding of tag %zz failed [+%zz -- id:whatever]
 327 Warning: cannot parse query: id:" (skipping)
 328 Warning: not an id query: tag:abc (skipping)
 329 Warning: cannot apply tags to missing message: missing_message_id
 330 Warning: cannot parse query: id:some)stuff (skipping)
 331 Warning: cannot parse query: id:some stuff (skipping)
 332 Warning: cannot apply tags to missing message: some"stuff
 333 Warning: cannot apply tags to missing message: a_message_id_with"_a_quote
 334 Warning: cannot apply tags to missing message: a message id with spaces
 335 Warning: cannot apply tags to missing message: an_id_with_leading_and_trailing_ws
 336 EOF
 337
 338 test_expect_equal_file EXPECTED OUTPUT
 339
 340 backup_database
 341 test_begin_subtest 'roundtripping random message-ids and tags'
 342
 343     ${TEST_DIRECTORY}/random-corpus --config-path=${NOTMUCH_CONFIG} \
 344                         --num-messages=100
 345
 346      notmuch dump --format=batch-tag| \
 347          sort > EXPECTED.$test_count
 348
 349      notmuch tag +this_tag_is_very_unlikely_to_be_random '*'
 350
 351      notmuch restore --format=batch-tag < EXPECTED.$test_count
 352
 353      notmuch dump --format=batch-tag| \
 354          sort > OUTPUT.$test_count
 355
 356 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 357 restore_database
 358
 359 test_done
 360