aboutsummaryrefslogtreecommitdiff
path: root/performance-test
diff options
context:
space:
mode:
authorDavid Bremner <bremner@debian.org>2018-06-12 22:39:33 -0300
committerDavid Bremner <bremner@debian.org>2018-06-12 22:39:33 -0300
commit045f0e455ac94e2393d0d729c9bbdf3459a4860f (patch)
tree8d8b46ecba2c3c128365f16ece54377b987dbe58 /performance-test
Import notmuch_0.27.orig.tar.gz
[dgit import orig notmuch_0.27.orig.tar.gz]
Diffstat (limited to 'performance-test')
-rw-r--r--performance-test/.gitignore4
-rwxr-xr-xperformance-test/M00-new.sh15
-rwxr-xr-xperformance-test/M01-dump-restore.sh15
-rwxr-xr-xperformance-test/M02-show.sh13
-rwxr-xr-xperformance-test/M03-search.sh13
-rwxr-xr-xperformance-test/M04-reply.sh15
-rwxr-xr-xperformance-test/M05-reindex.sh11
-rwxr-xr-xperformance-test/M06-insert.sh16
-rw-r--r--performance-test/Makefile7
-rw-r--r--performance-test/Makefile.local44
-rw-r--r--performance-test/README90
-rwxr-xr-xperformance-test/T00-new.sh15
-rwxr-xr-xperformance-test/T01-dump-restore.sh13
-rwxr-xr-xperformance-test/T02-tag.sh14
-rwxr-xr-xperformance-test/T03-reindex.sh13
-rwxr-xr-xperformance-test/T04-thread-subquery.sh13
-rw-r--r--performance-test/download/.gitignore2
-rw-r--r--performance-test/download/notmuch-email-corpus-0.3.tar.xz.asc9
-rw-r--r--performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc14
-rwxr-xr-xperformance-test/notmuch-memory-test22
-rwxr-xr-xperformance-test/notmuch-time-test22
-rw-r--r--performance-test/perf-test-lib.sh219
-rw-r--r--performance-test/version.sh3
23 files changed, 602 insertions, 0 deletions
diff --git a/performance-test/.gitignore b/performance-test/.gitignore
new file mode 100644
index 00000000..8a5dabf5
--- /dev/null
+++ b/performance-test/.gitignore
@@ -0,0 +1,4 @@
+/tmp.*/
+/log.*/
+/corpus/
+/notmuch.cache.*/
diff --git a/performance-test/M00-new.sh b/performance-test/M00-new.sh
new file mode 100755
index 00000000..aab36e69
--- /dev/null
+++ b/performance-test/M00-new.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='notmuch new'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+# ensure initial 'notmuch new' is run by memory_start
+uncache_database
+
+memory_start
+
+# run 'notmuch new' a second time, to test different code paths
+memory_run "notmuch new" "notmuch new"
+
+memory_done
diff --git a/performance-test/M01-dump-restore.sh b/performance-test/M01-dump-restore.sh
new file mode 100755
index 00000000..32ab8dc9
--- /dev/null
+++ b/performance-test/M01-dump-restore.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='dump and restore'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+memory_start
+
+memory_run 'load nmbug tags' 'notmuch restore --accumulate --input=corpus.tags/nmbug.sup-dump'
+memory_run 'dump *' 'notmuch dump --output=tags.sup'
+memory_run 'restore *' 'notmuch restore --input=tags.sup'
+memory_run 'dump --format=batch-tag *' 'notmuch dump --format=batch-tag --output=tags.bt'
+memory_run 'restore --format=batch-tag *' 'notmuch restore --format=batch-tag --input=tags.bt'
+
+memory_done
diff --git a/performance-test/M02-show.sh b/performance-test/M02-show.sh
new file mode 100755
index 00000000..2e218fd3
--- /dev/null
+++ b/performance-test/M02-show.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+test_description='show'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+memory_start
+
+memory_run 'show *' "notmuch show '*' 1>/dev/null"
+memory_run 'show --format=json *' "notmuch show --format=json '*' 1>/dev/null"
+memory_run 'show --format=sexp *' "notmuch show --format=sexp '*' 1>/dev/null"
+
+memory_done
diff --git a/performance-test/M03-search.sh b/performance-test/M03-search.sh
new file mode 100755
index 00000000..343f5c7c
--- /dev/null
+++ b/performance-test/M03-search.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+test_description='search'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+memory_start
+
+memory_run 'search *' "notmuch search '*' 1>/dev/null"
+memory_run 'search --format=json *' "notmuch search --format=json '*' 1>/dev/null"
+memory_run 'search --format=sexp *' "notmuch search --format=sexp '*' 1>/dev/null"
+
+memory_done
diff --git a/performance-test/M04-reply.sh b/performance-test/M04-reply.sh
new file mode 100755
index 00000000..3c1205db
--- /dev/null
+++ b/performance-test/M04-reply.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='search'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+memory_start
+
+for id in $(notmuch search --output=messages '*' | shuf -n 5); do
+ memory_run "reply $id" "notmuch reply \"$id\" 1>/dev/null"
+ memory_run "reply --format=json $id" "notmuch reply --format=json \"$id\" 1>/dev/null"
+ memory_run "reply --format=sexp $id" "notmuch reply --format=sexp \"$id\" 1>/dev/null"
+done
+
+memory_done
diff --git a/performance-test/M05-reindex.sh b/performance-test/M05-reindex.sh
new file mode 100755
index 00000000..17e2c824
--- /dev/null
+++ b/performance-test/M05-reindex.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+test_description='reindex'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+memory_start
+
+memory_run 'reindex *' "notmuch reindex '*'"
+
+memory_done
diff --git a/performance-test/M06-insert.sh b/performance-test/M06-insert.sh
new file mode 100755
index 00000000..5ae0656a
--- /dev/null
+++ b/performance-test/M06-insert.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+test_description='search'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+memory_start
+
+mkdir -p "$MAIL_DIR"/{cur,new,tmp}
+
+for count in {1..20}; do
+ generate_message "[file]=\"insert-$count\"" "[dir]='tmp/'"
+ memory_run "insert $count" "notmuch insert < $gen_msg_filename"
+done
+
+memory_done
diff --git a/performance-test/Makefile b/performance-test/Makefile
new file mode 100644
index 00000000..de492a7c
--- /dev/null
+++ b/performance-test/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+ $(MAKE) -C .. all
+
+.DEFAULT:
+ $(MAKE) -C .. $@
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
new file mode 100644
index 00000000..9dc260e3
--- /dev/null
+++ b/performance-test/Makefile.local
@@ -0,0 +1,44 @@
+# -*- makefile -*-
+
+dir := performance-test
+
+include $(srcdir)/$(dir)/version.sh
+
+TIME_TEST_SCRIPT := ${dir}/notmuch-time-test
+MEMORY_TEST_SCRIPT := ${dir}/notmuch-memory-test
+
+CORPUS_NAME := notmuch-email-corpus-$(PERFTEST_VERSION).tar.xz
+TXZFILE := ${dir}/download/${CORPUS_NAME}
+SIGFILE := ${TXZFILE}.asc
+DEFAULT_URL := https://notmuchmail.org/releases/${CORPUS_NAME}
+
+perf-test: time-test memory-test
+
+time-test: setup-perf-test all
+ @echo
+ $(TIME_TEST_SCRIPT) $(OPTIONS)
+
+memory-test: setup-perf-test all
+ @echo
+ $(MEMORY_TEST_SCRIPT) $(OPTIONS)
+
+
+.PHONY: download-corpus setup-perf-test
+
+# Note that this intentionally does not depend on download-corpus.
+setup-perf-test: $(TXZFILE)
+ gpg --verify $(SIGFILE)
+
+$(TXZFILE):
+ @printf "\nPlease download ${TXZFILE} using:\n\n"
+ @printf "\t%% make download-corpus\n\n"
+ @echo or see https://notmuchmail.org/corpus for download locations
+ @echo
+ @false
+
+download-corpus:
+ wget -O ${TXZFILE} ${DEFAULT_URL}
+
+CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/log.*
+DISTCLEAN := $(DISTCLEAN) $(dir)/corpus $(dir)/notmuch.cache.*
+DATACLEAN := $(DATACLEAN) $(TXZFILE)
diff --git a/performance-test/README b/performance-test/README
new file mode 100644
index 00000000..fbc61028
--- /dev/null
+++ b/performance-test/README
@@ -0,0 +1,90 @@
+Performance Tests
+-----------------
+
+This directory contains two kinds of performance tests: time tests,
+and memory tests. The former use gnu time, and the latter use
+valgrind.
+
+Pre-requisites
+--------------
+
+In addition to having notmuch, you need:
+
+- gpg
+- gnu tar
+- gnu time (for the time tests)
+- xz. Some speedup can be gotten by installing "pixz", but this is
+ probably only worthwhile if you are debugging the tests.
+- valgrind (for the memory tests)
+
+Getting set up to run tests:
+----------------------------
+
+First, you need to get the corpus. If you don't already have the gpg
+key for David Bremner, run
+
+ % gpg --search 'david@tethera.net'
+
+This should get you a key with fingerprint
+
+ 815B 6398 2A79 F8E7 C727 86C4 762B 57BB 7842 06AD
+
+(the last 8 digits are printed as the "key id").
+
+To fetch the actual corpus it should work to run
+
+ % make download-corpus
+
+In case that fails or is too slow, check
+
+ https://notmuchmail.org/corpus
+
+for a list of mirrors.
+
+Running tests
+-------------
+
+The easiest way to run performance tests is to say "make perf-test".
+This will run all time and memory tests. Be aware that the memory
+tests are quite time consuming when run on the full corpus, and that
+depending on your interests it may be more sensible to run "make
+time-test" or "make memory-test". You can also invoke one of the
+scripts notmuch-time-test or notmuch-memory-test or run a more
+specific subset of tests by simply invoking one of the executable
+scripts in this directory, (such as ./T00-new). Each test script
+supports the following arguments
+
+--small / --medium / --large Choose corpus size.
+--debug Enable debugging. In particular don't delete
+ temporary directories.
+
+When using the make targets, you can pass arguments to all test
+scripts by defining the make variable OPTIONS.
+
+Writing tests
+-------------
+
+Have a look at "T01-dump-restore" for an example time test and
+"M00-new" for an example memory test. In both cases sourcing
+"perf-test-lib.sh" is mandatory.
+
+Basics:
+
+- '(time|memory)_start' unpacks the mail corpus and calls notmuch new if it
+ cannot find a cache of the appropriate corpus.
+- '(time|memory)_run' runs the command under time or valgrind. Currently
+ "memory_run" does not support i/o redirection in the command.
+- '(time|memory)_done' does the cleanup; comment it out or pass --debug to the
+ script to leave the temporary files around.
+
+Utility functions include
+
+- 'add_email_corpus' unpacks a set of messages and tags
+- 'cache_database': makes a snapshot of the current database
+- 'uncache_database': forces the next '(time|memory)_start' to rebuild the
+ database.
+
+Scripts are run in the order specified in notmuch-perf-test. In the
+future this order might be chosen automatically so please follow the
+convention of starting the name with 'T' or 'M' followed by two digits
+to specify the order.
diff --git a/performance-test/T00-new.sh b/performance-test/T00-new.sh
new file mode 100755
index 00000000..68750129
--- /dev/null
+++ b/performance-test/T00-new.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+test_description='notmuch new'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+uncache_database
+
+time_start
+
+for i in $(seq 2 6); do
+ time_run "notmuch new #$i" 'notmuch new'
+done
+
+time_done
diff --git a/performance-test/T01-dump-restore.sh b/performance-test/T01-dump-restore.sh
new file mode 100755
index 00000000..12f12e66
--- /dev/null
+++ b/performance-test/T01-dump-restore.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+test_description='dump and restore'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+time_run 'load nmbug tags' 'notmuch restore --accumulate < corpus.tags/nmbug.sup-dump'
+time_run 'dump *' 'notmuch dump > tags.out'
+time_run 'restore *' 'notmuch restore < tags.out'
+
+time_done
diff --git a/performance-test/T02-tag.sh b/performance-test/T02-tag.sh
new file mode 100755
index 00000000..8c5dfd68
--- /dev/null
+++ b/performance-test/T02-tag.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+test_description='tagging'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
+time_run 'tag * +existing_tag' "notmuch tag +new_tag '*'"
+time_run 'tag * -existing_tag' "notmuch tag -new_tag '*'"
+time_run 'tag * -missing_tag' "notmuch tag -new_tag '*'"
+
+time_done
diff --git a/performance-test/T03-reindex.sh b/performance-test/T03-reindex.sh
new file mode 100755
index 00000000..d6d5c3c3
--- /dev/null
+++ b/performance-test/T03-reindex.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+test_description='tagging'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+time_run 'reindex *' "notmuch reindex '*'"
+time_run 'reindex *' "notmuch reindex '*'"
+time_run 'reindex *' "notmuch reindex '*'"
+
+time_done
diff --git a/performance-test/T04-thread-subquery.sh b/performance-test/T04-thread-subquery.sh
new file mode 100755
index 00000000..665d5a64
--- /dev/null
+++ b/performance-test/T04-thread-subquery.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+test_description='thread subqueries'
+
+. $(dirname "$0")/perf-test-lib.sh || exit 1
+
+time_start
+
+time_run "search thread:{} ..." "notmuch search thread:{date:2010} and thread:{from:linus}"
+time_run "search thread:{} ..." "notmuch search thread:{date:2010} and thread:{from:linus}"
+time_run "search thread:{} ..." "notmuch search thread:{date:2010} and thread:{from:linus}"
+
+time_done
diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore
new file mode 100644
index 00000000..5c356204
--- /dev/null
+++ b/performance-test/download/.gitignore
@@ -0,0 +1,2 @@
+/*.tar.gz
+/*.tar.xz
diff --git a/performance-test/download/notmuch-email-corpus-0.3.tar.xz.asc b/performance-test/download/notmuch-email-corpus-0.3.tar.xz.asc
new file mode 100644
index 00000000..f109e81d
--- /dev/null
+++ b/performance-test/download/notmuch-email-corpus-0.3.tar.xz.asc
@@ -0,0 +1,9 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iJwEAAECAAYFAlC9a90ACgkQTiiN/0Um85nAMAP+LCWdKzolcl/KW+JcCd0Dk+9v
+0vvtBVEhBes0TbK6iWrxCV2OIuYG/RhnFlJTZ4MjgaTRxzDubpC+JktaJdLmIQUN
+B7ZIDMjFduCwmtyLiuu/00CjxJKUXm7vx+ULGpvp0uxFE/vaqGP997BHwBjjfBVm
+YX6BlLX1SV6TfENkuRE=
+=Mks5
+-----END PGP SIGNATURE-----
diff --git a/performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc b/performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc
new file mode 100644
index 00000000..72dedd8b
--- /dev/null
+++ b/performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc
@@ -0,0 +1,14 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQGcBAABCAAGBQJSdaDkAAoJEPIClx2kp54sQ54L/ikkvF1fy88hjLitN59v6g2J
+vw85YNRifNHyp/UXI6nt2eXFzyWJiRHuvHFoBgmEsJVxauOKw61Gs2zd53x9Ear4
+MGcQWyiM1cnwX/nD7GvxRQNh33f+FEamTjg+QhG47K0A2YdLWcDC7r9GMatGT11x
+5KE24WQGOqtgQn/9qNtJvkiKIehpRiDTaW/QJ7mTCYeJFjIHJUY8dxyfiTtkJ0z7
+cJ6omehvWSw4STbEg65XJgqykxMdltNEavfvSbAT73FgmkkyXxul0s5hDZ/esd0n
+re3dyDxGt085POiAgPti05a4tJI5EQC2wLBUFri0s2JdMtazcD6yVuHNbVzZ4Do3
+nL/sgwKGUq5wRrPqPWp6HXtZ9zG+/V7hFNrr/l42qGrLqsSh0bqvEnUiwczZLBGy
+NEs4G8VjmfS2cMKePsWaekBAvFUtb47PSB6JIPwpCNvKXDrcCb28eOQVB2atgj1h
+9SktOtWYJhWIQp2YW9iae30Z6lhCcdPRRHTFMQq2nQ==
+=eSMY
+-----END PGP SIGNATURE-----
diff --git a/performance-test/notmuch-memory-test b/performance-test/notmuch-memory-test
new file mode 100755
index 00000000..047aac70
--- /dev/null
+++ b/performance-test/notmuch-memory-test
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Run tests
+#
+# Copyright (c) 2005 Junio C Hamano
+# Copyright (c) 2010 Notmuch Developers
+#
+# Adapted from a Makefile to a shell script by Carl Worth (2010)
+
+if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
+ echo "Error: The notmuch test suite requires a bash version >= 4.0"
+ echo "due to use of associative arrays within the test suite."
+ echo "Please try again with a newer bash (or help us fix the"
+ echo "test suite to be more portable). Thanks."
+ exit 1
+fi
+
+cd "$(dirname "$0")"
+
+for test in M*.sh; do
+ ./"$test" "$@"
+done
diff --git a/performance-test/notmuch-time-test b/performance-test/notmuch-time-test
new file mode 100755
index 00000000..4dd21fe1
--- /dev/null
+++ b/performance-test/notmuch-time-test
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Run tests
+#
+# Copyright (c) 2005 Junio C Hamano
+# Copyright (c) 2010 Notmuch Developers
+#
+# Adapted from a Makefile to a shell script by Carl Worth (2010)
+
+if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
+ echo "Error: The notmuch test suite requires a bash version >= 4.0"
+ echo "due to use of associative arrays within the test suite."
+ echo "Please try again with a newer bash (or help us fix the"
+ echo "test suite to be more portable). Thanks."
+ exit 1
+fi
+
+cd "$(dirname "$0")"
+
+for test in T*.sh; do
+ ./"$test" "$@"
+done
diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh
new file mode 100644
index 00000000..b70288cc
--- /dev/null
+++ b/performance-test/perf-test-lib.sh
@@ -0,0 +1,219 @@
+. $(dirname "$0")/version.sh || exit 1
+
+corpus_size=large
+
+while test "$#" -ne 0
+do
+ case "$1" in
+ -d|--debug)
+ debug=t;
+ shift
+ ;;
+ -s|--small)
+ corpus_size=small;
+ shift
+ ;;
+ -m|--medium)
+ corpus_size=medium;
+ shift
+ ;;
+ -l|--large)
+ corpus_size=large;
+ shift
+ ;;
+ *)
+ echo "error: unknown performance test option '$1'" >&2; exit 1 ;;
+ esac
+done
+
+# Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set.
+. $(dirname "$0")/../test/export-dirs.sh || exit 1
+
+# Where to run the tests
+TEST_DIRECTORY=$NOTMUCH_BUILDDIR/performance-test
+
+. "$NOTMUCH_SRCDIR/test/test-lib-common.sh" || exit 1
+
+set -e
+
+# It appears that people try to run tests without building...
+if [[ ! -x "$NOTMUCH_BUILDDIR/notmuch" ]]; then
+ echo >&2 'You do not seem to have built notmuch yet.'
+ exit 1
+fi
+
+DB_CACHE_DIR=${TEST_DIRECTORY}/notmuch.cache.$corpus_size
+
+add_email_corpus ()
+{
+ rm -rf ${MAIL_DIR}
+
+ CORPUS_DIR=${TEST_DIRECTORY}/corpus
+ mkdir -p "${CORPUS_DIR}"
+
+ MAIL_CORPUS="${CORPUS_DIR}/mail.${corpus_size}"
+ TAG_CORPUS="${CORPUS_DIR}/tags"
+
+ if command -v pixz > /dev/null; then
+ XZ=pixz
+ else
+ XZ=xz
+ fi
+
+ if [ ! -d "${CORPUS_DIR}/manifest" ]; then
+
+ printf "Unpacking manifests\n"
+ tar --extract --use-compress-program ${XZ} --strip-components=1 \
+ --directory ${TEST_DIRECTORY}/corpus \
+ --wildcards --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \
+ 'notmuch-email-corpus/manifest/*'
+ fi
+
+ file_list=$(mktemp file_listXXXXXX)
+ declare -a extract_dirs
+ if [ ! -d "$TAG_CORPUS" ] ; then
+ extract_dirs=("${extract_dirs[@]}" notmuch-email-corpus/tags)
+ fi
+
+ if [ ! -d "$MAIL_CORPUS" ] ; then
+ if [[ "$corpus_size" != "large" ]]; then
+ sed s,^,notmuch-email-corpus/, < \
+ ${TEST_DIRECTORY}/corpus/manifest/MANIFEST.${corpus_size} >> $file_list
+ else
+ extract_dirs=("${extract_dirs[@]}" notmuch-email-corpus/mail)
+ fi
+ fi
+
+ if [[ -s $file_list || -n "${extract_dirs[*]}" ]]; then
+
+ printf "Unpacking corpus\n"
+ tar --checkpoint=.5000 --extract --strip-components=1 \
+ --directory ${TEST_DIRECTORY}/corpus \
+ --use-compress-program ${XZ} \
+ --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \
+ --anchored --recursion \
+ --files-from $file_list "${extract_dirs[@]}"
+
+ printf "\n"
+
+ if [[ ! -d ${MAIL_CORPUS} ]]; then
+ printf "creating link farm\n"
+
+ if [[ "$corpus_size" = large ]]; then
+ cp -rl ${TEST_DIRECTORY}/corpus/mail ${MAIL_CORPUS}
+ else
+ while read -r file; do
+ tdir=${MAIL_CORPUS}/$(dirname $file)
+ mkdir -p $tdir
+ ln ${TEST_DIRECTORY}/corpus/$file $tdir
+ done <${TEST_DIRECTORY}/corpus/manifest/MANIFEST.${corpus_size}
+ fi
+ fi
+
+ fi
+
+ rm $file_list
+ cp -lr $TAG_CORPUS $TMP_DIRECTORY/corpus.tags
+ cp -lr $MAIL_CORPUS $MAIL_DIR
+}
+
+notmuch_new_with_cache ()
+{
+ if [ -d $DB_CACHE_DIR ]; then
+ cp -r $DB_CACHE_DIR ${MAIL_DIR}/.notmuch
+ else
+ "$1" 'Initial notmuch new' "notmuch new"
+ cache_database
+ fi
+}
+
+time_start ()
+{
+ add_email_corpus
+
+ print_header
+
+ notmuch_new_with_cache time_run
+}
+
+memory_start ()
+{
+ add_email_corpus
+
+ local timestamp=$(date +%Y%m%dT%H%M%S)
+ log_dir="${TEST_DIRECTORY}/log.$(basename $0)-$corpus_size-${timestamp}"
+ mkdir -p ${log_dir}
+
+ notmuch_new_with_cache memory_run
+}
+
+memory_run ()
+{
+ test_count=$(($test_count+1))
+
+ log_file=$log_dir/$test_count.log
+ talloc_log=$log_dir/$test_count.talloc
+
+ printf "[ %d ]\t%s\n" $test_count "$1"
+
+ NOTMUCH_TALLOC_REPORT="$talloc_log" eval "valgrind --leak-check=full --log-file='$log_file' $2"
+
+ awk '/LEAK SUMMARY/,/suppressed/ { sub(/^==[0-9]*==/," "); print }' "$log_file"
+ echo
+ sed -n -e 's/.*[(]total *\([^)]*\)[)]/talloced at exit: \1/p' $talloc_log
+ echo
+}
+
+memory_done ()
+{
+ time_done
+}
+
+cache_database ()
+{
+ if [ -d $MAIL_DIR/.notmuch ]; then
+ cp -r $MAIL_DIR/.notmuch $DB_CACHE_DIR
+ else
+ echo "Warning: No database found to cache"
+ fi
+}
+
+uncache_database ()
+{
+ rm -rf $DB_CACHE_DIR
+}
+
+print_header ()
+{
+ printf "\t\t\tWall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n"
+}
+
+time_run ()
+{
+ printf " %-22s" "$1"
+ test_count=$(($test_count+1))
+ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+ if ! eval >&3 "/usr/bin/time -f '%e\t%U\t%S\t%M\t%I/%O' $2" ; then
+ test_failure=$(($test_failure + 1))
+ return 1
+ fi
+ return 0
+}
+
+time_done ()
+{
+ if [ "$test_failure" = "0" ]; then
+ rm -rf "$remove_tmp"
+ exit 0
+ else
+ exit 1
+ fi
+}
+
+cd -P "$test" || error "Cannot set up test environment"
+test_failure=0
+test_count=0
+
+printf "\n%-55s [%s %s]\n" \
+ "$(basename "$0"): Testing ${test_description:-notmuch performance}" \
+ "${PERFTEST_VERSION}" "${corpus_size}"
diff --git a/performance-test/version.sh b/performance-test/version.sh
new file mode 100644
index 00000000..f02527a7
--- /dev/null
+++ b/performance-test/version.sh
@@ -0,0 +1,3 @@
+# this should be both a valid Makefile fragment and valid POSIX(ish) shell.
+
+PERFTEST_VERSION=0.4