git.notmuchmail.org Git - notmuch/blob - performance-test/perf-test-lib.sh

   1 . $(dirname "$0")/version.sh || exit 1
   2
   3 debug=""
   4 corpus_size=large
   5 perf_callgraph=lbr
   6 use_perf=0
   7
   8 while test "$#" -ne 0
   9 do
  10         case "$1" in
  11         -d|--debug)
  12                 debug=t;
  13                 shift
  14                 ;;
  15         -p|--perf)
  16                 use_perf=1;
  17                 shift
  18                 ;;
  19         -c|--call-graph)
  20                 shift
  21                 perf_callgraph=$1
  22                 shift
  23                 ;;
  24         -s|--small)
  25                 corpus_size=small;
  26                 shift
  27                 ;;
  28         -m|--medium)
  29                 corpus_size=medium;
  30                 shift
  31                 ;;
  32         -l|--large)
  33                 corpus_size=large;
  34                 shift
  35                 ;;
  36         *)
  37                 echo "error: unknown performance test option '$1'" >&2; exit 1 ;;
  38         esac
  39 done
  40
  41 # Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set.
  42 . $(dirname "$0")/../test/export-dirs.sh || exit 1
  43
  44 . "$NOTMUCH_SRCDIR/test/test-vars.sh" || exit 1
  45
  46 # Where to run the tests
  47 TEST_DIRECTORY=$NOTMUCH_BUILDDIR/performance-test
  48
  49 . "$NOTMUCH_SRCDIR/test/test-lib-common.sh" || exit 1
  50
  51 set -e
  52
  53 # It appears that people try to run tests without building...
  54 if [[ ! -x "$NOTMUCH_BUILDDIR/notmuch" ]]; then
  55         echo >&2 'You do not seem to have built notmuch yet.'
  56         exit 1
  57 fi
  58
  59 DB_CACHE_DIR=${TEST_DIRECTORY}/notmuch.cache.$corpus_size
  60
  61 add_email_corpus ()
  62 {
  63     rm -rf ${MAIL_DIR}
  64
  65     CORPUS_DIR=${TEST_DIRECTORY}/corpus
  66     mkdir -p "${CORPUS_DIR}"
  67
  68     MAIL_CORPUS="${CORPUS_DIR}/mail.${corpus_size}"
  69     TAG_CORPUS="${CORPUS_DIR}/tags"
  70
  71     if command -v pixz > /dev/null; then
  72         XZ=pixz
  73     else
  74         XZ=xz
  75     fi
  76
  77     if [ ! -d "${CORPUS_DIR}/manifest" ]; then
  78
  79         printf "Unpacking manifests\n"
  80         tar --extract --use-compress-program ${XZ} --strip-components=1 \
  81             --directory ${TEST_DIRECTORY}/corpus \
  82             --wildcards --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \
  83             'notmuch-email-corpus/manifest/*'
  84     fi
  85
  86     file_list=$(mktemp file_listXXXXXX)
  87     declare -a extract_dirs
  88     if [ ! -d "$TAG_CORPUS" ] ; then
  89         extract_dirs=("${extract_dirs[@]}" notmuch-email-corpus/tags)
  90     fi
  91
  92     if [ ! -d "$MAIL_CORPUS" ] ; then
  93         if [[ "$corpus_size" != "large" ]]; then
  94             sed s,^,notmuch-email-corpus/, < \
  95                 ${TEST_DIRECTORY}/corpus/manifest/MANIFEST.${corpus_size} >> $file_list
  96         else
  97             extract_dirs=("${extract_dirs[@]}" notmuch-email-corpus/mail)
  98         fi
  99     fi
 100
 101     if [[ -s $file_list || -n "${extract_dirs[*]}" ]]; then
 102
 103         printf "Unpacking corpus\n"
 104         tar --checkpoint=.5000 --extract --strip-components=1 \
 105             --directory ${TEST_DIRECTORY}/corpus \
 106             --use-compress-program ${XZ} \
 107             --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \
 108             --anchored --recursion \
 109             --files-from $file_list "${extract_dirs[@]}"
 110
 111         printf "\n"
 112
 113         if [[ ! -d ${MAIL_CORPUS} ]]; then
 114             printf "creating link farm\n"
 115
 116             if [[ "$corpus_size" = large ]]; then
 117                 cp -rl ${TEST_DIRECTORY}/corpus/mail ${MAIL_CORPUS}
 118             else
 119                 while read -r file; do
 120                     tdir=${MAIL_CORPUS}/$(dirname $file)
 121                     mkdir -p $tdir
 122                     ln ${TEST_DIRECTORY}/corpus/$file $tdir
 123                 done <${TEST_DIRECTORY}/corpus/manifest/MANIFEST.${corpus_size}
 124             fi
 125         fi
 126
 127     fi
 128
 129     rm $file_list
 130     cp -lr $TAG_CORPUS $TMP_DIRECTORY/corpus.tags
 131     cp -lr $MAIL_CORPUS $MAIL_DIR
 132 }
 133
 134 notmuch_new_with_cache ()
 135 {
 136     if [ -d $DB_CACHE_DIR ]; then
 137         cp -r $DB_CACHE_DIR ${MAIL_DIR}/.notmuch
 138     else
 139         "$1" 'Initial notmuch new' "notmuch new"
 140         cache_database
 141     fi
 142 }
 143
 144 make_log_dir () {
 145     local timestamp=$(date +%Y%m%dT%H%M%S)
 146     log_dir=${TEST_DIRECTORY}/log.$(basename $0)-$corpus_size-${timestamp}
 147     mkdir -p "${log_dir}"
 148 }
 149
 150 time_start ()
 151 {
 152     add_email_corpus
 153
 154     if [[ "$use_perf" = 1 ]]; then
 155         make_log_dir
 156     fi
 157
 158     print_header
 159
 160     notmuch_new_with_cache time_run
 161 }
 162
 163 memory_start ()
 164 {
 165     add_email_corpus
 166
 167     make_log_dir
 168
 169     notmuch_new_with_cache memory_run
 170 }
 171
 172 memory_run ()
 173 {
 174     test_count=$(($test_count+1))
 175
 176     log_file=$log_dir/$test_count.log
 177     talloc_log=$log_dir/$test_count.talloc
 178
 179     printf "[ %d ]\t%s\n" $test_count "$1"
 180
 181     NOTMUCH_TALLOC_REPORT="$talloc_log" eval "valgrind --leak-check=full --log-file='$log_file' $2"
 182
 183     awk '/LEAK SUMMARY/,/suppressed/ { sub(/^==[0-9]*==/," "); print }' "$log_file"
 184     echo
 185     sed -n -e 's/.*[(]total *\([^)]*\)[)]/talloced at exit: \1/p' $talloc_log
 186     echo
 187 }
 188
 189 memory_done ()
 190 {
 191     time_done
 192 }
 193
 194 cache_database ()
 195 {
 196     if [ -d $MAIL_DIR/.notmuch ]; then
 197         cp -r $MAIL_DIR/.notmuch $DB_CACHE_DIR
 198     else
 199         echo "Warning: No database found to cache"
 200     fi
 201 }
 202
 203 uncache_database ()
 204 {
 205     rm -rf $DB_CACHE_DIR
 206 }
 207
 208 print_header ()
 209 {
 210     printf "\t\t\tWall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n"
 211 }
 212
 213 print_emacs_header ()
 214 {
 215     printf "\t\t\tWall(s)\tGCs\tGC time(s)\n"
 216 }
 217
 218 time_run ()
 219 {
 220     printf "  %-22s" "$1"
 221     test_count=$(($test_count+1))
 222     if test "$verbose" != "t"; then exec 4>test.output 3>&4; else exec 3>&1; fi
 223     if [[ "$use_perf" = 1 ]]; then
 224         command_str="perf record --call-graph=${perf_callgraph} -o ${log_dir}/${test_count}.perf $2"
 225     else
 226         command_str="/usr/bin/time -f '%e\t%U\t%S\t%M\t%I/%O' $2"
 227     fi
 228
 229     if ! eval >&3 "$command_str" ; then
 230         test_failure=$(($test_failure + 1))
 231         return 1
 232     fi
 233     return 0
 234 }
 235
 236 time_done ()
 237 {
 238     if [ "$test_failure" = "0" ]; then
 239         rm -rf "$remove_tmp"
 240         exit 0
 241     else
 242         exit 1
 243     fi
 244 }
 245
 246 cd -P "$test" || error "Cannot set up test environment"
 247 test_failure=0
 248 test_count=0
 249
 250 printf "\n%-55s [%s %s]\n"  \
 251     "$(basename "$0"): Testing ${test_description:-notmuch performance}" \
 252     "${PERFTEST_VERSION}"  "${corpus_size}"