Merge tag 'debian/0.18.1-2' into wheezy-backports

uploaded to Debian unstable

Conflicts:
	debian/changelog

223 files changed, 6987 insertions, 4092 deletions

diff --git a/.gitignore b/.gitignore
index ef4f0748..1fb3a713 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .first-build-message
 Makefile.config
+version.stamp
 TAGS
 tags
 *cscope*
diff --git a/INSTALL b/INSTALL
index fce93528..b543c508 100644
--- a/INSTALL
+++ b/INSTALL
@@ -20,8 +20,8 @@ configure stage.
 
 Dependencies
 ------------
-Notmuch depends on three libraries: Xapian, GMime 2.4 or 2.6, and
-Talloc which are each described below:
+Notmuch depends on four libraries: Xapian, GMime 2.4 or 2.6,
+Talloc, and zlib which are each described below:
 
 	Xapian
 	------
@@ -60,16 +60,42 @@ Talloc which are each described below:
 
 	Talloc is available from http://talloc.samba.org/
 
+	zlib
+	----
+
+	zlib is an extremely popular compression library. It is used
+	by Xapian, so if you installed that you will already have
+	zlib. You may need to install the zlib headers separately.
+
+	Notmuch needs the transparent write feature of zlib introduced
+	in version 1.2.5.2 (Dec. 2011).
+
+	zlib is available from http://zlib.net
+
+Building Documentation
+----------------------
+
+By default the documentation for notmuch is built using sphinx.
+
+Sphinx is available from www.sphinx-doc.org.
+
+If you prefer, you can build the man pages using rst2man, from the
+python docutils package. See doc/INSTALL for details.
+
+
+Installing Dependencies from Packages
+-------------------------------------
+
 On a modern, package-based operating system you can install all of the
 dependencies with a simple simple command line. For example:
 
   For Debian and similar:
 
-        sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev
+        sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev python-sphinx
 
   For Fedora and similar:
 
-	sudo yum install xapian-core-devel gmime-devel libtalloc-devel
+	sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel python-sphinx
 
 On other systems, a similar command can be used, but the details of
 the package names may be different.
diff --git a/Makefile b/Makefile
index 0428160b..4c0e8c62 100644
--- a/Makefile
+++ b/Makefile
@@ -2,15 +2,6 @@
 # given explicitly on the command line) so mention it first.
 all:
 
-# List all subdirectories here. Each contains its own Makefile.local.
-# Use of '=', without '+=', seems to be required for out-of-tree
-# builds to work.
-subdirs = compat completion emacs lib man parse-time-string performance-test util test
-
-# We make all targets depend on the Makefiles themselves.
-global_deps = Makefile Makefile.config Makefile.local \
-	$(subdirs:%=%/Makefile) $(subdirs:%=%/Makefile.local)
-
 # Sub-directory Makefile.local fragments can append to these variables
 # to have directory-specific cflags as necessary.
 
@@ -26,6 +17,11 @@ extra_cxxflags :=
 srcdir ?= .
 
 include Makefile.config
+
+# We make all targets depend on the Makefiles themselves.
+global_deps = Makefile Makefile.config Makefile.local \
+	$(subdirs:%=%/Makefile) $(subdirs:%=%/Makefile.local)
+
 Makefile.config: $(srcdir)/configure
 ifeq ($(configure_options),)
 	@echo ""
diff --git a/Makefile.local b/Makefile.local
index 72524eb3..4f8f4640 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -22,6 +22,11 @@ VERSION:=$(shell cat ${srcdir}/version)
 ifeq ($(filter release release-message pre-release update-versions,$(MAKECMDGOALS)),)
 ifeq ($(IS_GIT),yes)
 VERSION:=$(shell git describe --match '[0-9.]*'|sed -e s/_/~/ -e s/-/+/ -e s/-/~/)
+# Write the file 'version.stamp' in case its contents differ from $(VERSION)
+FILE_VERSION:=$(shell test -f version.stamp && read vs < version.stamp || vs=; echo $$vs)
+ifneq ($(FILE_VERSION),$(VERSION))
+       $(shell echo "$(VERSION)" > version.stamp)
+endif
 endif
 endif
 
@@ -41,19 +46,20 @@ PV_FILE=bindings/python/notmuch/version.py
 # Smash together user's values with our extra values
 FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS)
 FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS)
-FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS)
+FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch
+ifeq ($(LIBDIR_IN_LDCONFIG),0)
+FINAL_NOTMUCH_LDFLAGS += $(RPATH_LDFLAGS)
+endif
+FINAL_NOTMUCH_LDFLAGS += $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS)
 FINAL_NOTMUCH_LINKER = CC
 ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)
 FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS)
 FINAL_NOTMUCH_LINKER = CXX
 endif
-ifeq ($(LIBDIR_IN_LDCONFIG),0)
-FINAL_NOTMUCH_LDFLAGS += $(RPATH_LDFLAGS)
-endif
 FINAL_LIBNOTMUCH_LDFLAGS = $(LDFLAGS) $(AS_NEEDED_LDFLAGS) $(CONFIGURE_LDFLAGS)
 
 .PHONY: all
-all: notmuch notmuch-shared
+all: notmuch notmuch-shared build-man
 ifeq ($(MAKECMDGOALS),)
 ifeq ($(shell cat .first-build-message 2>/dev/null),)
 	@NOTMUCH_FIRST_BUILD=1 $(MAKE) --no-print-directory all
@@ -69,9 +75,14 @@ ifeq ($(shell cat .first-build-message 2>/dev/null),)
 endif
 endif
 
+# Depend (also) on the file 'version'. In case of ifeq ($(IS_GIT),yes)
+# this file may already have been updated.
+version.stamp: $(srcdir)/version
+	echo $(VERSION) > $@
+
 $(TAR_FILE):
-	if git tag -v $(VERSION) >/dev/null 2>&1; then \
-           ref=$(VERSION); \
+	if git tag -v $(UPSTREAM_TAG) >/dev/null 2>&1; then \
+           ref=$(UPSTREAM_TAG); \
         else \
            ref="HEAD" ; \
 	   echo "Warning: No signed tag for $(VERSION)"; \
@@ -95,7 +106,7 @@ dist: $(TAR_FILE)
 
 .PHONY: update-versions
 
-update-versions: update-man-versions
+update-versions:
 	sed -i "s/^__VERSION__[[:blank:]]*=.*$$/__VERSION__ = \'${VERSION}\'/" $(PV_FILE)
 
 # We invoke make recursively only to force ordering of our phony
@@ -236,11 +247,11 @@ endif
 quiet ?= $($(shell echo $1 | sed -e s'/ .*//'))
 
 %.o: %.cc $(global_deps)
-	@mkdir -p .deps/$(@D)
+	@mkdir -p $(patsubst %/.,%,.deps/$(@D))
 	$(call quiet,CXX $(CPPFLAGS) $(CXXFLAGS)) -c $(FINAL_CXXFLAGS) $< -o $@ -MD -MP -MF .deps/$*.d
 
 %.o: %.c $(global_deps)
-	@mkdir -p .deps/$(@D)
+	@mkdir -p $(patsubst %/.,%,.deps/$(@D))
 	$(call quiet,CC $(CPPFLAGS) $(CFLAGS)) -c $(FINAL_CFLAGS) $< -o $@ -MD -MP -MF .deps/$*.d
 
 .PHONY : clean
@@ -280,6 +291,8 @@ notmuch_client_srcs =		\
 
 notmuch_client_modules = $(notmuch_client_srcs:.c=.o)
 
+notmuch.o: version.stamp
+
 notmuch: $(notmuch_client_modules) lib/libnotmuch.a util/libutil.a parse-time-string/libparse-time-string.a
 	$(call quiet,CXX $(CFLAGS)) $^ $(FINAL_LIBNOTMUCH_LDFLAGS) -o $@
 
@@ -318,10 +331,12 @@ install-desktop:
 	desktop-file-install --mode 0644 --dir "$(DESTDIR)$(desktop_dir)" notmuch.desktop
 
 SRCS  := $(SRCS) $(notmuch_client_srcs)
-CLEAN := $(CLEAN) notmuch notmuch-shared $(notmuch_client_modules) notmuch.elc
+CLEAN := $(CLEAN) notmuch notmuch-shared $(notmuch_client_modules) version.stamp
 
 DISTCLEAN := $(DISTCLEAN) .first-build-message Makefile.config
 
 DEPS := $(SRCS:%.c=.deps/%.d)
 DEPS := $(DEPS:%.cc=.deps/%.d)
 -include $(DEPS)
+
+.SUFFIXES: # Delete the default suffixes. Old-Fashioned Suffix Rules not used.
diff --git a/NEWS b/NEWS
index 28788d8d..eb8174cd 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,296 @@
+Notmuch 0.18.1 (2014-06-25)
+===========================
+
+This is a bug fix and portability release.
+
+Build System
+------------
+
+Add a workaround for systems without zlib.pc
+
+Make emacs install robust against the non-existence of emacs
+
+Put notmuch lib directory first in RPATH
+
+Fix handling of html_static_path in sphinx
+
+  Both the python bindings and the main docs had spurious settings of
+  this variable.
+
+Test Suite
+----------
+
+Use --quick when starting emacs
+
+  This avoids a hang in the T160-json tests.
+
+Allow pending break points in atomicity script
+
+  This allows the atomicity tests to run on several more architectures/OSes.
+
+Command-Line Interface
+----------------------
+
+To improve portability use fsync instead of fdatasync in
+`notmuch-dump`. There should be no functional difference.
+
+Library changes
+---------------
+
+Resurrect support for single-message mbox files
+
+  The removal introduced a bug with previously indexed single-message
+  mboxes.  This support remains deprecated.
+
+Fix for phrase indexing
+
+  There were several bugs where words intermingled from different
+  headers and MIME parts could match a single phrase query.  This fix
+  will affect only newly indexed messages.
+
+Emacs Interface
+---------------
+
+Make sure tagging on an empty query is harmless
+
+  Previously tagging an empty query could lead to tags being
+  unintentionally removed.
+
+Notmuch 0.18 (2014-05-06)
+=========================
+
+Overview
+--------
+
+This new release includes some enhancements to searching for messages
+by filesystem location (`folder:` and `path:` prefixes under *General*
+below).  Saved searches in *Emacs* have also been enhanced to allow
+distinct search orders for each one.  Another enhancement to the
+*Emacs* interface is that replies to encrypted messages are now
+encrypted, reducing the risk of unintentional information disclosure.
+The default dump output format has changed to the more robust
+`batch-tag` format. The previously deprecated parsing of single
+message mboxes has been removed. For detailed release notes, see
+below.
+
+General
+-------
+
+The `folder:` search prefix now requires an exact match
+
+  The `folder:` prefix has been changed to search for email messages
+  by the exact, case sensitive maildir or MH folder name. Wildcard
+  matching (`folder:foo*`) is no longer supported. The new behaviour
+  allows for more accurate mail folder based searches, makes it
+  possible to search for messages in the top-level folder, and should
+  lead to less surprising results than the old behaviour. Users are
+  advised to see the `notmuch-search-terms` manual page for details,
+  and review how the change affects their existing `folder:` searches.
+
+There is a new `path:` search prefix.
+
+  The new `path:` search prefix complements the `folder:` prefix. The
+  `path:` prefix searches for email messages that are in particular
+  directories within the mail store, optionally recursively using a
+  special syntax. See the `notmuch-search-terms` manual page for
+  details.
+
+Notmuch database upgrade due to `folder:` and `path:` changes
+
+  The above mentioned changes to the `folder:` prefix and the addition
+  of `path:` prefix require a Notmuch database upgrade. This will be
+  done automatically, without prompting on the next time `notmuch new`
+  is run after the upgrade. The upgrade is not reversible, and the
+  upgraded database will not be readable by older versions of
+  Notmuch. As a safeguard, a database dump will be created in the
+  `.notmuch` directory before upgrading.
+
+Library changes
+---------------
+
+Notmuch database upgrade
+
+  The libnotmuch consumers are reminded to handle database upgrades
+  properly, either by relying on running `notmuch new`, or checking
+  `notmuch_database_needs_upgrade()` and calling
+  `notmuch_database_upgrade()` as necessary. This has always been the
+  case, but in practise there have been no database upgrades in any
+  released version of Notmuch before now.
+
+Support for indexing mbox files has been dropped
+
+  There has never been proper support for mbox files containing
+  multiple messages, and the support for single-message mbox files has
+  been deprecated since Notmuch 0.15. The support has now been
+  dropped, and all mbox files will be rejected during indexing.
+
+Message header parsing changes
+
+  Notmuch previously had an internal parser for message headers. The
+  parser has now been dropped in favour of letting GMime parse both
+  the headers and the message MIME structure at the same pass. This is
+  mostly an internal change, but the GMime parser is stricter in its
+  interpretation of the headers. This may result in messages with
+  slightly malformed message headers being now rejected.
+
+Command-Line Interface
+----------------------
+
+`notmuch dump` now defaults to `batch-tag` format
+
+  The old format is still available with `--format=sup`.
+
+`notmuch new` has a --quiet option
+
+  This option suppresses the progress and summary reports.
+
+`notmuch insert` respects maildir.synchronize_flags config option
+
+  Do not synchronize tags to maildir flags in `notmuch insert` if the
+  user does not want it.
+
+The commands set consistent exit status codes on failures
+
+  The cli commands now consistently set exit status of 1 on failures,
+  except where explicitly otherwise noted. The notable expections are
+  the status codes for format version mismatches for commands that
+  support formatted output.
+
+Bug fix for checking configured new.tags for invalid tags
+
+  `notmuch new` and `notmuch insert` now check the user configured
+  new.tags for invalid tags, and refuse to apply them, similar to
+  `notmuch tag`. Invalid tags are currently the empty string and tags
+  starting with `-`.
+
+Emacs Interface
+---------------
+
+Init file
+
+  If the file pointed by new variable `notmuch-init-file` (typically
+  `~/.emacs.d/notmuch-config.el`) exists, it is loaded at the end of
+  `notmuch.el`. Users can put their personal notmuch emacs lisp based
+  configuration/customization items there instead of filling
+  `~/.emacs` with these.
+
+Changed format for saved searches
+
+  The format for `notmuch-saved-searches` has changed, but old style
+  saved searches are still supported. The new style means that a saved
+  search can store the desired sort order for the search, and it can
+  store a separate query to use for generating the count notmuch
+  shows.
+
+  The variable is fully customizable and any configuration done
+  through customize should *just work*, with the additional options
+  mentioned above. For manual customization see the documentation for
+  `notmuch-saved-searches`.
+
+  IMPORTANT: a new style notmuch-saved-searches variable will break
+  previous versions of notmuch-emacs (even search will not work); to
+  fix remove the customization for notmuch-saved-searches.
+
+  If you have a custom saved search sort function (not unsorted or
+  alphabetical) then the sort function will need to be
+  modified. Replacing (car saved-search) by (notmuch-saved-search-get
+  saved-search :name) and (cdr saved-search) by
+  (notmuch-saved-search-get saved-search :query) should be sufficient.
+
+The keys of `notmuch-tag-formats` are now regexps
+
+  Previously, the keys were literal strings.  Customized settings of
+  `notmuch-tag-formats` will continue to work as before unless tags
+  contain regexp special characters like `.` or `*`.
+
+Changed tags are now shown in the buffer
+
+  Previously tag changes made in a buffer were shown immediately. In
+  some cases (particularly automatic tag changes like marking read)
+  this made it hard to see what had happened (e.g., whether the
+  message had been unread).
+
+  The changes are now shown explicitly in the buffer: by default
+  deleted tags are displayed with red strike-through and added tags
+  are displayed underlined in green (inverse video is used for deleted
+  tags if the terminal does not support strike-through).
+
+  The variables `notmuch-tag-deleted-formats` and
+  `notmuch-tag-added-formats`, which have the same syntax as
+  `notmuch-tag-formats`, allow this to be customized.
+
+  Setting `notmuch-tag-deleted-formats` to `'((".*" nil))` and
+  `notmuch-tag-added-formats` to `'((".*" tag))` will give the old
+  behavior of hiding deleted tags and showing added tags identically
+  to tags already present.
+
+Version variable
+
+  The new, build-time generated variable `notmuch-emacs-version` is used
+  to distinguish between notmuch cli and notmuch emacs versions.
+  The function `notmuch-hello-versions` (bound to 'v' in notmuch-hello
+  window) prints both notmuch cli and notmuch emacs versions in case
+  these differ from each other.
+  This is especially useful when using notmuch remotely.
+
+Ido-completing-read initialization in Emacs 23
+
+  `ido-completing-read` in Emacs 23 versions 1 through 3 freezes unless
+  it is initialized. Defadvice-based *Ido* initialization is defined
+  for these Emacs versions.
+
+Bug fix for saved searches with newlines in them
+
+  Split lines confuse `notmuch count --batch`, so we remove embedded
+  newlines before calling notmuch count.
+
+Bug fixes for sender identities
+
+  Previously, Emacs would rewrite some sender identities in unexpected
+  and undesirable ways.  Now it will use identities exactly as
+  configured in `notmuch-identities`.
+
+Replies to encrypted messages will be encrypted by default
+
+  In the interest of maintaining confidentiality of communications,
+  the Notmuch Emacs interface now automatically adds the mml tag to
+  encrypt replies to encrypted messages. This should make it less
+  likely to accidentally reply to encrypted messages in plain text.
+
+Reply pushes mark before signature
+
+  We push mark and set point on reply so that the user can easily cut
+  the quoted text. The mark is now pushed before the signature, if
+  any, instead of end of buffer so the signature is preserved.
+
+Message piping uses the originating buffer's working directory
+
+  `notmuch-show-pipe-message` now uses the originating buffer's
+  current default directory instead of that of the `*notmuch-pipe*`
+  buffer's.
+
+nmbug
+-----
+
+nmbug adds a `clone` command for setting up the initial repository and
+uses `@{upstream}` instead of `FETCH_HEAD` to track upstream changes.
+
+  The `@{upstream}` change reduces ambiguity when fetching multiple
+  branches, but requires existing users update their `NMBGIT`
+  repository (usually `~/.nmbug`) to distinguish between local and
+  remote-tracking branches.  The easiest way to do this is:
+
+  1. If you have any purely local commits (i.e. they aren't in the
+     nmbug repository on nmbug.tethera.net), push them to a remote
+     repository.  We'll restore them from the backup in step 4.
+  2. Remove your `NMBGIT` repository (e.g. `mv .nmbug .nmbug.bak`).
+  3. Use the new `clone` command to create a fresh clone:
+
+        nmbug clone http://nmbug.tethera.net/git/nmbug-tags.git
+
+  4. If you had local commits in step 1, add a remote for that
+     repository and fetch them into the new repository.
+
 Notmuch 0.17 (2013-12-30)
 =========================
 
diff --git a/README b/README
index 3a003ad9..d92fcfdf 100644
--- a/README
+++ b/README
@@ -42,7 +42,7 @@ the libnotmuch library.
 Notmuch installs a full-featured email interface for use within
 emacs. To use this, first add the following line to your .emacs file:
 
-	(require 'notmuch)
+	(autoload 'notmuch "notmuch" "Notmuch mail" t)
 
 Then, either run "emacs -f notmuch" or execute the command "M-x
 notmuch" from within a running emacs.
diff --git a/bindings/python/docs/source/conf.py b/bindings/python/docs/source/conf.py
index 9db377f7..5107a96e 100644
--- a/bindings/python/docs/source/conf.py
+++ b/bindings/python/docs/source/conf.py
@@ -140,7 +140,7 @@ html_theme = 'default'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['../html']
+html_static_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
diff --git a/bindings/python/notmuch/version.py b/bindings/python/notmuch/version.py
index fa3f93b8..40f3e889 100644
--- a/bindings/python/notmuch/version.py
+++ b/bindings/python/notmuch/version.py
@@ -1,2 +1,2 @@
 # this file should be kept in sync with ../../../version
-__VERSION__ = '0.17'
+__VERSION__ = '0.18.1'
diff --git a/bindings/ruby/message.c b/bindings/ruby/message.c
index eed4b31b..4ff6097f 100644
--- a/bindings/ruby/message.c
+++ b/bindings/ruby/message.c
@@ -111,7 +111,7 @@ notmuch_rb_message_get_filename (VALUE self)
 }
 
 /*
- * call-seq: MESSAGE.filanames => FILENAMES
+ * call-seq: MESSAGE.filenames => FILENAMES
  *
  * Get all filenames for the email corresponding to MESSAGE.
  */
diff --git a/command-line-arguments.c b/command-line-arguments.c
index bf9aecab..844d6c3d 100644
--- a/command-line-arguments.c
+++ b/command-line-arguments.c
@@ -129,40 +129,41 @@ parse_option (const char *arg,
 
     const notmuch_opt_desc_t *try;
     for (try = options; try->opt_type != NOTMUCH_OPT_END; try++) {
-	if (try->name && strncmp (arg, try->name, strlen (try->name)) == 0) {
-	    char next = arg[strlen (try->name)];
-	    const char *value= arg+strlen(try->name)+1;
+	if (! try->name)
+	    continue;
 
-	    /* If we have not reached the end of the argument
-	       (i.e. the next character is not a space or delimiter)
-	       then the argument could still match a longer option
-	       name later in the option table.
-	    */
-	    if (next != '=' && next != ':' && next != '\0')
-		continue;
+	if (strncmp (arg, try->name, strlen (try->name)) != 0)
+	    continue;
 
-	    if (try->output_var == NULL)
-		INTERNAL_ERROR ("output pointer NULL for option %s", try->name);
+	char next = arg[strlen (try->name)];
+	const char *value = arg + strlen(try->name) + 1;
 
-	    switch (try->opt_type) {
-	    case NOTMUCH_OPT_KEYWORD:
-		return _process_keyword_arg (try, next, value);
-		break;
-	    case NOTMUCH_OPT_BOOLEAN:
-		return _process_boolean_arg (try, next, value);
-		break;
-	    case NOTMUCH_OPT_INT:
-		return _process_int_arg (try, next, value);
-		break;
-	    case NOTMUCH_OPT_STRING:
-		return _process_string_arg (try, next, value);
-		break;
-	    case NOTMUCH_OPT_POSITION:
-	    case NOTMUCH_OPT_END:
-	    default:
-		INTERNAL_ERROR ("unknown or unhandled option type %d", try->opt_type);
-		/*UNREACHED*/
-	    }
+	/*
+	 * If we have not reached the end of the argument (i.e. the
+	 * next character is not a space or delimiter) then the
+	 * argument could still match a longer option name later in
+	 * the option table.
+	 */
+	if (next != '=' && next != ':' && next != '\0')
+	    continue;
+
+	if (try->output_var == NULL)
+	    INTERNAL_ERROR ("output pointer NULL for option %s", try->name);
+
+	switch (try->opt_type) {
+	case NOTMUCH_OPT_KEYWORD:
+	    return _process_keyword_arg (try, next, value);
+	case NOTMUCH_OPT_BOOLEAN:
+	    return _process_boolean_arg (try, next, value);
+	case NOTMUCH_OPT_INT:
+	    return _process_int_arg (try, next, value);
+	case NOTMUCH_OPT_STRING:
+	    return _process_string_arg (try, next, value);
+	case NOTMUCH_OPT_POSITION:
+	case NOTMUCH_OPT_END:
+	default:
+	    INTERNAL_ERROR ("unknown or unhandled option type %d", try->opt_type);
+	    /*UNREACHED*/
 	}
     }
     fprintf (stderr, "Unrecognized option: --%s\n", arg);
diff --git a/compat/.gitignore b/compat/.gitignore
new file mode 100644
index 00000000..107ba17a
--- /dev/null
+++ b/compat/.gitignore
@@ -0,0 +1 @@
+zlib.pc
diff --git a/compat/Makefile.local b/compat/Makefile.local
index b0d5417f..bcb9f0ec 100644
--- a/compat/Makefile.local
+++ b/compat/Makefile.local
@@ -5,6 +5,10 @@ extra_cflags += -I$(srcdir)/$(dir)
 
 notmuch_compat_srcs :=
 
+ifneq ($(HAVE_CANONICALIZE_FILE_NAME),1)
+notmuch_compat_srcs += $(dir)/canonicalize_file_name.c
+endif
+
 ifneq ($(HAVE_GETLINE),1)
 notmuch_compat_srcs += $(dir)/getline.c $(dir)/getdelim.c
 endif
diff --git a/compat/canonicalize_file_name.c b/compat/canonicalize_file_name.c
new file mode 100644
index 00000000..e92c0f62
--- /dev/null
+++ b/compat/canonicalize_file_name.c
@@ -0,0 +1,18 @@
+#include "compat.h"
+#include <limits.h>
+#undef _GNU_SOURCE
+#include <stdlib.h>
+
+char *
+canonicalize_file_name (const char * path)
+{
+#ifdef PATH_MAX
+    char *resolved_path =  malloc (PATH_MAX+1);
+    if (resolved_path == NULL)
+	return NULL;
+
+    return realpath (path, resolved_path);
+#else
+#error undefined PATH_MAX _and_ missing canonicalize_file_name not supported
+#endif
+}
diff --git a/compat/compat.h b/compat/compat.h
index 5a402d5c..634d505b 100644
--- a/compat/compat.h
+++ b/compat/compat.h
@@ -37,6 +37,14 @@ extern "C" {
 #define _POSIX_PTHREAD_SEMANTICS 1
 #endif
 
+#if !HAVE_CANONICALIZE_FILE_NAME
+/* we only call this function from C, and this makes testing easier */
+#ifndef __cplusplus
+char *
+canonicalize_file_name (const char *path);
+#endif
+#endif
+
 #if !HAVE_GETLINE
 #include <stdio.h>
 #include <unistd.h>
diff --git a/compat/gen_zlib_pc.c b/compat/gen_zlib_pc.c
new file mode 100644
index 00000000..198a727c
--- /dev/null
+++ b/compat/gen_zlib_pc.c
@@ -0,0 +1,18 @@
+#include <stdio.h>
+#include <zlib.h>
+
+static const char *template =
+	"prefix=/usr\n"
+	"exec_prefix=${prefix}\n"
+	"libdir=${exec_prefix}/lib\n"
+	"\n"
+	"Name: zlib\n"
+	"Description: zlib compression library\n"
+	"Version: %s\n"
+	"Libs: -lz\n";
+
+int main(void)
+{
+	printf(template, ZLIB_VERSION);
+	return 0;
+}
diff --git a/compat/have_canonicalize_file_name.c b/compat/have_canonicalize_file_name.c
new file mode 100644
index 00000000..24c848ec
--- /dev/null
+++ b/compat/have_canonicalize_file_name.c
@@ -0,0 +1,10 @@
+#define _GNU_SOURCE
+#include <stdlib.h>
+
+int main()
+{
+    char *found;
+    char *string;
+
+    found = canonicalize_file_name (string);
+}
diff --git a/compat/have_d_type.c b/compat/have_d_type.c
new file mode 100644
index 00000000..9ca6c6e0
--- /dev/null
+++ b/compat/have_d_type.c
@@ -0,0 +1,10 @@
+#include <dirent.h>
+
+int main()
+{
+    struct dirent ent;
+
+    (void) ent.d_type;
+
+    return 0;
+}
diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index 04324bbb..d88c5e7d 100644
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -49,8 +49,19 @@ _notmuch_search_terms()
 	from:*)
 	    COMPREPLY=( $(compgen -P "from:" -W "`_notmuch_user_emails`" -- ${cur##from:}) )
 	    ;;
+	path:*)
+	    local path=`notmuch config get database.path`
+	    compopt -o nospace
+	    COMPREPLY=( $(compgen -d "$path/${cur##path:}" | sed "s|^$path/||" ) )
+	    ;;
+	folder:*)
+	    local path=`notmuch config get database.path`
+	    compopt -o nospace
+	    COMPREPLY=( $(compgen -d "$path/${cur##folder:}" | \
+		sed "s|^$path/||" | grep -v "\(^\|/\)\(cur\|new\|tmp\)$" ) )
+	    ;;
 	*)
-	    local search_terms="from: to: subject: attachment: tag: id: thread: folder: date:"
+	    local search_terms="from: to: subject: attachment: tag: id: thread: folder: path: date:"
 	    compopt -o nospace
 	    COMPREPLY=( $(compgen -W "${search_terms}" -- ${cur}) )
 	    ;;
@@ -67,7 +78,7 @@ _notmuch_compact()
     $split &&
     case "${prev}" in
 	--backup)
-	    _filedir
+	    _filedir -d
 	    return
 	    ;;
     esac
@@ -96,7 +107,7 @@ _notmuch_config()
 	    ;;
 	# these will also complete on config get, but we don't care
 	database.path)
-	    _filedir
+	    _filedir -d
 	    ;;
 	maildir.synchronize_flags)
 	    COMPREPLY=( $(compgen -W "true false" -- ${cur}) )
@@ -208,7 +219,7 @@ _notmuch_new()
 
     case "${cur}" in
 	-*)
-	    local options="--no-hooks"
+	    local options="--no-hooks --quiet"
 	    COMPREPLY=( $(compgen -W "${options}" -- ${cur}) )
 	    ;;
     esac
diff --git a/configure b/configure
index 13b60620..99ab74dc 100755
--- a/configure
+++ b/configure
@@ -19,11 +19,14 @@ readonly DEFAULT_IFS="$IFS"
 
 srcdir=$(dirname "$0")
 
+subdirs="util compat lib parse-time-string completion doc emacs"
+subdirs="${subdirs} performance-test test test/test-databases"
+
 # For a non-srcdir configure invocation (such as ../configure), create
 # the directory structure and copy Makefiles.
 if [ "$srcdir" != "." ]; then
 
-    for dir in . $(grep "^subdirs *=" "$srcdir"/Makefile | sed -e "s/subdirs *= *//"); do
+    for dir in . ${subdirs}; do
 	mkdir -p "$dir"
 	cp "$srcdir"/"$dir"/Makefile.local "$dir"
 	cp "$srcdir"/"$dir"/Makefile "$dir"
@@ -337,6 +340,27 @@ else
     errors=$((errors + 1))
 fi
 
+if ! pkg-config --exists zlib; then
+  ${CC} ${zlib_cflags} -o compat/gen_zlib_pc \
+	  "$srcdir"/compat/gen_zlib_pc.c ${zlib_ldflags} > /dev/null 2>&1 &&
+  compat/gen_zlib_pc > compat/zlib.pc &&
+  PKG_CONFIG_PATH="$PKG_CONFIG_PATH":compat &&
+  export PKG_CONFIG_PATH
+  rm -f compat/gen_zlib_pc
+fi
+
+printf "Checking for zlib (>= 1.2.5.2)... "
+have_zlib=0
+if pkg-config --atleast-version=1.2.5.2 zlib; then
+    printf "Yes.\n"
+    have_zlib=1
+    zlib_cflags=$(pkg-config --cflags zlib)
+    zlib_ldflags=$(pkg-config --libs zlib)
+else
+    printf "No.\n"
+    errors=$((errors + 1))
+fi
+
 printf "Checking for talloc development files... "
 if pkg-config --exists talloc; then
     printf "Yes.\n"
@@ -360,6 +384,14 @@ else
     have_valgrind=0
 fi
 
+printf "Checking for bash-completion (>= 1.90)... "
+if pkg-config --atleast-version=1.90 bash-completion; then
+    printf "Yes.\n"
+else
+    printf "No (will not install bash completion).\n"
+    WITH_BASH=0
+fi
+
 if [ -z "${EMACSLISPDIR}" ]; then
     if pkg-config --exists emacs; then
 	EMACSLISPDIR=$(pkg-config emacs --variable sitepkglispdir)
@@ -385,6 +417,25 @@ else
     have_emacs=0
 fi
 
+printf "Checking if sphinx is available and supports nroff output... "
+if hash sphinx-build > /dev/null 2>&1 && python -m sphinx.writers.manpage > /dev/null 2>&1 ; then
+    printf "Yes.\n"
+    have_sphinx=1
+    have_rst2man=0
+else
+    printf "No (falling back to rst2man).\n"
+    have_sphinx=0
+
+    printf "Checking if rst2man is available... "
+    if rst2man -V > /dev/null 2>&1; then
+       printf "Yes.\n"
+       have_rst2man=1
+    else
+       printf "No (so will not install man pages).\n"
+       have_rst2man=0
+    fi
+fi
+
 libdir_in_ldconfig=0
 
 printf "Checking which platform we are on... "
@@ -466,6 +517,11 @@ EOF
 	echo "	Xapian library (including development files such as headers)"
 	echo "	http://xapian.org/"
     fi
+    if [ $have_zlib -eq 0 ]; then
+	echo "	zlib library (>= version 1.2.5.2, including development files such as headers)"
+	echo "	http://zlib.net/"
+	echo
+    fi
     if [ $have_gmime -eq 0 ]; then
 	echo "	Either GMime 2.4 library" $GMIME_24_VERSION_CTR "or GMime 2.6 library" $GMIME_26_VERSION_CTR
 	echo "	(including development files such as headers)"
@@ -489,11 +545,11 @@ case a simple command will install everything you need. For example:
 
 On Debian and similar systems:
 
-	sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev
+	sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev
 
 Or on Fedora and similar systems:
 
-	sudo yum install xapian-core-devel gmime-devel libtalloc-devel
+	sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel
 
 On other systems, similar commands can be used, but the details of the
 package names may be different.
@@ -526,6 +582,18 @@ EOF
     exit 1
 fi
 
+printf "Checking for canonicalize_file_name... "
+if ${CC} -o compat/have_canonicalize_file_name "$srcdir"/compat/have_canonicalize_file_name.c > /dev/null 2>&1
+then
+    printf "Yes.\n"
+    have_canonicalize_file_name=1
+else
+    printf "No (will use our own instead).\n"
+    have_canonicalize_file_name=0
+fi
+rm -f compat/have_canonicalize_file_name
+
+
 printf "Checking for getline... "
 if ${CC} -o compat/have_getline "$srcdir"/compat/have_getline.c > /dev/null 2>&1
 then
@@ -570,6 +638,17 @@ else
 fi
 rm -f compat/have_timegm
 
+printf "Checking for dirent.d_type... "
+if ${CC} -o compat/have_d_type "$srcdir"/compat/have_d_type.c > /dev/null 2>&1
+then
+    printf "Yes.\n"
+    have_d_type="1"
+else
+    printf "No (will use stat instead).\n"
+    have_d_type="0"
+fi
+rm -f compat/have_d_type
+
 printf "Checking for standard version of getpwuid_r... "
 if ${CC} -o compat/check_getpwuid "$srcdir"/compat/check_getpwuid.c > /dev/null 2>&1
 then
@@ -660,6 +739,9 @@ cat > Makefile.config <<EOF
 # directory (the current directory at the time configure was run).
 srcdir = ${srcdir}
 
+# subdirectories to build
+subdirs = ${subdirs}
+
 configure_options = $@
 
 # We use vpath directives (rather than the VPATH variable) since the
@@ -675,8 +757,9 @@ configure_options = $@
 # files, (which is quite ugly).
 vpath %.c \$(srcdir)
 vpath %.cc \$(srcdir)
-vpath %.1 \$(srcdir)
 vpath Makefile.% \$(srcdir)
+vpath %.py \$(srcdir)
+vpath %.rst \$(srcdir)
 
 # The C compiler to use
 CC = ${CC}
@@ -740,6 +823,12 @@ emacsetcdir=${EMACSETCDIR}
 # Whether there's an emacs binary available for byte-compiling
 HAVE_EMACS = ${have_emacs}
 
+# Whether there's a sphinx-build binary available for building documentation
+HAVE_SPHINX=${have_sphinx}
+
+# Whether there's a rst2man binary available for building documentation
+HAVE_RST2MAN=${have_rst2man}
+
 # The directory to which desktop files should be installed
 desktop_dir = \$(prefix)/share/applications
 
@@ -749,6 +838,10 @@ bash_completion_dir = ${BASHCOMPLETIONDIR:=\$(sysconfdir)/bash_completion.d}
 # The directory to which zsh completions files should be installed
 zsh_completion_dir = ${ZSHCOMLETIONDIR:=\$(prefix)/share/zsh/functions/Completion/Unix}
 
+# Whether the canonicalize_file_name function is available (if not, then notmuch will
+# build its own version)
+HAVE_CANONICALIZE_FILE_NAME = ${have_canonicalize_file_name}
+
 # Whether the getline function is available (if not, then notmuch will
 # build its own version)
 HAVE_GETLINE = ${have_getline}
@@ -761,6 +854,9 @@ HAVE_STRCASESTR = ${have_strcasestr}
 # build its own version)
 HAVE_STRSEP = ${have_strsep}
 
+# Whether struct dirent has d_type (if not, then notmuch will use stat)
+HAVE_D_TYPE = ${have_d_type}
+
 # Whether the Xapian version in use supports compaction
 HAVE_XAPIAN_COMPACT = ${have_xapian_compact}
 
@@ -790,6 +886,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags}
 GMIME_CFLAGS = ${gmime_cflags}
 GMIME_LDFLAGS = ${gmime_ldflags}
 
+# Flags needed to compile and link against zlib
+ZLIB_CFLAGS = ${zlib_cflags}
+ZLIB_LDFLAGS = ${zlib_ldflags}
+
 # Flags needed to compile and link against talloc
 TALLOC_CFLAGS = ${talloc_cflags}
 TALLOC_LDFLAGS = ${talloc_ldflags}
@@ -817,24 +917,30 @@ WITH_ZSH = ${WITH_ZSH}
 
 # Combined flags for compiling and linking against all of the above
 CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)      \\
+		   -DHAVE_CANONICALIZE_FILE_NAME=\$(HAVE_CANONICALIZE_FILE_NAME) \\
+		   \$(ZLIB_CFLAGS)					 \\
 		   \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND)   \\
 		   \$(VALGRIND_CFLAGS)                                   \\
 		   -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR)                 \\
 		   -DHAVE_STRSEP=\$(HAVE_STRSEP)                         \\
+		   -DHAVE_D_TYPE=\$(HAVE_D_TYPE)                         \\
 		   -DSTD_GETPWUID=\$(STD_GETPWUID)                       \\
 		   -DSTD_ASCTIME=\$(STD_ASCTIME)                         \\
 		   -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT)	 \\
 		   -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
 CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)    \\
+		     -DHAVE_CANONICALIZE_FILE_NAME=\$(HAVE_CANONICALIZE_FILE_NAME) \\
+		     \$(ZLIB_CFLAGS)					 \\
 		     \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\
 		     \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS)             \\
 		     -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR)               \\
 		     -DHAVE_STRSEP=\$(HAVE_STRSEP)                       \\
+		     -DHAVE_D_TYPE=\$(HAVE_D_TYPE)                       \\
 		     -DSTD_GETPWUID=\$(STD_GETPWUID)                     \\
 		     -DSTD_ASCTIME=\$(STD_ASCTIME)                       \\
 		     -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT)       \\
 		     -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
-CONFIGURE_LDFLAGS =  \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS)
+CONFIGURE_LDFLAGS =  \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS)
 EOF
diff --git a/debian/NEWS b/debian/NEWS
index 8049a933..0ae08198 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,3 +1,13 @@
+notmuch (0.18~rc0-1) experimental; urgency=low
+
+  * This release of notmuch requires a non-reversable database upgrade
+    to support the new path: and updated folder: prefixes. Notmuch
+    will backup your tags for your before doing the upgrade, but it
+    never hurts to make your own backup with notmuch dump before
+    next running 'notmuch new'
+
+ -- David Bremner <bremner@debian.org>  Tue, 22 Apr 2014 09:32:11 +0900
+
 notmuch (0.17-1) unstable; urgency=low
 
   * Previously on big endian architectures like sparc and powerpc the
diff --git a/debian/changelog b/debian/changelog
index 5c8d6fba..d7af75e8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,100 @@
-notmuch (0.17-3~bpo70+1) wheezy-backports; urgency=medium
+notmuch (0.18.1-2) unstable; urgency=medium
 
-  * Rebuild for wheezy-backports.
+  * Update build-deps to use emacs24 on buildd (Closes: #756085)
+  * Disable gdb atomicity test for arm64 as gdb is currently broken on
+    the (unofficial) buildds
+  * Re-enable atomicity test on armhf; upstream fix seems to have
+    worked.
+
+ -- David Bremner <bremner@debian.org>  Sat, 09 Aug 2014 11:48:10 -0300
+
+notmuch (0.18.1-1) unstable; urgency=medium
+
+  * New upstream bug fix release
+    - Re-enable support for single-message mbox files
+    - Fix for phrase indexing
+    - Make tagging empty query in Emacs harmless
+
+ -- David Bremner <bremner@debian.org>  Wed, 25 Jun 2014 07:20:45 -0300
+
+notmuch (0.18.1~rc0-1) experimental; urgency=medium
+
+  * New upstream bug fix release (candidate)
+  * Tighten dependence of python packages on libnotmuch
+    (Closes: #749881).
+  * Redo emacsen-install script from sample in emacsen-common
+    (Closes: #739839).
+
+ -- David Bremner <bremner@debian.org>  Sat, 14 Jun 2014 07:50:28 -0300
+
+notmuch (0.18-3) unstable; urgency=medium
+
+  * Disable atomicity tests on armel.
+
+ -- David Bremner <bremner@debian.org>  Thu, 08 May 2014 14:26:45 +0900
+
+notmuch (0.18-2) unstable; urgency=medium
+
+  * Disable atomicity tests on armhf. These should be re-enabled when
+    upstream relases a fix for this (in progress).
+
+ -- David Bremner <bremner@debian.org>  Thu, 08 May 2014 08:28:33 +0900
+
+notmuch (0.18-1) unstable; urgency=medium
+
+  * New upstream release. For detailed release notes see
+    see /usr/share/doc/notmuch/NEWS.gz. Some highlights:
+    - Changes/enhancements to searching for messages by filesystem
+      location ('folder:' and 'path:' prefixes).
+    - Saved searches in Emacs have also been enhanced to allow
+      distinct search orders for each one.
+    - Another enhancement to the Emacs interface is that replies to
+      encrypted messages are now encrypted, reducing the risk of
+      unintentional information disclosure.
+    - The default dump output format has changed to the more robust
+      'batch-tag' format.
+    - The previously deprecated parsing of single message mboxes has
+      been removed.
 
- -- David Bremner <bremner@debian.org>  Wed, 29 Jan 2014 20:20:56 -0400
+ -- David Bremner <bremner@debian.org>  Tue, 06 May 2014 16:20:39 +0900
+
+notmuch (0.18~rc1-1) experimental; urgency=low
+
+  * Upstream release candidate
+    - include encoding fix for vim client.
+
+ -- David Bremner <bremner@debian.org>  Sun, 04 May 2014 07:29:51 +0900
+
+notmuch (0.18~rc0-1) experimental; urgency=low
+
+  * Upstream release candidate.
+  * Bug fix: "insufficient sanitization of arguments to notmuch CLI",
+    thanks to Antoine Beaupré (Closes: #737496).
+  * Bug fix: "notmuch(1) manpage: typo: int -> in", thanks to Jakub
+    Wilk (Closes: #739556).
+  * Bug fix: "get a quiet option", thanks to Joerg Jaspert (Closes:
+    #666027).
+  * Bug fix: "Please remove me from Uploaders", thanks to martin f
+    krafft (Closes: #719100).
+  * Bug fix: "M-x notmuch-show-reply on an encrypted message should
+    insert encryption tags into the mml buffer", thanks to Daniel Kahn
+    Gillmor (Closes: #704648).
+
+ -- David Bremner <bremner@debian.org>  Tue, 22 Apr 2014 09:27:29 +0900
+
+notmuch (0.17-5) unstable; urgency=medium
+
+  * Bug fix: "unowned directory after purge: /0755/", thanks to
+    Andreas Beckmann (Closes: #740325).
+
+ -- David Bremner <bremner@debian.org>  Mon, 03 Mar 2014 07:29:06 -0400
+
+notmuch (0.17-4) unstable; urgency=medium
+
+  * Bug fix: "Please update ruby binary extension install path",
+    thanks to Christian Hofstaedtler (Closes: #739120).
+
+ -- David Bremner <bremner@debian.org>  Tue, 18 Feb 2014 21:37:44 -0400
 
 notmuch (0.17-3) unstable; urgency=medium
 
@@ -59,6 +151,7 @@ notmuch (0.16-1~bpo70+1) wheezy-backports; urgency=low
   * Rebuild for wheezy-backports.
 
  -- David Bremner <bremner@debian.org>  Sun, 01 Sep 2013 19:04:32 -0300
+
 notmuch (0.16-1) unstable; urgency=low
 
   * New upstream feature release
diff --git a/debian/control b/debian/control
index a887263c..6fa1fa7b 100644
--- a/debian/control
+++ b/debian/control
@@ -4,8 +4,8 @@ Priority: optional
 Maintainer: Carl Worth <cworth@debian.org>
 Uploaders:
  Jameson Graef Rollins <jrollins@finestructure.net>,
- martin f. krafft <madduck@debian.org>,
  David Bremner <bremner@debian.org>
+Build-Conflicts: ruby1.8
 Build-Depends:
  debhelper (>= 9),
  pkg-config,
@@ -15,11 +15,13 @@ Build-Depends:
  libz-dev,
  python-all (>= 2.6.6-3~),
  python3-all (>= 3.1.2-7~),
+ python-sphinx (>= 1.0),
  ruby, ruby-dev (>>1:1.9.3~),
- emacs23-nox | emacs23 (>=23~) | emacs23-lucid (>=23~) |
- emacs24-nox | emacs24 (>=24~) | emacs24-lucid (>=24~),
- gdb [!s390x !ia64],
- dtach (>= 0.8)
+ emacs24-nox | emacs24 (>=24~) | emacs24-lucid (>=24~) |
+ emacs23-nox | emacs23 (>=23~) | emacs23-lucid (>=23~),
+ gdb [!s390x !ia64 !armel !arm64],
+ dtach (>= 0.8),
+ bash-completion (>=1.9.0~)
 Standards-Version: 3.9.4
 Homepage: http://notmuchmail.org/
 Vcs-Git: git://notmuchmail.org/git/notmuch
@@ -67,7 +69,7 @@ Description: thread-based email index, search and tagging (development)
 Package: python-notmuch
 Architecture: all
 Section: python
-Depends: ${misc:Depends}, ${python:Depends}, libnotmuch3
+Depends: ${misc:Depends}, ${python:Depends}, libnotmuch3 (>= ${source:Version})
 Description: python interface to the notmuch mail search and index library
  Notmuch is a system for indexing, searching, reading, and tagging
  large collections of email messages in maildir or mh format. It uses
@@ -80,7 +82,7 @@ Description: python interface to the notmuch mail search and index library
 Package: python3-notmuch
 Architecture: all
 Section: python
-Depends: ${misc:Depends}, ${python3:Depends}, libnotmuch3
+Depends: ${misc:Depends}, ${python3:Depends}, libnotmuch3 (>= ${source:Version})
 Description: Python 3 interface to the notmuch mail search and index library
  Notmuch is a system for indexing, searching, reading, and tagging
  large collections of email messages in maildir or mh format. It uses
@@ -108,10 +110,10 @@ Architecture: all
 Section: mail
 Breaks: notmuch (<<0.6~254~)
 Replaces: notmuch (<<0.6~254~)
-Conflicts: emacsen-common (<< 2.0.0)
 Depends: ${misc:Depends}, notmuch (>= ${source:Version}),
  emacs23 (>= 23~) | emacs23-nox (>=23~) | emacs23-lucid (>=23~) |
- emacs24 (>= 24~) | emacs24-nox (>=24~) | emacs24-lucid (>=24~)
+ emacs24 (>= 24~) | emacs24-nox (>=24~) | emacs24-lucid (>=24~),
+ emacsen-common (>= 2.0.8)
 Description: thread-based email index, search and tagging (emacs interface)
  Notmuch is a system for indexing, searching, reading, and tagging
  large collections of email messages in maildir or mh format. It uses
diff --git a/debian/notmuch-emacs.emacsen-install b/debian/notmuch-emacs.emacsen-install
index dfd8fda9..f35ef48e 100755
--- a/debian/notmuch-emacs.emacsen-install
+++ b/debian/notmuch-emacs.emacsen-install
@@ -1,45 +1,48 @@
-#! /bin/sh -e
+#!/bin/sh
 # /usr/lib/emacsen-common/packages/install/notmuch-emacs
-
-# Written by Jim Van Zandt <jrv@debian.org>, borrowing heavily
-# from the install scripts for gettext by Santiago Vila
-# <sanvila@ctv.es> and octave by Dirk Eddelbuettel <edd@debian.org>.
+set -e
 
 FLAVOR=$1
 PACKAGE=notmuch
 
-# We know that the notmuch emacs code doesn't work with emacs before emacs23
-if [ ${FLAVOR} = emacs21 ]; then exit 0; fi
-if [ ${FLAVOR} = emacs22 ]; then exit 0; fi
-if [ ${FLAVOR} = xemacs21 ]; then exit 0; fi
-if [ ${FLAVOR} = xemacs22 ]; then exit 0; fi
+case "${FLAVOR}" in
+    emacs)
+	return 0
+	;;
+    xemacs*|emacs2[12])
+	# patches welcome.
+	echo install/${PACKAGE}: skipping install for unsupported emacsen flavor ${FLAVOR}
+	exit 0
+	;;
+    *)
+	echo install/${PACKAGE}: Handling install for emacsen flavor ${FLAVOR}
+esac
+
+
+elc_dir=/usr/share/${FLAVOR}/site-lisp/${PACKAGE}
+el_dir=/usr/share/emacs/site-lisp/${PACKAGE}
 
-echo install/${PACKAGE}: Handling install for emacsen flavor ${FLAVOR}
+byte_compile_options="--quick --directory=${el_dir} -batch -f batch-byte-compile"
 
-#FLAVORTEST=`echo $FLAVOR | cut -c-6`
-#if [ ${FLAVORTEST} = xemacs ] ; then
-#    SITEFLAG="-no-site-file"
-#else
-#    SITEFLAG="--no-site-file"
-#fi
-#FLAGS="${SITEFLAG} -q -batch -l path.el -f batch-byte-compile"
-FLAGS="--no-site-file -q -batch -l path.el -f batch-byte-compile"
+echo install/${PACKAGE}: byte-compiling for ${FLAVOR}
 
-ELDIR=/usr/share/emacs/site-lisp/${PACKAGE}
-ELCDIR=/usr/share/${FLAVOR}/site-lisp/${PACKAGE}
+[ -d ${elc_dir} ] || mkdir ${elc_dir}
 
-install -m 755 -d ${ELCDIR}
-cd ${ELDIR}
-FILES=`echo *.el`
-cd ${ELCDIR}
-for file in ${FILES}; do
-	ln -sf ${ELDIR}/${file} .
-done
+# Create symlinks to the .el files (see section 6E in debian-emacs
+# polcy).  This makes complation easy, and also allows find-function
+# and find-library to work properly.
+(cd ${elc_dir} && cp -s ${el_dir}/*.el .)
 
-cat << EOF > path.el
-(setq load-path (cons "." load-path) byte-compile-warnings nil)
-EOF
-${FLAVOR} ${FLAGS} ${FILES}
-rm -f *.el
+# Byte compile them
+(cd ${elc_dir}
+ set +e
+ ${FLAVOR} ${byte_compile_options} *.el > Install.log 2>&1
+ if test $? -ne 0
+ then
+   cat Install.log
+   exit 1
+ fi
+ set -e
+ gzip -9 Install.log)
 
-exit 0
+exit 0;
diff --git a/debian/notmuch-emacs.emacsen-remove b/debian/notmuch-emacs.emacsen-remove
index 3b433ae2..a5553209 100755
--- a/debian/notmuch-emacs.emacsen-remove
+++ b/debian/notmuch-emacs.emacsen-remove
@@ -1,8 +1,34 @@
-#!/bin/sh -e
-# /usr/lib/emacsen-common/packages/remove/notmuch-emacs
+#!/bin/sh
+# /usr/lib/emacsen-common/packages/remove/notmuch
+
+set -e
 
 FLAVOR=$1
 PACKAGE=notmuch
+elc_dir=/usr/share/${FLAVOR}/site-lisp/${PACKAGE}
+
+case "${FLAVOR}" in
+    emacs)
+	return 0
+	;;
+    xemacs*|emacs2[12])
+	# patches welcome.
+	echo install/${PACKAGE}: skipping removal for unsupported emacsen flavor ${FLAVOR}
+	exit 0
+	;;
+    *)
+	echo remove/${PACKAGE}: Handling removal for emacsen flavor ${FLAVOR}
+esac
+
+echo remove/${PACKAGE}: Handling removal of emacsen flavor ${FLAVOR}
+
+echo emacsen-common: purging byte-compiled files for ${FLAVOR}
+rm -f ${elc_dir}/*.elc
+rm -f ${elc_dir}/*.el
+rm -f ${elc_dir}/Install.log*
+if test -e "${elc_dir}"
+then
+  rmdir --ignore-fail-on-non-empty "${elc_dir}"
+fi
 
-echo remove/${PACKAGE}: purging byte-compiled files for ${FLAVOR}
-rm -rf /usr/share/${FLAVOR}/site-lisp/${PACKAGE}
+exit 0;
diff --git a/debian/notmuch-emacs.postinst b/debian/notmuch-emacs.postinst
index 48ecf231..1237237d 100644
--- a/debian/notmuch-emacs.postinst
+++ b/debian/notmuch-emacs.postinst
@@ -1,4 +1,7 @@
 dir="/var/lib/emacsen-common/state/package/installed"
-mkdir -p 0755 ${dir}
+mkdir -p -m 0755 ${dir}
 touch ${dir}/notmuch-emacs
 #DEBHELPER#
+if [ -d /0755 ]; then
+   rmdir /0755 || true
+fi
diff --git a/debian/ruby-notmuch.install b/debian/ruby-notmuch.install
index 98e7050b..d3f2105c 100644
--- a/debian/ruby-notmuch.install
+++ b/debian/ruby-notmuch.install
@@ -1 +1 @@
-usr/lib/ruby/vendor_ruby/*/*/notmuch.so
+usr/lib/*/*ruby/*/*/notmuch.so
diff --git a/debian/source/options b/debian/source/options
index 7423a2df..7e95ec71 100644
--- a/debian/source/options
+++ b/debian/source/options
@@ -1 +1,3 @@
 single-debian-patch
+tar-ignore
+tar-ignore=performance-test/download/*.tar.xz
diff --git a/devel/gen-testdb.sh b/devel/gen-testdb.sh
new file mode 100755
index 00000000..621b31e6
--- /dev/null
+++ b/devel/gen-testdb.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+#
+# NAME
+#	gen-testdb.sh - generate test databases
+#
+# SYNOPSIS
+#	gen-testdb.sh -v NOTMUCH-VERSION [-c CORPUS-PATH] [-s TAR-SUFFIX]
+#
+# DESCRIPTION
+#	Generate a tarball containing the specified test corpus and
+#	the corresponding notmuch database, indexed using a specific
+#	version of notmuch, resulting in a specific version of the
+#	database.
+#
+#	The specific version of notmuch will be built on the fly.
+#	Therefore the script must be run within a git repository to be
+#	able to build the old versions of notmuch.
+#
+#	This script reuses the test infrastructure, and the script
+#	must be run from within the test directory.
+#
+#	The output tarballs, named database-<TAR-SUFFIX>.tar.gz, are
+#	placed in the test/test-databases directory.
+#
+# OPTIONS
+#	-v NOTMUCH-VERSION
+#		Notmuch version in terms of a git tag or commit to use
+#		for generating the database. Required.
+#
+#	-c CORPUS-PATH
+#		Path to a corpus to use for generating the
+#		database. Due to CWD changes within the test
+#		infrastructure, use absolute paths. Defaults to the
+#		test corpus.
+#
+#	-s TAR-SUFFIX
+#		Suffix for the tarball basename. Empty by default.
+#
+# EXAMPLE
+#
+#	Generate a database indexed with notmuch 0.17. Use the default
+#	test corpus. Name the tarball database-v1.tar.gz to reflect
+#	the fact that notmuch 0.17 used database version 1.
+#
+#	$ cd test
+#	$ ../devel/gen-testdb.sh -v 0.17 -s v1
+#
+# CAVEATS
+#	Test infrastructure options won't work.
+#
+#	Any existing databases with the same name will be overwritten.
+#
+#	It may not be possible to build old versions of notmuch with
+#	the set of dependencies that satisfy building the current
+#	version of notmuch.
+#
+# AUTHOR
+#	Jani Nikula <jani@nikula.org>
+#
+# LICENSE
+#	Same as notmuch test infrastructure (GPLv2+).
+#
+
+test_description="database generation abusing test infrastructure"
+
+# immediate exit on subtest failure; see test_failure_ in test-lib.sh
+immediate=t
+
+VERSION=
+CORPUS=
+SUFFIX=
+
+while getopts v:c:s: opt; do
+    case "$opt" in
+	v) VERSION="$OPTARG";;
+	c) CORPUS="$OPTARG";;
+	s) SUFFIX="-$OPTARG";;
+    esac
+done
+shift `expr $OPTIND - 1`
+
+. ./test-lib.sh
+
+SHORT_CORPUS=$(basename ${CORPUS:-database})
+DBNAME=${SHORT_CORPUS}${SUFFIX}
+TARBALLNAME=${DBNAME}.tar.xz
+
+CORPUS=${CORPUS:-${TEST_DIRECTORY}/corpus}
+
+test_expect_code 0 "notmuch version specified on the command line" \
+    "test -n ${VERSION}"
+
+test_expect_code 0 "the specified version ${VERSION} refers to a commit" \
+    "git show ${VERSION} >/dev/null 2>&1"
+
+BUILD_DIR="notmuch-${VERSION}"
+test_expect_code 0 "generate snapshot of notmuch version ${VERSION}" \
+    "git -C $TEST_DIRECTORY/.. archive --prefix=${BUILD_DIR}/ --format=tar ${VERSION} | tar x"
+
+# force version string
+git describe --match '[0-9.]*' ${VERSION} > ${BUILD_DIR}/version
+
+test_expect_code 0 "configure and build notmuch version ${VERSION}" \
+    "make -C ${BUILD_DIR}"
+
+# use the newly built notmuch
+export PATH=./${BUILD_DIR}:$PATH
+
+test_begin_subtest "verify the newly built notmuch version"
+test_expect_equal "`notmuch --version`" "notmuch `cat ${BUILD_DIR}/version`"
+
+# replace the existing mails, if any, with the specified corpus
+rm -rf ${MAIL_DIR}
+cp -a ${CORPUS} ${MAIL_DIR}
+
+test_expect_code 0 "index the corpus" \
+    "notmuch new"
+
+# wrap the resulting mail store and database in a tarball
+
+cp -a ${MAIL_DIR} ${TMP_DIRECTORY}/${DBNAME}
+tar Jcf ${TMP_DIRECTORY}/${TARBALLNAME} -C ${TMP_DIRECTORY} ${DBNAME}
+mkdir -p  ${TEST_DIRECTORY}/test-databases
+cp -a ${TMP_DIRECTORY}/${TARBALLNAME} ${TEST_DIRECTORY}/test-databases
+test_expect_code 0 "create the output tarball ${TARBALLNAME}" \
+    "test -f ${TEST_DIRECTORY}/test-databases/${TARBALLNAME}"
+
+# generate a checksum file
+test_expect_code 0 "compute checksum" \
+    "(cd ${TEST_DIRECTORY}/test-databases/ && sha256sum ${TARBALLNAME} > ${TARBALLNAME}.sha256)"
+test_done
diff --git a/devel/nmbug/nmbug b/devel/nmbug/nmbug
index 90d98b63..b18ded7b 100755
--- a/devel/nmbug/nmbug
+++ b/devel/nmbug/nmbug
@@ -26,6 +26,7 @@ my $ESCAPED_RX =	qr{$ESCAPE_CHAR([A-Fa-f0-9]{2})};
 my %command = (
 	     archive	=> \&do_archive,
 	     checkout	=> \&do_checkout,
+	     clone	=> \&do_clone,
 	     commit	=> \&do_commit,
 	     fetch	=> \&do_fetch,
 	     help	=> \&do_help,
@@ -125,6 +126,16 @@ sub do_archive {
   system ('git', "--git-dir=$NMBGIT", 'archive', 'HEAD');
 }
 
+sub do_clone {
+  my $repository = shift;
+
+  my $tempwork = tempdir ('/tmp/nmbug-clone.XXXXXX', CLEANUP => 1);
+  system ('git', 'clone', '--no-checkout', '--separate-git-dir', $NMBGIT,
+          $repository, $tempwork) == 0
+    or die "'git clone' exited with nonzero value\n";
+  git ('config', '--unset', 'core.worktree');
+  git ('config', 'core.bare', 'true');
+}
 
 sub is_committed {
   my $status = shift;
@@ -332,21 +343,24 @@ To discard your changes,  run 'nmbug checkout'
 
 sub do_pull {
   my $remote = shift || 'origin';
+  my $branch = shift || 'master';
 
   git ( 'fetch', $remote);
 
-  do_merge ();
+  do_merge ("$remote/$branch");
 }
 
 
 sub do_merge {
+  my $commit = shift || '@{upstream}';
+
   insist_committed ();
 
   my $tempwork = tempdir ('/tmp/nmbug-merge.XXXXXX', CLEANUP => 1);
 
   git ( { GIT_WORK_TREE => $tempwork }, 'checkout', '-f', 'HEAD');
 
-  git ( { GIT_WORK_TREE => $tempwork }, 'merge', 'FETCH_HEAD');
+  git ( { GIT_WORK_TREE => $tempwork }, 'merge', $commit);
 
   do_checkout ();
 }
@@ -407,11 +421,10 @@ sub do_status {
 
 
 sub is_unmerged {
+  my $commit = shift || '@{upstream}';
 
-  return 0 if (! -f $NMBGIT.'/FETCH_HEAD');
-
-  my $fetch_head = git ('rev-parse', 'FETCH_HEAD');
-  my $base = git ( 'merge-base', 'HEAD', 'FETCH_HEAD');
+  my $fetch_head = git ('rev-parse', $commit);
+  my $base = git ( 'merge-base', 'HEAD', $commit);
 
   return ($base ne $fetch_head);
 
@@ -473,7 +486,7 @@ sub diff_index {
 sub diff_refs {
   my $filter = shift;
   my $ref1 = shift || 'HEAD';
-  my $ref2 = shift || 'FETCH_HEAD';
+  my $ref2 = shift || '@{upstream}';
 
   my $fh= git_pipe ( 'diff', "--diff-filter=$filter", '--name-only',
 		 $ref1, $ref2);
@@ -561,10 +574,11 @@ git. Any extra arguments are used (one per line) as a commit message.
 
 push local nmbug git state to remote repo
 
-=item  B<pull> [remote]
+=item  B<pull> [remote] [branch]
 
 pull (merge) remote repo changes to notmuch. B<pull> is equivalent to
-B<fetch> followed by B<merge>.
+B<fetch> followed by B<merge>.  The default remote is C<origin>, and
+the default branch is C<master>.
 
 =back
 
@@ -572,6 +586,12 @@ B<fetch> followed by B<merge>.
 
 =over 8
 
+=item B<clone> repository
+
+Create a local nmbug repository from a remote source.  This wraps
+C<git clone>, adding some options to avoid creating a working tree
+while preserving remote-tracking branches and upstreams.
+
 =item B<checkout>
 
 Update the notmuch database from git. This is mainly useful to discard
@@ -589,12 +609,12 @@ print help [for subcommand]
 =item B<log> [parameters]
 
 A simple wrapper for git log. After running C<nmbug fetch>, you can
-inspect the changes with C<nmbug log HEAD..FETCH_HEAD>
+inspect the changes with C<nmbug log HEAD..@{upstream}>
 
-=item B<merge>
+=item B<merge> [commit]
 
-Merge changes from FETCH_HEAD into HEAD, and load the result into
-notmuch.
+Merge changes from C<commit> into HEAD, and load the result into
+notmuch.  The default commit is C<@{upstream}>.
 
 =item  B<status>
 
diff --git a/devel/nmbug/nmbug-status b/devel/nmbug/nmbug-status
index 934c895f..03621bd5 100755
--- a/devel/nmbug/nmbug-status
+++ b/devel/nmbug/nmbug-status
@@ -6,185 +6,330 @@
 #       - python 2.6 for json
 #       - argparse; either python 2.7, or install separately
 
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import codecs
+import collections
 import datetime
-import rfc822
-import urllib
+import email.utils
+try:  # Python 3
+    from urllib.parse import quote
+except ImportError:  # Python 2
+    from urllib import quote
 import json
 import argparse
 import os
+import re
 import sys
 import subprocess
+import xml.sax.saxutils
 
-# parse command line arguments
 
-parser = argparse.ArgumentParser()
-parser.add_argument('--text', help='output plain text format',
-                    action='store_true')
-parser.add_argument('--config', help='load config from given file')
-parser.add_argument('--list-views', help='list views',
-                    action='store_true')
-parser.add_argument('--get-query', help='get query for view')
+_ENCODING = 'UTF-8'
+_PAGES = {}
 
-args = parser.parse_args()
 
-# read config from json file
+if not hasattr(collections, 'OrderedDict'):  # Python 2.6 or earlier
+    class _OrderedDict (dict):
+        "Just enough of a stub to get through Page._get_threads"
+        def __init__(self, *args, **kwargs):
+            super(_OrderedDict, self).__init__(*args, **kwargs)
+            self._keys = []  # record key order
 
-if args.config != None:
-    fp = open(args.config)
-else:
-    nmbhome = os.getenv('NMBGIT', os.path.expanduser('~/.nmbug'))
+        def __setitem__(self, key, value):
+            super(_OrderedDict, self).__setitem__(key, value)
+            self._keys.append(key)
 
-    # read only the first line from the pipe
-    sha1 = subprocess.Popen(['git', '--git-dir', nmbhome,
-                             'show-ref', '-s', 'config'],
-                            stdout=subprocess.PIPE).stdout.readline()
+        def values(self):
+            for key in self._keys:
+                yield self[key]
 
-    sha1 = sha1.rstrip()
 
-    fp = subprocess.Popen(['git', '--git-dir', nmbhome,
-                           'cat-file', 'blob', sha1+':status-config.json'],
-                          stdout=subprocess.PIPE).stdout
+    collections.OrderedDict = _OrderedDict
 
-config = json.load(fp)
 
-if args.list_views:
-    for view in config['views']:
-        print view['title']
-    sys.exit(0)
-elif args.get_query != None:
-    for view in config['views']:
-        if args.get_query == view['title']:
-            print ' and '.join(view['query'])
-    sys.exit(0)
-else:
-    # only import notmuch if needed
-    import notmuch
+def read_config(path=None, encoding=None):
+    "Read config from json file"
+    if not encoding:
+        encoding = _ENCODING
+    if path:
+        fp = open(path)
+    else:
+        nmbhome = os.getenv('NMBGIT', os.path.expanduser('~/.nmbug'))
 
-if args.text:
-    output_format = 'text'
-else:
-    output_format = 'html'
+        # read only the first line from the pipe
+        sha1_bytes = subprocess.Popen(
+            ['git', '--git-dir', nmbhome, 'show-ref', '-s', 'config'],
+            stdout=subprocess.PIPE).stdout.readline()
+        sha1 = sha1_bytes.decode(encoding).rstrip()
 
-class Thread:
-    def __init__(self, last, lines):
-        self.last = last
-        self.lines = lines
+        fp_byte_stream = subprocess.Popen(
+            ['git', '--git-dir', nmbhome, 'cat-file', 'blob',
+             sha1+':status-config.json'],
+            stdout=subprocess.PIPE).stdout
+        fp = codecs.getreader(encoding=encoding)(stream=fp_byte_stream)
 
-    def join_utf8_with_newlines(self):
-        return '\n'.join( (line.encode('utf-8') for line in self.lines) )
+    return json.load(fp)
 
-def output_with_separator(threadlist, sep):
-    outputs = (thread.join_utf8_with_newlines() for thread in threadlist)
-    print sep.join(outputs)
 
-headers = ['date', 'from', 'subject']
+class Thread (list):
+    def __init__(self):
+        self.running_data = {}
 
-def print_view(title, query, comment):
 
-    query_string = ' and '.join(query)
-    q_new = notmuch.Query(db, query_string)
-    q_new.set_sort(notmuch.Query.SORT.OLDEST_FIRST)
+class Page (object):
+    def __init__(self, header=None, footer=None):
+        self.header = header
+        self.footer = footer
 
-    last_thread_id = ''
-    threads = {}
-    threadlist = []
-    out = {}
-    last = None
-    lines = None
+    def write(self, database, views, stream=None):
+        if not stream:
+            try:  # Python 3
+                byte_stream = sys.stdout.buffer
+            except AttributeError:  # Python 2
+                byte_stream = sys.stdout
+            stream = codecs.getwriter(encoding=_ENCODING)(stream=byte_stream)
+        self._write_header(views=views, stream=stream)
+        for view in views:
+            self._write_view(database=database, view=view, stream=stream)
+        self._write_footer(views=views, stream=stream)
 
-    if output_format == 'html':
-        print '<h3><a name="%s" />%s</h3>' % (title, title)
-        print comment
-        print 'The view is generated from the following query:'
-        print '<blockquote>'
-        print query_string
-        print '</blockquote>'
-        print '<table>\n'
+    def _write_header(self, views, stream):
+        if self.header:
+            stream.write(self.header)
 
-    for m in q_new.search_messages():
+    def _write_footer(self, views, stream):
+        if self.footer:
+            stream.write(self.footer)
 
-        thread_id = m.get_thread_id()
+    def _write_view(self, database, view, stream):
+        if 'query-string' not in view:
+            query = view['query']
+            view['query-string'] = ' and '.join(query)
+        q = notmuch.Query(database, view['query-string'])
+        q.set_sort(notmuch.Query.SORT.OLDEST_FIRST)
+        threads = self._get_threads(messages=q.search_messages())
+        self._write_view_header(view=view, stream=stream)
+        self._write_threads(threads=threads, stream=stream)
 
-        if thread_id != last_thread_id:
-            if threads.has_key(thread_id):
-                last = threads[thread_id].last
-                lines = threads[thread_id].lines
+    def _get_threads(self, messages):
+        threads = collections.OrderedDict()
+        for message in messages:
+            thread_id = message.get_thread_id()
+            if thread_id in threads:
+                thread = threads[thread_id]
             else:
-                last = {}
-                lines = []
-                thread = Thread(last, lines)
+                thread = Thread()
                 threads[thread_id] = thread
-                for h in headers:
-                    last[h] = ''
-                threadlist.append(thread)
-            last_thread_id = thread_id
+            thread.running_data, display_data = self._message_display_data(
+                running_data=thread.running_data, message=message)
+            thread.append(display_data)
+        return list(threads.values())
 
-        for header in headers:
-            val = m.get_header(header)
+    def _write_view_header(self, view, stream):
+        pass
 
-            if header == 'date':
-                val = str.join(' ', val.split(None)[1:4])
-                val = str(datetime.datetime.strptime(val, '%d %b %Y').date())
-            elif header == 'from':
-                (val, addr) = rfc822.parseaddr(val)
-                if val == '':
-                    val = addr.split('@')[0]
+    def _write_threads(self, threads, stream):
+        for thread in threads:
+            for message_display_data in thread:
+                stream.write(
+                    ('{date:10.10s} {from:20.20s} {subject:40.40s}\n'
+                     '{message-id-term:>72}\n'
+                     ).format(**message_display_data))
+            if thread != threads[-1]:
+                stream.write('\n')
 
-            if header != 'subject' and last[header] == val:
-                out[header] = ''
+    def _message_display_data(self, running_data, message):
+        headers = ('thread-id', 'message-id', 'date', 'from', 'subject')
+        data = {}
+        for header in headers:
+            if header == 'thread-id':
+                value = message.get_thread_id()
+            elif header == 'message-id':
+                value = message.get_message_id()
+                data['message-id-term'] = 'id:"{0}"'.format(value)
+            elif header == 'date':
+                value = str(datetime.datetime.utcfromtimestamp(
+                    message.get_date()).date())
             else:
-                out[header] = val
-                last[header] = val
+                value = message.get_header(header)
+            if header == 'from':
+                (value, addr) = email.utils.parseaddr(value)
+                if not value:
+                    value = addr.split('@')[0]
+            data[header] = value
+        next_running_data = data.copy()
+        for header, value in data.items():
+            if header in ['message-id', 'subject']:
+                continue
+            if value == running_data.get(header, None):
+                data[header] = ''
+        return (next_running_data, data)
 
-        mid = m.get_message_id()
-        out['id'] = 'id:"%s"' % mid
 
-        if output_format == 'html':
+class HtmlPage (Page):
+    _slug_regexp = re.compile('\W+')
 
-            out['subject'] = '<a href="http://mid.gmane.org/%s">%s</a>' \
-                % (urllib.quote(mid), out['subject'])
+    def _write_header(self, views, stream):
+        super(HtmlPage, self)._write_header(views=views, stream=stream)
+        stream.write('<ul>\n')
+        for view in views:
+            if 'id' not in view:
+                view['id'] = self._slug(view['title'])
+            stream.write(
+                '<li><a href="#{id}">{title}</a></li>\n'.format(**view))
+        stream.write('</ul>\n')
 
-            lines.append(' <tr><td>%s' % out['date'])
-            lines.append('</td><td>%s' % out['id'])
-            lines.append('</td></tr>')
-            lines.append(' <tr><td>%s' % out['from'])
-            lines.append('</td><td>%s' % out['subject'])
-            lines.append('</td></tr>')
-        else:
-            lines.append('%(date)-10.10s %(from)-20.20s %(subject)-40.40s\n%(id)72s' % out)
+    def _write_view_header(self, view, stream):
+        stream.write('<h3 id="{id}">{title}</h3>\n'.format(**view))
+        stream.write('<p>\n')
+        if 'comment' in view:
+            stream.write(view['comment'])
+            stream.write('\n')
+        for line in [
+                'The view is generated from the following query:',
+                '</p>',
+                '<p>',
+                '  <code>',
+                view['query-string'],
+                '  </code>',
+                '</p>',
+                ]:
+            stream.write(line)
+            stream.write('\n')
 
-    if output_format == 'html':
-        output_with_separator(threadlist,
-                              '\n<tr><td colspan="2"><br /></td></tr>\n')
-        print '</table>'
-    else:
-        output_with_separator(threadlist, '\n\n')
+    def _write_threads(self, threads, stream):
+        if not threads:
+            return
+        stream.write('<table>\n')
+        for thread in threads:
+            stream.write('  <tbody>\n')
+            for message_display_data in thread:
+                stream.write((
+                    '    <tr class="message-first">\n'
+                    '      <td>{date}</td>\n'
+                    '      <td><code>{message-id-term}</code></td>\n'
+                    '    </tr>\n'
+                    '    <tr class="message-last">\n'
+                    '      <td>{from}</td>\n'
+                    '      <td>{subject}</td>\n'
+                    '    </tr>\n'
+                    ).format(**message_display_data))
+            stream.write('  </tbody>\n')
+            if thread != threads[-1]:
+                stream.write(
+                    '  <tbody><tr><td colspan="2"><br /></td></tr></tbody>\n')
+        stream.write('</table>\n')
+
+    def _message_display_data(self, *args, **kwargs):
+        running_data, display_data = super(
+            HtmlPage, self)._message_display_data(
+                *args, **kwargs)
+        if 'subject' in display_data and 'message-id' in display_data:
+            d = {
+                'message-id': quote(display_data['message-id']),
+                'subject': xml.sax.saxutils.escape(display_data['subject']),
+                }
+            display_data['subject'] = (
+                '<a href="http://mid.gmane.org/{message-id}">{subject}</a>'
+                ).format(**d)
+        for key in ['message-id', 'from']:
+            if key in display_data:
+                display_data[key] = xml.sax.saxutils.escape(display_data[key])
+        return (running_data, display_data)
 
-# main program
+    def _slug(self, string):
+        return self._slug_regexp.sub('-', string)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--text', help='output plain text format',
+                    action='store_true')
+parser.add_argument('--config', help='load config from given file',
+                    metavar='PATH')
+parser.add_argument('--list-views', help='list views',
+                    action='store_true')
+parser.add_argument('--get-query', help='get query for view',
+                    metavar='VIEW')
+
+args = parser.parse_args()
 
-db = notmuch.Database(mode=notmuch.Database.MODE.READ_WRITE)
+config = read_config(path=args.config)
 
-if output_format == 'html':
-    print '''<?xml version="1.0" encoding="utf-8" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+_PAGES['text'] = Page()
+_PAGES['html'] = HtmlPage(
+    header='''<!DOCTYPE html>
+<html lang="en">
 <head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-<title>Notmuch Patches</title>
+  <meta http-equiv="Content-Type" content="text/html; charset={encoding}" />
+  <title>{title}</title>
+  <style media="screen" type="text/css">
+    table {{
+      border-spacing: 0;
+    }}
+    tr.message-first td {{
+      padding-top: {inter_message_padding};
+    }}
+    tr.message-last td {{
+      padding-bottom: {inter_message_padding};
+    }}
+    td {{
+      padding-left: {border_radius};
+      padding-right: {border_radius};
+    }}
+    tr:first-child td:first-child {{
+      border-top-left-radius: {border_radius};
+    }}
+    tr:first-child td:last-child {{
+      border-top-right-radius: {border_radius};
+    }}
+    tr:last-child td:first-child {{
+      border-bottom-left-radius: {border_radius};
+    }}
+    tr:last-child td:last-child {{
+      border-bottom-right-radius: {border_radius};
+    }}
+    tbody:nth-child(4n+1) tr td {{
+      background-color: #ffd96e;
+    }}
+    tbody:nth-child(4n+3) tr td {{
+      background-color: #bce;
+    }}
+  </style>
 </head>
-<body>'''
-    print '<h2>Notmuch Patches</h2>'
-    print 'Generated: %s<br />' % datetime.datetime.utcnow().date()
-    print 'For more infomation see <a href="http://notmuchmail.org/nmbug">nmbug</a>'
+<body>
+<h2>{title}</h2>
+<p>
+Generated: {date}<br />
+{blurb}
+</p>
+<h3>Views</h3>
+'''.format(date=datetime.datetime.utcnow().date(),
+           title=config['meta']['title'],
+           blurb=config['meta']['blurb'],
+           encoding=_ENCODING,
+           inter_message_padding='0.25em',
+           border_radius='0.5em'),
+    footer='</body>\n</html>\n',
+    )
 
-    print '<h3>Views</h3>'
-    print '<ul>'
+if args.list_views:
+    for view in config['views']:
+        print(view['title'])
+    sys.exit(0)
+elif args.get_query != None:
     for view in config['views']:
-        print '<li><a href="#%(title)s">%(title)s</a></li>' % view
-    print '</ul>'
+        if args.get_query == view['title']:
+            print(' and '.join(view['query']))
+    sys.exit(0)
+else:
+    # only import notmuch if needed
+    import notmuch
 
-for view in config['views']:
-    print_view(**view)
+if args.text:
+    page = _PAGES['text']
+else:
+    page = _PAGES['html']
 
-if output_format == 'html':
-    print '</body>\n</html>'
+db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY)
+page.write(database=db, views=config['views'])
diff --git a/devel/nmbug/status-config.json b/devel/nmbug/status-config.json
index 6b4934fa..b9269462 100644
--- a/devel/nmbug/status-config.json
+++ b/devel/nmbug/status-config.json
@@ -1,4 +1,9 @@
 {
+    "meta": {
+        "title": "Notmuch Patches",
+        "blurb": "For more information see <a href=\"http://notmuchmail.org/nmbug\">nmbug</a>"
+    },
+
     "views": [
 	{
 	    "comment": "Unresolved bugs (or just need tag updating).",
diff --git a/devel/release-checks.sh b/devel/release-checks.sh
index 8938905e..797d62ac 100755
--- a/devel/release-checks.sh
+++ b/devel/release-checks.sh
@@ -68,7 +68,7 @@ verfail ()
 
 echo -n "Checking that '$VERSION' is good with digits and periods... "
 case $VERSION in
-	*[^0-9.]*)
+	*[!0-9.]*)
 		verfail "'$VERSION' contains other characters than digits and periods" ;;
 	.*)	verfail "'$VERSION' begins with a period" ;;
 	*.)	verfail "'$VERSION' ends with a period" ;;
@@ -196,46 +196,6 @@ case $news_date in
 	append_emsg "Date '$news_date' in NEWS file is not in format (yyyy-mm-dd)"
 esac
 
-readonly DATE=${news_date//[()]/} # bash feature
-manthdata ()
-{
-	set x $*
-	if [ $# != 7 ]
-	then
-		append_emsg "'$mp' has too many '.TH' lines"
-		man_mismatch=1
-	fi
-	man_date=${5-} man_version=${7-}
-}
-
-echo -n "Checking that manual page dates and versions are $DATE and $VERSION... "
-manfiles=`find man -type f | sort`
-man_pages_ok=Yes
-for mp in $manfiles
-do
-	case $mp in
-		*.[0-9]) ;; # fall below this 'case ... esac'
-
-		*/Makefile.local | */Makefile ) continue ;;
-		*/.gitignore)	continue ;;
-		*.bak)		continue ;;
-
-		*)	append_emsg "'$mp': extra file"
-			man_pages_ok=No
-			continue
-	esac
-	manthdata `sed -n '/^[.]TH NOTMUCH/ { y/"/ /; p; }' "$mp"`
-	if [ "$man_version" != "$VERSION" ]
-	then	append_emsg "Version '$man_version' is not '$VERSION' in $mp"
-		mman_pages_ok=No
-	fi
-	if [ "$man_date" != "$DATE" ]
-	then	append_emsg "DATE '$man_date' is not '$DATE' in $mp"
-		man_pages_ok=No
-	fi
-done
-echo $man_pages_ok.
-
 if [ -n "$emsgs" ]
 then
 	echo
diff --git a/doc/.gitignore b/doc/.gitignore
new file mode 100644
index 00000000..a60fb31e
--- /dev/null
+++ b/doc/.gitignore
@@ -0,0 +1,2 @@
+docdeps.mk
+_build
diff --git a/doc/INSTALL b/doc/INSTALL
new file mode 100644
index 00000000..e37c2b9b
--- /dev/null
+++ b/doc/INSTALL
@@ -0,0 +1,24 @@
+This file contains some more detailed information about building and
+installing the documentation.
+
+Building with sphinx.
+---------------------
+
+- You need sphinx at least version 1.0.
+
+- You can build build and install man pages with 'make install-man'
+
+- You can build man, info, html, and pdf versions of the docs
+  (currently only the man pages) with
+
+     'make install-{man|info|html|pdf}'
+
+Building the man pages
+----------------------
+
+- You can build the man pages with rst2man (from python-docutils) with
+  'make rst2man'.
+
+- Currently there is no support to automagically install the resulting
+  nroff files, but it should work to modify the target install-man
+  in doc/Makefile.local.
diff --git a/man/Makefile b/doc/Makefile
index fa25832e..fa25832e 100644
--- a/man/Makefile
+++ b/doc/Makefile
diff --git a/doc/Makefile.local b/doc/Makefile.local
new file mode 100644
index 00000000..bbd46100
--- /dev/null
+++ b/doc/Makefile.local
@@ -0,0 +1,81 @@
+# -*- makefile -*-
+
+dir := doc
+
+# You can set these variables from the command line.
+SPHINXOPTS    := -q
+SPHINXBUILD   = sphinx-build
+DOCBUILDDIR      := $(dir)/_build
+
+prerst2man := python $(srcdir)/$(dir)/prerst2man.py
+mkdocdeps := python $(srcdir)/$(dir)/mkdocdeps.py
+
+# Internal variables.
+ALLSPHINXOPTS   := -d $(DOCBUILDDIR)/doctrees $(SPHINXOPTS) $(srcdir)/$(dir)
+
+.PHONY: sphinx-html sphinx-texinfo sphinx-info
+
+.PHONY: install-man build-man
+
+%.gz: %
+	rm -f $@ && gzip --stdout $^ > $@
+
+sphinx-html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(DOCBUILDDIR)/html
+
+sphinx-texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(DOCBUILDDIR)/texinfo
+
+sphinx-info: sphinx-texinfo
+	make -C $(DOCBUILDDIR)/texinfo info
+
+-include $(dir)/docdeps.mk
+
+MAN_GZIP_FILES := $(addsuffix .gz,${MAN_ROFF_FILES})
+
+# Use the man page converter that is available. We should never depend
+# on MAN_ROFF_FILES if a converter is not available.
+${MAN_ROFF_FILES}: $(DOCBUILDDIR)/.roff.stamp
+
+# By using $(DOCBUILDDIR)/.roff.stamp instead of ${MAN_ROFF_FILES}, we
+# convey to make that a single invocation of this recipe builds all
+# of the roff files.  This prevents parallel make from starting an
+# instance of this recipe for each roff file.
+$(DOCBUILDDIR)/.roff.stamp: ${MAN_RST_FILES}
+ifeq ($(HAVE_SPHINX),1)
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(DOCBUILDDIR)/man
+	for section in 1 5 7; do \
+	    mkdir -p $(DOCBUILDDIR)/man/man$${section}; \
+	    mv $(DOCBUILDDIR)/man/*.$${section} $(DOCBUILDDIR)/man/man$${section}; \
+	done
+else ifeq ($(HAVE_RST2MAN),1)
+	$(prerst2man) $(srcdir)/doc $(DOCBUILDDIR)/man
+else
+	@echo "Fatal: build dependency fail."
+	@false
+endif
+	touch ${MAN_ROFF_FILES} $@
+
+# Do not try to build or install man pages if a man page converter is
+# not available.
+ifeq ($(HAVE_SPHINX)$(HAVE_RST2MAN),00)
+build-man:
+install-man:
+	@echo "No sphinx or rst2man, will not install man pages."
+else
+build-man: ${MAN_GZIP_FILES}
+install-man: ${MAN_GZIP_FILES}
+	mkdir -p "$(DESTDIR)$(mandir)/man1"
+	mkdir -p "$(DESTDIR)$(mandir)/man5"
+	mkdir -p "$(DESTDIR)$(mandir)/man7"
+	install -m0644 $(DOCBUILDDIR)/man/man1/*.1.gz $(DESTDIR)/$(mandir)/man1
+	install -m0644 $(DOCBUILDDIR)/man/man5/*.5.gz $(DESTDIR)/$(mandir)/man5
+	install -m0644 $(DOCBUILDDIR)/man/man7/*.7.gz $(DESTDIR)/$(mandir)/man7
+	cd $(DESTDIR)/$(mandir)/man1 && ln -sf notmuch.1.gz notmuch-setup.1.gz
+endif
+
+$(dir)/docdeps.mk: $(dir)/conf.py $(dir)/mkdocdeps.py
+	$(mkdocdeps) $(srcdir)/doc $(DOCBUILDDIR) $@
+
+CLEAN := $(CLEAN) $(DOCBUILDDIR) $(dir)/docdeps.mk $(DOCBUILDDIR)/.roff.stamp
+CLEAN := $(CLEAN) $(MAN_GZIP_FILES) $(MAN_ROFF_FILES) $(dir)/conf.pyc
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 00000000..70ba1b8a
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,172 @@
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'notmuch'
+copyright = u'2014, Carl Worth and many others'
+
+location = os.path.dirname(__file__)
+
+for pathdir in ['.', '..']:
+    version_file = os.path.join(location,pathdir,'version')
+    if os.path.exists(version_file):
+        with open(version_file,'r') as infile:
+            version=infile.read().replace('\n','')
+
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build', 'notmuch-emacs.rst']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'notmuchdoc'
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+
+man_pages = [
+
+('man1/notmuch','notmuch',
+        u'thread-based email index, search, and tagging',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-compact','notmuch-compact',
+        u'compact the notmuch database',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-config','notmuch-config',
+        u'access notmuch configuration file',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-count','notmuch-count',
+        u'count messages matching the given search terms',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-dump','notmuch-dump',
+        u'creates a plain-text dump of the tags of each message',
+        [u'Carl Worth and many others'], 1),
+
+('man5/notmuch-hooks','notmuch-hooks',
+        u'hooks for notmuch',
+        [u'Carl Worth and many others'], 5),
+
+('man1/notmuch-insert','notmuch-insert',
+        u'add a message to the maildir and notmuch database',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-new','notmuch-new',
+        u'incorporate new mail into the notmuch database',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-reply','notmuch-reply',
+        u'constructs a reply template for a set of messages',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-restore','notmuch-restore',
+        u'restores the tags from the given file (see notmuch dump)',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-search','notmuch-search',
+        u'search for messages matching the given search terms',
+        [u'Carl Worth and many others'], 1),
+
+('man7/notmuch-search-terms','notmuch-search-terms',
+        u'syntax for notmuch queries',
+        [u'Carl Worth and many others'], 7),
+
+('man1/notmuch-show','notmuch-show',
+        u'show messages matching the given search terms',
+        [u'Carl Worth and many others'], 1),
+
+('man1/notmuch-tag','notmuch-tag',
+        u'add/remove tags for all messages matching the search terms',
+        [u'Carl Worth and many others'], 1),
+
+
+]
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+texinfo_no_detailmenu = True
+
+texinfo_documents = [
+ ('notmuch-emacs', 'notmuch-emacs', u'notmuch Documentation',
+   u'Carl Worth and many others', 'notmuch-emacs',
+   'emacs based front-end for notmuch', 'Miscellaneous'),
+('man1/notmuch','notmuch',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch',
+      'thread-based email index, search, and tagging','Miscellaneous'),
+('man1/notmuch-compact','notmuch-compact',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-compact',
+      'compact the notmuch database','Miscellaneous'),
+('man1/notmuch-config','notmuch-config',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-config',
+      'access notmuch configuration file','Miscellaneous'),
+('man1/notmuch-count','notmuch-count',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-count',
+      'count messages matching the given search terms','Miscellaneous'),
+('man1/notmuch-dump','notmuch-dump',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-dump',
+      'creates a plain-text dump of the tags of each message','Miscellaneous'),
+('man5/notmuch-hooks','notmuch-hooks',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-hooks',
+      'hooks for notmuch','Miscellaneous'),
+('man1/notmuch-insert','notmuch-insert',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-insert',
+      'add a message to the maildir and notmuch database','Miscellaneous'),
+('man1/notmuch-new','notmuch-new',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-new',
+      'incorporate new mail into the notmuch database','Miscellaneous'),
+('man1/notmuch-reply','notmuch-reply',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-reply',
+      'constructs a reply template for a set of messages','Miscellaneous'),
+('man1/notmuch-restore','notmuch-restore',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-restore',
+      'restores the tags from the given file (see notmuch dump)','Miscellaneous'),
+('man1/notmuch-search','notmuch-search',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-search',
+      'search for messages matching the given search terms','Miscellaneous'),
+('man7/notmuch-search-terms','notmuch-search-terms',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-search-terms',
+      'syntax for notmuch queries','Miscellaneous'),
+('man1/notmuch-show','notmuch-show',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-show',
+      'show messages matching the given search terms','Miscellaneous'),
+('man1/notmuch-tag','notmuch-tag',u'notmuch Documentation',
+      u'Carl Worth and many others', 'notmuch-tag',
+      'add/remove tags for all messages matching the search terms','Miscellaneous'),
+]
diff --git a/doc/doxygen.cfg b/doc/doxygen.cfg
new file mode 100644
index 00000000..bfbfcab3
--- /dev/null
+++ b/doc/doxygen.cfg
@@ -0,0 +1,304 @@
+# Doxyfile 1.8.4
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+DOXYFILE_ENCODING      = UTF-8
+PROJECT_NAME           = "Notmuch 0.18"
+PROJECT_NUMBER         =
+PROJECT_BRIEF          =
+PROJECT_LOGO           =
+OUTPUT_DIRECTORY       =
+CREATE_SUBDIRS         = NO
+OUTPUT_LANGUAGE        = English
+BRIEF_MEMBER_DESC      = YES
+REPEAT_BRIEF           = YES
+ABBREVIATE_BRIEF       =
+ALWAYS_DETAILED_SEC    = NO
+INLINE_INHERITED_MEMB  = NO
+FULL_PATH_NAMES        = NO
+STRIP_FROM_PATH        =
+STRIP_FROM_INC_PATH    =
+SHORT_NAMES            = NO
+JAVADOC_AUTOBRIEF      = YES
+QT_AUTOBRIEF           = NO
+MULTILINE_CPP_IS_BRIEF = NO
+INHERIT_DOCS           = YES
+SEPARATE_MEMBER_PAGES  = NO
+TAB_SIZE               = 8
+ALIASES                =
+TCL_SUBST              =
+OPTIMIZE_OUTPUT_FOR_C  = YES
+OPTIMIZE_OUTPUT_JAVA   = NO
+OPTIMIZE_FOR_FORTRAN   = NO
+OPTIMIZE_OUTPUT_VHDL   = NO
+EXTENSION_MAPPING      =
+MARKDOWN_SUPPORT       = YES
+AUTOLINK_SUPPORT       = YES
+BUILTIN_STL_SUPPORT    = NO
+CPP_CLI_SUPPORT        = NO
+SIP_SUPPORT            = NO
+IDL_PROPERTY_SUPPORT   = YES
+DISTRIBUTE_GROUP_DOC   = NO
+SUBGROUPING            = YES
+INLINE_GROUPED_CLASSES = NO
+INLINE_SIMPLE_STRUCTS  = NO
+TYPEDEF_HIDES_STRUCT   = YES
+LOOKUP_CACHE_SIZE      = 0
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+EXTRACT_ALL            = NO
+EXTRACT_PRIVATE        = NO
+EXTRACT_PACKAGE        = NO
+EXTRACT_STATIC         = NO
+EXTRACT_LOCAL_CLASSES  = YES
+EXTRACT_LOCAL_METHODS  = NO
+EXTRACT_ANON_NSPACES   = NO
+HIDE_UNDOC_MEMBERS     = NO
+HIDE_UNDOC_CLASSES     = NO
+HIDE_FRIEND_COMPOUNDS  = NO
+HIDE_IN_BODY_DOCS      = NO
+INTERNAL_DOCS          = NO
+CASE_SENSE_NAMES       = YES
+HIDE_SCOPE_NAMES       = NO
+SHOW_INCLUDE_FILES     = NO
+FORCE_LOCAL_INCLUDES   = NO
+INLINE_INFO            = YES
+SORT_MEMBER_DOCS       = NO
+SORT_BRIEF_DOCS        = NO
+SORT_MEMBERS_CTORS_1ST = NO
+SORT_GROUP_NAMES       = NO
+SORT_BY_SCOPE_NAME     = NO
+STRICT_PROTO_MATCHING  = NO
+GENERATE_TODOLIST      = NO
+GENERATE_TESTLIST      = NO
+GENERATE_BUGLIST       = NO
+GENERATE_DEPRECATEDLIST= NO
+ENABLED_SECTIONS       =
+MAX_INITIALIZER_LINES  = 30
+SHOW_USED_FILES        = NO
+SHOW_FILES             = NO
+SHOW_NAMESPACES        = NO
+FILE_VERSION_FILTER    =
+LAYOUT_FILE            =
+CITE_BIB_FILES         =
+#---------------------------------------------------------------------------
+# configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+QUIET                  = YES
+WARNINGS               = YES
+WARN_IF_UNDOCUMENTED   = YES
+WARN_IF_DOC_ERROR      = YES
+WARN_NO_PARAMDOC       = NO
+WARN_FORMAT            = "$file:$line: $text"
+WARN_LOGFILE           =
+#---------------------------------------------------------------------------
+# configuration options related to the input files
+#---------------------------------------------------------------------------
+INPUT                  = lib/notmuch.h
+INPUT_ENCODING         = UTF-8
+FILE_PATTERNS          =
+RECURSIVE              = NO
+EXCLUDE                =
+EXCLUDE_SYMLINKS       = NO
+EXCLUDE_PATTERNS       =
+EXCLUDE_SYMBOLS        =
+EXAMPLE_PATH           =
+EXAMPLE_PATTERNS       =
+EXAMPLE_RECURSIVE      = NO
+IMAGE_PATH             =
+INPUT_FILTER           =
+FILTER_PATTERNS        =
+FILTER_SOURCE_FILES    = NO
+FILTER_SOURCE_PATTERNS =
+USE_MDFILE_AS_MAINPAGE =
+#---------------------------------------------------------------------------
+# configuration options related to source browsing
+#---------------------------------------------------------------------------
+SOURCE_BROWSER         = NO
+INLINE_SOURCES         = NO
+STRIP_CODE_COMMENTS    = YES
+REFERENCED_BY_RELATION = NO
+REFERENCES_RELATION    = NO
+REFERENCES_LINK_SOURCE = YES
+USE_HTAGS              = NO
+VERBATIM_HEADERS       = NO
+#---------------------------------------------------------------------------
+# configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+ALPHABETICAL_INDEX     = NO
+COLS_IN_ALPHA_INDEX    = 5
+IGNORE_PREFIX          =
+#---------------------------------------------------------------------------
+# configuration options related to the HTML output
+#---------------------------------------------------------------------------
+GENERATE_HTML          = NO
+HTML_OUTPUT            = html
+HTML_FILE_EXTENSION    = .html
+HTML_HEADER            =
+HTML_FOOTER            =
+HTML_STYLESHEET        =
+HTML_EXTRA_STYLESHEET  =
+HTML_EXTRA_FILES       =
+HTML_COLORSTYLE_HUE    = 220
+HTML_COLORSTYLE_SAT    = 100
+HTML_COLORSTYLE_GAMMA  = 80
+HTML_TIMESTAMP         = YES
+HTML_DYNAMIC_SECTIONS  = NO
+HTML_INDEX_NUM_ENTRIES = 100
+GENERATE_DOCSET        = NO
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+DOCSET_PUBLISHER_NAME  = Publisher
+GENERATE_HTMLHELP      = NO
+CHM_FILE               =
+HHC_LOCATION           =
+GENERATE_CHI           = NO
+CHM_INDEX_ENCODING     =
+BINARY_TOC             = NO
+TOC_EXPAND             = NO
+GENERATE_QHP           = NO
+QCH_FILE               =
+QHP_NAMESPACE          = org.doxygen.Project
+QHP_VIRTUAL_FOLDER     = doc
+QHP_CUST_FILTER_NAME   =
+QHP_CUST_FILTER_ATTRS  =
+QHP_SECT_FILTER_ATTRS  =
+QHG_LOCATION           =
+GENERATE_ECLIPSEHELP   = NO
+ECLIPSE_DOC_ID         = org.doxygen.Project
+DISABLE_INDEX          = NO
+GENERATE_TREEVIEW      = NO
+ENUM_VALUES_PER_LINE   = 4
+TREEVIEW_WIDTH         = 250
+EXT_LINKS_IN_WINDOW    = NO
+FORMULA_FONTSIZE       = 10
+FORMULA_TRANSPARENT    = YES
+USE_MATHJAX            = NO
+MATHJAX_FORMAT         = HTML-CSS
+MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_EXTENSIONS     =
+MATHJAX_CODEFILE       =
+SEARCHENGINE           = YES
+SERVER_BASED_SEARCH    = NO
+EXTERNAL_SEARCH        = NO
+SEARCHENGINE_URL       =
+SEARCHDATA_FILE        = searchdata.xml
+EXTERNAL_SEARCH_ID     =
+EXTRA_SEARCH_MAPPINGS  =
+#---------------------------------------------------------------------------
+# configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+GENERATE_LATEX         = NO
+LATEX_OUTPUT           = latex
+LATEX_CMD_NAME         = latex
+MAKEINDEX_CMD_NAME     = makeindex
+COMPACT_LATEX          = NO
+PAPER_TYPE             = a4
+EXTRA_PACKAGES         =
+LATEX_HEADER           =
+LATEX_FOOTER           =
+LATEX_EXTRA_FILES      =
+PDF_HYPERLINKS         = YES
+USE_PDFLATEX           = YES
+LATEX_BATCHMODE        = NO
+LATEX_HIDE_INDICES     = NO
+LATEX_SOURCE_CODE      = NO
+LATEX_BIB_STYLE        = plain
+#---------------------------------------------------------------------------
+# configuration options related to the RTF output
+#---------------------------------------------------------------------------
+GENERATE_RTF           = NO
+RTF_OUTPUT             = rtf
+COMPACT_RTF            = NO
+RTF_HYPERLINKS         = NO
+RTF_STYLESHEET_FILE    =
+RTF_EXTENSIONS_FILE    =
+#---------------------------------------------------------------------------
+# configuration options related to the man page output
+#---------------------------------------------------------------------------
+GENERATE_MAN           = YES
+MAN_OUTPUT             = man
+MAN_EXTENSION          = .3
+MAN_LINKS              = NO
+#---------------------------------------------------------------------------
+# configuration options related to the XML output
+#---------------------------------------------------------------------------
+GENERATE_XML           = NO
+XML_OUTPUT             = xml
+XML_SCHEMA             =
+XML_DTD                =
+XML_PROGRAMLISTING     = YES
+#---------------------------------------------------------------------------
+# configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+GENERATE_DOCBOOK       = NO
+DOCBOOK_OUTPUT         = docbook
+#---------------------------------------------------------------------------
+# configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+GENERATE_AUTOGEN_DEF   = NO
+#---------------------------------------------------------------------------
+# configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+GENERATE_PERLMOD       = NO
+PERLMOD_LATEX          = NO
+PERLMOD_PRETTY         = YES
+PERLMOD_MAKEVAR_PREFIX =
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = NO
+EXPAND_ONLY_PREDEF     = NO
+SEARCH_INCLUDES        = NO
+INCLUDE_PATH           =
+INCLUDE_FILE_PATTERNS  =
+PREDEFINED             = __DOXYGEN__
+EXPAND_AS_DEFINED      =
+SKIP_FUNCTION_MACROS   = YES
+#---------------------------------------------------------------------------
+# Configuration::additions related to external references
+#---------------------------------------------------------------------------
+TAGFILES               =
+GENERATE_TAGFILE       =
+ALLEXTERNALS           = NO
+EXTERNAL_GROUPS        = NO
+EXTERNAL_PAGES         = NO
+PERL_PATH              = /usr/bin/perl
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+CLASS_DIAGRAMS         = NO
+MSCGEN_PATH            =
+HIDE_UNDOC_RELATIONS   = YES
+HAVE_DOT               = NO
+DOT_NUM_THREADS        = 0
+DOT_FONTNAME           = Helvetica
+DOT_FONTSIZE           = 10
+DOT_FONTPATH           =
+CLASS_GRAPH            = YES
+COLLABORATION_GRAPH    = YES
+GROUP_GRAPHS           = YES
+UML_LOOK               = NO
+UML_LIMIT_NUM_FIELDS   = 10
+TEMPLATE_RELATIONS     = NO
+INCLUDE_GRAPH          = NO
+INCLUDED_BY_GRAPH      = NO
+CALL_GRAPH             = NO
+CALLER_GRAPH           = NO
+GRAPHICAL_HIERARCHY    = NO
+DIRECTORY_GRAPH        = NO
+DOT_IMAGE_FORMAT       = png
+INTERACTIVE_SVG        = NO
+DOT_PATH               =
+DOTFILE_DIRS           =
+MSCFILE_DIRS           =
+DOT_GRAPH_MAX_NODES    = 50
+MAX_DOT_GRAPH_DEPTH    = 0
+DOT_TRANSPARENT        = NO
+DOT_MULTI_TARGETS      = YES
+GENERATE_LEGEND        = NO
+DOT_CLEANUP            = YES
diff --git a/doc/index.rst b/doc/index.rst
new file mode 100644
index 00000000..b33aa9fb
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1,30 @@
+
+Welcome to notmuch's documentation!
+===================================
+
+Contents:
+
+.. toctree::
+   :titlesonly:
+
+   man1/notmuch
+   man1/notmuch-compact
+   man1/notmuch-config
+   man1/notmuch-count
+   man1/notmuch-dump
+   man5/notmuch-hooks
+   man1/notmuch-insert
+   man1/notmuch-new
+   man1/notmuch-reply
+   man1/notmuch-restore
+   man1/notmuch-search
+   man7/notmuch-search-terms
+   man1/notmuch-show
+   man1/notmuch-tag
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/doc/man1/notmuch-compact.rst b/doc/man1/notmuch-compact.rst
new file mode 100644
index 00000000..e0109dce
--- /dev/null
+++ b/doc/man1/notmuch-compact.rst
@@ -0,0 +1,52 @@
+===============
+notmuch-compact
+===============
+
+SYNOPSIS
+========
+
+**notmuch** **compact** [--quiet] [--backup=<*directory*>]
+
+DESCRIPTION
+===========
+
+The **compact** command can be used to compact the notmuch database.
+This can both reduce the space required by the database and improve
+lookup performance.
+
+The compacted database is built in a temporary directory and is later
+moved into the place of the origin database. The original uncompacted
+database is discarded, unless the ``--backup=``\ <directory> option is
+used.
+
+Note that the database write lock will be held during the compaction
+process (which may be quite long) to protect data integrity.
+
+Supported options for **compact** include
+
+    ``--backup=``\ <directory>
+        Save the current database to the given directory before
+        replacing it with the compacted database. The backup directory
+        must not exist and it must reside on the same mounted filesystem
+        as the current database.
+
+    ``--quiet``
+        Do not report database compaction progress to stdout.
+
+ENVIRONMENT
+===========
+
+The following environment variables can be used to control the behavior
+of notmuch.
+
+**NOTMUCH\_CONFIG**
+    Specifies the location of the notmuch configuration file. Notmuch
+    will use ${HOME}/.notmuch-config if this variable is not set.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-count(1)**, **notmuch-dump(1)**,
+**notmuch-hooks(5)**, **notmuch-insert(1)**, **notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-config.rst b/doc/man1/notmuch-config.rst
new file mode 100644
index 00000000..3c9a5684
--- /dev/null
+++ b/doc/man1/notmuch-config.rst
@@ -0,0 +1,123 @@
+==============
+notmuch-config
+==============
+
+SYNOPSIS
+========
+
+**notmuch** **config** **get** <*section*>.<*item*>
+
+**notmuch** **config** **set** <*section*>.<*item*> [*value* ...]
+
+**notmuch** **config** **list**
+
+DESCRIPTION
+===========
+
+The **config** command can be used to get or set settings in the notmuch
+configuration file.
+
+    **get**
+        The value of the specified configuration item is printed to
+        stdout. If the item has multiple values (it is a list), each
+        value is separated by a newline character.
+
+    **set**
+        The specified configuration item is set to the given value. To
+        specify a multiple-value item (a list), provide each value as a
+        separate command-line argument.
+
+        If no values are provided, the specified configuration item will
+        be removed from the configuration file.
+
+    **list**
+        Every configuration item is printed to stdout, each on a
+        separate line of the form:
+
+        *section*.\ *item*\ =\ *value*
+
+        No additional whitespace surrounds the dot or equals sign
+        characters. In a multiple-value item (a list), the values are
+        separated by semicolon characters.
+
+The available configuration items are described below.
+
+    **database.path**
+        The top-level directory where your mail currently exists and to
+        where mail will be delivered in the future. Files should be
+        individual email messages. Notmuch will store its database
+        within a sub-directory of the path configured here named
+        ``.notmuch``.
+
+    **user.name**
+        Your full name.
+
+    **user.primary\_email**
+        Your primary email address.
+
+    **user.other\_email**
+        A list of other email addresses at which you receive email.
+
+    **new.tags**
+        A list of tags that will be added to all messages incorporated
+        by **notmuch new**.
+
+    **new.ignore**
+        A list of file and directory names, without path, that will not
+        be searched for messages by **notmuch new**. All the files and
+        directories matching any of the names specified here will be
+        ignored, regardless of the location in the mail store directory
+        hierarchy.
+
+    **search.exclude\_tags**
+        A list of tags that will be excluded from search results by
+        default. Using an excluded tag in a query will override that
+        exclusion.
+
+    **maildir.synchronize\_flags**
+        If true, then the following maildir flags (in message filenames)
+        will be synchronized with the corresponding notmuch tags:
+
+        +--------+-----------------------------------------------+
+        | Flag   | Tag                                           |
+        +========+===============================================+
+        | D      | draft                                         |
+        +--------+-----------------------------------------------+
+        | F      | flagged                                       |
+        +--------+-----------------------------------------------+
+        | P      | passed                                        |
+        +--------+-----------------------------------------------+
+        | R      | replied                                       |
+        +--------+-----------------------------------------------+
+        | S      | unread (added when 'S' flag is not present)   |
+        +--------+-----------------------------------------------+
+
+        The **notmuch new** command will notice flag changes in
+        filenames and update tags, while the **notmuch tag** and
+        **notmuch restore** commands will notice tag changes and update
+        flags in filenames.
+
+        If there have been any changes in the maildir (new messages
+        added, old ones removed or renamed, maildir flags changed,
+        etc.), it is advisable to run **notmuch new** before **notmuch
+        tag** or **notmuch restore** commands to ensure the tag changes
+        are properly synchronized to the maildir flags, as the commands
+        expect the database and maildir to be in sync.
+
+ENVIRONMENT
+===========
+
+The following environment variables can be used to control the behavior
+of notmuch.
+
+**NOTMUCH\_CONFIG**
+    Specifies the location of the notmuch configuration file. Notmuch
+    will use ${HOME}/.notmuch-config if this variable is not set.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-count(1)**, **notmuch-dump(1)**,
+**notmuch-hooks(5)**, **notmuch-insert(1)**, **notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-count.rst b/doc/man1/notmuch-count.rst
new file mode 100644
index 00000000..ca78c18b
--- /dev/null
+++ b/doc/man1/notmuch-count.rst
@@ -0,0 +1,60 @@
+=============
+notmuch-count
+=============
+
+SYNOPSIS
+========
+
+**notmuch** **count** [*option* ...] <*search-term*> ...
+
+DESCRIPTION
+===========
+
+Count messages matching the search terms.
+
+The number of matching messages (or threads) is output to stdout.
+
+With no search terms, a count of all messages (or threads) in the
+database will be displayed.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<search-terms>.
+
+Supported options for **count** include
+
+    ``--output=(messages|threads|files)``
+
+        **messages**
+            Output the number of matching messages. This is the default.
+
+        **threads**
+            Output the number of matching threads.
+
+        **files**
+            Output the number of files associated with matching
+            messages. This may be bigger than the number of matching
+            messages due to duplicates (i.e. multiple files having the
+            same message-id).
+
+    ``--exclude=(true|false)``
+        Specify whether to omit messages matching search.tag\_exclude
+        from the count (the default) or not.
+
+    ``--batch``
+        Read queries from a file (stdin by default), one per line, and
+        output the number of matching messages (or threads) to stdout,
+        one per line. On an empty input line the count of all messages
+        (or threads) in the database will be output. This option is not
+        compatible with specifying search terms on the command line.
+
+    ``--input=``\ <filename>
+        Read input from given file, instead of from stdin. Implies
+        ``--batch``.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-dump(1)**,
+**notmuch-hooks(5)**, **notmuch-insert(1)**, **notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
new file mode 100644
index 00000000..d94cb4f8
--- /dev/null
+++ b/doc/man1/notmuch-dump.rst
@@ -0,0 +1,75 @@
+============
+notmuch-dump
+============
+
+SYNOPSIS
+========
+
+**notmuch** **dump** [--format=(batch-tag|sup)] [--] [--output=<*file*>] [--] [<*search-term*> ...]
+
+DESCRIPTION
+===========
+
+Dump tags for messages matching the given search terms.
+
+Output is to the given filename, if any, or to stdout.
+
+These tags are the only data in the notmuch database that can't be
+recreated from the messages themselves. The output of notmuch dump is
+therefore the only critical thing to backup (and much more friendly to
+incremental backup than the native database files.)
+
+``--gzip``
+    Compress the output in a format compatible with **gzip(1)**.
+
+``--format=(sup|batch-tag)``
+    Notmuch restore supports two plain text dump formats, both with one
+    message-id per line, followed by a list of tags.
+
+    **batch-tag**
+        The default **batch-tag** dump format is intended to more robust
+        against malformed message-ids and tags containing whitespace or
+        non-\ **ascii(7)** characters. Each line has the form
+
+            +<*encoded-tag*\ > +<*encoded-tag*\ > ... --
+            id:<*quoted-message-id*\ >
+
+        Tags are hex-encoded by replacing every byte not matching the
+        regex **[A-Za-z0-9@=.,\_+-]** with **%nn** where nn is the two
+        digit hex encoding. The message ID is a valid Xapian query,
+        quoted using Xapian boolean term quoting rules: if the ID
+        contains whitespace or a close paren or starts with a double
+        quote, it must be enclosed in double quotes and double quotes
+        inside the ID must be doubled. The astute reader will notice
+        this is a special case of the batch input format for
+        **notmuch-tag(1)**; note that the single message-id query is
+        mandatory for **notmuch-restore(1)**.
+
+    **sup**
+        The **sup** dump file format is specifically chosen to be
+        compatible with the format of files produced by sup-dump. So if
+        you've previously been using sup for mail, then the **notmuch
+        restore** command provides you a way to import all of your tags
+        (or labels as sup calls them). Each line has the following form
+
+            <*message-id*\ > **(** <*tag*\ > ... **)**
+
+        with zero or more tags are separated by spaces. Note that
+        (malformed) message-ids may contain arbitrary non-null
+        characters. Note also that tags with spaces will not be
+        correctly restored with this format.
+
+    With no search terms, a dump of all messages in the database will be
+    generated. A "--" argument instructs notmuch that the remaining
+    arguments are search terms.
+
+    See **notmuch-search-terms(7)** for details of the supported syntax
+    for <search-terms>.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-hooks(5)**, **notmuch-insert(1)**, **notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-insert.rst b/doc/man1/notmuch-insert.rst
new file mode 100644
index 00000000..2be1a7b8
--- /dev/null
+++ b/doc/man1/notmuch-insert.rst
@@ -0,0 +1,58 @@
+==============
+notmuch-insert
+==============
+
+SYNOPSIS
+========
+
+**notmuch** **insert** [option ...] [+<*tag*>|-<*tag*> ...]
+
+DESCRIPTION
+===========
+
+**notmuch insert** reads a message from standard input and delivers it
+into the maildir directory given by configuration option
+**database.path**, then incorporates the message into the notmuch
+database. It is an alternative to using a separate tool to deliver the
+message then running **notmuch new** afterwards.
+
+The new message will be tagged with the tags specified by the
+**new.tags** configuration option, then by operations specified on the
+command-line: tags prefixed by '+' are added while those prefixed by '-'
+are removed.
+
+If the new message is a duplicate of an existing message in the database
+(it has same Message-ID), it will be added to the maildir folder and
+notmuch database, but the tags will not be changed.
+
+Option arguments must appear before any tag operation arguments.
+Supported options for **insert** include
+
+    ``--folder=<``\ folder\ **>**
+        Deliver the message to the specified folder, relative to the
+        top-level directory given by the value of **database.path**. The
+        default is to deliver to the top-level directory.
+
+    ``--create-folder``
+        Try to create the folder named by the ``--folder`` option, if it
+        does not exist. Otherwise the folder must already exist for mail
+        delivery to succeed.
+
+EXIT STATUS
+===========
+
+This command returns exit status 0 if the message was successfully added
+to the mail directory, even if the message could not be indexed and
+added to the notmuch database. In the latter case, a warning will be
+printed to standard error but the message file will be left on disk.
+
+If the message could not be written to disk then a non-zero exit status
+is returned.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-reply(1)**,
+**notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-new.rst b/doc/man1/notmuch-new.rst
new file mode 100644
index 00000000..787ed78f
--- /dev/null
+++ b/doc/man1/notmuch-new.rst
@@ -0,0 +1,52 @@
+===========
+notmuch-new
+===========
+
+SYNOPSIS
+========
+
+**notmuch** **new** [options]
+
+DESCRIPTION
+===========
+
+Find and import any new messages to the database.
+
+The **new** command scans all sub-directories of the database,
+performing full-text indexing on new messages that are found. Each new
+message will automatically be tagged with both the **inbox** and
+**unread** tags.
+
+You should run **notmuch new** once after first running **notmuch
+setup** to create the initial database. The first run may take a long
+time if you have a significant amount of mail (several hundred thousand
+messages or more). Subsequently, you should run **notmuch new** whenever
+new mail is delivered and you wish to incorporate it into the database.
+These subsequent runs will be much quicker than the initial run.
+
+Invoking ``notmuch`` with no command argument will run **new** if
+**notmuch setup** has previously been completed, but **notmuch new** has
+not previously been run.
+
+**notmuch new** updates tags according to maildir flag changes if the
+**maildir.synchronize\_flags** configuration option is enabled. See
+**notmuch-config(1)** for details.
+
+The **new** command supports hooks. See **notmuch-hooks(5)** for more
+details on hooks.
+
+Supported options for **new** include
+
+    ``--no-hooks``
+        Prevents hooks from being run.
+
+    ``--quiet``
+        Do not print progress or results.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-reply.rst b/doc/man1/notmuch-reply.rst
new file mode 100644
index 00000000..cfbd4ea8
--- /dev/null
+++ b/doc/man1/notmuch-reply.rst
@@ -0,0 +1,112 @@
+=============
+notmuch-reply
+=============
+
+SYNOPSIS
+========
+
+**notmuch** **reply** [option ...] <*search-term*> ...
+
+DESCRIPTION
+===========
+
+Constructs a reply template for a set of messages.
+
+To make replying to email easier, **notmuch reply** takes an existing
+set of messages and constructs a suitable mail template. The Reply-to:
+header (if any, otherwise From:) is used for the To: address. Unless
+``--reply-to=sender`` is specified, values from the To: and Cc: headers
+are copied, but not including any of the current user's email addresses
+(as configured in primary\_mail or other\_email in the .notmuch-config
+file) in the recipient list.
+
+It also builds a suitable new subject, including Re: at the front (if
+not already present), and adding the message IDs of the messages being
+replied to to the References list and setting the In-Reply-To: field
+correctly.
+
+Finally, the original contents of the emails are quoted by prefixing
+each line with '> ' and included in the body.
+
+The resulting message template is output to stdout.
+
+Supported options for **reply** include
+
+    ``--format=``\ (**default**\ \|\ **json**\ \|\ **sexp**\ \|\ **headers-only**)
+
+        **default**
+            Includes subject and quoted message body as an RFC 2822
+            message.
+
+        **json**
+            Produces JSON output containing headers for a reply message
+            and the contents of the original message. This output can be
+            used by a client to create a reply message intelligently.
+
+        **sexp**
+            Produces S-Expression output containing headers for a reply
+            message and the contents of the original message. This
+            output can be used by a client to create a reply message
+            intelligently.
+
+        **headers-only**
+            Only produces In-Reply-To, References, To, Cc, and Bcc
+            headers.
+
+    ``--format-version=N``
+        Use the specified structured output format version. This is
+        intended for programs that invoke **notmuch(1)** internally. If
+        omitted, the latest supported version will be used.
+
+    ``--reply-to=``\ (**all**\ \|\ **sender**)
+
+        **all** (default)
+            Replies to all addresses.
+
+        **sender**
+            Replies only to the sender. If replying to user's own
+            message (Reply-to: or From: header is one of the user's
+            configured email addresses), try To:, Cc:, and Bcc: headers
+            in this order, and copy values from the first that contains
+            something other than only the user's addresses.
+
+    ``--decrypt``
+        Decrypt any MIME encrypted parts found in the selected content
+        (ie. "multipart/encrypted" parts). Status of the decryption will
+        be reported (currently only supported with --format=json and
+        --format=sexp) and on successful decryption the
+        multipart/encrypted part will be replaced by the decrypted
+        content.
+
+        Decryption expects a functioning **gpg-agent(1)** to provide any
+        needed credentials. Without one, the decryption will fail.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<search-terms>.
+
+Note: It is most common to use **notmuch reply** with a search string
+matching a single message, (such as id:<message-id>), but it can be
+useful to reply to several messages at once. For example, when a series
+of patches are sent in a single thread, replying to the entire thread
+allows for the reply to comment on issues found in multiple patches. The
+default format supports replying to multiple messages at once, but the
+JSON and S-Expression formats do not.
+
+EXIT STATUS
+===========
+
+This command supports the following special exit status codes
+
+``20``
+    The requested format version is too old.
+
+``21``
+    The requested format version is too new.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-restore.rst b/doc/man1/notmuch-restore.rst
new file mode 100644
index 00000000..936b1383
--- /dev/null
+++ b/doc/man1/notmuch-restore.rst
@@ -0,0 +1,67 @@
+===============
+notmuch-restore
+===============
+
+SYNOPSIS
+========
+
+**notmuch** **restore** [--accumulate] [--format=(auto|batch-tag|sup)] [--input=<*filename*>]
+
+DESCRIPTION
+===========
+
+Restores the tags from the given file (see **notmuch dump**).
+
+The input is read from the given filename, if any, or from stdin.
+
+Supported options for **restore** include
+
+    ``--accumulate``
+        The union of the existing and new tags is applied, instead of
+        replacing each message's tags as they are read in from the dump
+        file.
+
+    ``--format=(sup|batch-tag|auto)``
+        Notmuch restore supports two plain text dump formats, with each
+        line specifying a message-id and a set of tags. For details of
+        the actual formats, see **notmuch-dump(1)**.
+
+        **sup**
+            The **sup** dump file format is specifically chosen to be
+            compatible with the format of files produced by sup-dump. So
+            if you've previously been using sup for mail, then the
+            **notmuch restore** command provides you a way to import all
+            of your tags (or labels as sup calls them).
+
+        **batch-tag**
+            The **batch-tag** dump format is intended to more robust
+            against malformed message-ids and tags containing whitespace
+            or non-\ **ascii(7)** characters. See **notmuch-dump(1)**
+            for details on this format.
+
+            **notmuch restore** updates the maildir flags according to
+            tag changes if the **maildir.synchronize\_flags**
+            configuration option is enabled. See **notmuch-config(1)**
+            for details.
+
+        **auto**
+            This option (the default) tries to guess the format from the
+            input. For correctly formed input in either supported
+            format, this heuristic, based the fact that batch-tag format
+            contains no parentheses, should be accurate.
+
+GZIPPED INPUT
+=============
+
+\ **notmuch restore** will detect if the input is compressed in
+**gzip(1)** format and automatically decompress it while reading. This
+detection does not depend on file naming and in particular works for
+standard input.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-search.rst b/doc/man1/notmuch-search.rst
new file mode 100644
index 00000000..90160f21
--- /dev/null
+++ b/doc/man1/notmuch-search.rst
@@ -0,0 +1,151 @@
+==============
+notmuch-search
+==============
+
+SYNOPSIS
+========
+
+**notmuch** **search** [*option* ...] <*search-term*> ...
+
+DESCRIPTION
+===========
+
+Search for messages matching the given search terms, and display as
+results the threads containing the matched messages.
+
+The output consists of one line per thread, giving a thread ID, the date
+of the newest (or oldest, depending on the sort option) matched message
+in the thread, the number of matched messages and total messages in the
+thread, the names of all participants in the thread, and the subject of
+the newest (or oldest) message.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<search-terms>.
+
+Supported options for **search** include
+
+    ``--format=``\ (**json**\ \|\ **sexp**\ \|\ **text**\ \|\ **text0**)
+        Presents the results in either JSON, S-Expressions, newline
+        character separated plain-text (default), or null character
+        separated plain-text (compatible with **xargs(1)** -0 option
+        where available).
+
+    ``--format-version=N``
+        Use the specified structured output format version. This is
+        intended for programs that invoke **notmuch(1)** internally. If
+        omitted, the latest supported version will be used.
+
+    ``--output=(summary|threads|messages|files|tags)``
+
+        **summary**
+            Output a summary of each thread with any message matching
+            the search terms. The summary includes the thread ID, date,
+            the number of messages in the thread (both the number
+            matched and the total number), the authors of the thread and
+            the subject.
+
+        **threads**
+            Output the thread IDs of all threads with any message
+            matching the search terms, either one per line
+            (--format=text), separated by null characters
+            (--format=text0), as a JSON array (--format=json), or an
+            S-Expression list (--format=sexp).
+
+        **messages**
+            Output the message IDs of all messages matching the search
+            terms, either one per line (--format=text), separated by
+            null characters (--format=text0), as a JSON array
+            (--format=json), or as an S-Expression list (--format=sexp).
+
+        **files**
+            Output the filenames of all messages matching the search
+            terms, either one per line (--format=text), separated by
+            null characters (--format=text0), as a JSON array
+            (--format=json), or as an S-Expression list (--format=sexp).
+
+            Note that each message may have multiple filenames
+            associated with it. All of them are included in the output
+            (unless limited with the --duplicate=N option). This may
+            be particularly confusing for **folder:** or **path:**
+            searches in a specified directory, as the messages may
+            have duplicates in other directories that are included in
+            the output, although these files alone would not match the
+            search.
+
+        **tags**
+            Output all tags that appear on any message matching the
+            search terms, either one per line (--format=text), separated
+            by null characters (--format=text0), as a JSON array
+            (--format=json), or as an S-Expression list (--format=sexp).
+
+    ``--sort=``\ (**newest-first**\ \|\ **oldest-first**)
+        This option can be used to present results in either
+        chronological order (**oldest-first**) or reverse chronological
+        order (**newest-first**).
+
+        Note: The thread order will be distinct between these two
+        options (beyond being simply reversed). When sorting by
+        **oldest-first** the threads will be sorted by the oldest
+        message in each thread, but when sorting by **newest-first** the
+        threads will be sorted by the newest message in each thread.
+
+        By default, results will be displayed in reverse chronological
+        order, (that is, the newest results will be displayed first).
+
+    ``--offset=[-]N``
+        Skip displaying the first N results. With the leading '-', start
+        at the Nth result from the end.
+
+    ``--limit=N``
+        Limit the number of displayed results to N.
+
+    ``--exclude=(true|false|all|flag)``
+        A message is called "excluded" if it matches at least one tag in
+        search.tag\_exclude that does not appear explicitly in the
+        search terms. This option specifies whether to omit excluded
+        messages in the search process.
+
+        The default value, **true**, prevents excluded messages from
+        matching the search terms.
+
+        **all** additionally prevents excluded messages from appearing
+        in displayed results, in effect behaving as though the excluded
+        messages do not exist.
+
+        **false** allows excluded messages to match search terms and
+        appear in displayed results. Excluded messages are still marked
+        in the relevant outputs.
+
+        **flag** only has an effect when ``--output=summary``. The
+        output is almost identical to **false**, but the "match count"
+        is the number of matching non-excluded messages in the thread,
+        rather than the number of matching messages.
+
+    ``--duplicate=N``
+        Effective with ``--output=files``, output the Nth filename
+        associated with each message matching the query (N is 1-based).
+        If N is greater than the number of files associated with the
+        message, don't print anything.
+
+        Note that this option is orthogonal with the **folder:** search
+        prefix. The prefix matches messages based on filenames. This
+        option filters filenames of the matching messages.
+
+EXIT STATUS
+===========
+
+This command supports the following special exit status codes
+
+``20``
+    The requested format version is too old.
+
+``21``
+    The requested format version is too new.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-restore(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-show.rst b/doc/man1/notmuch-show.rst
new file mode 100644
index 00000000..5aecbfd7
--- /dev/null
+++ b/doc/man1/notmuch-show.rst
@@ -0,0 +1,182 @@
+============
+notmuch-show
+============
+
+SYNOPSIS
+========
+
+**notmuch** **show** [*option* ...] <*search-term*> ...
+
+DESCRIPTION
+===========
+
+Shows all messages matching the search terms.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<search-terms>.
+
+The messages will be grouped and sorted based on the threading (all
+replies to a particular message will appear immediately after that
+message in date order). The output is not indented by default, but depth
+tags are printed so that proper indentation can be performed by a
+post-processor (such as the emacs interface to notmuch).
+
+Supported options for **show** include
+
+    ``--entire-thread=(true|false)``
+        If true, **notmuch show** outputs all messages in the thread of
+        any message matching the search terms; if false, it outputs only
+        the matching messages. For ``--format=json`` and
+        ``--format=sexp`` this defaults to true. For other formats, this
+        defaults to false.
+
+    ``--format=(text|json|sexp|mbox|raw)``
+
+        **text** (default for messages)
+            The default plain-text format has all text-content MIME
+            parts decoded. Various components in the output,
+            (**message**, **header**, **body**, **attachment**, and MIME
+            **part**), will be delimited by easily-parsed markers. Each
+            marker consists of a Control-L character (ASCII decimal 12),
+            the name of the marker, and then either an opening or
+            closing brace, ('{' or '}'), to either open or close the
+            component. For a multipart MIME message, these parts will be
+            nested.
+
+        **json**
+            The output is formatted with Javascript Object Notation
+            (JSON). This format is more robust than the text format for
+            automated processing. The nested structure of multipart MIME
+            messages is reflected in nested JSON output. By default JSON
+            output includes all messages in a matching thread; that is,
+            by default,
+            ``--format=json`` sets ``--entire-thread``. The caller can
+            disable this behaviour by setting ``--entire-thread=false``.
+            The JSON output is always encoded as UTF-8 and any message
+            content included in the output will be charset-converted to
+            UTF-8.
+
+        **sexp**
+            The output is formatted as the Lisp s-expression (sexp)
+            equivalent of the JSON format above. Objects are formatted
+            as property lists whose keys are keywords (symbols preceded
+            by a colon). True is formatted as ``t`` and both false and
+            null are formatted as ``nil``. As for JSON, the s-expression
+            output is always encoded as UTF-8.
+
+        **mbox**
+            All matching messages are output in the traditional, Unix
+            mbox format with each message being prefixed by a line
+            beginning with "From " and a blank line separating each
+            message. Lines in the message content beginning with "From "
+            (preceded by zero or more '>' characters) have an additional
+            '>' character added. This reversible escaping is termed
+            "mboxrd" format and described in detail here:
+
+	    http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/mail-mbox-formats.html
+
+        **raw** (default if --part is given)
+            Write the raw bytes of the given MIME part of a message to
+            standard out. For this format, it is an error to specify a
+            query that matches more than one message.
+
+            If the specified part is a leaf part, this outputs the
+            body of the part after performing content transfer
+            decoding (but no charset conversion). This is suitable for
+            saving attachments, for example.
+
+            For a multipart or message part, the output includes the
+            part headers as well as the body (including all child
+            parts). No decoding is performed because multipart and
+            message parts cannot have non-trivial content transfer
+            encoding. Consumers of this may need to implement MIME
+            decoding and similar functions.
+
+    ``--format-version=N``
+        Use the specified structured output format version. This is
+        intended for programs that invoke **notmuch(1)** internally. If
+        omitted, the latest supported version will be used.
+
+    ``--part=N``
+        Output the single decoded MIME part N of a single message. The
+        search terms must match only a single message. Message parts are
+        numbered in a depth-first walk of the message MIME structure,
+        and are identified in the 'json', 'sexp' or 'text' output
+        formats.
+
+        Note that even a message with no MIME structure or a single
+        body part still has two MIME parts: part 0 is the whole
+        message (headers and body) and part 1 is just the body.
+
+    ``--verify``
+        Compute and report the validity of any MIME cryptographic
+        signatures found in the selected content (ie. "multipart/signed"
+        parts). Status of the signature will be reported (currently only
+        supported with --format=json and --format=sexp), and the
+        multipart/signed part will be replaced by the signed data.
+
+    ``--decrypt``
+        Decrypt any MIME encrypted parts found in the selected content
+        (ie. "multipart/encrypted" parts). Status of the decryption will
+        be reported (currently only supported with --format=json and
+        --format=sexp) and on successful decryption the
+        multipart/encrypted part will be replaced by the decrypted
+        content.
+
+        Decryption expects a functioning **gpg-agent(1)** to provide any
+        needed credentials. Without one, the decryption will fail.
+
+        Implies --verify.
+
+    ``--exclude=(true|false)``
+        Specify whether to omit threads only matching
+        search.tag\_exclude from the search results (the default) or
+        not. In either case the excluded message will be marked with the
+        exclude flag (except when output=mbox when there is nowhere to
+        put the flag).
+
+        If --entire-thread is specified then complete threads are
+        returned regardless (with the excluded flag being set when
+        appropriate) but threads that only match in an excluded message
+        are not returned when ``--exclude=true.``
+
+        The default is ``--exclude=true.``
+
+    ``--body=(true|false)``
+        If true (the default) **notmuch show** includes the bodies of
+        the messages in the output; if false, bodies are omitted.
+        ``--body=false`` is only implemented for the json and sexp
+        formats and it is incompatible with ``--part > 0.``
+
+        This is useful if the caller only needs the headers as body-less
+        output is much faster and substantially smaller.
+
+    ``--include-html``
+        Include "text/html" parts as part of the output (currently only
+        supported with --format=json and --format=sexp). By default,
+        unless ``--part=N`` is used to select a specific part or
+        ``--include-html`` is used to include all "text/html" parts, no
+        part with content type "text/html" is included in the output.
+
+A common use of **notmuch show** is to display a single thread of email
+messages. For this, use a search term of "thread:<thread-id>" as can be
+seen in the first column of output from the **notmuch search** command.
+
+EXIT STATUS
+===========
+
+This command supports the following special exit status codes
+
+``20``
+    The requested format version is too old.
+
+``21``
+    The requested format version is too new.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-restore(1)**,
+**notmuch-search(1)**, **notmuch-search-terms(7)**, **notmuch-tag(1)**
diff --git a/doc/man1/notmuch-tag.rst b/doc/man1/notmuch-tag.rst
new file mode 100644
index 00000000..2e7e1d32
--- /dev/null
+++ b/doc/man1/notmuch-tag.rst
@@ -0,0 +1,107 @@
+===========
+notmuch-tag
+===========
+
+SYNOPSIS
+========
+
+**notmuch** **tag** [options ...] +<*tag*>|-<*tag*> [--] <*search-term*> ...
+
+**notmuch** **tag** **--batch** [--input=<*filename*>]
+
+DESCRIPTION
+===========
+
+Add/remove tags for all messages matching the search terms.
+
+See **notmuch-search-terms(7)** for details of the supported syntax for
+<*search-term*\ >.
+
+Tags prefixed by '+' are added while those prefixed by '-' are removed.
+For each message, tag changes are applied in the order they appear on
+the command line.
+
+The beginning of the search terms is recognized by the first argument
+that begins with neither '+' nor '-'. Support for an initial search term
+beginning with '+' or '-' is provided by allowing the user to specify a
+"--" argument to separate the tags from the search terms.
+
+**notmuch tag** updates the maildir flags according to tag changes if
+the **maildir.synchronize\_flags** configuration option is enabled. See
+**notmuch-config(1)** for details.
+
+Supported options for **tag** include
+
+    ``--remove-all``
+        Remove all tags from each message matching the search terms
+        before applying the tag changes appearing on the command line.
+        This means setting the tags of each message to the tags to be
+        added. If there are no tags to be added, the messages will have
+        no tags.
+
+    ``--batch``
+        Read batch tagging operations from a file (stdin by default).
+        This is more efficient than repeated **notmuch tag**
+        invocations. See `TAG FILE FORMAT <#tag_file_format>`__ below
+        for the input format. This option is not compatible with
+        specifying tagging on the command line.
+
+    ``--input=``\ <filename>
+        Read input from given file, instead of from stdin. Implies
+        ``--batch``.
+
+TAG FILE FORMAT
+===============
+
+The input must consist of lines of the format:
+
++<*tag*\ >\|-<*tag*\ > [...] [--] <*query*\ >
+
+Each line is interpreted similarly to **notmuch tag** command line
+arguments. The delimiter is one or more spaces ' '. Any characters in
+<*tag*\ > **may** be hex-encoded with %NN where NN is the hexadecimal
+value of the character. To hex-encode a character with a multi-byte
+UTF-8 encoding, hex-encode each byte. Any spaces in <tag> **must** be
+hex-encoded as %20. Any characters that are not part of <*tag*\ > **must
+not** be hex-encoded.
+
+In the future tag:"tag with spaces" style quoting may be supported for
+<*tag*\ > as well; for this reason all double quote characters in
+<*tag*\ > **should** be hex-encoded.
+
+The <*query*\ > should be quoted using Xapian boolean term quoting
+rules: if a term contains whitespace or a close paren or starts with a
+double quote, it must be enclosed in double quotes (not including any
+prefix) and double quotes inside the term must be doubled (see below for
+examples).
+
+Leading and trailing space ' ' is ignored. Empty lines and lines
+beginning with '#' are ignored.
+
+EXAMPLE
+-------
+
+The following shows a valid input to batch tagging. Note that only the
+isolated '\*' acts as a wildcard. Also note the two different quotings
+of the tag **space in tags**
+
+::
+
+    +winner *
+    +foo::bar%25 -- (One and Two) or (One and tag:winner)
+    +found::it -- tag:foo::bar%
+    # ignore this line and the next
+
+    +space%20in%20tags -- Two
+    # add tag '(tags)', among other stunts.
+    +crazy{ +(tags) +&are +#possible\ -- tag:"space in tags"
+    +match*crazy -- tag:crazy{
+    +some_tag -- id:"this is ""nauty)"""
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-restore(1)**,
+**notmuch-search(1)**, **notmuch-search-terms(7)**, **notmuch-show(1)**,
diff --git a/doc/man1/notmuch.rst b/doc/man1/notmuch.rst
new file mode 100644
index 00000000..97102943
--- /dev/null
+++ b/doc/man1/notmuch.rst
@@ -0,0 +1,143 @@
+=======
+notmuch
+=======
+
+SYNOPSIS
+========
+
+**notmuch** [option ...] **command** [arg ...]
+
+DESCRIPTION
+===========
+
+Notmuch is a command-line based program for indexing, searching,
+reading, and tagging large collections of email messages.
+
+This page describes how to get started using notmuch from the command
+line, and gives a brief overview of the commands available. For more
+information on e.g. **notmuch show** consult the **notmuch-show(1)** man
+page, also accessible via **notmuch help show**
+
+The quickest way to get started with Notmuch is to simply invoke the
+``notmuch`` command with no arguments, which will interactively guide
+you through the process of indexing your mail.
+
+NOTE
+====
+
+While the command-line program ``notmuch`` provides powerful
+functionality, it does not provide the most convenient interface for
+that functionality. More sophisticated interfaces are expected to be
+built on top of either the command-line interface, or more likely, on
+top of the notmuch library interface. See http://notmuchmail.org for
+more about alternate interfaces to notmuch. The emacs-based interface to
+notmuch (available under **emacs/** in the Notmuch source distribution)
+is probably the most widely used at this time.
+
+OPTIONS
+=======
+
+Supported global options for ``notmuch`` include
+
+    ``--help``
+        Print a synopsis of available commands and exit.
+
+    ``--version``
+        Print the installed version of notmuch, and exit.
+
+    ``--config=FILE``
+        Specify the configuration file to use. This overrides any
+        configuration file specified by ${NOTMUCH\_CONFIG}.
+
+COMMANDS
+========
+
+SETUP
+-----
+
+The **notmuch setup** command is used to configure Notmuch for first
+use, (or to reconfigure it later).
+
+The setup command will prompt for your full name, your primary email
+address, any alternate email addresses you use, and the directory
+containing your email archives. Your answers will be written to a
+configuration file in ${NOTMUCH\_CONFIG} (if set) or
+${HOME}/.notmuch-config . This configuration file will be created with
+descriptive comments, making it easy to edit by hand later to change the
+configuration. Or you can run **notmuch setup** again to change the
+configuration.
+
+The mail directory you specify can contain any number of sub-directories
+and should primarily contain only files with individual email messages
+(eg. maildir or mh archives are perfect). If there are other, non-email
+files (such as indexes maintained by other email programs) then notmuch
+will do its best to detect those and ignore them.
+
+Mail storage that uses mbox format, (where one mbox file contains many
+messages), will not work with notmuch. If that's how your mail is
+currently stored, it is recommended you first convert it to maildir
+format with a utility such as mb2md before running **notmuch setup .**
+
+Invoking ``notmuch`` with no command argument will run **setup** if the
+setup command has not previously been completed.
+
+OTHER COMMANDS
+--------------
+
+Several of the notmuch commands accept search terms with a common
+syntax. See **notmuch-search-terms**\ (7) for more details on the
+supported syntax.
+
+The **search**, **show** and **count** commands are used to query the
+email database.
+
+The **reply** command is useful for preparing a template for an email
+reply.
+
+The **tag** command is the only command available for manipulating
+database contents.
+
+The **dump** and **restore** commands can be used to create a textual
+dump of email tags for backup purposes, and to restore from that dump.
+
+The **config** command can be used to get or set settings in the notmuch
+configuration file.
+
+ENVIRONMENT
+===========
+
+The following environment variables can be used to control the behavior
+of notmuch.
+
+**NOTMUCH\_CONFIG**
+    Specifies the location of the notmuch configuration file. Notmuch
+    will use ${HOME}/.notmuch-config if this variable is not set.
+
+**NOTMUCH\_TALLOC\_REPORT**
+    Location to write a talloc memory usage report. See
+    **talloc\_enable\_leak\_report\_full** in **talloc(3)** for more
+    information.
+
+**NOTMUCH\_DEBUG\_QUERY**
+    If set to a non-empty value, the notmuch library will print (to
+    stderr) Xapian queries it constructs.
+
+SEE ALSO
+========
+
+**notmuch-config(1)**, **notmuch-count(1)**, **notmuch-dump(1)**,
+**notmuch-hooks(5)**, **notmuch-insert(1)**, **notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
+
+The notmuch website: **http://notmuchmail.org**
+
+CONTACT
+=======
+
+Feel free to send questions, comments, or kudos to the notmuch mailing
+list <notmuch@notmuchmail.org> . Subscription is not required before
+posting, but is available from the notmuchmail.org website.
+
+Real-time interaction with the Notmuch community is available via IRC
+(server: irc.freenode.net, channel: #notmuch).
diff --git a/doc/man5/notmuch-hooks.rst b/doc/man5/notmuch-hooks.rst
new file mode 100644
index 00000000..493abf20
--- /dev/null
+++ b/doc/man5/notmuch-hooks.rst
@@ -0,0 +1,44 @@
+=============
+notmuch-hooks
+=============
+
+SYNOPSIS
+========
+
+	$DATABASEDIR/.notmuch/hooks/*
+
+DESCRIPTION
+===========
+
+Hooks are scripts (or arbitrary executables or symlinks to such) that
+notmuch invokes before and after certain actions. These scripts reside
+in the .notmuch/hooks directory within the database directory and must
+have executable permissions.
+
+The currently available hooks are described below.
+
+    **pre-new**
+        This hook is invoked by the **new** command before scanning or
+        importing new messages into the database. If this hook exits
+        with a non-zero status, notmuch will abort further processing of
+        the **new** command.
+
+        Typically this hook is used for fetching or delivering new mail
+        to be imported into the database.
+
+    **post-new**
+        This hook is invoked by the **new** command after new messages
+        have been imported into the database and initial tags have been
+        applied. The hook will not be run if there have been any errors
+        during the scan or import.
+
+        Typically this hook is used to perform additional query-based
+        tagging on the imported messages.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-insert(1)**, **notmuch-new(1)**,
+**notmuch-reply(1)**, **notmuch-restore(1)**, **notmuch-search(1)**,
+**notmuch-search-terms(7)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/man7/notmuch-search-terms.rst b/doc/man7/notmuch-search-terms.rst
new file mode 100644
index 00000000..1acdaa0b
--- /dev/null
+++ b/doc/man7/notmuch-search-terms.rst
@@ -0,0 +1,252 @@
+====================
+notmuch-search-terms
+====================
+
+SYNOPSIS
+========
+
+**notmuch** **count** [option ...] <*search-term*> ...
+
+**notmuch** **dump** [--format=(batch-tag|sup)] [--] [--output=<*file*>] [--] [<*search-term*> ...]
+
+**notmuch** **search** [option ...] <*search-term*> ...
+
+**notmuch** **show** [option ...] <*search-term*> ...
+
+**notmuch** **tag** +<*tag*> ... -<*tag*> [--] <*search-term*> ...
+
+DESCRIPTION
+===========
+
+Several notmuch commands accept a common syntax for search terms.
+
+The search terms can consist of free-form text (and quoted phrases)
+which will match all messages that contain all of the given
+terms/phrases in the body, the subject, or any of the sender or
+recipient headers.
+
+As a special case, a search string consisting of exactly a single
+asterisk ("\*") will match all messages.
+
+In addition to free text, the following prefixes can be used to force
+terms to match against specific portions of an email, (where <brackets>
+indicate user-supplied values):
+
+-  from:<name-or-address>
+
+-  to:<name-or-address>
+
+-  subject:<word-or-quoted-phrase>
+
+-  attachment:<word>
+
+-  tag:<tag> (or is:<tag>)
+
+-  id:<message-id>
+
+-  thread:<thread-id>
+
+-  folder:<maildir-folder>
+
+-  path:<directory-path> or path:<directory-path>/**
+
+-  date:<since>..<until>
+
+The **from:** prefix is used to match the name or address of the sender
+of an email message.
+
+The **to:** prefix is used to match the names or addresses of any
+recipient of an email message, (whether To, Cc, or Bcc).
+
+Any term prefixed with **subject:** will match only text from the
+subject of an email. Searching for a phrase in the subject is supported
+by including quotation marks around the phrase, immediately following
+**subject:**.
+
+The **attachment:** prefix can be used to search for specific filenames
+(or extensions) of attachments to email messages.
+
+For **tag:** and **is:** valid tag values include **inbox** and
+**unread** by default for new messages added by **notmuch new** as well
+as any other tag values added manually with **notmuch tag**.
+
+For **id:**, message ID values are the literal contents of the
+Message-ID: header of email messages, but without the '<', '>'
+delimiters.
+
+The **thread:** prefix can be used with the thread ID values that are
+generated internally by notmuch (and do not appear in email messages).
+These thread ID values can be seen in the first column of output from
+**notmuch search**
+
+The **path:** prefix searches for email messages that are in
+particular directories within the mail store. The directory must be
+specified relative to the top-level maildir (and without the leading
+slash). By default, **path:** matches messages in the specified
+directory only. The "/\*\*" suffix can be used to match messages in
+the specified directory and all its subdirectories recursively.
+**path:""** matches messages in the root of the mail store and,
+likewise, **path:\*\*** matches all messages.
+
+The **folder:** prefix searches for email messages by maildir or MH
+folder. For MH-style folders, this is equivalent to **path:**. For
+maildir, this includes messages in the "new" and "cur"
+subdirectories. The exact syntax for maildir folders depends on your
+mail configuration. For maildir++, **folder:""** matches the inbox
+folder (which is the root in maildir++), other folder names always
+start with ".", and nested folders are separated by "."s, such as
+**folder:.classes.topology**. For "file system" maildir, the inbox is
+typically **folder:INBOX** and nested folders are separated by
+slashes, such as **folder:classes/topology**.
+
+Both **path:** and **folder:** will find a message if *any* copy of
+that message is in the specific directory/folder.
+
+The **date:** prefix can be used to restrict the results to only
+messages within a particular time range (based on the Date: header) with
+a range syntax of:
+
+date:<since>..<until>
+
+See **DATE AND TIME SEARCH** below for details on the range expression,
+and supported syntax for <since> and <until> date and time expressions.
+
+The time range can also be specified using timestamps with a syntax of:
+
+<initial-timestamp>..<final-timestamp>
+
+Each timestamp is a number representing the number of seconds since
+1970-01-01 00:00:00 UTC.
+
+In addition to individual terms, multiple terms can be combined with
+Boolean operators ( **and**, **or**, **not** , etc.). Each term in the
+query will be implicitly connected by a logical AND if no explicit
+operator is provided, (except that terms with a common prefix will be
+implicitly combined with OR until we get Xapian defect #402 fixed).
+
+Parentheses can also be used to control the combination of the Boolean
+operators, but will have to be protected from interpretation by the
+shell, (such as by putting quotation marks around any parenthesized
+expression).
+
+DATE AND TIME SEARCH
+====================
+
+notmuch understands a variety of standard and natural ways of expressing
+dates and times, both in absolute terms ("2012-10-24") and in relative
+terms ("yesterday"). Any number of relative terms can be combined ("1
+hour 25 minutes") and an absolute date/time can be combined with
+relative terms to further adjust it. A non-exhaustive description of the
+syntax supported for absolute and relative terms is given below.
+
+The range expression
+--------------------
+
+date:<since>..<until>
+
+The above expression restricts the results to only messages from <since>
+to <until>, based on the Date: header.
+
+<since> and <until> can describe imprecise times, such as "yesterday".
+In this case, <since> is taken as the earliest time it could describe
+(the beginning of yesterday) and <until> is taken as the latest time it
+could describe (the end of yesterday). Similarly, date:january..february
+matches from the beginning of January to the end of February.
+
+Currently, we do not support spaces in range expressions. You can
+replace the spaces with '\_', or (in most cases) '-', or (in some cases)
+leave the spaces out altogether. Examples in this man page use spaces
+for clarity.
+
+Open-ended ranges are supported (since Xapian 1.2.1), i.e. it's possible
+to specify date:..<until> or date:<since>.. to not limit the start or
+end time, respectively. Pre-1.2.1 Xapian does not report an error on
+open ended ranges, but it does not work as expected either.
+
+Entering date:expr without ".." (for example date:yesterday) won't work,
+as it's not interpreted as a range expression at all. You can achieve
+the expected result by duplicating the expr both sides of ".." (for
+example date:yesterday..yesterday).
+
+Relative date and time
+----------------------
+
+[N\|number]
+(years\|months\|weeks\|days\|hours\|hrs\|minutes\|mins\|seconds\|secs)
+[...]
+
+All refer to past, can be repeated and will be accumulated.
+
+Units can be abbreviated to any length, with the otherwise ambiguous
+single m being m for minutes and M for months.
+
+Number can also be written out one, two, ..., ten, dozen, hundred.
+Additionally, the unit may be preceded by "last" or "this" (e.g., "last
+week" or "this month").
+
+When combined with absolute date and time, the relative date and time
+specification will be relative from the specified absolute date and
+time.
+
+Examples: 5M2d, two weeks
+
+Supported absolute time formats
+-------------------------------
+
+-  H[H]:MM[:SS] [(am\|a.m.\|pm\|p.m.)]
+
+-  H[H] (am\|a.m.\|pm\|p.m.)
+
+-  HHMMSS
+
+-  now
+
+-  noon
+
+-  midnight
+
+-  Examples: 17:05, 5pm
+
+Supported absolute date formats
+-------------------------------
+
+-  YYYY-MM[-DD]
+
+-  DD-MM[-[YY]YY]
+
+-  MM-YYYY
+
+-  M[M]/D[D][/[YY]YY]
+
+-  M[M]/YYYY
+
+-  D[D].M[M][.[YY]YY]
+
+-  D[D][(st\|nd\|rd\|th)] Mon[thname] [YYYY]
+
+-  Mon[thname] D[D][(st\|nd\|rd\|th)] [YYYY]
+
+-  Wee[kday]
+
+Month names can be abbreviated at three or more characters.
+
+Weekday names can be abbreviated at three or more characters.
+
+Examples: 2012-07-31, 31-07-2012, 7/31/2012, August 3
+
+Time zones
+----------
+
+-  (+\|-)HH:MM
+
+-  (+\|-)HH[MM]
+
+Some time zone codes, e.g. UTC, EET.
+
+SEE ALSO
+========
+
+**notmuch(1)**, **notmuch-config(1)**, **notmuch-count(1)**,
+**notmuch-dump(1)**, **notmuch-hooks(5)**, **notmuch-insert(1)**,
+**notmuch-new(1)**, **notmuch-reply(1)**, **notmuch-restore(1)**,
+**notmuch-search(1)**, **notmuch-show(1)**, **notmuch-tag(1)**
diff --git a/doc/mkdocdeps.py b/doc/mkdocdeps.py
new file mode 100644
index 00000000..b87fe3e8
--- /dev/null
+++ b/doc/mkdocdeps.py
@@ -0,0 +1,18 @@
+import sys
+
+srcdir = sys.argv[1]
+builddir = sys.argv[2]
+outfile = sys.argv[3]
+
+sys.path.insert(0, srcdir)
+import conf
+
+roff_files = []
+rst_files = []
+for page in conf.man_pages:
+    rst_files = rst_files + ["{0:s}/{1:s}.rst".format(srcdir,page[0])]
+    roff_files = roff_files + ["{0:s}/man/{1:s}.{2:d}".format(builddir,page[0],page[4])]
+
+with open(outfile, 'w') as out:
+    out.write('MAN_ROFF_FILES := ' + ' \\\n\t'.join(roff_files) + '\n')
+    out.write('MAN_RST_FILES := ' + ' \\\n\t'.join(rst_files) + '\n')
diff --git a/doc/notmuch-emacs.rst b/doc/notmuch-emacs.rst
new file mode 100644
index 00000000..09579bf6
--- /dev/null
+++ b/doc/notmuch-emacs.rst
@@ -0,0 +1,204 @@
+=============
+notmuch-emacs
+=============
+
+About this Manual
+=================
+
+This manual covers only the Emacs interface to Notmuch. For information
+on the command line interface, see See section “Description” in Notmuch
+Manual Pager. To save typing, we will sometimes use *notmuch* in this
+manual to refer to the Emacs interface to Notmuch. If the distinction
+should every be important, we’ll refer to the Emacs interface as
+*notmuch-emacs*.
+
+Notmuch-emacs is highly customizable via the the Emacs customization
+framework (or just by setting the appropriate variables). We try to
+point out relevant variables in this manual, but in order to avoid
+duplication of information, but you can usually find the most detailed
+description in the variables docstring.
+
+notmuch-hello
+=============
+
+.. index::
+   single: notmuch-hello
+   single: notmuch
+
+``notmuch-hello`` is the main entry point for Notmuch. You can start it
+with ``M-x notmuch`` or ``M-x notmuch-hello``. The startup screen looks
+something like the following. There are some hints at the bottom of the
+screen. There are three main parts to the notmuch-hello screen,
+discussed below. The **bold** text indicates buttons you can click with
+a mouse or by positioning the cursor and pressing ``<return>``
+
+|   Welcome to **notmuch** You have 52 messages.
+|
+| Saved searches: **[edit]**
+|
+|	  52 **inbox**           52 **unread**
+|
+| Search: ____________________________________
+|
+| All tags: **[show]**
+|
+|	 Type a search query and hit RET to view matching threads.
+|		Edit saved searches with the ``edit`` button.
+| Hit RET or click on a saved search or tag name to view matching threads.
+|     ``=`` to refresh this screen. ``s`` to search messages. ``q`` to quit.
+|		    **Customize** this page.
+
+You can change the overall appearance of the notmuch-hello screen by
+customizing the variable :index:`notmuch-hello-sections`.
+
+
+
+notmuch-hello key bindings
+--------------------------
+
+``<tab>``
+    Move to the next widget (button or text entry field)
+
+``<backspace>``
+    Move to the previous widget.
+
+``<return>``
+    Activate the current widget.
+
+``=``
+    Refresh the buffer; mainly update the counts of messages for various
+    saved searches.
+
+``G``
+    Import mail, See :ref:`importing`
+
+``m``
+    Compose a message
+
+``s``
+    Search the notmuch database using :ref:`notmuch-search`
+
+``v``
+    Print notmuch version
+
+``q``
+    Quit
+
+.. _saved-searches:
+
+Saved Searches
+--------------
+
+Notmuch replaces the static assignment of messages with the more dynamic
+notion of searching. Notmuch-hello presents the user with a customizable
+set of saved searches. The initial defaults are ``tag:inbox`` and
+``tag:unread``, but you can customize the following variables
+
+:index:`notmuch-saved-searches`
+    A list of cons pairs, the first being the name to display, the
+    second being a query string for Notmuch. See section “Description”
+    in Notmuch Query Syntax.
+
+:index:`notmuch-saved-searches-sort-function`
+    This variable controls how saved searches should be sorted. A value
+    of ``nil`` displays the saved searches in the order they are stored
+    in ‘notmuch-saved-searches’.
+
+:index:`notmuch-column-control`
+    Controls the number of columns for displaying saved-searches/tags
+
+Search Box
+----------
+
+The search box lets the user enter a Notmuch query. See section
+“Description” in Notmuch Query Syntax, for more info on Notmuch query
+syntax. A history of recent searches is also displayed by default. The
+latter is controlled by the variable :index:`notmuch-hello-recent-searches-max`.
+
+Known Tags
+----------
+
+One special kind of saved search provided by default is for each
+individual tag defined in the database. This can be controlled via the
+following variables.
+
+:index:`notmuch-hello-tag-list-make-query`
+    Control how to construct a search (“virtual folder”) from a given
+    tag.
+
+:index:`notmuch-hello-hide-tags`
+    Which tags not to display at all.
+
+:index:`notmuch-column-control`
+    Controls the number of columns for displaying saved-searches/tags
+
+.. _notmuch-search:
+
+notmuch-search
+==============
+
+``notmuch-search-mode`` is used to display the results from executing
+a query via ``notmuch-search``. The syntax for these queries is the
+the same as :ref:`saved-searches`. For details of this syntax see
+info:notmuch-search-terms
+
+By default the output approximates that of the command line See section
+“Description” in notmuch search command.
+
+The main purpose of the ``notmuch-search-mode`` buffer is to act as a
+menu of results that the user can explore further by pressing
+``<return>`` on the appropriate line.
+
+``n,C-n,<down>``
+    Move to next line
+
+``p,C-p,<up>``
+    Move to previous line
+
+``<return>``
+    Open thread on current line in :ref:`notmuch-show` mode
+
+``?``
+    Display full set of key bindings
+
+The presentation of results can be controlled by the following
+variables.
+
+:index:`notmuch-search-result-format`
+    Control how each thread of messages is presented in the
+    ``notmuch-show-mode`` buffer
+
+:index:`notmuch-search-oldest-first`
+    Display the oldest threads at the top of the buffer
+
+.. _notmuch-show:
+
+notmuch-show
+============
+
+notmuch-tree
+============
+
+Configuration
+=============
+
+.. _importing:
+
+Importing Mail
+--------------
+
+:index:`notmuch-poll`
+
+:index:`notmuch-poll-script`
+
+Init File
+---------
+
+When Notmuch is loaded, it will read the ``notmuch-init-file``
+(``~/.emacs.d/notmuch-config`` by default) file. This is normal Emacs Lisp
+file and can be used to avoid cluttering your ``~/.emacs`` with Notmuch
+stuff. If the file with ``.elc``, ``.elc.gz``, ``.el`` or ``.el.gz``
+suffix exist it will be read instead (just one of these, chosen in this
+order). Most often users create ``~/.emacs.d/notmuch-config.el`` and just
+work with it. If Emacs was invoked with the ``-q`` or ``--no-init-file``
+options, ``notmuch-init-file`` is not read.
diff --git a/doc/prerst2man.py b/doc/prerst2man.py
new file mode 100644
index 00000000..437dea99
--- /dev/null
+++ b/doc/prerst2man.py
@@ -0,0 +1,63 @@
+from sys import argv
+from datetime import date
+from os.path import dirname, isdir
+from os import makedirs, system
+import re
+
+sourcedir = argv[1]
+outdir = argv[2]
+
+if not isdir(outdir):
+    makedirs(outdir, 0o755)
+
+execfile(sourcedir + "/conf.py")
+
+
+def header(file, startdocname, command, description, authors, section):
+    file.write("""
+{0:s}
+{1:s}
+{2:s}
+
+:Date:   {3:s}
+:Version: {4:s}
+:Manual section: {5:d}
+:Manual group: {6:s}
+
+""".format(
+'-' * len(description),
+description,
+'-' * len(description),
+date.today().isoformat(), release, section, project))
+
+blankre = re.compile("^\s*$")
+for page in man_pages:
+    outdirname = outdir + '/' + dirname(page[0])
+    if not isdir(outdirname):
+        makedirs(outdirname, 0o755)
+    filename = outdir + '/' + page[0] + '.rst'
+    outfile = open(filename, 'w')
+    infile = open(sourcedir + '/' + page[0] + '.rst', 'r')
+
+    # this is a crude hack. We look for the first blank line, and
+    # insert the rst2man header there.
+    #
+    # XXX consider really parsing input
+
+    count = 0
+    lines = infile.readlines()
+    for line in lines:
+        outfile.write(line)
+        if (blankre.match(line)):
+            break
+        count = count + 1
+
+    del lines[0:count + 1]
+
+    header(outfile, *page)
+
+    outfile.write("".join(lines))
+    outfile.close()
+
+    system('set -x; rst2man {0} {1}/{2}.{3}'
+           .format(filename, outdir, page[0], page[4]))
diff --git a/dump-restore-private.h b/dump-restore-private.h
deleted file mode 100644
index 896a0043..00000000
--- a/dump-restore-private.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef DUMP_RESTORE_PRIVATE_H
-#define DUMP_RESTORE_PRIVATE_H
-
-#include "hex-escape.h"
-#include "command-line-arguments.h"
-
-typedef enum dump_formats {
-    DUMP_FORMAT_AUTO,
-    DUMP_FORMAT_BATCH_TAG,
-    DUMP_FORMAT_SUP
-} dump_format_t;
-
-#endif
diff --git a/emacs/.gitignore b/emacs/.gitignore
index 5421301c..9fa1c44e 100644
--- a/emacs/.gitignore
+++ b/emacs/.gitignore
@@ -1,2 +1,3 @@
 .eldeps*
 *.elc
+notmuch-version.el
diff --git a/emacs/Makefile.local b/emacs/Makefile.local
index 92467a31..c0d6b190 100644
--- a/emacs/Makefile.local
+++ b/emacs/Makefile.local
@@ -17,7 +17,14 @@ emacs_sources := \
 	$(dir)/notmuch-crypto.el \
 	$(dir)/notmuch-tag.el \
 	$(dir)/coolj.el \
-	$(dir)/notmuch-print.el
+	$(dir)/notmuch-print.el \
+	$(dir)/notmuch-version.el
+
+$(dir)/notmuch-version.el: $(dir)/Makefile.local version.stamp
+$(dir)/notmuch-version.el: $(srcdir)/$(dir)/notmuch-version.el.tmpl
+	@sed -e 's/%AG%/Generated file (from $(<F)) -- do not edit!/' \
+	     -e 's/%VERSION%/"$(VERSION)"/' $< > $@
+
 
 emacs_images := \
 	$(srcdir)/$(dir)/notmuch-logo.png
@@ -29,26 +36,40 @@ emacs_bytecode = $(emacs_sources:.el=.elc)
 # the byte compiler may load an old .elc file when processing a
 # "require" or we may fail to rebuild a .elc that depended on a macro
 # from an updated file.
+ifeq ($(HAVE_EMACS),1)
 $(dir)/.eldeps: $(dir)/Makefile.local $(dir)/make-deps.el $(emacs_sources)
 	$(call quiet,EMACS) --directory emacs -batch -l make-deps.el \
 		-f batch-make-deps $(emacs_sources) > $@.tmp && \
-		(cmp -s $@.tmp $@ || mv $@.tmp $@)
--include $(dir)/.eldeps
-CLEAN+=$(dir)/.eldeps $(dir)/.eldeps.tmp
+		mv $@.tmp $@
+# We could include .eldeps directly, but that would cause a make
+# restart whenever any .el file was modified, even if dependencies
+# didn't change, because the mtime of .eldeps will change.  Instead,
+# we include a second file, .eldeps.x, which we ensure always has the
+# same content as .eldeps, but its mtime only changes when dependency
+# information changes, in which case a make restart is necessary
+# anyway.
+$(dir)/.eldeps.x: $(dir)/.eldeps
+	@cmp -s $^ $@ || cp $^ $@
+-include $(dir)/.eldeps.x
+endif
+CLEAN+=$(dir)/.eldeps $(dir)/.eldeps.tmp $(dir)/.eldeps.x
 
+ifeq ($(HAVE_EMACS),1)
 %.elc: %.el $(global_deps)
 	$(call quiet,EMACS) --directory emacs -batch -f batch-byte-compile $<
+endif
 
 ifeq ($(WITH_EMACS),1)
 ifeq ($(HAVE_EMACS),1)
 all: $(emacs_bytecode)
+install-emacs: $(emacs_bytecode)
 endif
 
 install: install-emacs
 endif
 
 .PHONY: install-emacs
-install-emacs:
+install-emacs: $(emacs_sources) $(emacs_images)
 	mkdir -p "$(DESTDIR)$(emacslispdir)"
 	install -m0644 $(emacs_sources) "$(DESTDIR)$(emacslispdir)"
 ifeq ($(HAVE_EMACS),1)
@@ -57,4 +78,4 @@ endif
 	mkdir -p "$(DESTDIR)$(emacsetcdir)"
 	install -m0644 $(emacs_images) "$(DESTDIR)$(emacsetcdir)"
 
-CLEAN := $(CLEAN) $(emacs_bytecode)
+CLEAN := $(CLEAN) $(emacs_bytecode) $(dir)/notmuch-version.el
diff --git a/emacs/notmuch-hello.el b/emacs/notmuch-hello.el
index 55c416ac..3de52386 100644
--- a/emacs/notmuch-hello.el
+++ b/emacs/notmuch-hello.el
@@ -29,6 +29,96 @@
 (declare-function notmuch-search "notmuch" (&optional query oldest-first target-thread target-line continuation))
 (declare-function notmuch-poll "notmuch" ())
 
+(defun notmuch-saved-search-get (saved-search field)
+  "Get FIELD from SAVED-SEARCH.
+
+If SAVED-SEARCH is a plist, this is just `plist-get', but for
+backwards compatibility, this also deals with the two other
+possible formats for SAVED-SEARCH: cons cells (NAME . QUERY) and
+lists (NAME QUERY COUNT-QUERY)."
+  (cond
+   ((keywordp (car saved-search))
+    (plist-get saved-search field))
+   ;; It is not a plist so it is an old-style entry.
+   ((consp (cdr saved-search)) ;; It is a list (NAME QUERY COUNT-QUERY)
+    (case field
+      (:name (first saved-search))
+      (:query (second saved-search))
+      (:count-query (third saved-search))
+      (t nil)))
+   (t  ;; It is a cons-cell (NAME . QUERY)
+    (case field
+      (:name (car saved-search))
+      (:query (cdr saved-search))
+      (t nil)))))
+
+(defun notmuch-hello-saved-search-to-plist (saved-search)
+  "Return a copy of SAVED-SEARCH in plist form.
+
+If saved search is a plist then just return a copy. In other
+cases, for backwards compatibility, convert to plist form and
+return that."
+  (if (keywordp (car saved-search))
+      (copy-seq saved-search)
+    (let ((fields (list :name :query :count-query))
+	  plist-search)
+      (dolist (field fields plist-search)
+	(let ((string (notmuch-saved-search-get saved-search field)))
+	  (when string
+	    (setq plist-search (append plist-search (list field string)))))))))
+
+(defun notmuch-hello--saved-searches-to-plist (symbol)
+  "Extract a saved-search variable into plist form.
+
+The new style saved search is just a plist, but for backwards
+compatibility we use this function to extract old style saved
+searches so they still work in customize."
+  (let ((saved-searches (default-value symbol)))
+    (mapcar #'notmuch-hello-saved-search-to-plist saved-searches)))
+
+(define-widget 'notmuch-saved-search-plist 'list
+  "A single saved search property list."
+  :tag "Saved Search"
+  :args '((list :inline t
+		:format "%v"
+		(group :format "%v" :inline t (const :format "   Name: " :name) (string :format "%v"))
+		(group :format "%v" :inline t (const :format "  Query: " :query) (string :format "%v")))
+	  (checklist :inline t
+		     :format "%v"
+		     (group :format "%v" :inline t (const :format "Count-Query: " :count-query) (string :format "%v"))
+		     (group :format "%v" :inline t (const :format "" :sort-order)
+			    (choice :tag " Sort Order"
+				    (const :tag "Default" nil)
+				    (const :tag "Oldest-first" oldest-first)
+				    (const :tag "Newest-first" newest-first))))))
+
+(defcustom notmuch-saved-searches '((:name "inbox" :query "tag:inbox")
+				    (:name "unread" :query "tag:unread"))
+  "A list of saved searches to display.
+
+The saved search can be given in 3 forms. The preferred way is as
+a plist. Supported properties are
+
+  :name            Name of the search (required).
+  :query           Search to run (required).
+  :count-query     Optional extra query to generate the count
+                   shown. If not present then the :query property
+                   is used.
+  :sort-order      Specify the sort order to be used for the search.
+                   Possible values are 'oldest-first 'newest-first or
+                   nil. Nil means use the default sort order.
+
+Other accepted forms are a cons cell of the form (NAME . QUERY)
+or a list of the form (NAME QUERY COUNT-QUERY)."
+;; The saved-search format is also used by the all-tags notmuch-hello
+;; section. This section generates its own saved-search list in one of
+;; the latter two forms.
+
+  :get 'notmuch-hello--saved-searches-to-plist
+  :type '(repeat notmuch-saved-search-plist)
+  :tag "List of Saved Searches"
+  :group 'notmuch-hello)
+
 (defcustom notmuch-hello-recent-searches-max 10
   "The number of recent searches to display."
   :type 'integer
@@ -39,9 +129,12 @@
   :type 'boolean
   :group 'notmuch-hello)
 
-(defun notmuch-sort-saved-searches (alist)
-  "Generate an alphabetically sorted saved searches alist."
-  (sort (copy-sequence alist) (lambda (a b) (string< (car a) (car b)))))
+(defun notmuch-sort-saved-searches (saved-searches)
+  "Generate an alphabetically sorted saved searches list."
+  (sort (copy-sequence saved-searches)
+	(lambda (a b)
+	  (string< (notmuch-saved-search-get a :name)
+		   (notmuch-saved-search-get b :name)))))
 
 (defcustom notmuch-saved-search-sort-function nil
   "Function used to sort the saved searches for the notmuch-hello view.
@@ -51,8 +144,10 @@ sorting (nil) displays the saved searches in the order they are
 stored in `notmuch-saved-searches'. Sort alphabetically sorts the
 saved searches in alphabetical order. Custom sort function should
 be a function or a lambda expression that takes the saved
-searches alist as a parameter, and returns a new saved searches
-alist to be used."
+searches list as a parameter, and returns a new saved searches
+list to be used. For compatibility with the various saved-search
+formats it should use notmuch-saved-search-get to access the
+fields of the search."
   :type '(choice (const :tag "No sorting" nil)
 		 (const :tag "Sort alphabetically" notmuch-sort-saved-searches)
 		 (function :tag "Custom sort function"
@@ -280,12 +375,12 @@ afterwards.")
     (setq notmuch-saved-searches
 	  (loop for elem in notmuch-saved-searches
 		if (not (equal name
-			       (car elem)))
+			       (notmuch-saved-search-get elem :name)))
 		collect elem))
     ;; Add the new one.
     (customize-save-variable 'notmuch-saved-searches
 			     (add-to-list 'notmuch-saved-searches
-					  (cons name search) t))
+					  (list :name name :query search) t))
     (message "Saved '%s' as '%s'." search name)
     (notmuch-hello-update)))
 
@@ -300,7 +395,7 @@ afterwards.")
 
 (defun notmuch-hello-longest-label (searches-alist)
   (or (loop for elem in searches-alist
-	    maximize (length (car elem)))
+	    maximize (length (notmuch-saved-search-get elem :name)))
       0))
 
 (defun notmuch-hello-reflect-generate-row (ncols nrows row list)
@@ -325,7 +420,8 @@ diagonal."
 (defun notmuch-hello-widget-search (widget &rest ignore)
   (notmuch-search (widget-get widget
 			      :notmuch-search-terms)
-		  notmuch-search-oldest-first))
+		  (widget-get widget
+			      :notmuch-search-oldest-first)))
 
 (defun notmuch-saved-search-count (search)
   (car (process-lines notmuch-command "count" search)))
@@ -379,30 +475,31 @@ Otherwise, FILTER is ignored.
     (concat "(" query ") and (" filter ")"))
    (t query)))
 
-(defun notmuch-hello-query-counts (query-alist &rest options)
-  "Compute list of counts of matched messages from QUERY-ALIST.
+(defun notmuch-hello-query-counts (query-list &rest options)
+  "Compute list of counts of matched messages from QUERY-LIST.
 
-QUERY-ALIST must be a list containing elements of the form (NAME . QUERY)
-or (NAME QUERY COUNT-QUERY). If the latter form is used,
-COUNT-QUERY specifies an alternate query to be used to generate
-the count for the associated query.
+QUERY-LIST must be a list of saved-searches. Ideally each of
+these is a plist but other options are available for backwards
+compatibility: see `notmuch-saved-searches' for details.
 
-The result is the list of elements of the form (NAME QUERY COUNT).
+The result is a list of plists each of which includes the
+properties :name NAME, :query QUERY and :count COUNT, together
+with any properties in the original saved-search.
 
 The values :show-empty-searches, :filter and :filter-count from
 options will be handled as specified for
 `notmuch-hello-insert-searches'."
   (with-temp-buffer
-    (dolist (elem query-alist nil)
-      (let ((count-query (if (consp (cdr elem))
-			     ;; do we have a different query for the message count?
-			     (third elem)
-			   (cdr elem))))
+    (dolist (elem query-list nil)
+      (let ((count-query (or (notmuch-saved-search-get elem :count-query)
+			     (notmuch-saved-search-get elem :query))))
 	(insert
-	 (notmuch-hello-filtered-query count-query
-				       (or (plist-get options :filter-count)
-					   (plist-get options :filter)))
-	 "\n")))
+	 (replace-regexp-in-string
+	  "\n" " "
+	  (notmuch-hello-filtered-query count-query
+					(or (plist-get options :filter-count)
+					    (plist-get options :filter))))
+	  "\n")))
 
     (unless (= (call-process-region (point-min) (point-max) notmuch-command
 				    t t nil "count" "--batch") 0)
@@ -417,26 +514,26 @@ the CLI and emacs interface."))
      #'identity
      (mapcar
       (lambda (elem)
-	(let ((name (car elem))
-	      (search-query (if (consp (cdr elem))
-				 ;; do we have a different query for the message count?
-				 (second elem)
-			      (cdr elem)))
-	      (message-count (prog1 (read (current-buffer))
+	(let* ((elem-plist (notmuch-hello-saved-search-to-plist elem))
+	       (search-query (plist-get elem-plist :query))
+	       (filtered-query (notmuch-hello-filtered-query
+				search-query (plist-get options :filter)))
+	       (message-count (prog1 (read (current-buffer))
 				(forward-line 1))))
-	  (and (or (plist-get options :show-empty-searches) (> message-count 0))
-	       (list name (notmuch-hello-filtered-query
-			   search-query (plist-get options :filter))
-		     message-count))))
-      query-alist))))
+	  (when (and filtered-query (or (plist-get options :show-empty-searches) (> message-count 0)))
+	    (setq elem-plist (plist-put elem-plist :query filtered-query))
+	    (plist-put elem-plist :count message-count))))
+      query-list))))
 
 (defun notmuch-hello-insert-buttons (searches)
   "Insert buttons for SEARCHES.
 
-SEARCHES must be a list containing lists of the form (NAME QUERY COUNT), where
-QUERY is the query to start when the button for the corresponding entry is
-activated. COUNT should be the number of messages matching the query.
-Such a list can be computed with `notmuch-hello-query-counts'."
+SEARCHES must be a list of plists each of which should contain at
+least the properties :name NAME :query QUERY and :count COUNT,
+where QUERY is the query to start when the button for the
+corresponding entry is activated, and COUNT should be the number
+of messages matching the query.  Such a plist can be computed
+with `notmuch-hello-query-counts'."
   (let* ((widest (notmuch-hello-longest-label searches))
 	 (tags-and-width (notmuch-hello-tags-per-line widest))
 	 (tags-per-line (car tags-and-width))
@@ -454,14 +551,19 @@ Such a list can be computed with `notmuch-hello-query-counts'."
 	    (when elem
 	      (if (> column-indent 0)
 		  (widget-insert (make-string column-indent ? )))
-	      (let* ((name (first elem))
-		     (query (second elem))
-		     (msg-count (third elem)))
+	      (let* ((name (plist-get elem :name))
+		     (query (plist-get elem :query))
+		     (oldest-first (case (plist-get elem :sort-order)
+				     (newest-first nil)
+				     (oldest-first t)
+				     (otherwise notmuch-search-oldest-first)))
+		     (msg-count (plist-get elem :count)))
 		(widget-insert (format "%8s "
 				       (notmuch-hello-nice-number msg-count)))
 		(widget-create 'push-button
 			       :notify #'notmuch-hello-widget-search
 			       :notmuch-search-terms query
+			       :notmuch-search-oldest-first oldest-first
 			       name)
 		(setq column-indent
 		      (1+ (max 0 (- column-width (length name)))))))
@@ -513,6 +615,18 @@ Such a list can be computed with `notmuch-hello-query-counts'."
       (remove-hook 'window-configuration-change-hook
 		   #'notmuch-hello-window-configuration-change))))
 
+;; the following variable is defined as being defconst in notmuch-version.el
+(defvar notmuch-emacs-version)
+
+(defun notmuch-hello-versions ()
+  "Display the notmuch version(s)"
+  (interactive)
+  (let ((notmuch-cli-version (notmuch-version)))
+    (message "notmuch version %s"
+	     (if (string= notmuch-emacs-version notmuch-cli-version)
+		 notmuch-cli-version
+	       (concat notmuch-cli-version
+		       " (emacs mua version " notmuch-emacs-version ")")))))
 
 (defvar notmuch-hello-mode-map
   (let ((map (if (fboundp 'make-composed-keymap)
@@ -523,8 +637,7 @@ Such a list can be computed with `notmuch-hello-query-counts'."
 	       ;; it's unlikely to change.
 	       (copy-keymap widget-keymap))))
     (set-keymap-parent map notmuch-common-keymap)
-    (define-key map "v" (lambda () "Display the notmuch version" (interactive)
-			  (message "notmuch version %s" (notmuch-version))))
+    (define-key map "v" 'notmuch-hello-versions)
     (define-key map (kbd "<C-tab>") 'widget-backward)
     map)
   "Keymap for \"notmuch hello\" buffers.")
@@ -687,13 +800,15 @@ Complete list of currently available key bindings:
       (indent-rigidly start (point) notmuch-hello-indent))
     nil))
 
-(defun notmuch-hello-insert-searches (title query-alist &rest options)
-  "Insert a section with TITLE showing a list of buttons made from QUERY-ALIST.
+(defun notmuch-hello-insert-searches (title query-list &rest options)
+  "Insert a section with TITLE showing a list of buttons made from QUERY-LIST.
 
-QUERY-ALIST must be a list containing elements of the form (NAME . QUERY)
-or (NAME QUERY COUNT-QUERY). If the latter form is used,
-COUNT-QUERY specifies an alternate query to be used to generate
-the count for the associated item.
+QUERY-LIST should ideally be a plist but for backwards
+compatibility other forms are also accepted (see
+`notmuch-saved-searches' for details).  The plist should
+contain keys :name and :query; if :count-query is also present
+then it specifies an alternate query to be used to generate the
+count for the associated search.
 
 Supports the following entries in OPTIONS as a plist:
 :initially-hidden - if non-nil, section will be hidden on startup
@@ -727,7 +842,7 @@ Supports the following entries in OPTIONS as a plist:
 		     "hide"))
     (widget-insert "\n")
     (when (not is-hidden)
-      (let ((searches (apply 'notmuch-hello-query-counts query-alist options)))
+      (let ((searches (apply 'notmuch-hello-query-counts query-list options)))
 	(when (or (not (plist-get options :hide-if-empty))
 		  searches)
 	  (widget-insert "\n")
@@ -788,6 +903,7 @@ following:
   "Run notmuch and display saved searches, known tags, etc."
   (interactive)
 
+  (notmuch-assert-cli-sane)
   ;; This may cause a window configuration change, so if the
   ;; auto-refresh hook is already installed, avoid recursive refresh.
   (let ((notmuch-hello-auto-refresh nil))
diff --git a/emacs/notmuch-lib.el b/emacs/notmuch-lib.el
index 49fe6445..2941da3e 100644
--- a/emacs/notmuch-lib.el
+++ b/emacs/notmuch-lib.el
@@ -107,12 +107,6 @@ Note that the recommended way of achieving the same is using
 (defvar notmuch-search-history nil
   "Variable to store notmuch searches history.")
 
-(defcustom notmuch-saved-searches '(("inbox" . "tag:inbox")
-				    ("unread" . "tag:unread"))
-  "A list of saved searches to display."
-  :type '(alist :key-type string :value-type string)
-  :group 'notmuch-hello)
-
 (defcustom notmuch-archive-tags '("-inbox")
   "List of tag changes to apply to a message or a thread when it is archived.
 
@@ -168,6 +162,24 @@ Otherwise the output will be returned"
       (notmuch-check-exit-status status (cons notmuch-command args) output)
       output)))
 
+(defvar notmuch--cli-sane-p nil
+  "Cache whether the CLI seems to be configured sanely.")
+
+(defun notmuch-cli-sane-p ()
+  "Return t if the cli seems to be configured sanely."
+  (unless notmuch--cli-sane-p
+    (let ((status (call-process notmuch-command nil nil nil
+				"config" "get" "user.primary_email")))
+      (setq notmuch--cli-sane-p (= status 0))))
+  notmuch--cli-sane-p)
+
+(defun notmuch-assert-cli-sane ()
+  (unless (notmuch-cli-sane-p)
+    (notmuch-logged-error
+     "notmuch cli seems misconfigured or unconfigured."
+"Perhaps you haven't run \"notmuch setup\" yet? Try running this
+on the command line, and then retry your notmuch command")))
+
 (defun notmuch-version ()
   "Return a string with the notmuch version number."
   (let ((long-string
@@ -180,8 +192,13 @@ Otherwise the output will be returned"
 
 (defun notmuch-config-get (item)
   "Return a value from the notmuch configuration."
-  ;; Trim off the trailing newline
-  (substring (notmuch-command-to-string "config" "get" item) 0 -1))
+  (let* ((val (notmuch-command-to-string "config" "get" item))
+	 (len (length val)))
+    ;; Trim off the trailing newline (if the value is empty or not
+    ;; configured, there will be no newline)
+    (if (and (> len 0) (= (aref val (- len 1)) ?\n))
+	(substring val 0 -1)
+      val)))
 
 (defun notmuch-database-path ()
   "Return the database.path value from the notmuch configuration."
@@ -197,7 +214,7 @@ Otherwise the output will be returned"
 
 (defun notmuch-user-other-email ()
   "Return the user.other_email value (as a list) from the notmuch configuration."
-  (split-string (notmuch-config-get "user.other_email") "\n"))
+  (split-string (notmuch-config-get "user.other_email") "\n" t))
 
 (defun notmuch-poll ()
   "Run \"notmuch new\" or an external script to import mail.
@@ -231,7 +248,8 @@ depending on the value of `notmuch-poll-script'."
   "Given a prefix key code, return a human-readable string representation.
 
 This is basically just `format-kbd-macro' but we also convert ESC to M-."
-  (let ((desc (format-kbd-macro (vector key))))
+  (let* ((key-vector (if (vectorp key) key (vector key)))
+	 (desc (format-kbd-macro key-vector)))
     (if (string= desc "ESC")
 	"M-"
       (concat desc " "))))
@@ -337,6 +355,28 @@ of its command symbol."
       (set-buffer-modified-p nil)
       (view-buffer (current-buffer) 'kill-buffer-if-not-modified))))
 
+(defun notmuch-subkeymap-help ()
+  "Show help for a subkeymap."
+  (interactive)
+  (let* ((key (this-command-keys-vector))
+	(prefix (make-vector (1- (length key)) nil))
+	(i 0))
+    (while (< i (length prefix))
+      (aset prefix i (aref key i))
+      (setq i (1+ i)))
+
+    (let* ((subkeymap (key-binding prefix))
+	   (ua-keys (where-is-internal 'universal-argument nil t))
+	   (prefix-string (notmuch-prefix-key-description prefix))
+	   (desc-alist (notmuch-describe-keymap subkeymap ua-keys subkeymap prefix-string))
+	   (desc-list (mapcar (lambda (arg) (concat (car arg) "\t" (cdr arg))) desc-alist))
+	   (desc (mapconcat #'identity desc-list "\n")))
+      (with-help-window (help-buffer)
+	(with-current-buffer standard-output
+	  (insert "\nPress 'q' to quit this window.\n\n")
+	  (insert desc)))
+      (pop-to-buffer (help-buffer)))))
+
 (defvar notmuch-buffer-refresh-function nil
   "Function to call to refresh the current buffer.")
 (make-variable-buffer-local 'notmuch-buffer-refresh-function)
@@ -380,7 +420,10 @@ user-friendly queries."
 
   (save-match-data
     (if (or (equal term "")
-	    (string-match "[ ()]\\|^\"" term))
+	    ;; To be pessimistic, only pass through terms composed
+	    ;; entirely of ASCII printing characters other than ", (,
+	    ;; and ).
+	    (string-match "[^!#-'*-~]" term))
 	;; Requires escaping
 	(concat "\"" (replace-regexp-in-string "\"" "\"\"" term t t) "\"")
       term)))
@@ -490,7 +533,8 @@ the given type."
 (if (>= emacs-major-version 24)
     (defadvice mm-shr (before load-gnus-arts activate)
       (require 'gnus-art nil t)
-      (ad-disable-advice 'mm-shr 'before 'load-gnus-arts)))
+      (ad-disable-advice 'mm-shr 'before 'load-gnus-arts)
+      (ad-activate 'mm-shr)))
 
 (defun notmuch-mm-display-part-inline (msg part nth content-type process-crypto)
   "Use the mm-decode/mm-view functions to display a part in the
@@ -531,23 +575,32 @@ single element face list."
       face
     (list face)))
 
-(defun notmuch-combine-face-text-property (start end face &optional below object)
-  "Combine FACE into the 'face text property between START and END.
+(defun notmuch-apply-face (object face &optional below start end)
+  "Combine FACE into the 'face text property of OBJECT between START and END.
 
 This function combines FACE with any existing faces between START
-and END in OBJECT (which defaults to the current buffer).
-Attributes specified by FACE take precedence over existing
-attributes unless BELOW is non-nil.  FACE must be a face name (a
-symbol or string), a property list of face attributes, or a list
-of these.  For convenience when applied to strings, this returns
-OBJECT."
+and END in OBJECT.  Attributes specified by FACE take precedence
+over existing attributes unless BELOW is non-nil.
+
+OBJECT may be a string, a buffer, or nil (which means the current
+buffer).  If object is a string, START and END are 0-based;
+otherwise they are buffer positions (integers or markers).  FACE
+must be a face name (a symbol or string), a property list of face
+attributes, or a list of these.  If START and/or END are omitted,
+they default to the beginning/end of OBJECT.  For convenience
+when applied to strings, this returns OBJECT."
 
   ;; A face property can have three forms: a face name (a string or
   ;; symbol), a property list, or a list of these two forms.  In the
   ;; list case, the faces will be combined, with the earlier faces
   ;; taking precedent.  Here we canonicalize everything to list form
   ;; to make it easy to combine.
-  (let ((pos start)
+  (let ((pos (cond (start start)
+		   ((stringp object) 0)
+		   (t 1)))
+	(end (cond (end end)
+		   ((stringp object) (length object))
+		   (t (1+ (buffer-size object)))))
 	(face-list (notmuch-face-ensure-list-form face)))
     (while (< pos end)
       (let* ((cur (get-text-property pos 'face object))
@@ -560,14 +613,6 @@ OBJECT."
 	(setq pos next))))
   object)
 
-(defun notmuch-combine-face-text-property-string (string face &optional below)
-  (notmuch-combine-face-text-property
-   0
-   (length string)
-   face
-   below
-   string))
-
 (defun notmuch-map-text-property (start end prop func &optional object)
   "Transform text property PROP using FUNC.
 
diff --git a/emacs/notmuch-mua.el b/emacs/notmuch-mua.el
index 00cd9808..95e4a4d3 100644
--- a/emacs/notmuch-mua.el
+++ b/emacs/notmuch-mua.el
@@ -115,6 +115,14 @@ list."
 	    (push header message-hidden-headers)))
 	notmuch-mua-hidden-headers))
 
+(defun notmuch-mua-reply-crypto (parts)
+  "Add mml sign-encrypt flag if any part of original message is encrypted."
+  (loop for part in parts
+	if (notmuch-match-content-type (plist-get part :content-type) "multipart/encrypted")
+	  do (mml-secure-message-sign-encrypt)
+	else if (notmuch-match-content-type (plist-get part :content-type) "multipart/*")
+	  do (notmuch-mua-reply-crypto (plist-get part :content))))
+
 (defun notmuch-mua-get-quotable-parts (parts)
   (loop for part in parts
 	if (notmuch-match-content-type (plist-get part :content-type) "multipart/alternative")
@@ -151,9 +159,10 @@ list."
 
 (defun notmuch-mua-reply (query-string &optional sender reply-all)
   (let ((args '("reply" "--format=sexp" "--format-version=1"))
+	(process-crypto notmuch-show-process-crypto)
 	reply
 	original)
-    (when notmuch-show-process-crypto
+    (when process-crypto
       (setq args (append args '("--decrypt"))))
 
     (if reply-all
@@ -224,28 +233,21 @@ list."
 	(set-mark (point))
 	(goto-char start)
 	;; Quote the original message according to the user's configured style.
-	(message-cite-original))))
+	(message-cite-original)))
 
-  (goto-char (point-max))
+    ;; Crypto processing based crypto content of the original message
+    (when process-crypto
+      (notmuch-mua-reply-crypto (plist-get original :body))))
+
+  ;; Push mark right before signature, if any.
+  (message-goto-signature)
+  (unless (eobp)
+    (end-of-line -1))
   (push-mark)
+
   (message-goto-body)
   (set-buffer-modified-p nil))
 
-(defun notmuch-mua-forward-message ()
-  (funcall (notmuch-mua-get-switch-function) (current-buffer))
-  (message-forward)
-
-  (when notmuch-mua-user-agent-function
-    (let ((user-agent (funcall notmuch-mua-user-agent-function)))
-      (when (not (string= "" user-agent))
-	(message-add-header (format "User-Agent: %s" user-agent)))))
-  (message-sort-headers)
-  (message-hide-headers)
-  (set-buffer-modified-p nil)
-  (notmuch-mua-maybe-set-window-dedicated)
-
-  (message-goto-to))
-
 (defun notmuch-mua-mail (&optional to subject other-headers &rest other-args)
   "Invoke the notmuch mail composition window.
 
@@ -287,31 +289,33 @@ the From: header is already filled in by notmuch."
 
 (defvar notmuch-mua-sender-history nil)
 
+;; Workaround: Running `ido-completing-read' in emacs 23.1, 23.2 and 23.3
+;; without some explicit initialization fill freeze the operation.
+;; Hence, we advice `ido-completing-read' to ensure required initialization
+;; is done.
+(if (and (= emacs-major-version 23) (< emacs-minor-version 4))
+    (defadvice ido-completing-read (before notmuch-ido-mode-init activate)
+      (ido-init-completion-maps)
+      (add-hook 'minibuffer-setup-hook 'ido-minibuffer-setup)
+      (add-hook 'choose-completion-string-functions
+		'ido-choose-completion-string)
+      (ad-disable-advice 'ido-completing-read 'before 'notmuch-ido-mode-init)
+      (ad-activate 'ido-completing-read)))
+
 (defun notmuch-mua-prompt-for-sender ()
-  (interactive)
-  (let (name addresses one-name-only)
-    ;; If notmuch-identities is non-nil, check if there is a fixed user name.
-    (if notmuch-identities
-	(let ((components (mapcar 'mail-extract-address-components notmuch-identities)))
-	  (setq name          (caar components)
-		addresses     (mapcar 'cadr components)
-		one-name-only (eval
-			       (cons 'and
-				     (mapcar (lambda (identity)
-					       (string-equal name (car identity)))
-					     components)))))
-      ;; If notmuch-identities is nil, use values from the notmuch configuration file.
-      (setq name          (notmuch-user-name)
-	    addresses     (cons (notmuch-user-primary-email) (notmuch-user-other-email))
-	    one-name-only t))
-    ;; Now prompt the user, either for an email address only or for a full identity.
-    (if one-name-only
-	(let ((address
-	       (ido-completing-read (concat "Sender address for " name ": ") addresses
-				    nil nil nil 'notmuch-mua-sender-history (car addresses))))
-	  (concat name " <" address ">"))
-      (ido-completing-read "Send mail From: " notmuch-identities
-			   nil nil nil 'notmuch-mua-sender-history (car notmuch-identities)))))
+  "Prompt for a sender from the user's configured identities."
+  (if notmuch-identities
+      (ido-completing-read "Send mail from: " notmuch-identities
+			   nil nil nil 'notmuch-mua-sender-history
+			   (car notmuch-identities))
+    (let* ((name (notmuch-user-name))
+	   (addrs (cons (notmuch-user-primary-email)
+			(notmuch-user-other-email)))
+	   (address
+	    (ido-completing-read (concat "Sender address for " name ": ") addrs
+				 nil nil nil 'notmuch-mua-sender-history
+				 (car addrs))))
+      (concat name " <" address ">"))))
 
 (put 'notmuch-mua-new-mail 'notmuch-prefix-doc "... and prompt for sender")
 (defun notmuch-mua-new-mail (&optional prompt-for-sender)
@@ -332,13 +336,17 @@ The current buffer must contain an RFC2822 message to forward.
 
 If PROMPT-FOR-SENDER is non-nil, the user will be prompted for
 the From: address first."
-  (if (or prompt-for-sender notmuch-always-prompt-for-sender)
-      (let* ((sender (notmuch-mua-prompt-for-sender))
-	     (address-components (mail-extract-address-components sender))
-	     (user-full-name (car address-components))
-	     (user-mail-address (cadr address-components)))
-	(notmuch-mua-forward-message))
-    (notmuch-mua-forward-message)))
+  (let* ((cur (current-buffer))
+	 (message-forward-decoded-p nil)
+	 (subject (message-make-forward-subject))
+	 (other-headers
+	  (when (or prompt-for-sender notmuch-always-prompt-for-sender)
+	    (list (cons 'From (notmuch-mua-prompt-for-sender))))))
+    (notmuch-mua-mail nil subject other-headers nil (notmuch-mua-get-switch-function))
+    (message-forward-make-body cur)
+    ;; `message-forward-make-body' shows the User-agent header.  Hide
+    ;; it again.
+    (message-hide-headers)))
 
 (defun notmuch-mua-new-reply (query-string &optional prompt-for-sender reply-all)
   "Compose a reply to the message identified by QUERY-STRING.
diff --git a/emacs/notmuch-show.el b/emacs/notmuch-show.el
index 784644cd..df10d4ba 100644
--- a/emacs/notmuch-show.el
+++ b/emacs/notmuch-show.el
@@ -171,7 +171,7 @@ each attachment handler is logged in buffers with names beginning
 (defcustom notmuch-show-stash-mlarchive-link-alist
   '(("Gmane" . "http://mid.gmane.org/")
     ("MARC" . "http://marc.info/?i=")
-    ("Mail Archive, The" . "http://mail-archive.com/search?l=mid&q=")
+    ("Mail Archive, The" . "http://mid.mail-archive.com/")
     ("LKML" . "http://lkml.kernel.org/r/")
     ;; FIXME: can these services be searched by `Message-Id' ?
     ;; ("MarkMail" . "http://markmail.org/")
@@ -344,7 +344,7 @@ operation on the contents of the current buffer."
     (if (re-search-forward "(\\([^()]*\\))$" (line-end-position) t)
 	(let ((inhibit-read-only t))
 	  (replace-match (concat "("
-				 (notmuch-tag-format-tags tags)
+				 (notmuch-tag-format-tags tags (notmuch-show-get-prop :orig-tags))
 				 ")"))))))
 
 (defun notmuch-clean-address (address)
@@ -423,7 +423,7 @@ message at DEPTH in the current thread."
 	    " ("
 	    date
 	    ") ("
-	    (notmuch-tag-format-tags tags)
+	    (notmuch-tag-format-tags tags tags)
 	    ")\n")
     (overlay-put (make-overlay start (point)) 'face 'notmuch-message-summary-face)))
 
@@ -785,7 +785,10 @@ message at DEPTH in the current thread."
     (while (and handlers
 		(not (condition-case err
 			 (funcall (car handlers) msg part content-type nth depth button)
-		       (error (progn
+		       ;; Specifying `debug' here lets the debugger
+		       ;; run if `debug-on-error' is non-nil.
+		       ((debug error)
+			(progn
 				(insert "!!! Bodypart insert error: ")
 				(insert (error-message-string err))
 				(insert " !!!\n") nil)))))
@@ -1145,6 +1148,7 @@ function is used."
     ;; Don't track undo information for this buffer
     (set 'buffer-undo-list t)
 
+    (notmuch-tag-clear-cache)
     (erase-buffer)
     (goto-char (point-min))
     (save-excursion
@@ -1167,6 +1171,8 @@ function is used."
 
       (jit-lock-register #'notmuch-show-buttonise-links)
 
+      (notmuch-show-mapc (lambda () (notmuch-show-set-prop :orig-tags (notmuch-show-get-tags))))
+
       ;; Set the header line to the subject of the first message.
       (setq header-line-format (notmuch-sanitize (notmuch-show-strip-re (notmuch-show-get-subject))))
 
@@ -1241,6 +1247,7 @@ reset based on the original query."
     (define-key map "t" 'notmuch-show-stash-to)
     (define-key map "l" 'notmuch-show-stash-mlarchive-link)
     (define-key map "L" 'notmuch-show-stash-mlarchive-link-and-go)
+    (define-key map "?" 'notmuch-subkeymap-help)
     map)
   "Submap for stash commands")
 (fset 'notmuch-show-stash-map notmuch-show-stash-map)
@@ -1251,6 +1258,7 @@ reset based on the original query."
     (define-key map "v" 'notmuch-show-view-part)
     (define-key map "o" 'notmuch-show-interactively-view-part)
     (define-key map "|" 'notmuch-show-pipe-part)
+    (define-key map "?" 'notmuch-subkeymap-help)
     map)
   "Submap for part commands")
 (fset 'notmuch-show-part-map notmuch-show-part-map)
@@ -1779,10 +1787,14 @@ message."
       (setq shell-command
 	    (concat notmuch-command " show --format=raw "
 		    (shell-quote-argument (notmuch-show-get-message-id)) " | " command)))
-    (let ((buf (get-buffer-create (concat "*notmuch-pipe*"))))
+    (let ((cwd default-directory)
+	  (buf (get-buffer-create (concat "*notmuch-pipe*"))))
       (with-current-buffer buf
 	(setq buffer-read-only nil)
 	(erase-buffer)
+	;; Use the originating buffer's working directory instead of
+	;; that of the pipe buffer.
+	(cd cwd)
 	(let ((exit-code (call-process-shell-command shell-command nil buf)))
 	  (goto-char (point-max))
 	  (set-buffer-modified-p nil)
diff --git a/emacs/notmuch-tag.el b/emacs/notmuch-tag.el
index b60f46c7..f54aa9d6 100644
--- a/emacs/notmuch-tag.el
+++ b/emacs/notmuch-tag.el
@@ -28,35 +28,9 @@
 (require 'crm)
 (require 'notmuch-lib)
 
-(defcustom notmuch-tag-formats
-  '(("unread" (propertize tag 'face '(:foreground "red")))
-    ("flagged" (propertize tag 'face '(:foreground "blue"))
-     (notmuch-tag-format-image-data tag (notmuch-tag-star-icon))))
-  "Custom formats for individual tags.
-
-This gives a list that maps from tag names to lists of formatting
-expressions.  The car of each element gives a tag name and the
-cdr gives a list of Elisp expressions that modify the tag.  If
-the list is empty, the tag will simply be hidden.  Otherwise,
-each expression will be evaluated in order: for the first
-expression, the variable `tag' will be bound to the tag name; for
-each later expression, the variable `tag' will be bound to the
-result of the previous expression.  In this way, each expression
-can build on the formatting performed by the previous expression.
-The result of the last expression will displayed in place of the
-tag.
-
-For example, to replace a tag with another string, simply use
-that string as a formatting expression.  To change the foreground
-of a tag to red, use the expression
-  (propertize tag 'face '(:foreground \"red\"))
-
-See also `notmuch-tag-format-image', which can help replace tags
-with images."
-
-  :group 'notmuch-search
-  :group 'notmuch-show
-  :type '(alist :key-type (string :tag "Tag")
+(define-widget 'notmuch-tag-format-type 'lazy
+  "Customize widget for notmuch-tag-format and friends"
+  :type '(alist :key-type (regexp :tag "Tag")
 		:extra-offset -3
 		:value-type
 		(radio :format "%v"
@@ -65,7 +39,7 @@ with images."
 			    (string :tag "Display as")
 			    (list :tag "Face" :extra-offset -4
 				  (const :format "" :inline t
-					 (propertize tag 'face))
+					 (notmuch-apply-face tag))
 				  (list :format "%v"
 					(const :format "" quote)
 					custom-face-edit))
@@ -82,6 +56,83 @@ with images."
 					  (string :tag "Custom")))
 			    (sexp :tag "Custom")))))
 
+(defcustom notmuch-tag-formats
+  '(("unread" (propertize tag 'face '(:foreground "red")))
+    ("flagged" (propertize tag 'face '(:foreground "blue"))
+     (notmuch-tag-format-image-data tag (notmuch-tag-star-icon))))
+  "Custom formats for individual tags.
+
+This is an association list that maps from tag name regexps to
+lists of formatting expressions.  The first entry whose car
+regexp-matches a tag will be used to format that tag.  The regexp
+is implicitly anchored, so to match a literal tag name, just use
+that tag name (if it contains special regexp characters like
+\".\" or \"*\", these have to be escaped).  The cdr of the
+matching entry gives a list of Elisp expressions that modify the
+tag.  If the list is empty, the tag will simply be hidden.
+Otherwise, each expression will be evaluated in order: for the
+first expression, the variable `tag' will be bound to the tag
+name; for each later expression, the variable `tag' will be bound
+to the result of the previous expression.  In this way, each
+expression can build on the formatting performed by the previous
+expression.  The result of the last expression will displayed in
+place of the tag.
+
+For example, to replace a tag with another string, simply use
+that string as a formatting expression.  To change the foreground
+of a tag to red, use the expression
+  (propertize tag 'face '(:foreground \"red\"))
+
+See also `notmuch-tag-format-image', which can help replace tags
+with images."
+  :group 'notmuch-search
+  :group 'notmuch-show
+  :group 'notmuch-faces
+  :type 'notmuch-tag-format-type)
+
+(defcustom notmuch-tag-deleted-formats
+  '(("unread" (notmuch-apply-face bare-tag
+				  (if (display-supports-face-attributes-p '(:strike-through "red"))
+				      '(:strike-through "red")
+				    '(:inverse-video t))))
+    (".*" (notmuch-apply-face tag
+			      (if (display-supports-face-attributes-p '(:strike-through "red"))
+				  '(:strike-through "red")
+				'(:inverse-video t)))))
+  "Custom formats for tags when deleted.
+
+For deleted tags the formats in `notmuch-tag-formats` are applied
+first and then these formats are applied on top; that is `tag'
+passed to the function is the tag with all these previous
+formattings applied. The formatted can access the original
+unformatted tag as `bare-tag'.
+
+By default this shows deleted tags with strike-through in red,
+unless strike-through is not available (e.g., emacs is running in
+a terminal) in which case it uses inverse video. To hide deleted
+tags completely set this to
+  '((\".*\" nil))
+
+See `notmuch-tag-formats' for full documentation."
+  :group 'notmuch-show
+  :group 'notmuch-faces
+  :type 'notmuch-tag-format-type)
+
+(defcustom notmuch-tag-added-formats
+  '((".*" (notmuch-apply-face tag '(:underline "green"))))
+  "Custom formats for tags when added.
+
+For added tags the formats in `notmuch-tag-formats` are applied
+first and then these formats are applied on top.
+
+To disable special formatting of added tags, set this variable to
+nil.
+
+See `notmuch-tag-formats' for full documentation."
+  :group 'notmuch-show
+  :group 'notmuch-faces
+  :type 'notmuch-tag-format-type)
+
 (defun notmuch-tag-format-image-data (tag data)
   "Replace TAG with image DATA, if available.
 
@@ -135,28 +186,81 @@ This can be used with `notmuch-tag-format-image-data'."
   </g>
 </svg>")
 
-(defun notmuch-tag-format-tag (tag)
-  "Format TAG by looking into `notmuch-tag-formats'."
-  (let ((formats (assoc tag notmuch-tag-formats)))
-    (cond
-     ((null formats)		;; - Tag not in `notmuch-tag-formats',
-      tag)			;;   the format is the tag itself.
-     ((null (cdr formats))	;; - Tag was deliberately hidden,
-      nil)			;;   no format must be returned
-     (t				;; - Tag was found and has formats,
-      (let ((tag tag))		;;   we must apply all the formats.
-	(dolist (format (cdr formats) tag)
-	  (setq tag (eval format))))))))
+(defvar notmuch-tag--format-cache (make-hash-table :test 'equal)
+  "Cache of tag format lookup.  Internal to `notmuch-tag-format-tag'.")
+
+(defun notmuch-tag-clear-cache ()
+  "Clear the internal cache of tag formats."
+  (clrhash notmuch-tag--format-cache))
+
+(defun notmuch-tag--get-formats (tag format-alist)
+  "Find the first item whose car regexp-matches TAG."
+  (save-match-data
+    ;; Don't use assoc-default since there's no way to distinguish a
+    ;; missing key from a present key with a null cdr.
+    (assoc* tag format-alist
+	    :test (lambda (tag key)
+		    (and (eq (string-match key tag) 0)
+			 (= (match-end 0) (length tag)))))))
+
+(defun notmuch-tag--do-format (tag formatted-tag formats)
+  "Apply a tag-formats entry to TAG."
+  (cond ((null formats)		;; - Tag not in `formats',
+	 formatted-tag)	       	;;   the format is the tag itself.
+	((null (cdr formats))	;; - Tag was deliberately hidden,
+	 nil)			;;   no format must be returned
+	(t
+	 ;; Tag was found and has formats, we must apply all the
+	 ;; formats.  TAG may be null so treat that as a special case.
+	 (let ((bare-tag tag)
+	       (tag (copy-sequence (or formatted-tag ""))))
+	   (dolist (format (cdr formats))
+	     (setq tag (eval format)))
+	   (if (and (null formatted-tag) (equal tag ""))
+	       nil
+	     tag)))))
+
+(defun notmuch-tag-format-tag (tags orig-tags tag)
+  "Format TAG according to `notmuch-tag-formats'.
+
+TAGS and ORIG-TAGS are lists of the current tags and the original
+tags; tags which have been deleted (i.e., are in ORIG-TAGS but
+are not in TAGS) are shown using formats from
+`notmuch-tag-deleted-formats'; tags which have been added (i.e.,
+are in TAGS but are not in ORIG-TAGS) are shown using formats
+from `notmuch-tag-added-formats' and tags which have not been
+changed (the normal case) are shown using formats from
+`notmuch-tag-formats'"
+  (let* ((tag-state (cond ((not (member tag tags)) 'deleted)
+			  ((not (member tag orig-tags)) 'added)))
+	 (formatted-tag (gethash (cons tag tag-state) notmuch-tag--format-cache 'missing)))
+    (when (eq formatted-tag 'missing)
+      (let ((base (notmuch-tag--get-formats tag notmuch-tag-formats))
+	    (over (case tag-state
+		    (deleted (notmuch-tag--get-formats
+			      tag notmuch-tag-deleted-formats))
+		    (added (notmuch-tag--get-formats
+			    tag notmuch-tag-added-formats))
+		    (otherwise nil))))
+	(setq formatted-tag (notmuch-tag--do-format tag tag base))
+	(setq formatted-tag (notmuch-tag--do-format tag formatted-tag over))
+
+	(puthash (cons tag tag-state) formatted-tag notmuch-tag--format-cache)))
+    formatted-tag))
 
-(defun notmuch-tag-format-tags (tags)
+(defun notmuch-tag-format-tags (tags orig-tags &optional face)
   "Return a string representing formatted TAGS."
-  (notmuch-combine-face-text-property-string
-   (mapconcat #'identity
-	      ;; nil indicated that the tag was deliberately hidden
-	      (delq nil (mapcar #'notmuch-tag-format-tag tags))
-	      " ")
-   'notmuch-tag-face
-   t))
+  (let ((face (or face 'notmuch-tag-face))
+	(all-tags (sort (delete-dups (append tags orig-tags nil)) #'string<)))
+    (notmuch-apply-face
+     (mapconcat #'identity
+		;; nil indicated that the tag was deliberately hidden
+		(delq nil (mapcar
+			   (apply-partially #'notmuch-tag-format-tag tags orig-tags)
+			   all-tags))
+		" ")
+     face
+     t)))
 
 (defcustom notmuch-before-tag-hook nil
   "Hooks that are run before tags of a message are modified.
@@ -283,6 +387,8 @@ notmuch-after-tag-hook will be run."
 	  (unless (string-match-p "^[-+]\\S-+$" tag-change)
 	    (error "Tag must be of the form `+this_tag' or `-that_tag'")))
 	tag-changes)
+  (unless query
+    (error "Nothing to tag!"))
   (unless (null tag-changes)
     (run-hooks 'notmuch-before-tag-hook)
     (if (<= (length query) notmuch-tag-argument-limit)
diff --git a/emacs/notmuch-tree.el b/emacs/notmuch-tree.el
index 8d59e65f..7d5f4750 100644
--- a/emacs/notmuch-tree.el
+++ b/emacs/notmuch-tree.el
@@ -70,8 +70,14 @@ Note the author string should not contain
   :group 'notmuch-tree)
 
 ;; Faces for messages that match the query.
-(defface notmuch-tree-match-date-face
+(defface notmuch-tree-match-face
   '((t :inherit default))
+  "Default face used in tree mode face for matching messages"
+  :group 'notmuch-tree
+  :group 'notmuch-faces)
+
+(defface notmuch-tree-match-date-face
+  nil
   "Face used in tree mode for the date in messages matching the query."
   :group 'notmuch-tree
   :group 'notmuch-faces)
@@ -90,13 +96,13 @@ Note the author string should not contain
   :group 'notmuch-faces)
 
 (defface notmuch-tree-match-subject-face
-  '((t :inherit default))
+  nil
   "Face used in tree mode for the subject in messages matching the query."
   :group 'notmuch-tree
   :group 'notmuch-faces)
 
 (defface notmuch-tree-match-tree-face
-  '((t :inherit default))
+  nil
   "Face used in tree mode for the thread tree block graphics in messages matching the query."
   :group 'notmuch-tree
   :group 'notmuch-faces)
@@ -115,32 +121,38 @@ Note the author string should not contain
   :group 'notmuch-faces)
 
 ;; Faces for messages that do not match the query.
-(defface notmuch-tree-no-match-date-face
+(defface notmuch-tree-no-match-face
   '((t (:foreground "gray")))
+  "Default face used in tree mode face for non-matching messages"
+  :group 'notmuch-tree
+  :group 'notmuch-faces)
+
+(defface notmuch-tree-no-match-date-face
+  nil
   "Face used in tree mode for non-matching dates."
   :group 'notmuch-tree
   :group 'notmuch-faces)
 
 (defface notmuch-tree-no-match-subject-face
-  '((t (:foreground "gray")))
+  nil
   "Face used in tree mode for non-matching subjects."
   :group 'notmuch-tree
   :group 'notmuch-faces)
 
 (defface notmuch-tree-no-match-tree-face
-  '((t (:foreground "gray")))
+  nil
   "Face used in tree mode for the thread tree block graphics in messages matching the query."
   :group 'notmuch-tree
   :group 'notmuch-faces)
 
 (defface notmuch-tree-no-match-author-face
-  '((t (:foreground "gray")))
+  nil
   "Face used in tree mode for the date in messages matching the query."
   :group 'notmuch-tree
   :group 'notmuch-faces)
 
 (defface notmuch-tree-no-match-tag-face
-  '((t (:foreground "gray")))
+  nil
   "Face used in tree mode face for non-matching tags."
   :group 'notmuch-tree
   :group 'notmuch-faces)
@@ -319,11 +331,13 @@ correct message properties."
   "Return the tags of the current message."
   (notmuch-tree-get-prop :tags))
 
-(defun notmuch-tree-get-message-id ()
+(defun notmuch-tree-get-message-id (&optional bare)
   "Return the message id of the current message."
   (let ((id (notmuch-tree-get-prop :id)))
     (if id
-	(notmuch-id-to-query id)
+	(if bare
+	    id
+	  (notmuch-id-to-query id))
       nil)))
 
 (defun notmuch-tree-get-match ()
@@ -687,20 +701,22 @@ unchanged ADDRESS if parsing fails."
 
      ((string-equal field "tags")
       (let ((tags (plist-get msg :tags))
+	    (orig-tags (plist-get msg :orig-tags))
 	    (face (if match
 		      'notmuch-tree-match-tag-face
 		    'notmuch-tree-no-match-tag-face)))
-	(propertize (format format-string
-			    (mapconcat #'identity tags ", "))
-		    'face face))))))
-
+	(format format-string (notmuch-tag-format-tags tags orig-tags face)))))))
 
 (defun notmuch-tree-format-field-list (field-list msg)
   "Format fields of MSG according to FIELD-LIST and return string"
-  (let (result-string)
+  (let ((face (if (plist-get msg :match)
+		  'notmuch-tree-match-face
+		'notmuch-tree-no-match-face))
+	(result-string))
     (dolist (spec field-list result-string)
       (let ((field-string (notmuch-tree-format-field (car spec) (cdr spec) msg)))
-	(setq result-string (concat result-string field-string))))))
+	(setq result-string (concat result-string field-string))))
+    (notmuch-apply-face result-string face t)))
 
 (defun notmuch-tree-insert-msg (msg)
   "Insert the message MSG according to notmuch-tree-result-format"
@@ -751,8 +767,10 @@ message together with all its descendents."
 	(push "├" tree-status)))
 
       (push (concat (if replies "┬" "─") "►") tree-status)
-      (plist-put msg :first (and first (eq 0 depth)))
-      (notmuch-tree-goto-and-insert-msg (plist-put msg :tree-status tree-status))
+      (setq msg (plist-put msg :first (and first (eq 0 depth))))
+      (setq msg (plist-put msg :tree-status tree-status))
+      (setq msg (plist-put msg :orig-tags (plist-get msg :tags)))
+      (notmuch-tree-goto-and-insert-msg msg)
       (pop tree-status)
       (pop tree-status)
 
@@ -866,6 +884,7 @@ the same as for the function notmuch-tree."
 	 (message-arg "--entire-thread"))
     (if (equal (car (process-lines notmuch-command "count" search-args)) "0")
 	(setq search-args basic-query))
+    (notmuch-tag-clear-cache)
     (let ((proc (notmuch-start-notmuch
 		 "notmuch-tree" (current-buffer) #'notmuch-tree-process-sentinel
 		 "show" "--body=false" "--format=sexp"
diff --git a/emacs/notmuch-version.el.tmpl b/emacs/notmuch-version.el.tmpl
new file mode 100644
index 00000000..236aaf7d
--- /dev/null
+++ b/emacs/notmuch-version.el.tmpl
@@ -0,0 +1,23 @@
+;; -*- emacs-lisp -*-
+;;
+;; %AG%
+;;
+;; This file is part of Notmuch.
+;;
+;; Notmuch is free software: you can redistribute it and/or modify it
+;; under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+;;
+;; Notmuch is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;; General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with Notmuch.  If not, see <http://www.gnu.org/licenses/>.
+
+(defconst notmuch-emacs-version %VERSION%
+  "Version of Notmuch Emacs MUA.")
+
+(provide 'notmuch-version)
diff --git a/emacs/notmuch.el b/emacs/notmuch.el
index c9bc2f22..1adea9c2 100644
--- a/emacs/notmuch.el
+++ b/emacs/notmuch.el
@@ -36,7 +36,7 @@
 ;;
 ;; Then, to actually run it, add:
 ;;
-;;	(require 'notmuch)
+;;	(autoload 'notmuch "notmuch" "Notmuch mail" t)
 ;;
 ;; to your ~/.emacs file, and then run "M-x notmuch" from within emacs,
 ;; or run:
@@ -61,6 +61,10 @@
 (require 'notmuch-message)
 (require 'notmuch-parser)
 
+(unless (require 'notmuch-version nil t)
+  (defconst notmuch-emacs-version "unknown"
+    "Placeholder variable when notmuch-version.el[c] is not available."))
+
 (defcustom notmuch-search-result-format
   `(("date" . "%12s ")
     ("count" . "%-7s ")
@@ -81,6 +85,18 @@ To enter a line break in customize, press \\[quoted-insert] C-j."
   :type '(alist :key-type (string) :value-type (string))
   :group 'notmuch-search)
 
+;; The name of this variable `notmuch-init-file' is consistent with the
+;; convention used in e.g. emacs and gnus. The value, `notmuch-config[.el[c]]'
+;; is consistent with notmuch cli configuration file `~/.notmuch-config'.
+(defcustom notmuch-init-file (locate-user-emacs-file "notmuch-config")
+  "Your Notmuch Emacs-Lisp configuration file name.
+If a file with one of the suffixes defined by `get-load-suffixes' exists,
+it will be read instead.
+This file is read once when notmuch is loaded; the notmuch hooks added
+there will be called at other points of notmuch execution."
+  :type 'file
+  :group 'notmuch)
+
 (defvar notmuch-query-history nil
   "Variable to store minibuffer history for notmuch queries")
 
@@ -165,6 +181,7 @@ To enter a line break in customize, press \\[quoted-insert] C-j."
 (defvar notmuch-search-stash-map
   (let ((map (make-sparse-keymap)))
     (define-key map "i" 'notmuch-search-stash-thread-id)
+    (define-key map "?" 'notmuch-subkeymap-help)
     map)
   "Submap for stash commands")
 (fset 'notmuch-search-stash-map notmuch-search-stash-map)
@@ -411,14 +428,16 @@ matched and unmatched messages in the current thread."
   "Return the stable query for the current region.
 
 If ONLY-MATCHED is non-nil, include only matched messages.  If it
-is nil, include both matched and unmatched messages."
+is nil, include both matched and unmatched messages. If there are
+no messages in the region then return nil."
   (let ((query-list nil) (all (not only-matched)))
     (dolist (queries (notmuch-search-properties-in-region :query beg end))
       (when (first queries)
 	(push (first queries) query-list))
       (when (and all (second queries))
 	(push (second queries) query-list)))
-    (concat "(" (mapconcat 'identity query-list ") or (") ")")))
+    (when query-list
+      (concat "(" (mapconcat 'identity query-list ") or (") ")"))))
 
 (defun notmuch-search-find-authors ()
   "Return the authors for the current thread"
@@ -648,7 +667,7 @@ foreground and blue background."
 	  (let ((tag (car elem))
 		(attributes (cdr elem)))
 	    (when (member tag line-tag-list)
-	      (notmuch-combine-face-text-property start end attributes))))
+	      (notmuch-apply-face nil attributes nil start end))))
 	;; Reverse the list so earlier entries take precedence
 	(reverse notmuch-search-line-faces)))
 
@@ -752,24 +771,33 @@ non-authors is found, assume that all of the authors match."
      format-string (notmuch-sanitize (plist-get result :authors))))
 
    ((string-equal field "tags")
-    (let ((tags (plist-get result :tags)))
-      (insert (format format-string (notmuch-tag-format-tags tags)))))))
+    (let ((tags (plist-get result :tags))
+	  (orig-tags (plist-get result :orig-tags)))
+      (insert (format format-string (notmuch-tag-format-tags tags orig-tags)))))))
 
-(defun notmuch-search-show-result (result &optional pos)
-  "Insert RESULT at POS or the end of the buffer if POS is null."
+(defun notmuch-search-show-result (result pos)
+  "Insert RESULT at POS."
   ;; Ignore excluded matches
   (unless (= (plist-get result :matched) 0)
-    (let ((beg (or pos (point-max))))
-      (save-excursion
-	(goto-char beg)
-	(dolist (spec notmuch-search-result-format)
-	  (notmuch-search-insert-field (car spec) (cdr spec) result))
-	(insert "\n")
-	(notmuch-search-color-line beg (point) (plist-get result :tags))
-	(put-text-property beg (point) 'notmuch-search-result result))
-      (when (string= (plist-get result :thread) notmuch-search-target-thread)
-	(setq notmuch-search-target-thread "found")
-	(goto-char beg)))))
+    (save-excursion
+      (goto-char pos)
+      (dolist (spec notmuch-search-result-format)
+	(notmuch-search-insert-field (car spec) (cdr spec) result))
+      (insert "\n")
+      (notmuch-search-color-line pos (point) (plist-get result :tags))
+      (put-text-property pos (point) 'notmuch-search-result result))))
+
+(defun notmuch-search-append-result (result)
+  "Insert RESULT at the end of the buffer.
+
+This is only called when a result is first inserted so it also
+sets the :orig-tag property."
+  (let ((new-result (plist-put result :orig-tags (plist-get result :tags)))
+	(pos (point-max)))
+    (notmuch-search-show-result new-result pos)
+    (when (string= (plist-get result :thread) notmuch-search-target-thread)
+      (setq notmuch-search-target-thread "found")
+      (goto-char pos))))
 
 (defun notmuch-search-process-filter (proc string)
   "Process and filter the output of \"notmuch search\""
@@ -783,7 +811,7 @@ non-authors is found, assume that all of the authors match."
 	(save-excursion
 	  (goto-char (point-max))
 	  (insert string))
-	(notmuch-sexp-parse-partial-list 'notmuch-search-show-result
+	(notmuch-sexp-parse-partial-list 'notmuch-search-append-result
 					 results-buf)))))
 
 (defun notmuch-search-tag-all (tag-changes)
@@ -801,14 +829,14 @@ See `notmuch-tag' for information on the format of TAG-CHANGES."
 	  (let (longest
 		(longest-length 0))
 	    (loop for tuple in notmuch-saved-searches
-		  if (let ((quoted-query (regexp-quote (cdr tuple))))
+		  if (let ((quoted-query (regexp-quote (notmuch-saved-search-get tuple :query))))
 		       (and (string-match (concat "^" quoted-query) query)
 			    (> (length (match-string 0 query))
 			       longest-length)))
 		  do (setq longest tuple))
 	    longest))
-	 (saved-search-name (car saved-search))
-	 (saved-search-query (cdr saved-search)))
+	 (saved-search-name (notmuch-saved-search-get saved-search :name))
+	 (saved-search-query (notmuch-saved-search-get saved-search :query)))
     (cond ((and saved-search (equal saved-search-query query))
 	   ;; Query is the same as saved search (ignoring case)
 	   (concat "*notmuch-saved-search-" saved-search-name "*"))
@@ -828,7 +856,7 @@ See `notmuch-tag' for information on the format of TAG-CHANGES."
 PROMPT is the string to prompt with."
   (lexical-let
       ((completions
-	(append (list "folder:" "thread:" "id:" "date:" "from:" "to:"
+	(append (list "folder:" "path:" "thread:" "id:" "date:" "from:" "to:"
 		      "subject:" "attachment:")
 		(mapcar (lambda (tag)
 			  (concat "tag:" (notmuch-escape-boolean-term tag)))
@@ -887,6 +915,7 @@ the configured default sort order."
     (set 'notmuch-search-oldest-first oldest-first)
     (set 'notmuch-search-target-thread target-thread)
     (set 'notmuch-search-target-line target-line)
+    (notmuch-tag-clear-cache)
     (let ((proc (get-buffer-process (current-buffer)))
 	  (inhibit-read-only t))
       (if proc
@@ -1002,3 +1031,9 @@ notmuch buffers exist, run `notmuch'."
 (setq mail-user-agent 'notmuch-user-agent)
 
 (provide 'notmuch)
+
+;; After provide to avoid loops if notmuch was require'd via notmuch-init-file.
+(if init-file-user ; don't load init file if the -q option was used.
+    (let ((init-file (locate-file notmuch-init-file '("/")
+				  (get-load-suffixes))))
+      (if init-file (load init-file nil t t))))
diff --git a/hooks.c b/hooks.c
index 44ee4198..662629a9 100644
--- a/hooks.c
+++ b/hooks.c
@@ -50,6 +50,9 @@ notmuch_run_hook (const char *db_path, const char *hook)
 	goto DONE;
     }
 
+    /* Flush any buffered output before forking. */
+    fflush (stdout);
+
     pid = fork();
     if (pid == -1) {
 	fprintf (stderr, "Error: %s hook fork failed: %s\n", hook,
diff --git a/lib/database.cc b/lib/database.cc
index f395061e..1efb14d4 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -42,7 +42,7 @@ typedef struct {
     const char *prefix;
 } prefix_t;
 
-#define NOTMUCH_DATABASE_VERSION 1
+#define NOTMUCH_DATABASE_VERSION 2
 
 #define STRINGIFY(s) _SUB_STRINGIFY(s)
 #define _SUB_STRINGIFY(s) #s
@@ -100,8 +100,8 @@ typedef struct {
  * In addition, terms from the content of the message are added with
  * "from", "to", "attachment", and "subject" prefixes for use by the
  * user in searching. Similarly, terms from the path of the mail
- * message are added with a "folder" prefix. But the database doesn't
- * really care itself about any of these.
+ * message are added with "folder" and "path" prefixes. But the
+ * database doesn't really care itself about any of these.
  *
  * The data portion of a mail document is empty.
  *
@@ -208,7 +208,15 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "thread",			"G" },
     { "tag",			"K" },
     { "is",			"K" },
-    { "id",			"Q" }
+    { "id",			"Q" },
+    { "path",			"P" },
+    /*
+     * Without the ":", since this is a multi-letter prefix, Xapian
+     * will add a colon itself if the first letter of the path is
+     * upper-case ASCII. Including the ":" forces there to always be a
+     * colon, which keeps our own logic simpler.
+     */
+    { "folder",			"XFOLDER:" },
 };
 
 static prefix_t PROBABILISTIC_PREFIX[]= {
@@ -216,7 +224,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= {
     { "to",			"XTO" },
     { "attachment",		"XATTACHMENT" },
     { "subject",		"XSUBJECT"},
-    { "folder",			"XFOLDER"}
 };
 
 const char *
@@ -1167,6 +1174,40 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
 	}
     }
 
+    /*
+     * Prior to version 2, the "folder:" prefix was probabilistic and
+     * stemmed. Change it to the current boolean prefix. Add "path:"
+     * prefixes while at it.
+     */
+    if (version < 2) {
+	notmuch_query_t *query = notmuch_query_create (notmuch, "");
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+
+	count = 0;
+	total = notmuch_query_count_messages (query);
+
+	for (messages = notmuch_query_search_messages (query);
+	     notmuch_messages_valid (messages);
+	     notmuch_messages_move_to_next (messages)) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    _notmuch_message_upgrade_folder (message);
+	    _notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+    }
+
     db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
     db->flush ();
 
@@ -1930,15 +1971,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     if (ret)
 	goto DONE;
 
-    notmuch_message_file_restrict_headers (message_file,
-					   "date",
-					   "from",
-					   "in-reply-to",
-					   "message-id",
-					   "references",
-					   "subject",
-					   "to",
-					   (char *) NULL);
+    /* Parse message up front to get better error status. */
+    ret = _notmuch_message_file_parse (message_file);
+    if (ret)
+	goto DONE;
 
     try {
 	/* Before we do any real work, (especially before doing a
@@ -2025,7 +2061,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	    date = notmuch_message_file_get_header (message_file, "date");
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
-	    ret = _notmuch_message_index_file (message, filename);
+	    ret = _notmuch_message_index_file (message, message_file);
 	    if (ret)
 		goto DONE;
 	} else {
diff --git a/lib/gen-version-script.sh b/lib/gen-version-script.sh
index 76670d57..64a73749 100644
--- a/lib/gen-version-script.sh
+++ b/lib/gen-version-script.sh
@@ -23,6 +23,6 @@ while read sym; do
 	    ;;
     esac
 done
-nm $* | awk '$1 ~ "^[0-9a-fA-F][0-9a-fA-F]*$" && $2 == "T" && $3 ~ "^get(line|delim)$" {print $3 ";"}'
+nm $* | awk '$1 ~ "^[0-9a-fA-F][0-9a-fA-F]*$" && $2 == "T" && $3 ~ "^(getline|getdelim|canonicalize_file_name)$" {print $3 ";"}'
 sed  -n 's/^[[:space:]]*\(notmuch_[a-z_]*\)[[:space:]]*(.*/ \1;/p' $HEADER
 printf "local: *;\n};\n"
diff --git a/lib/index.cc b/lib/index.cc
index 78c18cf3..1a2e63df 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -231,26 +231,22 @@ _index_address_mailbox (notmuch_message_t *message,
 			InternetAddress *address)
 {
     InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
-    const char *name, *addr;
+    const char *name, *addr, *combined;
     void *local = talloc_new (message);
 
     name = internet_address_get_name (address);
     addr = internet_address_mailbox_get_addr (mailbox);
 
-    /* In the absence of a name, we'll strip the part before the @
-     * from the address. */
-    if (! name) {
-	const char *at;
+    /* Combine the name and address and index them as a phrase. */
+    if (name && addr)
+	combined = talloc_asprintf (local, "%s %s", name, addr);
+    else if (name)
+	combined = name;
+    else
+	combined = addr;
 
-	at = strchr (addr, '@');
-	if (at)
-	    name = talloc_strndup (local, addr, at - addr);
-    }
-
-    if (name)
-	_notmuch_message_gen_terms (message, prefix_name, name);
-    if (addr)
-	_notmuch_message_gen_terms (message, prefix_name, addr);
+    if (combined)
+	_notmuch_message_gen_terms (message, prefix_name, combined);
 
     talloc_free (local);
 }
@@ -425,63 +421,17 @@ _index_mime_part (notmuch_message_t *message,
 
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
-			     const char *filename)
+			     notmuch_message_file_t *message_file)
 {
-    GMimeStream *stream = NULL;
-    GMimeParser *parser = NULL;
-    GMimeMessage *mime_message = NULL;
+    GMimeMessage *mime_message;
     InternetAddressList *addresses;
-    FILE *file = NULL;
     const char *from, *subject;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
-    static int initialized = 0;
-    char from_buf[5];
-    bool is_mbox = false;
-    static bool mbox_warning = false;
-
-    if (! initialized) {
-	g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS);
-	initialized = 1;
-    }
-
-    file = fopen (filename, "r");
-    if (! file) {
-	fprintf (stderr, "Error opening %s: %s\n", filename, strerror (errno));
-	ret = NOTMUCH_STATUS_FILE_ERROR;
-	goto DONE;
-    }
-
-    /* Is this mbox? */
-    if (fread (from_buf, sizeof (from_buf), 1, file) == 1 &&
-	strncmp (from_buf, "From ", 5) == 0)
-	is_mbox = true;
-    rewind (file);
-
-    /* Evil GMime steals my FILE* here so I won't fclose it. */
-    stream = g_mime_stream_file_new (file);
+    notmuch_status_t status;
 
-    parser = g_mime_parser_new_with_stream (stream);
-    g_mime_parser_set_scan_from (parser, is_mbox);
-
-    mime_message = g_mime_parser_construct_message (parser);
-
-    if (is_mbox) {
-	if (!g_mime_parser_eos (parser)) {
-	    /* This is a multi-message mbox. */
-	    ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
-	    goto DONE;
-	}
-	/* For historical reasons, we support single-message mboxes,
-	 * but this behavior is likely to change in the future, so
-	 * warn. */
-	if (!mbox_warning) {
-	    mbox_warning = true;
-	    fprintf (stderr, "\
-Warning: %s is an mbox containing a single message,\n\
-likely caused by misconfigured mail delivery.  Support for single-message\n\
-mboxes is deprecated and may be removed in the future.\n", filename);
-	}
-    }
+    status = _notmuch_message_file_get_mime_message (message_file,
+						     &mime_message);
+    if (status)
+	return status;
 
     from = g_mime_message_get_sender (mime_message);
 
@@ -502,15 +452,5 @@ mboxes is deprecated and may be removed in the future.\n", filename);
 
     _index_mime_part (message, g_mime_message_get_mime_part (mime_message));
 
-  DONE:
-    if (mime_message)
-	g_object_unref (mime_message);
-
-    if (parser)
-	g_object_unref (parser);
-
-    if (stream)
-	g_object_unref (stream);
-
-    return ret;
+    return NOTMUCH_STATUS_SUCCESS;
 }
diff --git a/lib/message-file.c b/lib/message-file.c
index a2850c27..483ba1e9 100644
--- a/lib/message-file.c
+++ b/lib/message-file.c
@@ -26,30 +26,15 @@
 
 #include <glib.h> /* GHashTable */
 
-typedef struct {
-    char *str;
-    size_t size;
-    size_t len;
-} header_value_closure_t;
-
 struct _notmuch_message_file {
     /* File object */
     FILE *file;
+    char *filename;
 
-    /* Header storage */
-    int restrict_headers;
+    /* Cache for decoded headers */
     GHashTable *headers;
-    int broken_headers;
-    int good_headers;
-    size_t header_size; /* Length of full message header in bytes. */
-
-    /* Parsing state */
-    char *line;
-    size_t line_size;
-    header_value_closure_t value;
 
-    int parsing_started;
-    int parsing_finished;
+    GMimeMessage *message;
 };
 
 static int
@@ -76,15 +61,12 @@ strcase_hash (const void *ptr)
 static int
 _notmuch_message_file_destructor (notmuch_message_file_t *message)
 {
-    if (message->line)
-	free (message->line);
-
-    if (message->value.size)
-	free (message->value.str);
-
     if (message->headers)
 	g_hash_table_destroy (message->headers);
 
+    if (message->message)
+	g_object_unref (message->message);
+
     if (message->file)
 	fclose (message->file);
 
@@ -102,20 +84,17 @@ _notmuch_message_file_open_ctx (void *ctx, const char *filename)
     if (unlikely (message == NULL))
 	return NULL;
 
+    /* Only needed for error messages during parsing. */
+    message->filename = talloc_strdup (message, filename);
+    if (message->filename == NULL)
+	goto FAIL;
+
     talloc_set_destructor (message, _notmuch_message_file_destructor);
 
     message->file = fopen (filename, "r");
     if (message->file == NULL)
 	goto FAIL;
 
-    message->headers = g_hash_table_new_full (strcase_hash,
-					      strcase_equal,
-					      free,
-					      g_free);
-
-    message->parsing_started = 0;
-    message->parsing_finished = 0;
-
     return message;
 
   FAIL:
@@ -137,264 +116,222 @@ notmuch_message_file_close (notmuch_message_file_t *message)
     talloc_free (message);
 }
 
-void
-notmuch_message_file_restrict_headersv (notmuch_message_file_t *message,
-					va_list va_headers)
+static notmuch_bool_t
+_is_mbox (FILE *file)
 {
-    char *header;
+    char from_buf[5];
+    notmuch_bool_t ret = FALSE;
 
-    if (message->parsing_started)
-	INTERNAL_ERROR ("notmuch_message_file_restrict_headers called after parsing has started");
+    /* Is this mbox? */
+    if (fread (from_buf, sizeof (from_buf), 1, file) == 1 &&
+	strncmp (from_buf, "From ", 5) == 0)
+	ret = TRUE;
 
-    while (1) {
-	header = va_arg (va_headers, char*);
-	if (header == NULL)
-	    break;
-	g_hash_table_insert (message->headers,
-			     xstrdup (header), NULL);
-    }
+    rewind (file);
 
-    message->restrict_headers = 1;
+    return ret;
 }
 
-void
-notmuch_message_file_restrict_headers (notmuch_message_file_t *message, ...)
+notmuch_status_t
+_notmuch_message_file_parse (notmuch_message_file_t *message)
 {
-    va_list va_headers;
+    GMimeStream *stream;
+    GMimeParser *parser;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    static int initialized = 0;
+    notmuch_bool_t is_mbox;
 
-    va_start (va_headers, message);
+    if (message->message)
+	return NOTMUCH_STATUS_SUCCESS;
 
-    notmuch_message_file_restrict_headersv (message, va_headers);
-}
+    is_mbox = _is_mbox (message->file);
 
-static void
-copy_header_unfolding (header_value_closure_t *value,
-		       const char *chunk)
-{
-    char *last;
+    if (! initialized) {
+	g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS);
+	initialized = 1;
+    }
 
-    if (chunk == NULL)
-	return;
+    message->headers = g_hash_table_new_full (strcase_hash, strcase_equal,
+					      free, g_free);
+    if (! message->headers)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
 
-    while (*chunk == ' ' || *chunk == '\t')
-	chunk++;
+    stream = g_mime_stream_file_new (message->file);
 
-    if (value->len + 1 + strlen (chunk) + 1 > value->size) {
-	unsigned int new_size = value->size;
-	if (value->size == 0)
-	    new_size = strlen (chunk) + 1;
-	else
-	    while (value->len + 1 + strlen (chunk) + 1 > new_size)
-		new_size *= 2;
-	value->str = xrealloc (value->str, new_size);
-	value->size = new_size;
+    /* We'll own and fclose the FILE* ourselves. */
+    g_mime_stream_file_set_owner (GMIME_STREAM_FILE (stream), FALSE);
+
+    parser = g_mime_parser_new_with_stream (stream);
+    g_mime_parser_set_scan_from (parser, is_mbox);
+
+    message->message = g_mime_parser_construct_message (parser);
+    if (! message->message) {
+	status = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	goto DONE;
     }
 
-    last = value->str + value->len;
-    if (value->len) {
-	*last = ' ';
-	last++;
-	value->len++;
+    if (is_mbox) {
+	if (! g_mime_parser_eos (parser)) {
+	    /* This is a multi-message mbox. */
+	    status = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	    goto DONE;
+	}
+	/*
+	 * For historical reasons, we support single-message mboxes,
+	 * but this behavior is likely to change in the future, so
+	 * warn.
+	 */
+	static notmuch_bool_t mbox_warning = FALSE;
+	if (! mbox_warning) {
+	    mbox_warning = TRUE;
+	    fprintf (stderr, "\
+Warning: %s is an mbox containing a single message,\n\
+likely caused by misconfigured mail delivery.  Support for single-message\n\
+mboxes is deprecated and may be removed in the future.\n", message->filename);
+	}
     }
 
-    strcpy (last, chunk);
-    value->len += strlen (chunk);
+  DONE:
+    g_object_unref (stream);
+    g_object_unref (parser);
+
+    if (status) {
+	g_hash_table_destroy (message->headers);
+	message->headers = NULL;
 
-    last = value->str + value->len - 1;
-    if (*last == '\n') {
-	*last = '\0';
-	value->len--;
+	if (message->message) {
+	    g_object_unref (message->message);
+	    message->message = NULL;
+	}
+
+	rewind (message->file);
     }
+
+    return status;
 }
 
-/* As a special-case, a value of NULL for header_desired will force
- * the entire header to be parsed if it is not parsed already. This is
- * used by the _notmuch_message_file_get_headers_end function.
- * Another special case is the Received: header. For this header we
- * want to concatenate all instances of the header instead of just
- * hashing the first instance as we use this when analyzing the path
- * the mail has taken from sender to recipient.
- */
-const char *
-notmuch_message_file_get_header (notmuch_message_file_t *message,
-				 const char *header_desired)
+notmuch_status_t
+_notmuch_message_file_get_mime_message (notmuch_message_file_t *message,
+					GMimeMessage **mime_message)
 {
-    int contains;
-    char *header, *decoded_value, *header_sofar, *combined_header;
-    const char *s, *colon;
-    int match, newhdr, hdrsofar, is_received;
-    static int initialized = 0;
+    notmuch_status_t status;
 
-    is_received = (strcmp(header_desired,"received") == 0);
+    status = _notmuch_message_file_parse (message);
+    if (status)
+	return status;
 
-    if (! initialized) {
-	g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS);
-	initialized = 1;
-    }
-
-    message->parsing_started = 1;
+    *mime_message = message->message;
 
-    if (header_desired == NULL)
-	contains = 0;
-    else
-	contains = g_hash_table_lookup_extended (message->headers,
-						 header_desired, NULL,
-						 (gpointer *) &decoded_value);
+    return NOTMUCH_STATUS_SUCCESS;
+}
 
-    if (contains && decoded_value)
-	return decoded_value;
+/*
+ * Get all instances of a header decoded and concatenated.
+ *
+ * The result must be freed using g_free().
+ *
+ * Return NULL on errors, empty string for non-existing headers.
+ */
+static char *
+_notmuch_message_file_get_combined_header (notmuch_message_file_t *message,
+					   const char *header)
+{
+    GMimeHeaderList *headers;
+    GMimeHeaderIter *iter;
+    char *combined = NULL;
 
-    if (message->parsing_finished)
-	return "";
+    headers = g_mime_object_get_header_list (GMIME_OBJECT (message->message));
+    if (! headers)
+	return NULL;
 
-#define NEXT_HEADER_LINE(closure)				\
-    while (1) {							\
-	ssize_t bytes_read = getline (&message->line,		\
-				      &message->line_size,	\
-				      message->file);		\
-	if (bytes_read == -1) {					\
-	    message->parsing_finished = 1;			\
-	    break;						\
-	}							\
-	if (*message->line == '\n') {				\
-	    message->parsing_finished = 1;			\
-	    break;						\
-	}							\
-	if (closure &&						\
-	    (*message->line == ' ' || *message->line == '\t'))	\
-	{							\
-	    copy_header_unfolding ((closure), message->line);	\
-	}							\
-	if (*message->line == ' ' || *message->line == '\t')	\
-	    message->header_size += strlen (message->line);	\
-	else							\
-	    break;						\
-    }
+    iter = g_mime_header_iter_new ();
+    if (! iter)
+	return NULL;
 
-    if (message->line == NULL)
-	NEXT_HEADER_LINE (NULL);
+    if (! g_mime_header_list_get_iter (headers, iter))
+	goto DONE;
 
-    while (1) {
+    do {
+	const char *value;
+	char *decoded;
 
-	if (message->parsing_finished)
-	    break;
+	if (strcasecmp (g_mime_header_iter_get_name (iter), header) != 0)
+	    continue;
 
-	colon = strchr (message->line, ':');
+	/* Note that GMime retains ownership of value... */
+	value = g_mime_header_iter_get_value (iter);
 
-	if (colon == NULL) {
-	    message->broken_headers++;
-	    /* A simple heuristic for giving up on things that just
-	     * don't look like mail messages. */
-	    if (message->broken_headers >= 10 &&
-		message->good_headers < 5)
-	    {
-		message->parsing_finished = 1;
-		break;
+	/* ... while decoded needs to be freed with g_free(). */
+	decoded = g_mime_utils_header_decode_text (value);
+	if (! decoded) {
+	    if (combined) {
+		g_free (combined);
+		combined = NULL;
 	    }
-	    NEXT_HEADER_LINE (NULL);
-	    continue;
+	    goto DONE;
 	}
 
-	message->header_size += strlen (message->line);
-
-	message->good_headers++;
-
-	header = xstrndup (message->line, colon - message->line);
+	if (combined) {
+	    char *tmp = g_strdup_printf ("%s %s", combined, decoded);
+	    g_free (decoded);
+	    g_free (combined);
+	    if (! tmp) {
+		combined = NULL;
+		goto DONE;
+	    }
 
-	if (message->restrict_headers &&
-	    ! g_hash_table_lookup_extended (message->headers,
-					    header, NULL, NULL))
-	{
-	    free (header);
-	    NEXT_HEADER_LINE (NULL);
-	    continue;
+	    combined = tmp;
+	} else {
+	    combined = decoded;
 	}
+    } while (g_mime_header_iter_next (iter));
 
-	s = colon + 1;
-	while (*s == ' ' || *s == '\t')
-	    s++;
-
-	message->value.len = 0;
-	copy_header_unfolding (&message->value, s);
+    /* Return empty string for non-existing headers. */
+    if (! combined)
+	combined = g_strdup ("");
 
-	NEXT_HEADER_LINE (&message->value);
+  DONE:
+    g_mime_header_iter_free (iter);
 
-	if (header_desired == NULL)
-	    match = 0;
-	else
-	    match = (strcasecmp (header, header_desired) == 0);
+    return combined;
+}
 
-	decoded_value = g_mime_utils_header_decode_text (message->value.str);
-	header_sofar = (char *)g_hash_table_lookup (message->headers, header);
-	/* we treat the Received: header special - we want to concat ALL of 
-	 * the Received: headers we encounter.
-	 * for everything else we return the first instance of a header */
-	if (strcasecmp(header, "received") == 0) {
-	    if (header_sofar == NULL) {
-		/* first Received: header we encountered; just add it */
-		g_hash_table_insert (message->headers, header, decoded_value);
-	    } else {
-		/* we need to add the header to those we already collected */
-		newhdr = strlen(decoded_value);
-		hdrsofar = strlen(header_sofar);
-		combined_header = g_malloc(hdrsofar + newhdr + 2);
-		strncpy(combined_header,header_sofar,hdrsofar);
-		*(combined_header+hdrsofar) = ' ';
-		strncpy(combined_header+hdrsofar+1,decoded_value,newhdr+1);
-		g_free (decoded_value);
-		g_hash_table_insert (message->headers, header, combined_header);
-	    }
-	} else {
-	    if (header_sofar == NULL) {
-		/* Only insert if we don't have a value for this header, yet. */
-		g_hash_table_insert (message->headers, header, decoded_value);
-	    } else {
-		free (header);
-		g_free (decoded_value);
-		decoded_value = header_sofar;
-	    }
-	}
-	/* if we found a match we can bail - unless of course we are
-	 * collecting all the Received: headers */
-	if (match && !is_received)
-	    return decoded_value;
-    }
+const char *
+notmuch_message_file_get_header (notmuch_message_file_t *message,
+				 const char *header)
+{
+    const char *value;
+    char *decoded;
 
-    if (message->parsing_finished) {
-        fclose (message->file);
-        message->file = NULL;
-    }
+    if (_notmuch_message_file_parse (message))
+	return NULL;
 
-    if (message->line)
-	free (message->line);
-    message->line = NULL;
+    /* If we have a cached decoded value, use it. */
+    value = g_hash_table_lookup (message->headers, header);
+    if (value)
+	return value;
 
-    if (message->value.size) {
-	free (message->value.str);
-	message->value.str = NULL;
-	message->value.size = 0;
-	message->value.len = 0;
+    if (strcasecmp (header, "received") == 0) {
+	/*
+	 * The Received: header is special. We concatenate all
+	 * instances of the header as we use this when analyzing the
+	 * path the mail has taken from sender to recipient.
+	 */
+	decoded = _notmuch_message_file_get_combined_header (message, header);
+    } else {
+	value = g_mime_object_get_header (GMIME_OBJECT (message->message),
+					  header);
+	if (value)
+	    decoded = g_mime_utils_header_decode_text (value);
+	else
+	    decoded = g_strdup ("");
     }
 
-    /* For the Received: header we actually might end up here even
-     * though we found the header (as we force continued parsing
-     * in that case). So let's check if that's the header we were
-     * looking for and return the value that we found (if any)
-     */
-    if (is_received)
-	return (char *)g_hash_table_lookup (message->headers, "received");
+    if (! decoded)
+	return NULL;
 
-    /* We've parsed all headers and never found the one we're looking
-     * for. It's probably just not there, but let's check that we
-     * didn't make a mistake preventing us from seeing it. */
-    if (message->restrict_headers && header_desired &&
-	! g_hash_table_lookup_extended (message->headers,
-					header_desired, NULL, NULL))
-    {
-	INTERNAL_ERROR ("Attempt to get header \"%s\" which was not\n"
-			"included in call to notmuch_message_file_restrict_headers\n",
-			header_desired);
-    }
+    /* Cache the decoded value. We also own the strings. */
+    g_hash_table_insert (message->headers, xstrdup (header), decoded);
 
-    return "";
+    return decoded;
 }
diff --git a/lib/message.cc b/lib/message.cc
index 1b463795..d0b7351e 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -412,19 +412,27 @@ _notmuch_message_ensure_message_file (notmuch_message_t *message)
 const char *
 notmuch_message_get_header (notmuch_message_t *message, const char *header)
 {
-    std::string value;
+    try {
+	    std::string value;
+
+	    /* Fetch header from the appropriate xapian value field if
+	     * available */
+	    if (strcasecmp (header, "from") == 0)
+		value = message->doc.get_value (NOTMUCH_VALUE_FROM);
+	    else if (strcasecmp (header, "subject") == 0)
+		value = message->doc.get_value (NOTMUCH_VALUE_SUBJECT);
+	    else if (strcasecmp (header, "message-id") == 0)
+		value = message->doc.get_value (NOTMUCH_VALUE_MESSAGE_ID);
 
-    /* Fetch header from the appropriate xapian value field if
-     * available */
-    if (strcasecmp (header, "from") == 0)
-	value = message->doc.get_value (NOTMUCH_VALUE_FROM);
-    else if (strcasecmp (header, "subject") == 0)
-	value = message->doc.get_value (NOTMUCH_VALUE_SUBJECT);
-    else if (strcasecmp (header, "message-id") == 0)
-	value = message->doc.get_value (NOTMUCH_VALUE_MESSAGE_ID);
+	    if (!value.empty())
+		return talloc_strdup (message, value.c_str ());
 
-    if (!value.empty())
-	return talloc_strdup (message, value.c_str ());
+    } catch (Xapian::Error &error) {
+	fprintf (stderr, "A Xapian exception occurred when reading header: %s\n",
+		 error.get_msg().c_str());
+	message->notmuch->exception_reported = TRUE;
+	return NULL;
+    }
 
     /* Otherwise fall back to parsing the file */
     _notmuch_message_ensure_message_file (message);
@@ -473,6 +481,153 @@ notmuch_message_get_replies (notmuch_message_t *message)
     return _notmuch_messages_create (message->replies);
 }
 
+static void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
+{
+    Xapian::TermIterator i;
+    size_t prefix_len = strlen (prefix);
+
+    while (1) {
+	i = message->doc.termlist_begin ();
+	i.skip_to (prefix);
+
+	/* Terminate loop when no terms remain with desired prefix. */
+	if (i == message->doc.termlist_end () ||
+	    strncmp ((*i).c_str (), prefix, prefix_len))
+	    break;
+
+	try {
+	    message->doc.remove_term ((*i));
+	} catch (const Xapian::InvalidArgumentError) {
+	    /* Ignore failure to remove non-existent term. */
+	}
+    }
+}
+
+/* Return true if p points at "new" or "cur". */
+static bool is_maildir (const char *p)
+{
+    return strcmp (p, "cur") == 0 || strcmp (p, "new") == 0;
+}
+
+/* Add "folder:" term for directory. */
+static notmuch_status_t
+_notmuch_message_add_folder_terms (notmuch_message_t *message,
+				   const char *directory)
+{
+    char *folder, *last;
+
+    folder = talloc_strdup (NULL, directory);
+    if (! folder)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    /*
+     * If the message file is in a leaf directory named "new" or
+     * "cur", presume maildir and index the parent directory. Thus a
+     * "folder:" prefix search matches messages in the specified
+     * maildir folder, i.e. in the specified directory and its "new"
+     * and "cur" subdirectories.
+     *
+     * Note that this means the "folder:" prefix can't be used for
+     * distinguishing between message files in "new" or "cur". The
+     * "path:" prefix needs to be used for that.
+     *
+     * Note the deliberate difference to _filename_is_in_maildir(). We
+     * don't want to index different things depending on the existence
+     * or non-existence of all maildir sibling directories "new",
+     * "cur", and "tmp". Doing so would be surprising, and difficult
+     * for the user to fix in case all subdirectories were not in
+     * place during indexing.
+     */
+    last = strrchr (folder, '/');
+    if (last) {
+	if (is_maildir (last + 1))
+	    *last = '\0';
+    } else if (is_maildir (folder)) {
+	*folder = '\0';
+    }
+
+    _notmuch_message_add_term (message, "folder", folder);
+
+    talloc_free (folder);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+#define RECURSIVE_SUFFIX "/**"
+
+/* Add "path:" terms for directory. */
+static notmuch_status_t
+_notmuch_message_add_path_terms (notmuch_message_t *message,
+				 const char *directory)
+{
+    /* Add exact "path:" term. */
+    _notmuch_message_add_term (message, "path", directory);
+
+    if (strlen (directory)) {
+	char *path, *p;
+
+	path = talloc_asprintf (NULL, "%s%s", directory, RECURSIVE_SUFFIX);
+	if (! path)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+	/* Add recursive "path:" terms for directory and all parents. */
+	for (p = path + strlen (path) - 1; p > path; p--) {
+	    if (*p == '/') {
+		strcpy (p, RECURSIVE_SUFFIX);
+		_notmuch_message_add_term (message, "path", path);
+	    }
+	}
+
+	talloc_free (path);
+    }
+
+    /* Recursive all-matching path:** for consistency. */
+    _notmuch_message_add_term (message, "path", "**");
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Add directory based terms for all filenames of the message. */
+static notmuch_status_t
+_notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
+{
+    const char *direntry_prefix = _find_prefix ("file-direntry");
+    int direntry_prefix_len = strlen (direntry_prefix);
+    Xapian::TermIterator i = message->doc.termlist_begin ();
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    for (i.skip_to (direntry_prefix); i != message->doc.termlist_end (); i++) {
+	unsigned int directory_id;
+	const char *direntry, *directory;
+	char *colon;
+
+	/* Terminate loop at first term without desired prefix. */
+	if (strncmp ((*i).c_str (), direntry_prefix, direntry_prefix_len))
+	    break;
+
+	/* Indicate that there are filenames remaining. */
+	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	direntry = (*i).c_str ();
+	direntry += direntry_prefix_len;
+
+	directory_id = strtol (direntry, &colon, 10);
+
+	if (colon == NULL || *colon != ':')
+	    INTERNAL_ERROR ("malformed direntry");
+
+	directory = _notmuch_database_get_directory_path (ctx,
+							  message->notmuch,
+							  directory_id);
+
+	_notmuch_message_add_folder_terms (message, directory);
+	_notmuch_message_add_path_terms (message, directory);
+    }
+
+    return status;
+}
+
 /* Add an additional 'filename' for 'message'.
  *
  * This change will not be reflected in the database until the next
@@ -504,8 +659,8 @@ _notmuch_message_add_filename (notmuch_message_t *message,
      * notmuch_directory_get_child_files() . */
     _notmuch_message_add_term (message, "file-direntry", direntry);
 
-    /* New terms allow user to search with folder: specification. */
-    _notmuch_message_gen_terms (message, "folder", directory);
+    _notmuch_message_add_folder_terms (message, directory);
+    _notmuch_message_add_path_terms (message, directory);
 
     talloc_free (local);
 
@@ -528,17 +683,10 @@ notmuch_status_t
 _notmuch_message_remove_filename (notmuch_message_t *message,
 				  const char *filename)
 {
-    const char *direntry_prefix = _find_prefix ("file-direntry");
-    int direntry_prefix_len = strlen (direntry_prefix);
-    const char *folder_prefix = _find_prefix ("folder");
-    int folder_prefix_len = strlen (folder_prefix);
     void *local = talloc_new (message);
-    char *zfolder_prefix = talloc_asprintf(local, "Z%s", folder_prefix);
-    int zfolder_prefix_len = strlen (zfolder_prefix);
     char *direntry;
     notmuch_private_status_t private_status;
     notmuch_status_t status;
-    Xapian::TermIterator i, last;
 
     status = _notmuch_database_filename_to_direntry (
 	local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
@@ -553,83 +701,36 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
     if (status)
 	return status;
 
-    /* Re-synchronize "folder:" terms for this message. This requires:
-     *  1. removing all "folder:" terms
-     *  2. removing all "folder:" stemmed terms
-     *  3. adding back terms for all remaining filenames of the message. */
-
-    /* 1. removing all "folder:" terms */
-    while (1) {
-	i = message->doc.termlist_begin ();
-	i.skip_to (folder_prefix);
-
-	/* Terminate loop when no terms remain with desired prefix. */
-	if (i == message->doc.termlist_end () ||
-	    strncmp ((*i).c_str (), folder_prefix, folder_prefix_len))
-	{
-	    break;
-	}
-
-	try {
-	    message->doc.remove_term ((*i));
-	} catch (const Xapian::InvalidArgumentError) {
-	    /* Ignore failure to remove non-existent term. */
-	}
-    }
-
-    /* 2. removing all "folder:" stemmed terms */
-    while (1) {
-	i = message->doc.termlist_begin ();
-	i.skip_to (zfolder_prefix);
-
-	/* Terminate loop when no terms remain with desired prefix. */
-	if (i == message->doc.termlist_end () ||
-	    strncmp ((*i).c_str (), zfolder_prefix, zfolder_prefix_len))
-	{
-	    break;
-	}
-
-	try {
-	    message->doc.remove_term ((*i));
-	} catch (const Xapian::InvalidArgumentError) {
-	    /* Ignore failure to remove non-existent term. */
-	}
-    }
+    /* Re-synchronize "folder:" and "path:" terms for this message. */
 
-    /* 3. adding back terms for all remaining filenames of the message. */
-    i = message->doc.termlist_begin ();
-    i.skip_to (direntry_prefix);
+    /* Remove all "folder:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("folder"));
 
-    for (; i != message->doc.termlist_end (); i++) {
-	unsigned int directory_id;
-	const char *direntry, *directory;
-	char *colon;
+    /* Remove all "path:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("path"));
 
-	/* Terminate loop at first term without desired prefix. */
-	if (strncmp ((*i).c_str (), direntry_prefix, direntry_prefix_len))
-	    break;
-
-	/* Indicate that there are filenames remaining. */
-	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-
-	direntry = (*i).c_str ();
-	direntry += direntry_prefix_len;
+    /* Add back terms for all remaining filenames of the message. */
+    status = _notmuch_message_add_directory_terms (local, message);
 
-	directory_id = strtol (direntry, &colon, 10);
+    talloc_free (local);
 
-	if (colon == NULL || *colon != ':')
-	    INTERNAL_ERROR ("malformed direntry");
+    return status;
+}
 
-	directory = _notmuch_database_get_directory_path (local,
-							  message->notmuch,
-							  directory_id);
-	if (strlen (directory))
-	    _notmuch_message_gen_terms (message, "folder", directory);
-    }
+/* Upgrade the "folder:" prefix from V1 to V2. */
+#define FOLDER_PREFIX_V1       "XFOLDER"
+#define ZFOLDER_PREFIX_V1      "Z" FOLDER_PREFIX_V1
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message)
+{
+    /* Remove all old "folder:" terms. */
+    _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1);
 
-    talloc_free (local);
+    /* Remove all old "folder:" stemmed terms. */
+    _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1);
 
-    return status;
+    /* Add new boolean "folder:" and "path:" terms. */
+    _notmuch_message_add_directory_terms (message, message);
 }
 
 char *
@@ -766,7 +867,9 @@ notmuch_message_get_date (notmuch_message_t *message)
     try {
 	value = message->doc.get_value (NOTMUCH_VALUE_TIMESTAMP);
     } catch (Xapian::Error &error) {
-	INTERNAL_ERROR ("Failed to read timestamp value from document.");
+	fprintf (stderr, "A Xapian exception occurred when reading date: %s\n",
+		 error.get_msg().c_str());
+	message->notmuch->exception_reported = TRUE;
 	return 0;
     }
 
@@ -920,16 +1023,21 @@ _notmuch_message_gen_terms (notmuch_message_t *message,
 	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
 
     term_gen->set_document (message->doc);
-    term_gen->set_termpos (message->termpos);
 
     if (prefix_name) {
 	const char *prefix = _find_prefix (prefix_name);
 
+	term_gen->set_termpos (message->termpos);
 	term_gen->index_text (text, 1, prefix);
-	message->termpos = term_gen->get_termpos ();
+	/* Create a gap between this an the next terms so they don't
+	 * appear to be a phrase. */
+	message->termpos = term_gen->get_termpos () + 100;
     }
 
+    term_gen->set_termpos (message->termpos);
     term_gen->index_text (text);
+    /* Create a term gap, as above. */
+    message->termpos = term_gen->get_termpos () + 100;
 
     return NOTMUCH_PRIVATE_STATUS_SUCCESS;
 }
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index af185c7c..703ae7bb 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -46,6 +46,8 @@ NOTMUCH_BEGIN_DECLS
 
 #include <talloc.h>
 
+#include <gmime/gmime.h>
+
 #include "xutil.h"
 #include "error_util.h"
 
@@ -263,6 +265,9 @@ _notmuch_message_gen_terms (notmuch_message_t *message,
 void
 _notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
 
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message);
+
 notmuch_status_t
 _notmuch_message_add_filename (notmuch_message_t *message,
 			       const char *filename);
@@ -317,13 +322,6 @@ notmuch_message_set_author (notmuch_message_t *message, const char *author);
 const char *
 notmuch_message_get_author (notmuch_message_t *message);
 
-
-/* index.cc */
-
-notmuch_status_t
-_notmuch_message_index_file (notmuch_message_t *message,
-			     const char *filename);
-
 /* message-file.c */
 
 /* XXX: I haven't decided yet whether these will actually get exported
@@ -349,32 +347,32 @@ _notmuch_message_file_open_ctx (void *ctx, const char *filename);
 void
 notmuch_message_file_close (notmuch_message_file_t *message);
 
-/* Restrict 'message' to only save the named headers.
+/* Parse the message.
  *
- * When the caller is only interested in a short list of headers,
- * known in advance, calling this function can avoid wasted time and
- * memory parsing/saving header values that will never be needed.
+ * This will be done automatically as necessary on other calls
+ * depending on it, but an explicit call allows for better error
+ * status reporting.
+ */
+notmuch_status_t
+_notmuch_message_file_parse (notmuch_message_file_t *message);
+
+/* Get the gmime message of a message file.
  *
- * The variable arguments should be a list of const char * with a
- * final '(const char *) NULL' to terminate the list.
+ * The message file is parsed as necessary.
  *
- * If this function is called, it must be called before any calls to
- * notmuch_message_get_header for this message.
+ * The GMimeMessage* is set to *mime_message on success (which the
+ * caller must not unref).
  *
- * After calling this function, if notmuch_message_get_header is
- * called with a header name not in this list, then NULL will be
- * returned even if that header exists in the actual message.
+ * XXX: Would be nice to not have to expose GMimeMessage here.
  */
-void
-notmuch_message_file_restrict_headers (notmuch_message_file_t *message, ...);
-
-/* Identical to notmuch_message_restrict_headers but accepting a va_list. */
-void
-notmuch_message_file_restrict_headersv (notmuch_message_file_t *message,
-					va_list va_headers);
+notmuch_status_t
+_notmuch_message_file_get_mime_message (notmuch_message_file_t *message,
+					GMimeMessage **mime_message);
 
 /* Get the value of the specified header from the message as a UTF-8 string.
  *
+ * The message file is parsed as necessary.
+ *
  * The header name is case insensitive.
  *
  * The Received: header is special - for it all Received: headers in
@@ -384,13 +382,19 @@ notmuch_message_file_restrict_headersv (notmuch_message_file_t *message,
  * only until the message is closed. The caller should copy it if
  * needing to modify the value or to hold onto it for longer.
  *
- * Returns NULL if the message does not contain a header line matching
- * 'header'.
+ * Returns NULL on errors, empty string if the message does not
+ * contain a header line matching 'header'.
  */
 const char *
 notmuch_message_file_get_header (notmuch_message_file_t *message,
 				 const char *header);
 
+/* index.cc */
+
+notmuch_status_t
+_notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_message_file_t *message_file);
+
 /* messages.c */
 
 typedef struct _notmuch_message_node {
diff --git a/lib/notmuch.h b/lib/notmuch.h
index d30768d9..350bed8b 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -18,9 +18,19 @@
  * Author: Carl Worth <cworth@cworth.org>
  */
 
+/**
+ * @defgroup notmuch The notmuch API
+ *
+ * Not much of an email library, (just index and search)
+ *
+ * @{
+ */
+
 #ifndef NOTMUCH_H
 #define NOTMUCH_H
 
+#ifndef __DOXYGEN__
+
 #ifdef  __cplusplus
 # define NOTMUCH_BEGIN_DECLS  extern "C" {
 # define NOTMUCH_END_DECLS    }
@@ -49,19 +59,28 @@ NOTMUCH_BEGIN_DECLS
 #define LIBNOTMUCH_MINOR_VERSION	1
 #define LIBNOTMUCH_MICRO_VERSION	0
 
-/*
+#endif /* __DOXYGEN__ */
+
+/**
  * Check the version of the notmuch library being compiled against.
  *
  * Return true if the library being compiled against is of the
  * specified version or above. For example:
  *
+ * @code
  * #if LIBNOTMUCH_CHECK_VERSION(3, 1, 0)
  *     (code requiring libnotmuch 3.1.0 or above)
  * #endif
+ * @endcode
  *
- * LIBNOTMUCH_CHECK_VERSION has been defined since version 3.1.0; you
- * can use #if !defined(NOTMUCH_CHECK_VERSION) to check for versions
- * prior to that.
+ * LIBNOTMUCH_CHECK_VERSION has been defined since version 3.1.0; to
+ * check for versions prior to that, use:
+ *
+ * @code
+ * #if !defined(NOTMUCH_CHECK_VERSION)
+ *     (code requiring libnotmuch prior to 3.1.0)
+ * #endif
+ * @endcode
  */
 #define LIBNOTMUCH_CHECK_VERSION (major, minor, micro)			\
     (LIBNOTMUCH_MAJOR_VERSION > (major) ||					\
@@ -69,72 +88,86 @@ NOTMUCH_BEGIN_DECLS
      (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION == (minor) && \
       LIBNOTMUCH_MICRO_VERSION >= (micro)))
 
+/**
+ * Notmuch boolean type.
+ */
 typedef int notmuch_bool_t;
 
-/* Status codes used for the return values of most functions.
+/**
+ * Status codes used for the return values of most functions.
  *
  * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function
- * completed without error. Any other value indicates an error as
- * follows:
- *
- * NOTMUCH_STATUS_SUCCESS: No error occurred.
- *
- * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory
- *
- * XXX: We don't really want to expose this lame XAPIAN_EXCEPTION
- * value. Instead we should map to things like DATABASE_LOCKED or
- * whatever.
- *
- * NOTMUCH_STATUS_READ_ONLY_DATABASE: An attempt was made to write to
- *	a database opened in read-only mode.
- *
- * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
- *
- * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to read or
- *	write to a file (this could be file not found, permission
- *	denied, etc.)
- *
- * NOTMUCH_STATUS_FILE_NOT_EMAIL: A file was presented that doesn't
- *	appear to be an email message.
- *
- * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: A file contains a message ID
- *	that is identical to a message already in the database.
- *
- * NOTMUCH_STATUS_NULL_POINTER: The user erroneously passed a NULL
- *	pointer to a notmuch function.
- *
- * NOTMUCH_STATUS_TAG_TOO_LONG: A tag value is too long (exceeds
- *	NOTMUCH_TAG_MAX)
- *
- * NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: The notmuch_message_thaw
- *	function has been called more times than notmuch_message_freeze.
- *
- * NOTMUCH_STATUS_UNBALANCED_ATOMIC: notmuch_database_end_atomic has
- *	been called more times than notmuch_database_begin_atomic.
- *
- * And finally:
- *
- * NOTMUCH_STATUS_LAST_STATUS: Not an actual status value. Just a way
- *	to find out how many valid status values there are.
+ * completed without error. Any other value indicates an error.
  */
 typedef enum _notmuch_status {
+    /**
+     * No error occurred.
+     */
     NOTMUCH_STATUS_SUCCESS = 0,
+    /**
+     * Out of memory.
+     */
     NOTMUCH_STATUS_OUT_OF_MEMORY,
+    /**
+     * An attempt was made to write to a database opened in read-only
+     * mode.
+     */
     NOTMUCH_STATUS_READ_ONLY_DATABASE,
+    /**
+     * A Xapian exception occurred.
+     */
     NOTMUCH_STATUS_XAPIAN_EXCEPTION,
+    /**
+     * An error occurred trying to read or write to a file (this could
+     * be file not found, permission denied, etc.)
+     *
+     * @todo We don't really want to expose this lame XAPIAN_EXCEPTION
+     * value. Instead we should map to things like DATABASE_LOCKED or
+     * whatever.
+     */
     NOTMUCH_STATUS_FILE_ERROR,
+    /**
+     * A file was presented that doesn't appear to be an email
+     * message.
+     */
     NOTMUCH_STATUS_FILE_NOT_EMAIL,
+    /**
+     * A file contains a message ID that is identical to a message
+     * already in the database.
+     */
     NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID,
+    /**
+     * The user erroneously passed a NULL pointer to a notmuch
+     * function.
+     */
     NOTMUCH_STATUS_NULL_POINTER,
+    /**
+     * A tag value is too long (exceeds NOTMUCH_TAG_MAX).
+     */
     NOTMUCH_STATUS_TAG_TOO_LONG,
+    /**
+     * The notmuch_message_thaw function has been called more times
+     * than notmuch_message_freeze.
+     */
     NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
+    /**
+     * notmuch_database_end_atomic has been called more times than
+     * notmuch_database_begin_atomic.
+     */
     NOTMUCH_STATUS_UNBALANCED_ATOMIC,
+    /**
+     * The operation is not supported.
+     */
     NOTMUCH_STATUS_UNSUPPORTED_OPERATION,
-
+    /**
+     * Not an actual status value. Just a way to find out how many
+     * valid status values there are.
+     */
     NOTMUCH_STATUS_LAST_STATUS
 } notmuch_status_t;
 
-/* Get a string representation of a notmuch_status_t value.
+/**
+ * Get a string representation of a notmuch_status_t value.
  *
  * The result is read-only.
  */
@@ -143,6 +176,7 @@ notmuch_status_to_string (notmuch_status_t status);
 
 /* Various opaque data types. For each notmuch_<foo>_t see the various
  * notmuch_<foo> functions below. */
+#ifndef __DOXYGEN__
 typedef struct _notmuch_database notmuch_database_t;
 typedef struct _notmuch_query notmuch_query_t;
 typedef struct _notmuch_threads notmuch_threads_t;
@@ -152,8 +186,10 @@ typedef struct _notmuch_message notmuch_message_t;
 typedef struct _notmuch_tags notmuch_tags_t;
 typedef struct _notmuch_directory notmuch_directory_t;
 typedef struct _notmuch_filenames notmuch_filenames_t;
+#endif /* __DOXYGEN__ */
 
-/* Create a new, empty notmuch database located at 'path'.
+/**
+ * Create a new, empty notmuch database located at 'path'.
  *
  * The path should be a top-level directory to a collection of
  * plain-text email messages (one message per file). This call will
@@ -189,12 +225,22 @@ typedef struct _notmuch_filenames notmuch_filenames_t;
 notmuch_status_t
 notmuch_database_create (const char *path, notmuch_database_t **database);
 
+/**
+ * Database open mode for notmuch_database_open.
+ */
 typedef enum {
+    /**
+     * Open database for reading only.
+     */
     NOTMUCH_DATABASE_MODE_READ_ONLY = 0,
+    /**
+     * Open database for reading and writing.
+     */
     NOTMUCH_DATABASE_MODE_READ_WRITE
 } notmuch_database_mode_t;
 
-/* Open an existing notmuch database located at 'path'.
+/**
+ * Open an existing notmuch database located at 'path'.
  *
  * The database should have been created at some time in the past,
  * (not necessarily by this process), by calling
@@ -230,7 +276,8 @@ notmuch_database_open (const char *path,
 		       notmuch_database_mode_t mode,
 		       notmuch_database_t **database);
 
-/* Close the given notmuch database.
+/**
+ * Close the given notmuch database.
  *
  * After notmuch_database_close has been called, calls to other
  * functions on objects derived from this database may either behave
@@ -244,12 +291,14 @@ notmuch_database_open (const char *path,
 void
 notmuch_database_close (notmuch_database_t *database);
 
-/* A callback invoked by notmuch_database_compact to notify the user
+/**
+ * A callback invoked by notmuch_database_compact to notify the user
  * of the progress of the compaction process.
  */
 typedef void (*notmuch_compact_status_cb_t)(const char *message, void *closure);
 
-/* Compact a notmuch database, backing up the original database to the
+/**
+ * Compact a notmuch database, backing up the original database to the
  * given path.
  *
  * The database will be opened with NOTMUCH_DATABASE_MODE_READ_WRITE
@@ -265,33 +314,41 @@ notmuch_database_compact (const char* path,
 			  notmuch_compact_status_cb_t status_cb,
 			  void *closure);
 
-/* Destroy the notmuch database, closing it if necessary and freeing
+/**
+ * Destroy the notmuch database, closing it if necessary and freeing
  * all associated resources.
  */
 void
 notmuch_database_destroy (notmuch_database_t *database);
 
-/* Return the database path of the given database.
+/**
+ * Return the database path of the given database.
  *
  * The return value is a string owned by notmuch so should not be
- * modified nor freed by the caller. */
+ * modified nor freed by the caller.
+ */
 const char *
 notmuch_database_get_path (notmuch_database_t *database);
 
-/* Return the database format version of the given database. */
+/**
+ * Return the database format version of the given database.
+ */
 unsigned int
 notmuch_database_get_version (notmuch_database_t *database);
 
-/* Does this database need to be upgraded before writing to it?
+/**
+ * Does this database need to be upgraded before writing to it?
  *
  * If this function returns TRUE then no functions that modify the
  * database (notmuch_database_add_message, notmuch_message_add_tag,
  * notmuch_directory_set_mtime, etc.) will work unless the function
- * notmuch_database_upgrade is called successfully first. */
+ * notmuch_database_upgrade is called successfully first.
+ */
 notmuch_bool_t
 notmuch_database_needs_upgrade (notmuch_database_t *database);
 
-/* Upgrade the current database.
+/**
+ * Upgrade the current database.
  *
  * After opening a database in read-write mode, the client should
  * check if an upgrade is needed (notmuch_database_needs_upgrade) and
@@ -310,7 +367,8 @@ notmuch_database_upgrade (notmuch_database_t *database,
 						   double progress),
 			  void *closure);
 
-/* Begin an atomic database operation.
+/**
+ * Begin an atomic database operation.
  *
  * Any modifications performed between a successful begin and a
  * notmuch_database_end_atomic will be applied to the database
@@ -331,7 +389,8 @@ notmuch_database_upgrade (notmuch_database_t *database,
 notmuch_status_t
 notmuch_database_begin_atomic (notmuch_database_t *notmuch);
 
-/* Indicate the end of an atomic database operation.
+/**
+ * Indicate the end of an atomic database operation.
  *
  * Return value:
  *
@@ -346,7 +405,8 @@ notmuch_database_begin_atomic (notmuch_database_t *notmuch);
 notmuch_status_t
 notmuch_database_end_atomic (notmuch_database_t *notmuch);
 
-/* Retrieve a directory object from the database for 'path'.
+/**
+ * Retrieve a directory object from the database for 'path'.
  *
  * Here, 'path' should be a path relative to the path of 'database'
  * (see notmuch_database_get_path), or else should be an absolute path
@@ -355,6 +415,10 @@ notmuch_database_end_atomic (notmuch_database_t *notmuch);
  * If this directory object does not exist in the database, this
  * returns NOTMUCH_STATUS_SUCCESS and sets *directory to NULL.
  *
+ * Otherwise the returned directory object is owned by the database
+ * and as such, will only be valid until notmuch_database_destroy is
+ * called.
+ *
  * Return value:
  *
  * NOTMUCH_STATUS_SUCCESS: Successfully retrieved directory.
@@ -369,7 +433,8 @@ notmuch_database_get_directory (notmuch_database_t *database,
 				const char *path,
 				notmuch_directory_t **directory);
 
-/* Add a new message to the given notmuch database or associate an
+/**
+ * Add a new message to the given notmuch database or associate an
  * additional filename with an existing message.
  *
  * Here, 'filename' should be a path relative to the path of
@@ -420,7 +485,8 @@ notmuch_database_add_message (notmuch_database_t *database,
 			      const char *filename,
 			      notmuch_message_t **message);
 
-/* Remove a message filename from the given notmuch database. If the
+/**
+ * Remove a message filename from the given notmuch database. If the
  * message has no more filenames, remove the message.
  *
  * If the same message (as determined by the message ID) is still
@@ -448,7 +514,8 @@ notmuch_status_t
 notmuch_database_remove_message (notmuch_database_t *database,
 				 const char *filename);
 
-/* Find a message with the given message_id.
+/**
+ * Find a message with the given message_id.
  *
  * If a message with the given message_id is found then, on successful return
  * (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to a message
@@ -475,7 +542,8 @@ notmuch_database_find_message (notmuch_database_t *database,
 			       const char *message_id,
 			       notmuch_message_t **message);
 
-/* Find a message with the given filename.
+/**
+ * Find a message with the given filename.
  *
  * If the database contains a message with the given filename then, on
  * successful return (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to
@@ -502,7 +570,8 @@ notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
 					   const char *filename,
 					   notmuch_message_t **message);
 
-/* Return a list of all tags found in the database.
+/**
+ * Return a list of all tags found in the database.
  *
  * This function creates a list of all tags found in the database. The
  * resulting list contains all tags from all messages found in the database.
@@ -512,7 +581,8 @@ notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
 notmuch_tags_t *
 notmuch_database_get_all_tags (notmuch_database_t *db);
 
-/* Create a new query for 'database'.
+/**
+ * Create a new query for 'database'.
  *
  * Here, 'database' should be an open database, (see
  * notmuch_database_open and notmuch_database_create).
@@ -540,19 +610,36 @@ notmuch_query_t *
 notmuch_query_create (notmuch_database_t *database,
 		      const char *query_string);
 
-/* Sort values for notmuch_query_set_sort */
+/**
+ * Sort values for notmuch_query_set_sort.
+ */
 typedef enum {
+    /**
+     * Oldest first.
+     */
     NOTMUCH_SORT_OLDEST_FIRST,
+    /**
+     * Newest first.
+     */
     NOTMUCH_SORT_NEWEST_FIRST,
+    /**
+     * Sort by message-id.
+     */
     NOTMUCH_SORT_MESSAGE_ID,
+    /**
+     * Do not sort.
+     */
     NOTMUCH_SORT_UNSORTED
 } notmuch_sort_t;
 
-/* Return the query_string of this query. See notmuch_query_create. */
+/**
+ * Return the query_string of this query. See notmuch_query_create.
+ */
 const char *
 notmuch_query_get_query_string (notmuch_query_t *query);
 
-/* Exclude values for notmuch_query_set_omit_excluded. The strange
+/**
+ * Exclude values for notmuch_query_set_omit_excluded. The strange
  * order is to maintain backward compatibility: the old FALSE/TRUE
  * options correspond to the new
  * NOTMUCH_EXCLUDE_FLAG/NOTMUCH_EXCLUDE_TRUE options.
@@ -564,7 +651,8 @@ typedef enum {
     NOTMUCH_EXCLUDE_ALL
 } notmuch_exclude_t;
 
-/* Specify whether to omit excluded results or simply flag them.  By
+/**
+ * Specify whether to omit excluded results or simply flag them.  By
  * default, this is set to TRUE.
  *
  * If set to TRUE or ALL, notmuch_query_search_messages will omit excluded
@@ -594,21 +682,29 @@ void
 notmuch_query_set_omit_excluded (notmuch_query_t *query,
 				 notmuch_exclude_t omit_excluded);
 
-/* Specify the sorting desired for this query. */
+/**
+ * Specify the sorting desired for this query.
+ */
 void
 notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort);
 
-/* Return the sort specified for this query. See notmuch_query_set_sort. */
+/**
+ * Return the sort specified for this query. See
+ * notmuch_query_set_sort.
+ */
 notmuch_sort_t
 notmuch_query_get_sort (notmuch_query_t *query);
 
-/* Add a tag that will be excluded from the query results by default.
+/**
+ * Add a tag that will be excluded from the query results by default.
  * This exclusion will be overridden if this tag appears explicitly in
- * the query. */
+ * the query.
+ */
 void
 notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag);
 
-/* Execute a query for threads, returning a notmuch_threads_t object
+/**
+ * Execute a query for threads, returning a notmuch_threads_t object
  * which can be used to iterate over the results. The returned threads
  * object is owned by the query and as such, will only be valid until
  * notmuch_query_destroy.
@@ -649,7 +745,8 @@ notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag);
 notmuch_threads_t *
 notmuch_query_search_threads (notmuch_query_t *query);
 
-/* Execute a query for messages, returning a notmuch_messages_t object
+/**
+ * Execute a query for messages, returning a notmuch_messages_t object
  * which can be used to iterate over the results. The returned
  * messages object is owned by the query and as such, will only be
  * valid until notmuch_query_destroy.
@@ -690,7 +787,8 @@ notmuch_query_search_threads (notmuch_query_t *query);
 notmuch_messages_t *
 notmuch_query_search_messages (notmuch_query_t *query);
 
-/* Destroy a notmuch_query_t along with any associated resources.
+/**
+ * Destroy a notmuch_query_t along with any associated resources.
  *
  * This will in turn destroy any notmuch_threads_t and
  * notmuch_messages_t objects generated by this query, (and in
@@ -701,19 +799,23 @@ notmuch_query_search_messages (notmuch_query_t *query);
 void
 notmuch_query_destroy (notmuch_query_t *query);
 
-/* Is the given 'threads' iterator pointing at a valid thread.
+/**
+ * Is the given 'threads' iterator pointing at a valid thread.
  *
  * When this function returns TRUE, notmuch_threads_get will return a
  * valid object. Whereas when this function returns FALSE,
  * notmuch_threads_get will return NULL.
  *
+ * If passed a NULL pointer, this function returns FALSE
+ *
  * See the documentation of notmuch_query_search_threads for example
  * code showing how to iterate over a notmuch_threads_t object.
  */
 notmuch_bool_t
 notmuch_threads_valid (notmuch_threads_t *threads);
 
-/* Get the current thread from 'threads' as a notmuch_thread_t.
+/**
+ * Get the current thread from 'threads' as a notmuch_thread_t.
  *
  * Note: The returned thread belongs to 'threads' and has a lifetime
  * identical to it (and the query to which it belongs).
@@ -727,7 +829,8 @@ notmuch_threads_valid (notmuch_threads_t *threads);
 notmuch_thread_t *
 notmuch_threads_get (notmuch_threads_t *threads);
 
-/* Move the 'threads' iterator to the next thread.
+/**
+ * Move the 'threads' iterator to the next thread.
  *
  * If 'threads' is already pointing at the last thread then the
  * iterator will be moved to a point just beyond that last thread,
@@ -740,7 +843,8 @@ notmuch_threads_get (notmuch_threads_t *threads);
 void
 notmuch_threads_move_to_next (notmuch_threads_t *threads);
 
-/* Destroy a notmuch_threads_t object.
+/**
+ * Destroy a notmuch_threads_t object.
  *
  * It's not strictly necessary to call this function. All memory from
  * the notmuch_threads_t object will be reclaimed when the
@@ -749,7 +853,8 @@ notmuch_threads_move_to_next (notmuch_threads_t *threads);
 void
 notmuch_threads_destroy (notmuch_threads_t *threads);
 
-/* Return an estimate of the number of messages matching a search
+/**
+ * Return an estimate of the number of messages matching a search.
  *
  * This function performs a search and returns Xapian's best
  * guess as to number of matching messages.
@@ -759,8 +864,9 @@ notmuch_threads_destroy (notmuch_threads_t *threads);
  */
 unsigned
 notmuch_query_count_messages (notmuch_query_t *query);
- 
-/* Return the number of threads matching a search.
+
+/**
+ * Return the number of threads matching a search.
  *
  * This function performs a search and returns the number of unique thread IDs
  * in the matching messages. This is the same as number of threads matching a
@@ -774,7 +880,8 @@ notmuch_query_count_messages (notmuch_query_t *query);
 unsigned
 notmuch_query_count_threads (notmuch_query_t *query);
 
-/* Get the thread ID of 'thread'.
+/**
+ * Get the thread ID of 'thread'.
  *
  * The returned string belongs to 'thread' and as such, should not be
  * modified by the caller and will only be valid for as long as the
@@ -784,7 +891,8 @@ notmuch_query_count_threads (notmuch_query_t *query);
 const char *
 notmuch_thread_get_thread_id (notmuch_thread_t *thread);
 
-/* Get the total number of messages in 'thread'.
+/**
+ * Get the total number of messages in 'thread'.
  *
  * This count consists of all messages in the database belonging to
  * this thread. Contrast with notmuch_thread_get_matched_messages() .
@@ -792,7 +900,8 @@ notmuch_thread_get_thread_id (notmuch_thread_t *thread);
 int
 notmuch_thread_get_total_messages (notmuch_thread_t *thread);
 
-/* Get a notmuch_messages_t iterator for the top-level messages in
+/**
+ * Get a notmuch_messages_t iterator for the top-level messages in
  * 'thread' in oldest-first order.
  *
  * This iterator will not necessarily iterate over all of the messages
@@ -804,7 +913,8 @@ notmuch_thread_get_total_messages (notmuch_thread_t *thread);
 notmuch_messages_t *
 notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread);
 
-/* Get a notmuch_thread_t iterator for all messages in 'thread' in
+/**
+ * Get a notmuch_thread_t iterator for all messages in 'thread' in
  * oldest-first order.
  *
  * The returned list will be destroyed when the thread is destroyed.
@@ -812,7 +922,8 @@ notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread);
 notmuch_messages_t *
 notmuch_thread_get_messages (notmuch_thread_t *thread);
 
-/* Get the number of messages in 'thread' that matched the search.
+/**
+ * Get the number of messages in 'thread' that matched the search.
  *
  * This count includes only the messages in this thread that were
  * matched by the search from which the thread was created and were
@@ -823,7 +934,8 @@ notmuch_thread_get_messages (notmuch_thread_t *thread);
 int
 notmuch_thread_get_matched_messages (notmuch_thread_t *thread);
 
-/* Get the authors of 'thread' as a UTF-8 string.
+/**
+ * Get the authors of 'thread' as a UTF-8 string.
  *
  * The returned string is a comma-separated list of the names of the
  * authors of mail messages in the query results that belong to this
@@ -837,7 +949,8 @@ notmuch_thread_get_matched_messages (notmuch_thread_t *thread);
 const char *
 notmuch_thread_get_authors (notmuch_thread_t *thread);
 
-/* Get the subject of 'thread' as a UTF-8 string.
+/**
+ * Get the subject of 'thread' as a UTF-8 string.
  *
  * The subject is taken from the first message (according to the query
  * order---see notmuch_query_set_sort) in the query results that
@@ -851,17 +964,20 @@ notmuch_thread_get_authors (notmuch_thread_t *thread);
 const char *
 notmuch_thread_get_subject (notmuch_thread_t *thread);
 
-/* Get the date of the oldest message in 'thread' as a time_t value.
+/**
+ * Get the date of the oldest message in 'thread' as a time_t value.
  */
 time_t
 notmuch_thread_get_oldest_date (notmuch_thread_t *thread);
 
-/* Get the date of the newest message in 'thread' as a time_t value.
+/**
+ * Get the date of the newest message in 'thread' as a time_t value.
  */
 time_t
 notmuch_thread_get_newest_date (notmuch_thread_t *thread);
 
-/* Get the tags for 'thread', returning a notmuch_tags_t object which
+/**
+ * Get the tags for 'thread', returning a notmuch_tags_t object which
  * can be used to iterate over all tags.
  *
  * Note: In the Notmuch database, tags are stored on individual
@@ -884,7 +1000,7 @@ notmuch_thread_get_newest_date (notmuch_thread_t *thread);
  *
  *     for (tags = notmuch_thread_get_tags (thread);
  *          notmuch_tags_valid (tags);
- *          notmuch_result_move_to_next (tags))
+ *          notmuch_tags_move_to_next (tags))
  *     {
  *         tag = notmuch_tags_get (tags);
  *         ....
@@ -900,11 +1016,14 @@ notmuch_thread_get_newest_date (notmuch_thread_t *thread);
 notmuch_tags_t *
 notmuch_thread_get_tags (notmuch_thread_t *thread);
 
-/* Destroy a notmuch_thread_t object. */
+/**
+ * Destroy a notmuch_thread_t object.
+ */
 void
 notmuch_thread_destroy (notmuch_thread_t *thread);
 
-/* Is the given 'messages' iterator pointing at a valid message.
+/**
+ * Is the given 'messages' iterator pointing at a valid message.
  *
  * When this function returns TRUE, notmuch_messages_get will return a
  * valid object. Whereas when this function returns FALSE,
@@ -916,7 +1035,8 @@ notmuch_thread_destroy (notmuch_thread_t *thread);
 notmuch_bool_t
 notmuch_messages_valid (notmuch_messages_t *messages);
 
-/* Get the current message from 'messages' as a notmuch_message_t.
+/**
+ * Get the current message from 'messages' as a notmuch_message_t.
  *
  * Note: The returned message belongs to 'messages' and has a lifetime
  * identical to it (and the query to which it belongs).
@@ -930,7 +1050,8 @@ notmuch_messages_valid (notmuch_messages_t *messages);
 notmuch_message_t *
 notmuch_messages_get (notmuch_messages_t *messages);
 
-/* Move the 'messages' iterator to the next message.
+/**
+ * Move the 'messages' iterator to the next message.
  *
  * If 'messages' is already pointing at the last message then the
  * iterator will be moved to a point just beyond that last message,
@@ -943,7 +1064,8 @@ notmuch_messages_get (notmuch_messages_t *messages);
 void
 notmuch_messages_move_to_next (notmuch_messages_t *messages);
 
-/* Destroy a notmuch_messages_t object.
+/**
+ * Destroy a notmuch_messages_t object.
  *
  * It's not strictly necessary to call this function. All memory from
  * the notmuch_messages_t object will be reclaimed when the containing
@@ -952,7 +1074,8 @@ notmuch_messages_move_to_next (notmuch_messages_t *messages);
 void
 notmuch_messages_destroy (notmuch_messages_t *messages);
 
-/* Return a list of tags from all messages.
+/**
+ * Return a list of tags from all messages.
  *
  * The resulting list is guaranteed not to contain duplicated tags.
  *
@@ -967,7 +1090,8 @@ notmuch_messages_destroy (notmuch_messages_t *messages);
 notmuch_tags_t *
 notmuch_messages_collect_tags (notmuch_messages_t *messages);
 
-/* Get the message ID of 'message'.
+/**
+ * Get the message ID of 'message'.
  *
  * The returned string belongs to 'message' and as such, should not be
  * modified by the caller and will only be valid for as long as the
@@ -981,7 +1105,8 @@ notmuch_messages_collect_tags (notmuch_messages_t *messages);
 const char *
 notmuch_message_get_message_id (notmuch_message_t *message);
 
-/* Get the thread ID of 'message'.
+/**
+ * Get the thread ID of 'message'.
  *
  * The returned string belongs to 'message' and as such, should not be
  * modified by the caller and will only be valid for as long as the
@@ -995,7 +1120,8 @@ notmuch_message_get_message_id (notmuch_message_t *message);
 const char *
 notmuch_message_get_thread_id (notmuch_message_t *message);
 
-/* Get a notmuch_messages_t iterator for all of the replies to
+/**
+ * Get a notmuch_messages_t iterator for all of the replies to
  * 'message'.
  *
  * Note: This call only makes sense if 'message' was ultimately
@@ -1015,7 +1141,8 @@ notmuch_message_get_thread_id (notmuch_message_t *message);
 notmuch_messages_t *
 notmuch_message_get_replies (notmuch_message_t *message);
 
-/* Get a filename for the email corresponding to 'message'.
+/**
+ * Get a filename for the email corresponding to 'message'.
  *
  * The returned filename is an absolute filename, (the initial
  * component will match notmuch_database_get_path() ).
@@ -1033,7 +1160,8 @@ notmuch_message_get_replies (notmuch_message_t *message);
 const char *
 notmuch_message_get_filename (notmuch_message_t *message);
 
-/* Get all filenames for the email corresponding to 'message'.
+/**
+ * Get all filenames for the email corresponding to 'message'.
  *
  * Returns a notmuch_filenames_t iterator listing all the filenames
  * associated with 'message'. These files may not have identical
@@ -1045,31 +1173,40 @@ notmuch_message_get_filename (notmuch_message_t *message);
 notmuch_filenames_t *
 notmuch_message_get_filenames (notmuch_message_t *message);
 
-/* Message flags */
+/**
+ * Message flags.
+ */
 typedef enum _notmuch_message_flag {
     NOTMUCH_MESSAGE_FLAG_MATCH,
     NOTMUCH_MESSAGE_FLAG_EXCLUDED
 } notmuch_message_flag_t;
 
-/* Get a value of a flag for the email corresponding to 'message'. */
+/**
+ * Get a value of a flag for the email corresponding to 'message'.
+ */
 notmuch_bool_t
 notmuch_message_get_flag (notmuch_message_t *message,
 			  notmuch_message_flag_t flag);
 
-/* Set a value of a flag for the email corresponding to 'message'. */
+/**
+ * Set a value of a flag for the email corresponding to 'message'.
+ */
 void
 notmuch_message_set_flag (notmuch_message_t *message,
 			  notmuch_message_flag_t flag, notmuch_bool_t value);
 
-/* Get the date of 'message' as a time_t value.
+/**
+ * Get the date of 'message' as a time_t value.
  *
  * For the original textual representation of the Date header from the
  * message call notmuch_message_get_header() with a header value of
- * "date". */
+ * "date".
+ */
 time_t
 notmuch_message_get_date  (notmuch_message_t *message);
 
-/* Get the value of the specified header from 'message' as a UTF-8 string.
+/**
+ * Get the value of the specified header from 'message' as a UTF-8 string.
  *
  * Common headers are stored in the database when the message is
  * indexed and will be returned from the database.  Other headers will
@@ -1087,7 +1224,8 @@ notmuch_message_get_date  (notmuch_message_t *message);
 const char *
 notmuch_message_get_header (notmuch_message_t *message, const char *header);
 
-/* Get the tags for 'message', returning a notmuch_tags_t object which
+/**
+ * Get the tags for 'message', returning a notmuch_tags_t object which
  * can be used to iterate over all tags.
  *
  * The tags object is owned by the message and as such, will only be
@@ -1104,7 +1242,7 @@ notmuch_message_get_header (notmuch_message_t *message, const char *header);
  *
  *     for (tags = notmuch_message_get_tags (message);
  *          notmuch_tags_valid (tags);
- *          notmuch_result_move_to_next (tags))
+ *          notmuch_tags_move_to_next (tags))
  *     {
  *         tag = notmuch_tags_get (tags);
  *         ....
@@ -1120,10 +1258,13 @@ notmuch_message_get_header (notmuch_message_t *message, const char *header);
 notmuch_tags_t *
 notmuch_message_get_tags (notmuch_message_t *message);
 
-/* The longest possible tag value. */
+/**
+ * The longest possible tag value.
+ */
 #define NOTMUCH_TAG_MAX 200
 
-/* Add a tag to the given message.
+/**
+ * Add a tag to the given message.
  *
  * Return value:
  *
@@ -1140,7 +1281,8 @@ notmuch_message_get_tags (notmuch_message_t *message);
 notmuch_status_t
 notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
 
-/* Remove a tag from the given message.
+/**
+ * Remove a tag from the given message.
  *
  * Return value:
  *
@@ -1157,7 +1299,8 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
 notmuch_status_t
 notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
 
-/* Remove all tags from the given message.
+/**
+ * Remove all tags from the given message.
  *
  * See notmuch_message_freeze for an example showing how to safely
  * replace tag values.
@@ -1168,7 +1311,8 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
 notmuch_status_t
 notmuch_message_remove_all_tags (notmuch_message_t *message);
 
-/* Add/remove tags according to maildir flags in the message filename(s)
+/**
+ * Add/remove tags according to maildir flags in the message filename(s).
  *
  * This function examines the filenames of 'message' for maildir
  * flags, and adds or removes tags on 'message' as follows when these
@@ -1202,7 +1346,8 @@ notmuch_message_remove_all_tags (notmuch_message_t *message);
 notmuch_status_t
 notmuch_message_maildir_flags_to_tags (notmuch_message_t *message);
 
-/* Rename message filename(s) to encode tags as maildir flags
+/**
+ * Rename message filename(s) to encode tags as maildir flags.
  *
  * Specifically, for each filename corresponding to this message:
  *
@@ -1238,7 +1383,8 @@ notmuch_message_maildir_flags_to_tags (notmuch_message_t *message);
 notmuch_status_t
 notmuch_message_tags_to_maildir_flags (notmuch_message_t *message);
 
-/* Freeze the current state of 'message' within the database.
+/**
+ * Freeze the current state of 'message' within the database.
  *
  * This means that changes to the message state, (via
  * notmuch_message_add_tag, notmuch_message_remove_tag, and
@@ -1281,7 +1427,8 @@ notmuch_message_tags_to_maildir_flags (notmuch_message_t *message);
 notmuch_status_t
 notmuch_message_freeze (notmuch_message_t *message);
 
-/* Thaw the current 'message', synchronizing any changes that may have
+/**
+ * Thaw the current 'message', synchronizing any changes that may have
  * occurred while 'message' was frozen into the notmuch database.
  *
  * See notmuch_message_freeze for an example of how to use this
@@ -1304,7 +1451,8 @@ notmuch_message_freeze (notmuch_message_t *message);
 notmuch_status_t
 notmuch_message_thaw (notmuch_message_t *message);
 
-/* Destroy a notmuch_message_t object.
+/**
+ * Destroy a notmuch_message_t object.
  *
  * It can be useful to call this function in the case of a single
  * query object with many messages in the result, (such as iterating
@@ -1315,7 +1463,8 @@ notmuch_message_thaw (notmuch_message_t *message);
 void
 notmuch_message_destroy (notmuch_message_t *message);
 
-/* Is the given 'tags' iterator pointing at a valid tag.
+/**
+ * Is the given 'tags' iterator pointing at a valid tag.
  *
  * When this function returns TRUE, notmuch_tags_get will return a
  * valid string. Whereas when this function returns FALSE,
@@ -1327,7 +1476,8 @@ notmuch_message_destroy (notmuch_message_t *message);
 notmuch_bool_t
 notmuch_tags_valid (notmuch_tags_t *tags);
 
-/* Get the current tag from 'tags' as a string.
+/**
+ * Get the current tag from 'tags' as a string.
  *
  * Note: The returned string belongs to 'tags' and has a lifetime
  * identical to it (and the query to which it ultimately belongs).
@@ -1338,7 +1488,8 @@ notmuch_tags_valid (notmuch_tags_t *tags);
 const char *
 notmuch_tags_get (notmuch_tags_t *tags);
 
-/* Move the 'tags' iterator to the next tag.
+/**
+ * Move the 'tags' iterator to the next tag.
  *
  * If 'tags' is already pointing at the last tag then the iterator
  * will be moved to a point just beyond that last tag, (where
@@ -1351,7 +1502,8 @@ notmuch_tags_get (notmuch_tags_t *tags);
 void
 notmuch_tags_move_to_next (notmuch_tags_t *tags);
 
-/* Destroy a notmuch_tags_t object.
+/**
+ * Destroy a notmuch_tags_t object.
  *
  * It's not strictly necessary to call this function. All memory from
  * the notmuch_tags_t object will be reclaimed when the containing
@@ -1360,7 +1512,8 @@ notmuch_tags_move_to_next (notmuch_tags_t *tags);
 void
 notmuch_tags_destroy (notmuch_tags_t *tags);
 
-/* Store an mtime within the database for 'directory'.
+/**
+ * Store an mtime within the database for 'directory'.
  *
  * The 'directory' should be an object retrieved from the database
  * with notmuch_database_get_directory for a particular path.
@@ -1400,35 +1553,44 @@ notmuch_status_t
 notmuch_directory_set_mtime (notmuch_directory_t *directory,
 			     time_t mtime);
 
-/* Get the mtime of a directory, (as previously stored with
+/**
+ * Get the mtime of a directory, (as previously stored with
  * notmuch_directory_set_mtime).
  *
  * Returns 0 if no mtime has previously been stored for this
- * directory.*/
+ * directory.
+ */
 time_t
 notmuch_directory_get_mtime (notmuch_directory_t *directory);
 
-/* Get a notmuch_filenames_t iterator listing all the filenames of
+/**
+ * Get a notmuch_filenames_t iterator listing all the filenames of
  * messages in the database within the given directory.
  *
  * The returned filenames will be the basename-entries only (not
- * complete paths). */
+ * complete paths).
+ */
 notmuch_filenames_t *
 notmuch_directory_get_child_files (notmuch_directory_t *directory);
 
-/* Get a notmuch_filenams_t iterator listing all the filenames of
+/**
+ * Get a notmuch_filenams_t iterator listing all the filenames of
  * sub-directories in the database within the given directory.
  *
  * The returned filenames will be the basename-entries only (not
- * complete paths). */
+ * complete paths).
+ */
 notmuch_filenames_t *
 notmuch_directory_get_child_directories (notmuch_directory_t *directory);
 
-/* Destroy a notmuch_directory_t object. */
+/**
+ * Destroy a notmuch_directory_t object.
+ */
 void
 notmuch_directory_destroy (notmuch_directory_t *directory);
 
-/* Is the given 'filenames' iterator pointing at a valid filename.
+/**
+ * Is the given 'filenames' iterator pointing at a valid filename.
  *
  * When this function returns TRUE, notmuch_filenames_get will return
  * a valid string. Whereas when this function returns FALSE,
@@ -1440,7 +1602,8 @@ notmuch_directory_destroy (notmuch_directory_t *directory);
 notmuch_bool_t
 notmuch_filenames_valid (notmuch_filenames_t *filenames);
 
-/* Get the current filename from 'filenames' as a string.
+/**
+ * Get the current filename from 'filenames' as a string.
  *
  * Note: The returned string belongs to 'filenames' and has a lifetime
  * identical to it (and the directory to which it ultimately belongs).
@@ -1451,7 +1614,8 @@ notmuch_filenames_valid (notmuch_filenames_t *filenames);
 const char *
 notmuch_filenames_get (notmuch_filenames_t *filenames);
 
-/* Move the 'filenames' iterator to the next filename.
+/**
+ * Move the 'filenames' iterator to the next filename.
  *
  * If 'filenames' is already pointing at the last filename then the
  * iterator will be moved to a point just beyond that last filename,
@@ -1464,7 +1628,8 @@ notmuch_filenames_get (notmuch_filenames_t *filenames);
 void
 notmuch_filenames_move_to_next (notmuch_filenames_t *filenames);
 
-/* Destroy a notmuch_filenames_t object.
+/**
+ * Destroy a notmuch_filenames_t object.
  *
  * It's not strictly necessary to call this function. All memory from
  * the notmuch_filenames_t object will be reclaimed when the
@@ -1476,6 +1641,8 @@ notmuch_filenames_move_to_next (notmuch_filenames_t *filenames);
 void
 notmuch_filenames_destroy (notmuch_filenames_t *filenames);
 
+/* @} */
+
 NOTMUCH_END_DECLS
 
 #endif
diff --git a/lib/query.cc b/lib/query.cc
index ec60e2e4..60ff8bd9 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -462,6 +462,9 @@ notmuch_threads_valid (notmuch_threads_t *threads)
 {
     unsigned int doc_id;
 
+    if (! threads)
+	return FALSE;
+
     while (threads->doc_id_pos < threads->doc_ids->len) {
 	doc_id = g_array_index (threads->doc_ids, unsigned int,
 				threads->doc_id_pos);
diff --git a/lib/thread.cc b/lib/thread.cc
index 4dcf7053..8f53e122 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -524,7 +524,7 @@ _notmuch_thread_create (void *ctx,
     _resolve_thread_relationships (thread);
 
     /* Commit to returning thread. */
-    talloc_steal (ctx, thread);
+    (void) talloc_steal (ctx, thread);
 
   DONE:
     talloc_free (local);
diff --git a/man/.gitignore b/man/.gitignore
deleted file mode 100644
index 26ead201..00000000
--- a/man/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-# ignore gzipped man pages
-*.[0-9].gz
diff --git a/man/Makefile.local b/man/Makefile.local
deleted file mode 100644
index 57910b76..00000000
--- a/man/Makefile.local
+++ /dev/null
@@ -1,55 +0,0 @@
-# -*- Makefile -*-
-
-dir := man
-
-# this variable seems to be needed to prevent lazy evaluation causing
-# problems with $(dir) changing values.
-MAIN_PAGE := $(dir)/man1/notmuch.1
-
-MAN1 := \
-	$(MAIN_PAGE) \
-	$(dir)/man1/notmuch-compact.1 \
-	$(dir)/man1/notmuch-config.1 \
-	$(dir)/man1/notmuch-count.1 \
-	$(dir)/man1/notmuch-dump.1 \
-	$(dir)/man1/notmuch-restore.1 \
-	$(dir)/man1/notmuch-insert.1 \
-	$(dir)/man1/notmuch-new.1 \
-	$(dir)/man1/notmuch-reply.1 \
-	$(dir)/man1/notmuch-search.1 \
-	$(dir)/man1/notmuch-show.1 \
-	$(dir)/man1/notmuch-tag.1
-
-MAN5 := $(dir)/man5/notmuch-hooks.5
-MAN7 := $(dir)/man7/notmuch-search-terms.7
-
-MAN1_GZ := $(addsuffix .gz,$(MAN1))
-MAN5_GZ := $(addsuffix .gz,$(MAN5))
-MAN7_GZ := $(addsuffix .gz,$(MAN7))
-
-MAN_SOURCE := $(MAN1) $(MAN5) $(MAN7)
-MAN_BACKUP := $(addsuffix .bak,$(MAN_SOURCE))
-COMPRESSED_MAN := $(MAN1_GZ) $(MAN5_GZ) $(MAN7_GZ)
-
-%.gz: %
-	gzip --stdout $^ > $@
-
-.PHONY: install-man update-man-versions
-
-install-man: $(COMPRESSED_MAN)
-	mkdir -p "$(DESTDIR)$(mandir)/man1"
-	mkdir -p "$(DESTDIR)$(mandir)/man5"
-	mkdir -p "$(DESTDIR)$(mandir)/man7"
-	install -m0644 $(MAN1_GZ) $(DESTDIR)/$(mandir)/man1
-	install -m0644 $(MAN5_GZ) $(DESTDIR)/$(mandir)/man5
-	install -m0644 $(MAN7_GZ) $(DESTDIR)/$(mandir)/man7
-	cd $(DESTDIR)/$(mandir)/man1 && ln -sf notmuch.1.gz notmuch-setup.1.gz
-
-update-man-versions: $(MAN_SOURCE)
-	for file in $(MAN_SOURCE); do \
-	    cp $$file $$file.bak ; \
-	    sed "s/^.TH NOTMUCH\([^[:blank:]]*\) \([1-9]\) .*$$/.TH NOTMUCH\1 \2 ${DATE} \"Notmuch ${VERSION}\"/" \
-	        < $$file.bak > $$file; \
-	done
-
-CLEAN := $(CLEAN) $(COMPRESSED_MAN) $(MAN_BACKUP)
diff --git a/man/man1/notmuch-compact.1 b/man/man1/notmuch-compact.1
deleted file mode 100644
index ea6218fb..00000000
--- a/man/man1/notmuch-compact.1
+++ /dev/null
@@ -1,62 +0,0 @@
-.TH NOTMUCH-COMPACT 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-compact \- compact the notmuch database
-.SH SYNOPSIS
-
-.B notmuch compact
-.RI "[ --quiet ]"
-.RI "[ --backup=<" directory "> ]"
-
-.SH DESCRIPTION
-
-The
-.B compact
-command can be used to compact the notmuch database. This can both reduce
-the space required by the database and improve lookup performance.
-
-The compacted database is built in a temporary directory and is later
-moved into the place of the origin database. The original uncompacted
-database is discarded, unless the
-.BR "\-\-backup=" <directory>
-option is used.
-
-Note that the database write lock will be held during the compaction
-process (which may be quite long) to protect data integrity.
-
-Supported options for
-.B compact
-include
-
-.RS 4
-.TP 4
-.BR "\-\-backup=" <directory>
-
-Save the current database to the given directory before replacing it
-with the compacted database. The backup directory must not exist and
-it must reside on the same mounted filesystem as the current database.
-
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-quiet
-
-Do not report database compaction progress to stdout.
-
-.RE
-
-.RE
-.SH ENVIRONMENT
-The following environment variables can be used to control the
-behavior of notmuch.
-.TP
-.B NOTMUCH_CONFIG
-Specifies the location of the notmuch configuration file. Notmuch will
-use ${HOME}/.notmuch\-config if this variable is not set.
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-count\fR(1), \fBnotmuch-dump\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-config.1 b/man/man1/notmuch-config.1
deleted file mode 100644
index 00a420f9..00000000
--- a/man/man1/notmuch-config.1
+++ /dev/null
@@ -1,158 +0,0 @@
-.TH NOTMUCH-CONFIG 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-config \- access notmuch configuration file
-.SH SYNOPSIS
-
-.B notmuch config get
-.RI  "<" section ">.<" item ">"
-
-.B notmuch config set
-.RI  "<" section ">.<" item "> [" value " ...]"
-
-.B notmuch config list
-
-.SH DESCRIPTION
-
-The
-.B config
-command can be used to get or set settings in the notmuch
-configuration file.
-
-.RS 4
-.TP 4
-.B get
-The value of the specified configuration item is printed to stdout. If
-the item has multiple values (it is a list), each value is separated
-by a newline character.
-.RE
-
-.RS 4
-.TP 4
-.B set
-The specified configuration item is set to the given value. To specify
-a multiple-value item (a list), provide each value as a separate
-command-line argument.
-
-If no values are provided, the specified configuration item will be
-removed from the configuration file.
-.RE
-
-.RS 4
-.TP 4
-.B list
-Every configuration item is printed to stdout, each on a separate line
-of the form:
-
-.RI  "" section "." item "=" value
-
-No additional whitespace surrounds the dot or equals sign characters. In a
-multiple-value item (a list), the values are separated by semicolon characters.
-.RE
-
-The available configuration items are described below.
-
-.RS 4
-.TP 4
-.B database.path
-The top-level directory where your mail currently exists and to where
-mail will be delivered in the future. Files should be individual email
-messages. Notmuch will store its database within a sub-directory of
-the path configured here named
-.BR ".notmuch".
-.RE
-
-.RS 4
-.TP 4
-.B user.name
-Your full name.
-.RE
-
-.RS 4
-.TP 4
-.B user.primary_email
-Your primary email address.
-.RE
-
-.RS 4
-.TP 4
-.B user.other_email
-A list of other email addresses at which you receive email.
-.RE
-
-.RS 4
-.TP 4
-.B new.tags
-A list of tags that will be added to all messages incorporated by
-.BR "notmuch new".
-.RE
-
-.RS 4
-.TP 4
-.B new.ignore
-A list of file and directory names, without path, that will not be
-searched for messages by
-.BR "notmuch new".
-All the files and directories matching any of the names specified here
-will be ignored, regardless of the location in the mail store
-directory hierarchy.
-.RE
-
-.RS 4
-.TP 4
-.B search.exclude_tags
-A list of tags that will be excluded from search results by
-default. Using an excluded tag in a query will override that
-exclusion.
-.RE
-
-.RS 4
-.TP 4
-.B maildir.synchronize_flags
-If true, then the following maildir flags (in message filenames) will
-be synchronized with the corresponding notmuch tags:
-
-  Flag    Tag
-  ----    -------
-  D       draft
-  F       flagged
-  P       passed
-  R       replied
-  S       unread (added when 'S' flag is not present)
-
-The
-.B notmuch new
-command will notice flag changes in filenames and update tags, while
-the
-.B notmuch tag
-and
-.B notmuch restore
-commands will notice tag changes and update flags in filenames.
-
-If there have been any changes in the maildir (new messages added, old
-ones removed or renamed, maildir flags changed, etc.), it is advisable
-to run
-.B notmuch new
-before
-.B notmuch tag
-or
-.B notmuch restore
-commands to ensure the tag changes are properly synchronized to the
-maildir flags, as the commands expect the database and maildir to be
-in sync.
-.RE
-
-.RE
-.SH ENVIRONMENT
-The following environment variables can be used to control the
-behavior of notmuch.
-.TP
-.B NOTMUCH_CONFIG
-Specifies the location of the notmuch configuration file. Notmuch will
-use ${HOME}/.notmuch\-config if this variable is not set.
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-count\fR(1), \fBnotmuch-dump\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-count.1 b/man/man1/notmuch-count.1
deleted file mode 100644
index 562dde15..00000000
--- a/man/man1/notmuch-count.1
+++ /dev/null
@@ -1,86 +0,0 @@
-.TH NOTMUCH-COUNT 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-count \- count messages matching the given search terms
-.SH SYNOPSIS
-
-.B notmuch count
-.RI  [ options "... ] <" search-term ">..."
-
-.SH DESCRIPTION
-
-Count messages matching the search terms.
-
-The number of matching messages (or threads) is output to stdout.
-
-With no search terms, a count of all messages (or threads) in the database will
-be displayed.
-
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
-
-Supported options for
-.B count
-include
-.RS 4
-.TP 4
-.B \-\-output=(messages|threads|files)
-
-.RS 4
-.TP 4
-.B messages
-
-Output the number of matching messages. This is the default.
-.RE
-.RS 4
-.TP 4
-.B threads
-
-Output the number of matching threads.
-.RE
-.RS 4
-.TP 4
-.B files
-
-Output the number of files associated with matching messages. This may
-be bigger than the number of matching messages due to duplicates
-(i.e. multiple files having the same message-id).
-.RE
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-exclude=(true|false)
-
-Specify whether to omit messages matching search.tag_exclude from the
-count (the default) or not.
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-batch
-
-Read queries from a file (stdin by default), one per line, and output
-the number of matching messages (or threads) to stdout, one per
-line. On an empty input line the count of all messages (or threads) in
-the database will be output. This option is not compatible with
-specifying search terms on the command line.
-.RE
-
-.RS 4
-.TP 4
-.BR "\-\-input=" <filename>
-
-Read input from given file, instead of from stdin. Implies
-.BR --batch .
-.RE
-
-.RE
-.RE
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-dump\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-dump.1 b/man/man1/notmuch-dump.1
deleted file mode 100644
index 0c52d1b7..00000000
--- a/man/man1/notmuch-dump.1
+++ /dev/null
@@ -1,98 +0,0 @@
-.TH NOTMUCH-DUMP 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-dump \- creates a plain-text dump of the tags of each message
-
-.SH SYNOPSIS
-
-.B "notmuch dump"
-.RB  [ "\-\-format=(sup|batch-tag)"  "] [--]"
-.RI "[ --output=<" filename "> ] [--]"
-.RI "[ <" search-term ">...]"
-
-.SH DESCRIPTION
-
-Dump tags for messages matching the given search terms.
-
-Output is to the given filename, if any, or to stdout.
-
-These tags are the only data in the notmuch database that can't be
-recreated from the messages themselves.  The output of notmuch dump is
-therefore the only critical thing to backup (and much more friendly to
-incremental backup than the native database files.)
-
-.TP 4
-.B \-\-format=(sup|batch-tag)
-
-Notmuch restore supports two plain text dump formats, both with one message-id
-per line, followed by a list of tags.
-
-.RS 4
-.TP 4
-.B sup
-
-The
-.B sup
-dump file format is specifically chosen to be
-compatible with the format of files produced by sup-dump.
-So if you've previously been using sup for mail, then the
-.B "notmuch restore"
-command provides you a way to import all of your tags (or labels as
-sup calls them).
-Each line has the following form
-
-.RS 4
-.RI < message-id >
-.B (
-.RI < tag "> ..."
-.B )
-
-with zero or more tags are separated by spaces. Note that (malformed)
-message-ids may contain arbitrary non-null characters. Note also
-that tags with spaces will not be correctly restored with this format.
-
-.RE
-
-.RE
-.RS 4
-.TP 4
-.B batch-tag
-
-The
-.B batch-tag
-dump format is intended to more robust against malformed message-ids
-and tags containing whitespace or non-\fBascii\fR(7) characters.
-Each line has the form
-
-.RS 4
-.RI "+<" "encoded-tag" "> " "" "+<" "encoded-tag" "> ... -- " "" " id:<" quoted-message-id >
-
-Tags are hex-encoded by replacing every byte not matching the regex
-.B [A-Za-z0-9@=.,_+-]
-with
-.B %nn
-where nn is the two digit hex encoding.  The message ID is a valid Xapian
-query, quoted using Xapian boolean term quoting rules: if the ID contains
-whitespace or a close paren or starts with a double quote, it must be
-enclosed in double quotes and double quotes inside the ID must be doubled.
-The astute reader will notice this is a special case of the batch input
-format for \fBnotmuch-tag\fR(1); note that the single message-id query is
-mandatory for \fBnotmuch-restore\fR(1).
-
-.RE
-
-
-With no search terms, a dump of all messages in the database will be
-generated.  A "--" argument instructs notmuch that the
-remaining arguments are search terms.
-
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
-
-.RE
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-insert.1 b/man/man1/notmuch-insert.1
deleted file mode 100644
index 8ce8413e..00000000
--- a/man/man1/notmuch-insert.1
+++ /dev/null
@@ -1,75 +0,0 @@
-.TH NOTMUCH-INSERT 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-insert \- add a message to the maildir and notmuch database
-.SH SYNOPSIS
-
-.B notmuch insert
-.RI "[" options "]"
-.RI "[ +<" tag> "|\-<" tag "> ... ]"
-
-.SH DESCRIPTION
-
-.B notmuch insert
-reads a message from standard input
-and delivers it into the maildir directory given by configuration option
-.BR database.path ,
-then incorporates the message into the notmuch database.
-It is an alternative to using a separate tool to deliver
-the message then running
-.B notmuch new
-afterwards.
-
-The new message will be tagged with the tags specified by the
-.B new.tags
-configuration option, then by operations specified on the command-line:
-tags prefixed by '+' are added while
-those prefixed by '\-' are removed.
-
-If the new message is a duplicate of an existing message in the database
-(it has same Message-ID), it will be added to the maildir folder and
-notmuch database, but the tags will not be changed.
-
-Option arguments must appear before any tag operation arguments.
-Supported options for
-.B insert
-include
-.RS 4
-.TP 4
-.BI "--folder=<" folder ">"
-
-Deliver the message to the specified folder,
-relative to the top-level directory given by the value of
-\fBdatabase.path\fR.
-The default is to deliver to the top-level directory.
-
-.RE
-
-.RS 4
-.TP 4
-.B "--create-folder"
-
-Try to create the folder named by the
-.B "--folder"
-option, if it does not exist.
-Otherwise the folder must already exist for mail
-delivery to succeed.
-
-.RE
-.SH EXIT STATUS
-
-This command returns exit status 0 if the message was successfully
-added to the mail directory, even if the message could not be indexed
-and added to the notmuch database.  In the latter case, a warning will
-be printed to standard error but the message file will be left on disk.
-
-If the message could not be written to disk then a non-zero exit
-status is returned.
-
-.RE
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-reply\fR(1),
-\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-new.1 b/man/man1/notmuch-new.1
deleted file mode 100644
index 5725b7d8..00000000
--- a/man/man1/notmuch-new.1
+++ /dev/null
@@ -1,70 +0,0 @@
-.TH NOTMUCH-NEW 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-new \- incorporate new mail into the notmuch database
-.SH SYNOPSIS
-
-.B notmuch new
-.RB "[" --no-hooks "]"
-
-.SH DESCRIPTION
-
-Find and import any new messages to the database.
-
-The
-.B new
-command scans all sub-directories of the database, performing
-full-text indexing on new messages that are found. Each new message
-will automatically be tagged with both the
-.BR inbox " and " unread
-tags.
-
-You should run
-.B "notmuch new"
-once after first running
-.B "notmuch setup"
-to create the initial database. The first run may take a long time if
-you have a significant amount of mail (several hundred thousand
-messages or more). Subsequently, you should run
-.B "notmuch new"
-whenever new mail is delivered and you wish to incorporate it into the
-database. These subsequent runs will be much quicker than the initial
-run.
-
-Invoking
-.B notmuch
-with no command argument will run
-.B new
-if
-.B "notmuch setup"
-has previously been completed, but
-.B "notmuch new"
-has not previously been run.
-
-.B "notmuch new"
-updates tags according to maildir flag changes if the
-.B "maildir.synchronize_flags"
-configuration option is enabled. See \fBnotmuch-config\fR(1) for
-details.
-
-The
-.B new
-command supports hooks. See  \fBnotmuch-hooks(5)\fR
-for more details on hooks.
-
-Supported options for
-.B new
-include
-.RS 4
-.TP 4
-.BR \-\-no\-hooks
-
-Prevents hooks from being run.
-.RE
-.RE
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5), \fBnotmuch-insert\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-reply.1 b/man/man1/notmuch-reply.1
deleted file mode 100644
index 93f90673..00000000
--- a/man/man1/notmuch-reply.1
+++ /dev/null
@@ -1,133 +0,0 @@
-.TH NOTMUCH-REPLY 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-reply \- constructs a reply template for a set of messages
-
-.SH SYNOPSIS
-
-.B notmuch reply
-.RI "[" options "...] <" search-term ">..."
-
-.SH DESCRIPTION
-
-Constructs a reply template for a set of messages.
-
-To make replying to email easier,
-.B notmuch reply
-takes an existing set of messages and constructs a suitable mail
-template. The Reply-to: header (if any, otherwise From:) is used for
-the To: address. Unless
-.BR \-\-reply-to=sender
-is specified, values from the To: and Cc: headers are copied, but not
-including any of the current user's email addresses (as configured in
-primary_mail or other_email in the .notmuch\-config file) in the
-recipient list.
-
-It also builds a suitable new subject, including Re: at the front (if
-not already present), and adding the message IDs of the messages being
-replied to to the References list and setting the In\-Reply\-To: field
-correctly.
-
-Finally, the original contents of the emails are quoted by prefixing
-each line with '> ' and included in the body.
-
-The resulting message template is output to stdout.
-
-Supported options for
-.B reply
-include
-.RS
-.TP 4
-.BR \-\-format= ( default | json | sexp | headers\-only )
-.RS
-.TP 4
-.BR default
-Includes subject and quoted message body as an RFC 2822 message.
-.TP
-.BR json
-Produces JSON output containing headers for a reply message and the
-contents of the original message. This output can be used by a client
-to create a reply message intelligently.
-.TP
-.BR sexp
-Produces S-Expression output containing headers for a reply message and
-the contents of the original message. This output can be used by a client
-to create a reply message intelligently.
-.TP
-.BR headers\-only
-Only produces In\-Reply\-To, References, To, Cc, and Bcc headers.
-.RE
-.RE
-
-.RS
-.TP 4
-.BR \-\-format-version=N
-
-Use the specified structured output format version.  This is intended
-for programs that invoke \fBnotmuch\fR(1) internally.  If omitted, the
-latest supported version will be used.
-.RE
-
-.RS
-.TP 4
-.BR \-\-reply\-to= ( all | sender )
-.RS
-.TP 4
-.BR all " (default)"
-Replies to all addresses.
-.TP 4
-.BR sender
-Replies only to the sender. If replying to user's own message
-(Reply-to: or From: header is one of the user's configured email
-addresses), try To:, Cc:, and Bcc: headers in this order, and copy
-values from the first that contains something other than only the
-user's addresses.
-.RE
-.RE
-.RS
-.TP 4
-.B \-\-decrypt
-
-Decrypt any MIME encrypted parts found in the selected content
-(ie. "multipart/encrypted" parts). Status of the decryption will be
-reported (currently only supported with --format=json and
---format=sexp) and on successful decryption the multipart/encrypted
-part will be replaced by the decrypted content.
-
-Decryption expects a functioning \fBgpg-agent\fR(1) to provide any
-needed credentials. Without one, the decryption will fail.
-.RE
-
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
-
-Note: It is most common to use
-.B "notmuch reply"
-with a search string matching a single message, (such as
-id:<message-id>), but it can be useful to reply to several messages at
-once. For example, when a series of patches are sent in a single
-thread, replying to the entire thread allows for the reply to comment
-on issues found in multiple patches. The default format supports
-replying to multiple messages at once, but the JSON and S-Expression
-formats do not.
-.RE
-.RE
-
-.SH EXIT STATUS
-
-This command supports the following special exit status codes
-
-.TP
-.B 20
-The requested format version is too old.
-.TP
-.B 21
-The requested format version is too new.
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-restore.1 b/man/man1/notmuch-restore.1
deleted file mode 100644
index 4cb02e3e..00000000
--- a/man/man1/notmuch-restore.1
+++ /dev/null
@@ -1,91 +0,0 @@
-.TH NOTMUCH-RESTORE 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-restore \- restores the tags from the given file (see notmuch dump)
-
-.SH SYNOPSIS
-
-.B "notmuch restore"
-.RB [ "--accumulate" ]
-.RB [ "--format=(auto|batch-tag|sup)" ]
-.RI "[ --input=<" filename "> ]"
-
-.SH DESCRIPTION
-
-Restores the tags from the given file (see
-.BR "notmuch dump" ")."
-
-The input is read from the given filename, if any, or from stdin.
-
-
-Supported options for
-.B restore
-include
-.RS 4
-.TP 4
-.B \-\-accumulate
-
-The union of the existing and new tags is applied, instead of
-replacing each message's tags as they are read in from the dump file.
-
-.RE
-.RS 4
-.TP 4
-.B \-\-format=(sup|batch-tag|auto)
-
-Notmuch restore supports two plain text dump formats, with each line
-specifying a message-id and a set of tags.
-For details of the actual formats, see \fBnotmuch-dump\fR(1).
-
-.RS 4
-.TP 4
-.B sup
-
-The
-.B sup
-dump file format is specifically chosen to be
-compatible with the format of files produced by sup-dump.
-So if you've previously been using sup for mail, then the
-.B "notmuch restore"
-command provides you a way to import all of your tags (or labels as
-sup calls them).
-
-.RE
-.RS 4
-.TP 4
-.B batch-tag
-
-The
-.B batch-tag
-dump format is intended to more robust against malformed message-ids
-and tags containing whitespace or non-\fBascii\fR(7) characters.  See
-\fBnotmuch-dump\fR(1) for details on this format.
-
-.B "notmuch restore"
-updates the maildir flags according to tag changes if the
-.B "maildir.synchronize_flags"
-configuration option is enabled. See \fBnotmuch-config\fR(1) for
-details.
-
-.RE
-
-.RS 4
-.TP 4
-.B auto
-
-This option (the default) tries to guess the format from the
-input. For correctly formed input in either supported format, this
-heuristic, based the fact that batch-tag format contains no parentheses,
-should be accurate.
-
-.RE
-
-.RE
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-search.1 b/man/man1/notmuch-search.1
deleted file mode 100644
index 55a81e79..00000000
--- a/man/man1/notmuch-search.1
+++ /dev/null
@@ -1,199 +0,0 @@
-.TH NOTMUCH-SEARCH 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-search \- search for messages matching the given search terms
-.SH SYNOPSIS
-
-.B notmuch search
-.RI  [  options "...] <" search-term ">..."
-
-.SH DESCRIPTION
-
-Search for messages matching the given search terms, and display as
-results the threads containing the matched messages.
-
-The output consists of one line per thread, giving a thread ID, the
-date of the newest (or oldest, depending on the sort option) matched
-message in the thread, the number of matched messages and total
-messages in the thread, the names of all participants in the thread,
-and the subject of the newest (or oldest) message.
-
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
-
-Supported options for
-.B search
-include
-.RS 4
-.TP 4
-.BR \-\-format= ( json | sexp | text | text0 )
-
-Presents the results in either JSON, S-Expressions, newline character
-separated plain-text (default), or null character separated plain-text
-(compatible with \fBxargs\fR(1) -0 option where available).
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-format-version=N
-
-Use the specified structured output format version.  This is intended
-for programs that invoke \fBnotmuch\fR(1) internally.  If omitted, the
-latest supported version will be used.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-output=(summary|threads|messages|files|tags)
-
-.RS 4
-.TP 4
-.B summary
-
-Output a summary of each thread with any message matching the search
-terms. The summary includes the thread ID, date, the number of
-messages in the thread (both the number matched and the total number),
-the authors of the thread and the subject.
-.RE
-.RS 4
-.TP 4
-.B threads
-
-Output the thread IDs of all threads with any message matching the
-search terms, either one per line (\-\-format=text), separated by null
-characters (\-\-format=text0), as a JSON array (\-\-format=json), or
-an S-Expression list (\-\-format=sexp).
-.RE
-.RS 4
-.TP 4
-.B messages
-
-Output the message IDs of all messages matching the search terms,
-either one per line (\-\-format=text), separated by null characters
-(\-\-format=text0), as a JSON array (\-\-format=json), or as an
-S-Expression list (\-\-format=sexp).
-.RE
-.RS 4
-.TP 4
-.B files
-
-Output the filenames of all messages matching the search terms, either
-one per line (\-\-format=text), separated by null characters
-(\-\-format=text0), as a JSON array (\-\-format=json), or as an
-S-Expression list (\-\-format=sexp).
-
-Note that each message may have multiple filenames associated with it.
-All of them are included in the output, unless limited with the
-\-\-duplicate=N option.
-.RE
-.RS 4
-.TP 4
-.B tags
-
-Output all tags that appear on any message matching the search terms,
-either one per line (\-\-format=text), separated by null characters
-(\-\-format=text0), as a JSON array (\-\-format=json), or as an
-S-Expression list (\-\-format=sexp).
-.RE
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-sort= ( newest\-first | oldest\-first )
-
-This option can be used to present results in either chronological order
-.RB ( oldest\-first )
-or reverse chronological order
-.RB ( newest\-first ).
-
-Note: The thread order will be distinct between these two options
-(beyond being simply reversed). When sorting by
-.B oldest\-first
-the threads will be sorted by the oldest message in each thread, but
-when sorting by
-.B newest\-first
-the threads will be sorted by the newest message in each thread.
-
-By default, results will be displayed in reverse chronological order,
-(that is, the newest results will be displayed first).
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-offset=[\-]N
-
-Skip displaying the first N results. With the leading '\-', start at the Nth
-result from the end.
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-limit=N
-
-Limit the number of displayed results to N.
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-exclude=(true|false|all|flag)
-
-A message is called "excluded" if it matches at least one tag in
-search.tag_exclude that does not appear explicitly in the search terms.
-This option specifies whether to omit excluded messages in the search
-process.
-
-The default value,
-.BR true ,
-prevents excluded messages from matching the search terms.
-
-.B all
-additionally prevents excluded messages from appearing in displayed
-results, in effect behaving as though the excluded messages do not exist.
-
-.B false
-allows excluded messages to match search terms and appear in displayed
-results. Excluded messages are still marked in the relevant outputs.
-
-.B flag
-only has an effect when
-.BR --output=summary .
-The output is almost identical to
-.BR false ,
-but the "match count" is the number of matching non-excluded messages in the
-thread, rather than the number of matching messages.
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-duplicate=N
-
-Effective with
-.BR --output=files ,
-output the Nth filename associated with each message matching the
-query (N is 1-based). If N is greater than the number of files
-associated with the message, don't print anything.
-
-Note that this option is orthogonal with the
-.BR folder:
-search prefix. The prefix matches messages based on filenames. This
-option filters filenames of the matching messages.
-.RE
-
-.SH EXIT STATUS
-
-This command supports the following special exit status codes
-
-.TP
-.B 20
-The requested format version is too old.
-.TP
-.B 21
-The requested format version is too new.
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-setup.1 b/man/man1/notmuch-setup.1
deleted file mode 120000
index 5c78dc87..00000000
--- a/man/man1/notmuch-setup.1
+++ /dev/null
@@ -1 +0,0 @@
-notmuch.1
\ No newline at end of file
diff --git a/man/man1/notmuch-show.1 b/man/man1/notmuch-show.1
deleted file mode 100644
index 7eefdec6..00000000
--- a/man/man1/notmuch-show.1
+++ /dev/null
@@ -1,250 +0,0 @@
-.TH NOTMUCH-SHOW 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-show \- show messages matching the given search terms
-.SH SYNOPSIS
-
-.B notmuch show
-.RI "[" options "...] <" search-term ">..."
-
-.SH DESCRIPTION
-
-Shows all messages matching the search terms.
-
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for <search-terms>.
-
-The messages will be grouped and sorted based on the threading (all
-replies to a particular message will appear immediately after that
-message in date order). The output is not indented by default, but
-depth tags are printed so that proper indentation can be performed by
-a post-processor (such as the emacs interface to notmuch).
-
-Supported options for
-.B show
-include
-.RS 4
-.TP 4
-.B \-\-entire\-thread=(true|false)
-
-If true,
-.B notmuch show
-outputs all messages in the thread of any message matching the search
-terms; if false, it outputs only the matching messages. For
-.B --format=json
-and
-.B --format=sexp
-this defaults to true.  For other formats, this defaults to false.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-format=(text|json|sexp|mbox|raw)
-
-.RS 4
-.TP 4
-.BR text " (default for messages)"
-
-The default plain-text format has all text-content MIME parts
-decoded. Various components in the output,
-.RB ( message ", " header ", " body ", " attachment ", and MIME " part ),
-will be delimited by easily-parsed markers. Each marker consists of a
-Control-L character (ASCII decimal 12), the name of the marker, and
-then either an opening or closing brace, ('{' or '}'), to either open
-or close the component. For a multipart MIME message, these parts will
-be nested.
-.RE
-.RS 4
-.TP 4
-.B json
-
-The output is formatted with Javascript Object Notation (JSON). This
-format is more robust than the text format for automated
-processing. The nested structure of multipart MIME messages is
-reflected in nested JSON output. By default JSON output includes all
-messages in a matching thread; that is, by default,
-
-.B \-\-format=json
-sets
-.B "\-\-entire\-thread"
-The caller can disable this behaviour by setting
-.B \-\-entire\-thread=false
-.RE
-.RS 4
-.TP 4
-.B sexp
-
-The output is formatted as an S-Expression (sexp). This
-format is more robust than the text format for automated
-processing. The nested structure of multipart MIME messages is
-reflected in nested S-Expression output. By default,
-S-Expression output includes all messages in a matching thread;
-that is, by default,
-
-.B \-\-format=sexp
-sets
-.B "\-\-entire\-thread"
-The caller can disable this behaviour by setting
-.B \-\-entire\-thread=false
-
-.RE
-.RS 4
-.TP 4
-.B mbox
-
-All matching messages are output in the traditional, Unix mbox format
-with each message being prefixed by a line beginning with "From " and
-a blank line separating each message. Lines in the message content
-beginning with "From " (preceded by zero or more '>' characters) have
-an additional '>' character added. This reversible escaping
-is termed "mboxrd" format and described in detail here:
-
-.nf
-.nh
-http://homepage.ntlworld.com/jonathan.deboynepollard/FGA/mail-mbox-formats.html
-.hy
-.fi
-.
-.RE
-.RS 4
-.TP 4
-.BR raw " (default for a single part, see \-\-part)"
-
-For a message or an attached message part, the original, raw content
-of the email message is output. Consumers of this format should expect
-to implement MIME decoding and similar functions.
-
-For a single part (\-\-part) the raw part content is output after
-performing any necessary MIME decoding.  Note that messages with a
-simple body still have two parts: part 0 is the whole message and part
-1 is the body.
-
-For a multipart part, the part headers and body (including all child
-parts) is output.
-
-The raw format must only be used with search terms matching single
-message.
-.RE
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-format-version=N
-
-Use the specified structured output format version.  This is intended
-for programs that invoke \fBnotmuch\fR(1) internally.  If omitted, the
-latest supported version will be used.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-part=N
-
-Output the single decoded MIME part N of a single message.  The search
-terms must match only a single message.  Message parts are numbered in
-a depth-first walk of the message MIME structure, and are identified
-in the 'json', 'sexp' or 'text' output formats.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-verify
-
-Compute and report the validity of any MIME cryptographic signatures
-found in the selected content (ie. "multipart/signed" parts). Status
-of the signature will be reported (currently only supported with
---format=json and --format=sexp), and the multipart/signed part
-will be replaced by the signed data.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-decrypt
-
-Decrypt any MIME encrypted parts found in the selected content
-(ie. "multipart/encrypted" parts). Status of the decryption will be
-reported (currently only supported with --format=json and
---format=sexp) and on successful decryption the multipart/encrypted
-part will be replaced by the decrypted content.
-
-Decryption expects a functioning \fBgpg-agent\fR(1) to provide any
-needed credentials. Without one, the decryption will fail.
-
-Implies --verify.
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-exclude=(true|false)
-
-Specify whether to omit threads only matching search.tag_exclude from
-the search results (the default) or not. In either case the excluded
-message will be marked with the exclude flag (except when output=mbox
-when there is nowhere to put the flag).
-
-If --entire-thread is specified then complete threads are returned
-regardless (with the excluded flag being set when appropriate) but
-threads that only match in an excluded message are not returned when
-.B --exclude=true.
-
-The default is
-.B --exclude=true.
-
-.RE
-
-.RS 4
-.TP 4
-.B \-\-body=(true|false)
-
-If true (the default)
-.B notmuch show
-includes the bodies of the messages in the output; if false,
-bodies are omitted.
-.B --body=false
-is only implemented for the json and sexp formats and it is incompatible with
-.B --part > 0.
-
-This is useful if the caller only needs the headers as body-less
-output is much faster and substantially smaller.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-include-html
-
-Include "text/html" parts as part of the output (currently only supported with
---format=json and --format=sexp).
-By default, unless
-.B --part=N
-is used to select a specific part or
-.B --include-html
-is used to include all "text/html" parts, no part with content type "text/html"
-is included in the output.
-.RE
-
-A common use of
-.B notmuch show
-is to display a single thread of email messages. For this, use a
-search term of "thread:<thread-id>" as can be seen in the first
-column of output from the
-.B notmuch search
-command.
-
-.SH EXIT STATUS
-
-This command supports the following special exit status codes
-
-.TP
-.B 20
-The requested format version is too old.
-.TP
-.B 21
-The requested format version is too new.
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1),
-\fBnotmuch-search\fR(1), \fBnotmuch-search-terms\fR(7),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man1/notmuch-tag.1 b/man/man1/notmuch-tag.1
deleted file mode 100644
index 710fae6a..00000000
--- a/man/man1/notmuch-tag.1
+++ /dev/null
@@ -1,142 +0,0 @@
-.TH NOTMUCH-TAG 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch-tag \- add/remove tags for all messages matching the search terms
-
-.SH SYNOPSIS
-.B notmuch tag
-.RI [ options "...] +<" tag ">|\-<" tag "> [...] [\-\-] <" search-term "> [...]"
-
-.B notmuch tag
-.RI "--batch"
-.RI "[ --input=<" filename "> ]"
-
-
-.SH DESCRIPTION
-
-Add/remove tags for all messages matching the search terms.
-
-See \fBnotmuch-search-terms\fR(7)
-for details of the supported syntax for
-.RI < search-term >.
-
-Tags prefixed by '+' are added while those prefixed by '\-' are
-removed. For each message, tag changes are applied in the order they
-appear on the command line.
-
-The beginning of the search terms is recognized by the first
-argument that begins with neither '+' nor '\-'. Support for
-an initial search term beginning with '+' or '\-' is provided
-by allowing the user to specify a "\-\-" argument to separate
-the tags from the search terms.
-
-.B "notmuch tag"
-updates the maildir flags according to tag changes if the
-.B "maildir.synchronize_flags"
-configuration option is enabled. See \fBnotmuch-config\fR(1) for
-details.
-
-Supported options for
-.B tag
-include
-.RS 4
-.TP 4
-.BR \-\-remove\-all
-
-Remove all tags from each message matching the search terms before
-applying the tag changes appearing on the command line. This means
-setting the tags of each message to the tags to be added. If there are
-no tags to be added, the messages will have no tags.
-.RE
-
-.RS 4
-.TP 4
-.BR \-\-batch
-
-Read batch tagging operations from a file (stdin by default). This is more
-efficient than repeated
-.B notmuch tag
-invocations. See
-.B TAG FILE FORMAT
-below for the input format. This option is not compatible with
-specifying tagging on the command line.
-.RE
-
-.RS 4
-.TP 4
-.BR "\-\-input=" <filename>
-
-Read input from given file, instead of from stdin. Implies
-.BR --batch .
-
-.SH TAG FILE FORMAT
-
-The input must consist of lines of the format:
-
-.RI "+<" tag ">|\-<" tag "> [...] [\-\-] <" query ">"
-
-Each line is interpreted similarly to
-.B notmuch tag
-command line arguments. The delimiter is one or more spaces ' '. Any
-characters in
-.RI < tag >
-.B may
-be hex-encoded with %NN where NN is the hexadecimal value of the
-character. To hex-encode a character with a multi-byte UTF-8 encoding,
-hex-encode each byte.
-Any spaces in <tag>
-.B must
-be hex-encoded as %20. Any characters that are not
-part of
-.RI  < tag >
-.B must not
-be hex-encoded.
-
-In the future tag:"tag with spaces" style quoting may be supported for
-.RI < tag >
-as well;
-for this reason all double quote characters in
-.RI < tag >
-.B should
-be hex-encoded.
-
-The
-.RI < query >
-should be quoted using Xapian boolean term quoting rules: if a term
-contains whitespace or a close paren or starts with a double quote, it
-must be enclosed in double quotes (not including any prefix) and
-double quotes inside the term must be doubled (see below for
-examples).
-
-Leading and trailing space ' ' is ignored. Empty lines and lines
-beginning with '#' are ignored.
-
-.SS EXAMPLE
-
-The following shows a valid input to batch tagging. Note that only the
-isolated '*' acts as a wildcard. Also note the two different quotings
-of the tag
-.B space in tags
-.
-.RS
-.nf
-+winner *
-+foo::bar%25 -- (One and Two) or (One and tag:winner)
-+found::it -- tag:foo::bar%
-# ignore this line and the next
-
-+space%20in%20tags -- Two
-# add tag '(tags)', among other stunts.
-+crazy{ +(tags) +&are +#possible\ -- tag:"space in tags"
-+match*crazy -- tag:crazy{
-+some_tag -- id:"this is ""nauty)"""
-.fi
-.RE
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1),
-\fBnotmuch-search\fR(1), \fBnotmuch-search-terms\fR(7),
-\fBnotmuch-show\fR(1),
diff --git a/man/man1/notmuch.1 b/man/man1/notmuch.1
deleted file mode 100644
index 605b5146..00000000
--- a/man/man1/notmuch.1
+++ /dev/null
@@ -1,190 +0,0 @@
-.\" notmuch - Not much of an email program, (just index, search and tagging)
-.\"
-.\" Copyright © 2009 Carl Worth
-.\"
-.\" Notmuch is free software: you can redistribute it and/or modify
-.\" it under the terms of the GNU General Public License as published by
-.\" the Free Software Foundation, either version 3 of the License, or
-.\" (at your option) any later version.
-.\"
-.\" Notmuch is distributed in the hope that it will be useful,
-.\" but WITHOUT ANY WARRANTY; without even the implied warranty of
-.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-.\" GNU General Public License for more details.
-.\"
-.\" You should have received a copy of the GNU General Public License
-.\" along with this program.  If not, see http://www.gnu.org/licenses/ .
-.\"
-.\" Author: Carl Worth <cworth@cworth.org>
-.TH NOTMUCH 1 2013-12-30 "Notmuch 0.17"
-.SH NAME
-notmuch \- thread-based email index, search, and tagging
-.SH SYNOPSIS
-.B notmuch
-.RI "[" option " ...] " command  " [" arg " ...]"
-.SH DESCRIPTION
-Notmuch is a command-line based program for indexing, searching,
-reading, and tagging large collections of email messages.
-
-This page describes how to get started using notmuch from the command
-line, and gives a brief overview of the commands available. For more
-information on e.g.
-.B notmuch show
-consult the \fBnotmuch-show\fR(1) man page, also accessible via
-.B notmuch help show
-
-The quickest way to get started with Notmuch is to simply invoke the
-.B notmuch
-command with no arguments, which will interactively guide you through
-the process of indexing your mail.
-.SH NOTE
-While the command-line program
-.B notmuch
-provides powerful functionality, it does not provide the most
-convenient interface for that functionality. More sophisticated
-interfaces are expected to be built on top of either the command-line
-interface, or more likely, on top of the notmuch library
-interface. See http://notmuchmail.org for more about alternate
-interfaces to notmuch. The emacs-based interface to notmuch (available under
-.B emacs/
-in the Notmuch source distribution) is probably the most widely used at
-this time.
-
-.SH OPTIONS
-
-Supported global options for
-.B notmuch
-include
-
-.RS 4
-.TP 4
-.B \-\-help
-
-Print a synopsis of available commands and exit.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-version
-
-Print the installed version of notmuch, and exit.
-.RE
-
-.RS 4
-.TP 4
-.B \-\-config=FILE
-
-Specify the configuration file to use. This overrides any
-configuration file specified by ${NOTMUCH_CONFIG}.
-
-.RE
-
-.SH COMMANDS
-
-
-.SS SETUP
-
-The
-.B notmuch setup
-command is used to configure Notmuch for first use, (or to reconfigure
-it later).
-
-The setup command will prompt for your full name, your primary email
-address, any alternate email addresses you use, and the directory
-containing your email archives. Your answers will be written to a
-configuration file in ${NOTMUCH_CONFIG} (if set) or
-${HOME}/.notmuch-config . This configuration file will be created with
-descriptive comments, making it easy to edit by hand later to change the
-configuration. Or you can run
-.B "notmuch setup"
-again to change the configuration.
-
-The mail directory you specify can contain any number of
-sub-directories and should primarily contain only files with individual
-email messages (eg. maildir or mh archives are perfect). If there are
-other, non-email files (such as indexes maintained by other email
-programs) then notmuch will do its best to detect those and ignore
-them.
-
-Mail storage that uses mbox format, (where one mbox file contains many
-messages), will not work with notmuch. If that's how your mail is
-currently stored, it is recommended you first convert it to maildir
-format with a utility such as mb2md before running
-.B "notmuch setup" .
-
-Invoking
-.B notmuch
-with no command argument will run
-.B setup
-if the setup command has not previously been completed.
-.RE
-
-.SS OTHER COMMANDS
-
-Several of the notmuch commands accept search terms with a common
-syntax. See \fNnotmuch-search-terms\fR(7)
-for more details on the supported syntax.
-
-The
-.BR search ", " show " and " count
-commands are used to query the email database.
-
-The
-.B reply
-command is useful for preparing a template for an email reply.
-
-The
-.B tag
-command is the only command available for manipulating database
-contents.
-
-
-The
-.BR dump " and " restore
-commands can be used to create a textual dump of email tags for backup
-purposes, and to restore from that dump.
-
-The
-.B config
-command can be used to get or set settings int the notmuch
-configuration file.
-
-.SH ENVIRONMENT
-The following environment variables can be used to control the
-behavior of notmuch.
-.TP
-.B NOTMUCH_CONFIG
-Specifies the location of the notmuch configuration file. Notmuch will
-use ${HOME}/.notmuch\-config if this variable is not set.
-
-.TP
-.B NOTMUCH_TALLOC_REPORT
-Location to write a talloc memory usage report. See
-.B talloc_enable_leak_report_full
-in \fBtalloc\fR(3)
-for more information.
-
-.TP
-.B NOTMUCH_DEBUG_QUERY
-If set to a non-empty value, the notmuch library will print (to stderr) Xapian
-queries it constructs.
-
-.SH SEE ALSO
-
-\fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1),
-\fBnotmuch-search\fR(1), \fBnotmuch-search-terms\fR(7),
-\fBnotmuch-show\fR(1), \fBnotmuch-tag\fR(1)
-
-
-The notmuch website:
-.B http://notmuchmail.org
-.SH CONTACT
-Feel free to send questions, comments, or kudos to the notmuch mailing
-list <notmuch@notmuchmail.org> . Subscription is not required before
-posting, but is available from the notmuchmail.org website.
-
-Real-time interaction with the Notmuch community is available via IRC
-(server: irc.freenode.net, channel: #notmuch).
diff --git a/man/man5/notmuch-hooks.5 b/man/man5/notmuch-hooks.5
deleted file mode 100644
index 11c55ddd..00000000
--- a/man/man5/notmuch-hooks.5
+++ /dev/null
@@ -1,48 +0,0 @@
-.TH NOTMUCH-HOOKS 5 2013-12-30 "Notmuch 0.17"
-
-.SH NAME
-notmuch-hooks \- hooks for notmuch
-
-.SH SYNOPSIS
- $DATABASEDIR/.notmuch/hooks/*
-
-.SH DESCRIPTION
-Hooks are scripts (or arbitrary executables or symlinks to such) that notmuch
-invokes before and after certain actions. These scripts reside in
-the .notmuch/hooks directory within the database directory and must have
-executable permissions.
-
-The currently available hooks are described below.
-.RS 4
-.TP 4
-.B pre\-new
-This hook is invoked by the
-.B new
-command before scanning or importing new messages into the database. If this
-hook exits with a non-zero status, notmuch will abort further processing of the
-.B new
-command.
-
-Typically this hook is used for fetching or delivering new mail to be imported
-into the database.
-.RE
-.RS 4
-.TP 4
-.B post\-new
-This hook is invoked by the
-.B new
-command after new messages have been imported into the database and initial tags
-have been applied. The hook will not be run if there have been any errors during
-the scan or import.
-
-Typically this hook is used to perform additional query\-based tagging on the
-imported messages.
-.RE
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1), \fBnotmuch-search\fR(1),
-\fBnotmuch-search-terms\fR(7), \fBnotmuch-show\fR(1),
-\fBnotmuch-tag\fR(1)
diff --git a/man/man7/notmuch-search-terms.7 b/man/man7/notmuch-search-terms.7
deleted file mode 100644
index a768b630..00000000
--- a/man/man7/notmuch-search-terms.7
+++ /dev/null
@@ -1,269 +0,0 @@
-.TH NOTMUCH-SEARCH-TERMS 7 2013-12-30 "Notmuch 0.17"
-
-.SH NAME
-notmuch-search-terms \- syntax for notmuch queries
-
-.SH SYNOPSIS
-
-.B notmuch count
-.RI  [ options... ]
-.RI  < search-term ">..."
-
-.B "notmuch dump"
-.RI "[ <" filename "> ] [--]"
-.RI "[ <" search-term ">...]"
-
-.B notmuch search
-.RI  [  options "...] <" search-term ">..."
-
-.B notmuch show
-.RI "[" options "...] <" search-term ">..."
-
-.B notmuch tag
-.RI  "+<" tag> "|\-<" tag "> [...] [\-\-] <" search-term ">..."
-
-
-.SH DESCRIPTION
-Several notmuch commands accept a common syntax for search terms.
-
-The search terms can consist of free-form text (and quoted phrases)
-which will match all messages that contain all of the given
-terms/phrases in the body, the subject, or any of the sender or
-recipient headers.
-
-As a special case, a search string consisting of exactly a single
-asterisk ("*") will match all messages.
-
-In addition to free text, the following prefixes can be used to force
-terms to match against specific portions of an email, (where
-<brackets> indicate user-supplied values):
-
-	from:<name-or-address>
-
-	to:<name-or-address>
-
-	subject:<word-or-quoted-phrase>
-
-	attachment:<word>
-
-	tag:<tag> (or is:<tag>)
-
-	id:<message-id>
-
-	thread:<thread-id>
-
-	folder:<directory-path>
-
-	date:<since>..<until>
-
-The
-.B from:
-prefix is used to match the name or address of the sender of an email
-message.
-
-The
-.B to:
-prefix is used to match the names or addresses of any recipient of an
-email message, (whether To, Cc, or Bcc).
-
-Any term prefixed with
-.B subject:
-will match only text from the subject of an email. Searching for a
-phrase in the subject is supported by including quotation marks around
-the phrase, immediately following
-.BR subject: .
-
-The
-.B attachment:
-prefix can be used to search for specific filenames (or extensions) of
-attachments to email messages.
-
-For
-.BR tag: " and " is:
-valid tag values include
-.BR inbox " and " unread
-by default for new messages added by
-.B notmuch new
-as well as any other tag values added manually with
-.BR "notmuch tag" .
-
-For
-.BR id: ,
-message ID values are the literal contents of the Message\-ID: header
-of email messages, but without the '<', '>' delimiters.
-
-The
-.B thread:
-prefix can be used with the thread ID values that are generated
-internally by notmuch (and do not appear in email messages). These
-thread ID values can be seen in the first column of output from
-.B "notmuch search"
-
-The
-.B folder:
-prefix can be used to search for email message files that are
-contained within particular directories within the mail store. If the
-same email message has multiple message files associated with it, it's
-sufficient for a match that at least one of the files is contained
-within a matching directory. Only the directory components below the
-top-level mail database path are available to be searched.
-
-The
-.B date:
-prefix can be used to restrict the results to only messages within a
-particular time range (based on the Date: header) with a range syntax
-of:
-
-	date:<since>..<until>
-
-See \fBDATE AND TIME SEARCH\fR below for details on the range
-expression, and supported syntax for <since> and <until> date and time
-expressions.
-
-The time range can also be specified using timestamps with a syntax
-of:
-
-	<initial-timestamp>..<final-timestamp>
-
-Each timestamp is a number representing the number of seconds since
-1970\-01\-01 00:00:00 UTC.
-
-In addition to individual terms, multiple terms can be
-combined with Boolean operators (
-.BR and ", " or ", " not
-, etc.). Each term in the query will be implicitly connected by a
-logical AND if no explicit operator is provided, (except that terms
-with a common prefix will be implicitly combined with OR until we get
-Xapian defect #402 fixed).
-
-Parentheses can also be used to control the combination of the Boolean
-operators, but will have to be protected from interpretation by the
-shell, (such as by putting quotation marks around any parenthesized
-expression).
-
-.SH DATE AND TIME SEARCH
-
-notmuch understands a variety of standard and natural ways of
-expressing dates and times, both in absolute terms ("2012-10-24") and
-in relative terms ("yesterday"). Any number of relative terms can be
-combined ("1 hour 25 minutes") and an absolute date/time can be
-combined with relative terms to further adjust it. A non-exhaustive
-description of the syntax supported for absolute and relative terms is
-given below.
-
-.RS 4
-.TP 4
-.B The range expression
-
-date:<since>..<until>
-
-The above expression restricts the results to only messages from
-<since> to <until>, based on the Date: header.
-
-<since> and <until> can describe imprecise times, such as "yesterday".
-In this case, <since> is taken as the earliest time it could describe
-(the beginning of yesterday) and <until> is taken as the latest time
-it could describe (the end of yesterday). Similarly,
-date:january..february matches from the beginning of January to the
-end of February.
-
-Currently, we do not support spaces in range expressions. You can
-replace the spaces with '_', or (in most cases) '-', or (in some
-cases) leave the spaces out altogether. Examples in this man page use
-spaces for clarity.
-
-Open-ended ranges are supported (since Xapian 1.2.1), i.e. it's
-possible to specify date:..<until> or date:<since>.. to not limit the
-start or end time, respectively. Pre-1.2.1 Xapian does not report an
-error on open ended ranges, but it does not work as expected either.
-
-Entering date:expr without ".." (for example date:yesterday) won't
-work, as it's not interpreted as a range expression at all. You can
-achieve the expected result by duplicating the expr both sides of ".."
-(for example date:yesterday..yesterday).
-.RE
-
-.RS 4
-.TP 4
-.B Relative date and time
-[N|number] (years|months|weeks|days|hours|hrs|minutes|mins|seconds|secs) [...]
-
-All refer to past, can be repeated and will be accumulated.
-
-Units can be abbreviated to any length, with the otherwise ambiguous
-single m being m for minutes and M for months.
-
-Number can also be written out one, two, ..., ten, dozen,
-hundred. Additionally, the unit may be preceded by "last" or "this"
-(e.g., "last week" or "this month").
-
-When combined with absolute date and time, the relative date and time
-specification will be relative from the specified absolute date and
-time.
-
-Examples: 5M2d, two weeks
-.RE
-
-.RS 4
-.TP 4
-.B Supported absolute time formats
-H[H]:MM[:SS] [(am|a.m.|pm|p.m.)]
-
-H[H] (am|a.m.|pm|p.m.)
-
-HHMMSS
-
-now
-
-noon
-
-midnight
-
-Examples: 17:05, 5pm
-.RE
-
-.RS 4
-.TP 4
-.B Supported absolute date formats
-YYYY-MM[-DD]
-
-DD-MM[-[YY]YY]
-
-MM-YYYY
-
-M[M]/D[D][/[YY]YY]
-
-M[M]/YYYY
-
-D[D].M[M][.[YY]YY]
-
-D[D][(st|nd|rd|th)] Mon[thname] [YYYY]
-
-Mon[thname] D[D][(st|nd|rd|th)] [YYYY]
-
-Wee[kday]
-
-Month names can be abbreviated at three or more characters.
-
-Weekday names can be abbreviated at three or more characters.
-
-Examples: 2012-07-31, 31-07-2012, 7/31/2012, August 3
-.RE
-
-.RS 4
-.TP 4
-.B Time zones
-(+|-)HH:MM
-
-(+|-)HH[MM]
-
-Some time zone codes, e.g. UTC, EET.
-.RE
-
-.SH SEE ALSO
-
-\fBnotmuch\fR(1), \fBnotmuch-config\fR(1), \fBnotmuch-count\fR(1),
-\fBnotmuch-dump\fR(1), \fBnotmuch-hooks\fR(5),
-\fBnotmuch-insert\fR(1), \fBnotmuch-new\fR(1),
-\fBnotmuch-reply\fR(1), \fBnotmuch-restore\fR(1),
-\fBnotmuch-search\fR(1), \fBnotmuch-show\fR(1), \fBnotmuch-tag\fR(1)
diff --git a/notmuch-client.h b/notmuch-client.h
index 278b498a..e1efbe0c 100644
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -441,5 +441,18 @@ mime_node_child (mime_node_t *parent, int child);
 mime_node_t *
 mime_node_seek_dfs (mime_node_t *node, int n);
 
+typedef enum dump_formats {
+    DUMP_FORMAT_AUTO,
+    DUMP_FORMAT_BATCH_TAG,
+    DUMP_FORMAT_SUP
+} dump_format_t;
+
+int
+notmuch_database_dump (notmuch_database_t *notmuch,
+		       const char *output_file_name,
+		       const char *query_str,
+		       dump_format_t output_format,
+		       notmuch_bool_t gzip_output);
+
 #include "command-line-arguments.h"
 #endif
diff --git a/notmuch-compact.c b/notmuch-compact.c
index 8b820c0d..2fc012a9 100644
--- a/notmuch-compact.c
+++ b/notmuch-compact.c
@@ -32,7 +32,7 @@ notmuch_compact_command (notmuch_config_t *config, int argc, char *argv[])
     const char *path = notmuch_config_get_database_path (config);
     const char *backup_path = NULL;
     notmuch_status_t ret;
-    notmuch_bool_t quiet;
+    notmuch_bool_t quiet = FALSE;
     int opt_index;
 
     notmuch_opt_desc_t options[] = {
@@ -42,7 +42,7 @@ notmuch_compact_command (notmuch_config_t *config, int argc, char *argv[])
 
     opt_index = parse_arguments (argc, argv, options, 1);
     if (opt_index < 0)
-	return 1;
+	return EXIT_FAILURE;
 
     if (! quiet)
 	printf ("Compacting database...\n");
@@ -50,7 +50,7 @@ notmuch_compact_command (notmuch_config_t *config, int argc, char *argv[])
 				    quiet ? NULL : status_update_cb, NULL);
     if (ret) {
 	fprintf (stderr, "Compaction failed: %s\n", notmuch_status_to_string (ret));
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (! quiet) {
@@ -60,5 +60,5 @@ notmuch_compact_command (notmuch_config_t *config, int argc, char *argv[])
 	printf ("Done.\n");
     }
 
-    return 0;
+    return EXIT_SUCCESS;
 }
diff --git a/notmuch-config.c b/notmuch-config.c
index 6845e3c3..4886d366 100644
--- a/notmuch-config.c
+++ b/notmuch-config.c
@@ -454,7 +454,7 @@ notmuch_config_save (notmuch_config_t *config)
     }
 
     /* Try not to overwrite symlinks. */
-    filename = realpath (config->filename, NULL);
+    filename = canonicalize_file_name (config->filename);
     if (! filename) {
 	if (errno == ENOENT) {
 	    filename = strdup (config->filename);
@@ -496,6 +496,32 @@ notmuch_config_is_new (notmuch_config_t *config)
     return config->is_new;
 }
 
+static const char *
+_config_get (notmuch_config_t *config, char **field,
+	     const char *group, const char *key)
+{
+    /* read from config file and cache value, if not cached already */
+    if (*field == NULL) {
+	char *value;
+	value = g_key_file_get_string (config->key_file, group, key, NULL);
+	if (value) {
+	    *field = talloc_strdup (config, value);
+	    free (value);
+	}
+    }
+    return *field;
+}
+
+static void
+_config_set (notmuch_config_t *config, char **field,
+	     const char *group, const char *key, const char *value)
+{
+    g_key_file_set_string (config->key_file, group, key, value);
+
+    /* drop the cached value */
+    talloc_free (*field);
+    *field = NULL;
+}
 
 static const char **
 _config_get_list (notmuch_config_t *config,
@@ -504,6 +530,7 @@ _config_get_list (notmuch_config_t *config,
 {
     assert(outlist);
 
+    /* read from config file and cache value, if not cached already */
     if (*outlist == NULL) {
 
 	char **inlist = g_key_file_get_string_list (config->key_file,
@@ -535,6 +562,8 @@ _config_set_list (notmuch_config_t *config,
 		  size_t length, const char ***config_var )
 {
     g_key_file_set_string_list (config->key_file, group, name, list, length);
+
+    /* drop the cached value */
     talloc_free (*config_var);
     *config_var = NULL;
 }
@@ -542,85 +571,40 @@ _config_set_list (notmuch_config_t *config,
 const char *
 notmuch_config_get_database_path (notmuch_config_t *config)
 {
-    char *path;
-
-    if (config->database_path == NULL) {
-	path = g_key_file_get_string (config->key_file,
-				      "database", "path", NULL);
-	if (path) {
-	    config->database_path = talloc_strdup (config, path);
-	    free (path);
-	}
-    }
-
-    return config->database_path;
+    return _config_get (config, &config->database_path, "database", "path");
 }
 
 void
 notmuch_config_set_database_path (notmuch_config_t *config,
 				  const char *database_path)
 {
-    g_key_file_set_string (config->key_file,
-			   "database", "path", database_path);
-
-    talloc_free (config->database_path);
-    config->database_path = NULL;
+    _config_set (config, &config->database_path, "database", "path", database_path);
 }
 
 const char *
 notmuch_config_get_user_name (notmuch_config_t *config)
 {
-    char *name;
-
-    if (config->user_name == NULL) {
-	name = g_key_file_get_string (config->key_file,
-				      "user", "name", NULL);
-	if (name) {
-	    config->user_name = talloc_strdup (config, name);
-	    free (name);
-	}
-    }
-
-    return config->user_name;
+    return _config_get (config, &config->user_name, "user", "name");
 }
 
 void
 notmuch_config_set_user_name (notmuch_config_t *config,
 			      const char *user_name)
 {
-    g_key_file_set_string (config->key_file,
-			   "user", "name", user_name);
-
-    talloc_free (config->user_name);
-    config->user_name = NULL;
+    _config_set (config, &config->user_name, "user", "name", user_name);
 }
 
 const char *
 notmuch_config_get_user_primary_email (notmuch_config_t *config)
 {
-    char *email;
-
-    if (config->user_primary_email == NULL) {
-	email = g_key_file_get_string (config->key_file,
-				       "user", "primary_email", NULL);
-	if (email) {
-	    config->user_primary_email = talloc_strdup (config, email);
-	    free (email);
-	}
-    }
-
-    return config->user_primary_email;
+    return _config_get (config, &config->user_primary_email, "user", "primary_email");
 }
 
 void
 notmuch_config_set_user_primary_email (notmuch_config_t *config,
 				       const char *primary_email)
 {
-    g_key_file_set_string (config->key_file,
-			   "user", "primary_email", primary_email);
-
-    talloc_free (config->user_primary_email);
-    config->user_primary_email = NULL;
+    _config_set (config, &config->user_primary_email, "user", "primary_email", primary_email);
 }
 
 const char **
@@ -839,34 +823,39 @@ notmuch_config_command_list (notmuch_config_t *config)
 int
 notmuch_config_command (notmuch_config_t *config, int argc, char *argv[])
 {
+    int ret;
+
     argc--; argv++; /* skip subcommand argument */
 
     if (argc < 1) {
 	fprintf (stderr, "Error: notmuch config requires at least one argument.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (strcmp (argv[0], "get") == 0) {
 	if (argc != 2) {
 	    fprintf (stderr, "Error: notmuch config get requires exactly "
 		     "one argument.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
-	return notmuch_config_command_get (config, argv[1]);
+	ret = notmuch_config_command_get (config, argv[1]);
     } else if (strcmp (argv[0], "set") == 0) {
 	if (argc < 2) {
 	    fprintf (stderr, "Error: notmuch config set requires at least "
 		     "one argument.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
-	return notmuch_config_command_set (config, argv[1], argc - 2, argv + 2);
+	ret = notmuch_config_command_set (config, argv[1], argc - 2, argv + 2);
     } else if (strcmp (argv[0], "list") == 0) {
-	return notmuch_config_command_list (config);
+	ret = notmuch_config_command_list (config);
+    } else {
+	fprintf (stderr, "Unrecognized argument for notmuch config: %s\n",
+		 argv[0]);
+	return EXIT_FAILURE;
     }
 
-    fprintf (stderr, "Unrecognized argument for notmuch config: %s\n",
-	     argv[0]);
-    return 1;
+    return ret ? EXIT_FAILURE : EXIT_SUCCESS;
+
 }
 
 notmuch_bool_t
diff --git a/notmuch-count.c b/notmuch-count.c
index 01e4e301..6058f7c9 100644
--- a/notmuch-count.c
+++ b/notmuch-count.c
@@ -150,10 +150,8 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-
-    if (opt_index < 0) {
-	return 1;
-    }
+    if (opt_index < 0)
+	return EXIT_FAILURE;
 
     if (input_file_name) {
 	batch = TRUE;
@@ -161,23 +159,23 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
 	if (input == NULL) {
 	    fprintf (stderr, "Error opening %s for reading: %s\n",
 		     input_file_name, strerror (errno));
-	    return 1;
+	    return EXIT_FAILURE;
 	}
     }
 
     if (batch && opt_index != argc) {
 	fprintf (stderr, "--batch and query string are not compatible\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_ONLY, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
     query_str = query_string_from_args (config, argc-opt_index, argv+opt_index);
     if (query_str == NULL) {
 	fprintf (stderr, "Out of memory.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (exclude == EXCLUDE_TRUE) {
@@ -197,5 +195,5 @@ notmuch_count_command (notmuch_config_t *config, int argc, char *argv[])
     if (input != stdin)
 	fclose (input);
 
-    return ret;
+    return ret ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/notmuch-dump.c b/notmuch-dump.c
index 2024e303..887a2082 100644
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -19,67 +19,27 @@
  */
 
 #include "notmuch-client.h"
-#include "dump-restore-private.h"
+#include "hex-escape.h"
 #include "string-util.h"
+#include <zlib.h>
 
-int
-notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
+
+static int
+database_dump_file (notmuch_database_t *notmuch, gzFile output,
+		    const char *query_str, int output_format)
 {
-    notmuch_database_t *notmuch;
     notmuch_query_t *query;
-    FILE *output = stdout;
     notmuch_messages_t *messages;
     notmuch_message_t *message;
     notmuch_tags_t *tags;
-    const char *query_str = "";
-
-    if (notmuch_database_open (notmuch_config_get_database_path (config),
-			       NOTMUCH_DATABASE_MODE_READ_ONLY, &notmuch))
-	return 1;
-
-    char *output_file_name = NULL;
-    int opt_index;
-
-    int output_format = DUMP_FORMAT_SUP;
-
-    notmuch_opt_desc_t options[] = {
-	{ NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
-	  (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
-				  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
-				  { 0, 0 } } },
-	{ NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
-	{ 0, 0, 0, 0, 0 }
-    };
 
-    opt_index = parse_arguments (argc, argv, options, 1);
-
-    if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
-    }
-
-    if (output_file_name) {
-	output = fopen (output_file_name, "w");
-	if (output == NULL) {
-	    fprintf (stderr, "Error opening %s for writing: %s\n",
-		     output_file_name, strerror (errno));
-	    return 1;
-	}
-    }
-
-
-    if (opt_index < argc) {
-	query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
-	if (query_str == NULL) {
-	    fprintf (stderr, "Out of memory.\n");
-	    return 1;
-	}
-    }
+    if (! query_str)
+	query_str = "";
 
     query = notmuch_query_create (notmuch, query_str);
     if (query == NULL) {
 	fprintf (stderr, "Out of memory\n");
-	return 1;
+	return EXIT_FAILURE;
     }
     /* Don't ask xapian to sort by Message-ID. Xapian optimizes returning the
      * first results quickly at the expense of total time.
@@ -111,7 +71,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
 	}
 
 	if (output_format == DUMP_FORMAT_SUP) {
-	    fprintf (output, "%s (", message_id);
+	    gzprintf (output, "%s (", message_id);
 	}
 
 	for (tags = notmuch_message_get_tags (message);
@@ -120,43 +80,173 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
 	    const char *tag_str = notmuch_tags_get (tags);
 
 	    if (! first)
-		fputs (" ", output);
+		gzputs (output, " ");
 
 	    first = 0;
 
 	    if (output_format == DUMP_FORMAT_SUP) {
-		fputs (tag_str, output);
+		gzputs (output, tag_str);
 	    } else {
 		if (hex_encode (notmuch, tag_str,
 				&buffer, &buffer_size) != HEX_SUCCESS) {
 		    fprintf (stderr, "Error: failed to hex-encode tag %s\n",
 			     tag_str);
-		    return 1;
+		    return EXIT_FAILURE;
 		}
-		fprintf (output, "+%s", buffer);
+		gzprintf (output, "+%s", buffer);
 	    }
 	}
 
 	if (output_format == DUMP_FORMAT_SUP) {
-	    fputs (")\n", output);
+	    gzputs (output, ")\n");
 	} else {
 	    if (make_boolean_term (notmuch, "id", message_id,
 				   &buffer, &buffer_size)) {
 		    fprintf (stderr, "Error quoting message id %s: %s\n",
 			     message_id, strerror (errno));
-		    return 1;
+		    return EXIT_FAILURE;
 	    }
-	    fprintf (output, " -- %s\n", buffer);
+	    gzprintf (output, " -- %s\n", buffer);
 	}
 
 	notmuch_message_destroy (message);
     }
 
-    if (output != stdout)
-	fclose (output);
-
     notmuch_query_destroy (query);
+
+    return EXIT_SUCCESS;
+}
+
+/* Dump database into output_file_name if it's non-NULL, stdout
+ * otherwise.
+ */
+int
+notmuch_database_dump (notmuch_database_t *notmuch,
+		       const char *output_file_name,
+		       const char *query_str,
+		       dump_format_t output_format,
+		       notmuch_bool_t gzip_output)
+{
+    gzFile output = NULL;
+    const char *mode = gzip_output ? "w9" : "wT";
+    const char *name_for_error = output_file_name ? output_file_name : "stdout";
+
+    char *tempname = NULL;
+    int outfd = -1;
+
+    int ret = -1;
+
+    if (output_file_name) {
+	tempname = talloc_asprintf (notmuch, "%s.XXXXXX", output_file_name);
+	outfd = mkstemp (tempname);
+    } else {
+	outfd = dup (STDOUT_FILENO);
+    }
+
+    if (outfd < 0) {
+	fprintf (stderr, "Bad output file %s\n", name_for_error);
+	goto DONE;
+    }
+
+    output = gzdopen (outfd, mode);
+
+    if (output == NULL) {
+	fprintf (stderr, "Error opening %s for (gzip) writing: %s\n",
+		 name_for_error, strerror (errno));
+	if (close (outfd))
+	    fprintf (stderr, "Error closing %s during shutdown: %s\n",
+		 name_for_error, strerror (errno));
+	goto DONE;
+    }
+
+    ret = database_dump_file (notmuch, output, query_str, output_format);
+    if (ret) goto DONE;
+
+    ret = gzflush (output, Z_FINISH);
+    if (ret) {
+	fprintf (stderr, "Error flushing output: %s\n", gzerror (output, NULL));
+	goto DONE;
+    }
+
+    if (output_file_name) {
+	ret = fsync (outfd);
+	if (ret) {
+	    fprintf (stderr, "Error syncing %s to disk: %s\n",
+		     name_for_error, strerror (errno));
+	    goto DONE;
+	}
+    }
+
+    if (gzclose_w (output) != Z_OK) {
+	fprintf (stderr, "Error closing %s: %s\n", name_for_error,
+		 gzerror (output, NULL));
+	ret = EXIT_FAILURE;
+	output = NULL;
+	goto DONE;
+    }
+
+    if (output_file_name) {
+	ret = rename (tempname, output_file_name);
+	if (ret) {
+	    fprintf (stderr, "Error renaming %s to %s: %s\n",
+		     tempname, output_file_name, strerror (errno));
+	    goto DONE;
+	}
+
+    }
+ DONE:
+    if (ret != EXIT_SUCCESS && output)
+	(void) gzclose_w (output);
+
+    if (ret != EXIT_SUCCESS && output_file_name)
+	(void) unlink (tempname);
+
+    return ret;
+}
+
+int
+notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
+{
+    notmuch_database_t *notmuch;
+    const char *query_str = NULL;
+    int ret;
+
+    if (notmuch_database_open (notmuch_config_get_database_path (config),
+			       NOTMUCH_DATABASE_MODE_READ_ONLY, &notmuch))
+	return EXIT_FAILURE;
+
+    char *output_file_name = NULL;
+    int opt_index;
+
+    int output_format = DUMP_FORMAT_BATCH_TAG;
+    notmuch_bool_t gzip_output = 0;
+
+    notmuch_opt_desc_t options[] = {
+	{ NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
+	  (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
+				  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
+				  { 0, 0 } } },
+	{ NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
+	{ NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
+	{ 0, 0, 0, 0, 0 }
+    };
+
+    opt_index = parse_arguments (argc, argv, options, 1);
+    if (opt_index < 0)
+	return EXIT_FAILURE;
+
+    if (opt_index < argc) {
+	query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
+	if (query_str == NULL) {
+	    fprintf (stderr, "Out of memory.\n");
+	    return EXIT_FAILURE;
+	}
+    }
+
+    ret = notmuch_database_dump (notmuch, output_file_name, query_str,
+				 output_format, gzip_output);
+
     notmuch_database_destroy (notmuch);
 
-    return 0;
+    return ret;
 }
diff --git a/notmuch-insert.c b/notmuch-insert.c
index 2207b1e8..6752fc8d 100644
--- a/notmuch-insert.c
+++ b/notmuch-insert.c
@@ -295,7 +295,7 @@ copy_stdin (int fdin, int fdout)
  * The file is renamed to encode notmuch tags as maildir flags. */
 static void
 add_file_to_database (notmuch_database_t *notmuch, const char *path,
-		      tag_op_list_t *tag_ops)
+		      tag_op_list_t *tag_ops, notmuch_bool_t synchronize_flags)
 {
     notmuch_message_t *message;
     notmuch_status_t status;
@@ -323,11 +323,15 @@ add_file_to_database (notmuch_database_t *notmuch, const char *path,
 
     if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) {
 	/* Don't change tags of an existing message. */
-	status = notmuch_message_tags_to_maildir_flags (message);
-	if (status != NOTMUCH_STATUS_SUCCESS)
-	    fprintf (stderr, "Error: failed to sync tags to maildir flags\n");
+	if (synchronize_flags) {
+	    status = notmuch_message_tags_to_maildir_flags (message);
+	    if (status != NOTMUCH_STATUS_SUCCESS)
+		fprintf (stderr, "Error: failed to sync tags to maildir flags\n");
+	}
     } else {
-	tag_op_list_apply (message, tag_ops, TAG_FLAG_MAILDIR_SYNC);
+	tag_op_flag_t flags = synchronize_flags ? TAG_FLAG_MAILDIR_SYNC : 0;
+
+	tag_op_list_apply (message, tag_ops, flags);
     }
 
     notmuch_message_destroy (message);
@@ -335,7 +339,8 @@ add_file_to_database (notmuch_database_t *notmuch, const char *path,
 
 static notmuch_bool_t
 insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
-		const char *dir, tag_op_list_t *tag_ops)
+		const char *dir, tag_op_list_t *tag_ops,
+		notmuch_bool_t synchronize_flags)
 {
     char *tmppath;
     char *newpath;
@@ -377,7 +382,7 @@ insert_message (void *ctx, notmuch_database_t *notmuch, int fdin,
 
     /* Even if adding the message to the notmuch database fails,
      * the message is on disk and we consider the delivery completed. */
-    add_file_to_database (notmuch, newpath, tag_ops);
+    add_file_to_database (notmuch, newpath, tag_ops, synchronize_flags);
 
     return TRUE;
 
@@ -400,6 +405,7 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
     char *query_string = NULL;
     const char *folder = NULL;
     notmuch_bool_t create_folder = FALSE;
+    notmuch_bool_t synchronize_flags;
     const char *maildir;
     int opt_index;
     unsigned int i;
@@ -412,32 +418,39 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-
-    if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
-    }
+    if (opt_index < 0)
+	return EXIT_FAILURE;
 
     db_path = notmuch_config_get_database_path (config);
     new_tags = notmuch_config_get_new_tags (config, &new_tags_length);
+    synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config);
 
     tag_ops = tag_op_list_create (config);
     if (tag_ops == NULL) {
 	fprintf (stderr, "Out of memory.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
     for (i = 0; i < new_tags_length; i++) {
+	const char *error_msg;
+
+	error_msg = illegal_tag (new_tags[i], FALSE);
+	if (error_msg) {
+	    fprintf (stderr, "Error: tag '%s' in new.tags: %s\n",
+		     new_tags[i],  error_msg);
+	    return EXIT_FAILURE;
+	}
+
 	if (tag_op_list_append (tag_ops, new_tags[i], FALSE))
-	    return 1;
+	    return EXIT_FAILURE;
     }
 
     if (parse_tag_command_line (config, argc - opt_index, argv + opt_index,
 				&query_string, tag_ops))
-	return 1;
+	return EXIT_FAILURE;
 
     if (*query_string != '\0') {
 	fprintf (stderr, "Error: unexpected query string: %s\n", query_string);
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (folder == NULL) {
@@ -445,17 +458,17 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
     } else {
 	if (! check_folder_name (folder)) {
 	    fprintf (stderr, "Error: bad folder name: %s\n", folder);
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 	maildir = talloc_asprintf (config, "%s/%s", db_path, folder);
 	if (! maildir) {
 	    fprintf (stderr, "Out of memory\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 	if (create_folder && ! maildir_create_folder (config, maildir)) {
 	    fprintf (stderr, "Error: creating maildir %s: %s\n",
 		     maildir, strerror (errno));
-	    return 1;
+	    return EXIT_FAILURE;
 	}
     }
 
@@ -469,11 +482,12 @@ notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[])
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_WRITE, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
-    ret = insert_message (config, notmuch, STDIN_FILENO, maildir, tag_ops);
+    ret = insert_message (config, notmuch, STDIN_FILENO, maildir, tag_ops,
+			  synchronize_flags);
 
     notmuch_database_destroy (notmuch);
 
-    return (ret) ? 0 : 1;
+    return ret ? EXIT_SUCCESS : EXIT_FAILURE;
 }
diff --git a/notmuch-new.c b/notmuch-new.c
index ba05cb41..d269c7cd 100644
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -19,6 +19,7 @@
  */
 
 #include "notmuch-client.h"
+#include "tag-util.h"
 
 #include <unistd.h>
 
@@ -34,9 +35,15 @@ typedef struct _filename_list {
     _filename_node_t **tail;
 } _filename_list_t;
 
+enum verbosity {
+    VERBOSITY_QUIET,
+    VERBOSITY_NORMAL,
+    VERBOSITY_VERBOSE,
+};
+
 typedef struct {
     int output_is_a_tty;
-    notmuch_bool_t verbose;
+    enum verbosity verbosity;
     notmuch_bool_t debug;
     const char **new_tags;
     size_t new_tags_length;
@@ -167,7 +174,7 @@ dirent_type (const char *path, const struct dirent *entry)
     char *abspath;
     int err, saved_errno;
 
-#ifdef _DIRENT_HAVE_D_TYPE
+#if HAVE_D_TYPE
     /* Mapping from d_type to stat mode_t.  We omit DT_LNK so that
      * we'll fall through to stat and get the real file type. */
     static const mode_t modes[] = {
@@ -240,6 +247,60 @@ _entry_in_ignore_list (const char *entry, add_files_state_t *state)
     return FALSE;
 }
 
+/* Add a single file to the database. */
+static notmuch_status_t
+add_file (notmuch_database_t *notmuch, const char *filename,
+	  add_files_state_t *state)
+{
+    notmuch_message_t *message = NULL;
+    const char **tag;
+    notmuch_status_t status;
+
+    status = notmuch_database_begin_atomic (notmuch);
+    if (status)
+	goto DONE;
+
+    status = notmuch_database_add_message (notmuch, filename, &message);
+    switch (status) {
+    /* Success. */
+    case NOTMUCH_STATUS_SUCCESS:
+	state->added_messages++;
+	notmuch_message_freeze (message);
+	for (tag = state->new_tags; *tag != NULL; tag++)
+	    notmuch_message_add_tag (message, *tag);
+	if (state->synchronize_flags)
+	    notmuch_message_maildir_flags_to_tags (message);
+	notmuch_message_thaw (message);
+	break;
+    /* Non-fatal issues (go on to next file). */
+    case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
+	if (state->synchronize_flags)
+	    notmuch_message_maildir_flags_to_tags (message);
+	break;
+    case NOTMUCH_STATUS_FILE_NOT_EMAIL:
+	fprintf (stderr, "Note: Ignoring non-mail file: %s\n", filename);
+	break;
+    /* Fatal issues. Don't process anymore. */
+    case NOTMUCH_STATUS_READ_ONLY_DATABASE:
+    case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
+    case NOTMUCH_STATUS_OUT_OF_MEMORY:
+	fprintf (stderr, "Error: %s. Halting processing.\n",
+		 notmuch_status_to_string (status));
+	goto DONE;
+    default:
+	INTERNAL_ERROR ("add_message returned unexpected value: %d", status);
+	goto DONE;
+    }
+
+    status = notmuch_database_end_atomic (notmuch);
+
+  DONE:
+    if (message)
+	notmuch_message_destroy (message);
+
+    return status;
+}
+
 /* Examine 'path' recursively as follows:
  *
  *   o Ask the filesystem for the mtime of 'path' (fs_mtime)
@@ -291,7 +352,6 @@ add_files (notmuch_database_t *notmuch,
     char *next = NULL;
     time_t fs_mtime, db_mtime;
     notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS;
-    notmuch_message_t *message = NULL;
     struct dirent **fs_entries = NULL;
     int i, num_fs_entries = 0, entry_type;
     notmuch_directory_t *directory;
@@ -300,7 +360,6 @@ add_files (notmuch_database_t *notmuch,
     time_t stat_time;
     struct stat st;
     notmuch_bool_t is_maildir;
-    const char **tag;
 
     if (stat (path, &st)) {
 	fprintf (stderr, "Error reading directory %s: %s\n",
@@ -514,76 +573,23 @@ add_files (notmuch_database_t *notmuch,
 
 	state->processed_files++;
 
-	if (state->verbose) {
+	if (state->verbosity >= VERBOSITY_VERBOSE) {
 	    if (state->output_is_a_tty)
 		printf("\r\033[K");
 
-	    printf ("%i/%i: %s",
-		    state->processed_files,
-		    state->total_files,
+	    printf ("%i/%i: %s", state->processed_files, state->total_files,
 		    next);
 
 	    putchar((state->output_is_a_tty) ? '\r' : '\n');
 	    fflush (stdout);
 	}
 
-	status = notmuch_database_begin_atomic (notmuch);
+	status = add_file (notmuch, next, state);
 	if (status) {
 	    ret = status;
 	    goto DONE;
 	}
 
-	status = notmuch_database_add_message (notmuch, next, &message);
-	switch (status) {
-	/* success */
-	case NOTMUCH_STATUS_SUCCESS:
-	    state->added_messages++;
-	    notmuch_message_freeze (message);
-	    for (tag=state->new_tags; *tag != NULL; tag++)
-	        notmuch_message_add_tag (message, *tag);
-	    if (state->synchronize_flags == TRUE)
-		notmuch_message_maildir_flags_to_tags (message);
-	    notmuch_message_thaw (message);
-	    break;
-	/* Non-fatal issues (go on to next file) */
-	case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
-	    if (state->synchronize_flags == TRUE)
-		notmuch_message_maildir_flags_to_tags (message);
-	    break;
-	case NOTMUCH_STATUS_FILE_NOT_EMAIL:
-	    fprintf (stderr, "Note: Ignoring non-mail file: %s\n",
-		     next);
-	    break;
-	/* Fatal issues. Don't process anymore. */
-	case NOTMUCH_STATUS_READ_ONLY_DATABASE:
-	case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
-	case NOTMUCH_STATUS_OUT_OF_MEMORY:
-	    fprintf (stderr, "Error: %s. Halting processing.\n",
-		     notmuch_status_to_string (status));
-	    ret = status;
-	    goto DONE;
-	default:
-	case NOTMUCH_STATUS_FILE_ERROR:
-	case NOTMUCH_STATUS_NULL_POINTER:
-	case NOTMUCH_STATUS_TAG_TOO_LONG:
-	case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
-	case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
-	case NOTMUCH_STATUS_LAST_STATUS:
-	    INTERNAL_ERROR ("add_message returned unexpected value: %d",  status);
-	    goto DONE;
-	}
-
-	status = notmuch_database_end_atomic (notmuch);
-	if (status) {
-	    ret = status;
-	    goto DONE;
-	}
-
-	if (message) {
-	    notmuch_message_destroy (message);
-	    message = NULL;
-	}
-
 	if (do_print_progress) {
 	    do_print_progress = 0;
 	    generic_print_progress ("Processed", "files", state->tv_start,
@@ -701,10 +707,9 @@ count_files (const char *path, int *count, add_files_state_t *state)
 {
     struct dirent *entry = NULL;
     char *next;
-    struct stat st;
     struct dirent **fs_entries = NULL;
     int num_fs_entries = scandir (path, &fs_entries, 0, dirent_sort_inode);
-    int i = 0;
+    int entry_type, i;
 
     if (num_fs_entries == -1) {
 	fprintf (stderr, "Warning: failed to open directory %s: %s\n",
@@ -712,11 +717,8 @@ count_files (const char *path, int *count, add_files_state_t *state)
 	goto DONE;
     }
 
-    while (!interrupted) {
-        if (i == num_fs_entries)
-	    break;
-
-        entry = fs_entries[i++];
+    for (i = 0; i < num_fs_entries && ! interrupted; i++) {
+        entry = fs_entries[i];
 
 	/* Ignore special directories to avoid infinite recursion.
 	 * Also ignore the .notmuch directory and files/directories
@@ -727,7 +729,7 @@ count_files (const char *path, int *count, add_files_state_t *state)
 	    strcmp (entry->d_name, ".notmuch") == 0 ||
 	    _entry_in_ignore_list (entry->d_name, state))
 	{
-	    if (_entry_in_ignore_list (entry->d_name, state) && state->debug)
+	    if (state->debug && _entry_in_ignore_list (entry->d_name, state))
 		printf ("(D) count_files: explicitly ignoring %s/%s\n",
 			path,
 			entry->d_name);
@@ -741,15 +743,14 @@ count_files (const char *path, int *count, add_files_state_t *state)
 	    continue;
 	}
 
-	stat (next, &st);
-
-	if (S_ISREG (st.st_mode)) {
+	entry_type = dirent_type (path, entry);
+	if (entry_type == S_IFREG) {
 	    *count = *count + 1;
-	    if (*count % 1000 == 0) {
+	    if (*count % 1000 == 0 && state->verbosity >= VERBOSITY_NORMAL) {
 		printf ("Found %d files so far.\r", *count);
 		fflush (stdout);
 	    }
-	} else if (S_ISDIR (st.st_mode)) {
+	} else if (entry_type == S_IFDIR) {
 	    count_files (next, count, state);
 	}
 
@@ -868,13 +869,49 @@ _remove_directory (void *ctx,
     return status;
 }
 
+static void
+print_results (const add_files_state_t *state)
+{
+    double elapsed;
+    struct timeval tv_now;
+
+    gettimeofday (&tv_now, NULL);
+    elapsed = notmuch_time_elapsed (state->tv_start, tv_now);
+
+    if (state->processed_files) {
+	printf ("Processed %d %s in ", state->processed_files,
+		state->processed_files == 1 ? "file" : "total files");
+	notmuch_time_print_formatted_seconds (elapsed);
+	if (elapsed > 1)
+	    printf (" (%d files/sec.).\033[K\n",
+		    (int) (state->processed_files / elapsed));
+	else
+	    printf (".\033[K\n");
+    }
+
+    if (state->added_messages)
+	printf ("Added %d new %s to the database.", state->added_messages,
+		state->added_messages == 1 ? "message" : "messages");
+    else
+	printf ("No new mail.");
+
+    if (state->removed_messages)
+	printf (" Removed %d %s.", state->removed_messages,
+		state->removed_messages == 1 ? "message" : "messages");
+
+    if (state->renamed_messages)
+	printf (" Detected %d file %s.", state->renamed_messages,
+		state->renamed_messages == 1 ? "rename" : "renames");
+
+    printf ("\n");
+}
+
 int
 notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
 {
     notmuch_database_t *notmuch;
     add_files_state_t add_files_state;
-    double elapsed;
-    struct timeval tv_now, tv_start;
+    struct timeval tv_start;
     int ret = 0;
     struct stat st;
     const char *db_path;
@@ -882,36 +919,53 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     struct sigaction action;
     _filename_node_t *f;
     int opt_index;
-    int i;
+    unsigned int i;
     notmuch_bool_t timer_is_active = FALSE;
     notmuch_bool_t no_hooks = FALSE;
+    notmuch_bool_t quiet = FALSE, verbose = FALSE;
 
-    add_files_state.verbose = FALSE;
+    add_files_state.verbosity = VERBOSITY_NORMAL;
     add_files_state.debug = FALSE;
     add_files_state.output_is_a_tty = isatty (fileno (stdout));
 
     notmuch_opt_desc_t options[] = {
-	{ NOTMUCH_OPT_BOOLEAN,  &add_files_state.verbose, "verbose", 'v', 0 },
+	{ NOTMUCH_OPT_BOOLEAN,  &quiet, "quiet", 'q', 0 },
+	{ NOTMUCH_OPT_BOOLEAN,  &verbose, "verbose", 'v', 0 },
 	{ NOTMUCH_OPT_BOOLEAN,  &add_files_state.debug, "debug", 'd', 0 },
 	{ NOTMUCH_OPT_BOOLEAN,  &no_hooks, "no-hooks", 'n', 0 },
 	{ 0, 0, 0, 0, 0 }
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-    if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
-    }
+    if (opt_index < 0)
+	return EXIT_FAILURE;
+
+    /* quiet trumps verbose */
+    if (quiet)
+	add_files_state.verbosity = VERBOSITY_QUIET;
+    else if (verbose)
+	add_files_state.verbosity = VERBOSITY_VERBOSE;
 
     add_files_state.new_tags = notmuch_config_get_new_tags (config, &add_files_state.new_tags_length);
     add_files_state.new_ignore = notmuch_config_get_new_ignore (config, &add_files_state.new_ignore_length);
     add_files_state.synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config);
     db_path = notmuch_config_get_database_path (config);
 
+    for (i = 0; i < add_files_state.new_tags_length; i++) {
+	const char *error_msg;
+
+	error_msg = illegal_tag (add_files_state.new_tags[i], FALSE);
+	if (error_msg) {
+	    fprintf (stderr, "Error: tag '%s' in new.tags: %s\n",
+		     add_files_state.new_tags[i], error_msg);
+	    return EXIT_FAILURE;
+	}
+    }
+
     if (!no_hooks) {
 	ret = notmuch_run_hook (db_path, "pre-new");
 	if (ret)
-	    return ret;
+	    return EXIT_FAILURE;
     }
 
     dot_notmuch_path = talloc_asprintf (config, "%s/%s", db_path, ".notmuch");
@@ -922,23 +976,54 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
 	count = 0;
 	count_files (db_path, &count, &add_files_state);
 	if (interrupted)
-	    return 1;
+	    return EXIT_FAILURE;
 
-	printf ("Found %d total files (that's not much mail).\n", count);
+	if (add_files_state.verbosity >= VERBOSITY_NORMAL)
+	    printf ("Found %d total files (that's not much mail).\n", count);
 	if (notmuch_database_create (db_path, &notmuch))
-	    return 1;
+	    return EXIT_FAILURE;
 	add_files_state.total_files = count;
     } else {
 	if (notmuch_database_open (db_path, NOTMUCH_DATABASE_MODE_READ_WRITE,
 				   &notmuch))
-	    return 1;
+	    return EXIT_FAILURE;
 
 	if (notmuch_database_needs_upgrade (notmuch)) {
-	    printf ("Welcome to a new version of notmuch! Your database will now be upgraded.\n");
+	    time_t now = time (NULL);
+	    struct tm *gm_time = gmtime (&now);
+
+	    /* since dump files are written atomically, the amount of
+	     * harm from overwriting one within a second seems
+	     * relatively small. */
+
+	    const char *backup_name =
+		talloc_asprintf (notmuch, "%s/dump-%04d%02d%02dT%02d%02d%02d.gz",
+				 dot_notmuch_path,
+				 gm_time->tm_year + 1900,
+				 gm_time->tm_mon + 1,
+				 gm_time->tm_mday,
+				 gm_time->tm_hour,
+				 gm_time->tm_min,
+				 gm_time->tm_sec);
+
+	    if (add_files_state.verbosity >= VERBOSITY_NORMAL) {
+		printf ("Welcome to a new version of notmuch! Your database will now be upgraded.\n");
+		printf ("This process is safe to interrupt.\n");
+		printf ("Backing up tags to %s...\n", backup_name);
+	    }
+
+	    if (notmuch_database_dump (notmuch, backup_name, "",
+				       DUMP_FORMAT_BATCH_TAG, TRUE)) {
+		fprintf (stderr, "Backup failed. Aborting upgrade.");
+		return EXIT_FAILURE;
+	    }
+
 	    gettimeofday (&add_files_state.tv_start, NULL);
-	    notmuch_database_upgrade (notmuch, upgrade_print_progress,
+	    notmuch_database_upgrade (notmuch,
+				      add_files_state.verbosity >= VERBOSITY_NORMAL ? upgrade_print_progress : NULL,
 				      &add_files_state);
-	    printf ("Your notmuch database has now been upgraded to database format version %u.\n",
+	    if (add_files_state.verbosity >= VERBOSITY_NORMAL)
+		printf ("Your notmuch database has now been upgraded to database format version %u.\n",
 		    notmuch_database_get_version (notmuch));
 	}
 
@@ -946,7 +1031,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     }
 
     if (notmuch == NULL)
-	return 1;
+	return EXIT_FAILURE;
 
     /* Setup our handler for SIGINT. We do this after having
      * potentially done a database upgrade we this interrupt handler
@@ -969,8 +1054,8 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     add_files_state.removed_directories = _filename_list_create (config);
     add_files_state.directory_mtimes = _filename_list_create (config);
 
-    if (! debugger_is_active () && add_files_state.output_is_a_tty
-	&& ! add_files_state.verbose) {
+    if (add_files_state.verbosity == VERBOSITY_NORMAL &&
+	add_files_state.output_is_a_tty && ! debugger_is_active ()) {
 	setup_progress_printing_timer ();
 	timer_is_active = TRUE;
     }
@@ -1023,45 +1108,8 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     if (timer_is_active)
 	stop_progress_printing_timer ();
 
-    gettimeofday (&tv_now, NULL);
-    elapsed = notmuch_time_elapsed (add_files_state.tv_start,
-				    tv_now);
-
-    if (add_files_state.processed_files) {
-	printf ("Processed %d %s in ", add_files_state.processed_files,
-		add_files_state.processed_files == 1 ?
-		"file" : "total files");
-	notmuch_time_print_formatted_seconds (elapsed);
-	if (elapsed > 1) {
-	    printf (" (%d files/sec.).\033[K\n",
-		    (int) (add_files_state.processed_files / elapsed));
-	} else {
-	    printf (".\033[K\n");
-	}
-    }
-
-    if (add_files_state.added_messages) {
-	printf ("Added %d new %s to the database.",
-		add_files_state.added_messages,
-		add_files_state.added_messages == 1 ?
-		"message" : "messages");
-    } else {
-	printf ("No new mail.");
-    }
-
-    if (add_files_state.removed_messages) {
-	printf (" Removed %d %s.",
-		add_files_state.removed_messages,
-		add_files_state.removed_messages == 1 ? "message" : "messages");
-    }
-
-    if (add_files_state.renamed_messages) {
-	printf (" Detected %d file %s.",
-		add_files_state.renamed_messages,
-		add_files_state.renamed_messages == 1 ? "rename" : "renames");
-    }
-
-    printf ("\n");
+    if (add_files_state.verbosity >= VERBOSITY_NORMAL)
+	print_results (&add_files_state);
 
     if (ret)
 	fprintf (stderr, "Note: A fatal error was encountered: %s\n",
@@ -1072,5 +1120,5 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
     if (!no_hooks && !ret && !interrupted)
 	ret = notmuch_run_hook (db_path, "post-new");
 
-    return ret || interrupted;
+    return ret || interrupted ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/notmuch-reply.c b/notmuch-reply.c
index 9d6f8436..7c1c8095 100644
--- a/notmuch-reply.c
+++ b/notmuch-reply.c
@@ -21,6 +21,7 @@
  */
 
 #include "notmuch-client.h"
+#include "string-util.h"
 #include "sprinter.h"
 
 static void
@@ -369,78 +370,44 @@ add_recipients_from_message (GMimeMessage *reply,
     return from_addr;
 }
 
+/*
+ * Look for the user's address in " for <email@add.res>" in the
+ * received headers.
+ *
+ * Return the address that was found, if any, and NULL otherwise.
+ */
 static const char *
-guess_from_received_header (notmuch_config_t *config, notmuch_message_t *message)
+guess_from_in_received_for (notmuch_config_t *config, const char *received)
 {
-    const char *addr, *received, *by;
-    char *mta,*ptr,*token;
-    char *domain=NULL;
-    char *tld=NULL;
-    const char *delim=". \t";
-    size_t i;
-
-    const char *to_headers[] = {
-	"Envelope-to",
-	"X-Original-To",
-	"Delivered-To",
-    };
-
-    /* sadly, there is no standard way to find out to which email
-     * address a mail was delivered - what is in the headers depends
-     * on the MTAs used along the way. So we are trying a number of
-     * heuristics which hopefully will answer this question.
+    const char *ptr;
 
-     * We only got here if none of the users email addresses are in
-     * the To: or Cc: header. From here we try the following in order:
-     * 1) check for an Envelope-to: header
-     * 2) check for an X-Original-To: header
-     * 3) check for a Delivered-To: header
-     * 4) check for a (for <email@add.res>) clause in Received: headers
-     * 5) check for the domain part of known email addresses in the
-     *    'by' part of Received headers
-     * If none of these work, we give up and return NULL
-     */
-    for (i = 0; i < ARRAY_SIZE (to_headers); i++) {
-	const char *tohdr = notmuch_message_get_header (message, to_headers[i]);
-
-	/* Note: tohdr potentially contains a list of email addresses. */
-	addr = user_address_in_string (tohdr, config);
-	if (addr)
-	    return addr;
-    }
-
-    /* We get the concatenated Received: headers and search from the
-     * front (last Received: header added) and try to extract from
-     * them indications to which email address this message was
-     * delivered.
-     * The Received: header is special in our get_header function
-     * and is always concatenated.
-     */
-    received = notmuch_message_get_header (message, "received");
-    if (received == NULL)
+    ptr = strstr (received, " for ");
+    if (! ptr)
 	return NULL;
 
-    /* First we look for a " for <email@add.res>" in the received
-     * header
-     */
-    ptr = strstr (received, " for ");
+    return user_address_in_string (ptr, config);
+}
 
-    /* Note: ptr potentially contains a list of email addresses. */
-    addr = user_address_in_string (ptr, config);
-    if (addr)
-	return addr;
+/*
+ * Parse all the " by MTA ..." parts in received headers to guess the
+ * email address that this was originally delivered to.
+ *
+ * Extract just the MTA here by removing leading whitespace and
+ * assuming that the MTA name ends at the next whitespace. Test for
+ * *(by+4) to be non-'\0' to make sure there's something there at all
+ * - and then assume that the first whitespace delimited token that
+ * follows is the receiving system in this step of the receive chain.
+ *
+ * Return the address that was found, if any, and NULL otherwise.
+ */
+static const char *
+guess_from_in_received_by (notmuch_config_t *config, const char *received)
+{
+    const char *addr;
+    const char *by = received;
+    char *domain, *tld, *mta, *ptr, *token;
 
-    /* Finally, we parse all the " by MTA ..." headers to guess the
-     * email address that this was originally delivered to.
-     * We extract just the MTA here by removing leading whitespace and
-     * assuming that the MTA name ends at the next whitespace.
-     * We test for *(by+4) to be non-'\0' to make sure there's
-     * something there at all - and then assume that the first
-     * whitespace delimited token that follows is the receiving
-     * system in this step of the receive chain
-     */
-    by = received;
-    while((by = strstr (by, " by ")) != NULL) {
+    while ((by = strstr (by, " by ")) != NULL) {
 	by += 4;
 	if (*by == '\0')
 	    break;
@@ -450,11 +417,12 @@ guess_from_received_header (notmuch_config_t *config, notmuch_message_t *message
 	    free (mta);
 	    break;
 	}
-	/* Now extract the last two components of the MTA host name
-	 * as domain and tld.
+	/*
+	 * Now extract the last two components of the MTA host name as
+	 * domain and tld.
 	 */
 	domain = tld = NULL;
-	while ((ptr = strsep (&token, delim)) != NULL) {
+	while ((ptr = strsep (&token, ". \t")) != NULL) {
 	    if (*ptr == '\0')
 		continue;
 	    domain = tld;
@@ -462,13 +430,14 @@ guess_from_received_header (notmuch_config_t *config, notmuch_message_t *message
 	}
 
 	if (domain) {
-	    /* Recombine domain and tld and look for it among the configured
-	     * email addresses.
-	     * This time we have a known domain name and nothing else - so
-	     * the test is the other way around: we check if this is a
-	     * substring of one of the email addresses.
+	    /*
+	     * Recombine domain and tld and look for it among the
+	     * configured email addresses. This time we have a known
+	     * domain name and nothing else - so the test is the other
+	     * way around: we check if this is a substring of one of
+	     * the email addresses.
 	     */
-	    *(tld-1) = '.';
+	    *(tld - 1) = '.';
 
 	    addr = string_in_user_address (domain, config);
 	    if (addr) {
@@ -482,6 +451,70 @@ guess_from_received_header (notmuch_config_t *config, notmuch_message_t *message
     return NULL;
 }
 
+/*
+ * Get the concatenated Received: headers and search from the front
+ * (last Received: header added) and try to extract from them
+ * indications to which email address this message was delivered.
+ *
+ * The Received: header is special in our get_header function and is
+ * always concatenated.
+ *
+ * Return the address that was found, if any, and NULL otherwise.
+ */
+static const char *
+guess_from_in_received_headers (notmuch_config_t *config,
+				notmuch_message_t *message)
+{
+    const char *received, *addr;
+    char *sanitized;
+
+    received = notmuch_message_get_header (message, "received");
+    if (! received)
+	return NULL;
+
+    sanitized = sanitize_string (NULL, received);
+    if (! sanitized)
+	return NULL;
+
+    addr = guess_from_in_received_for (config, sanitized);
+    if (! addr)
+	addr = guess_from_in_received_by (config, sanitized);
+
+    talloc_free (sanitized);
+
+    return addr;
+}
+
+/*
+ * Try to find user's email address in one of the extra To-like
+ * headers: Envelope-To, X-Original-To, and Delivered-To (searched in
+ * that order).
+ *
+ * Return the address that was found, if any, and NULL otherwise.
+ */
+static const char *
+get_from_in_to_headers (notmuch_config_t *config, notmuch_message_t *message)
+{
+    size_t i;
+    const char *tohdr, *addr;
+    const char *to_headers[] = {
+	"Envelope-to",
+	"X-Original-To",
+	"Delivered-To",
+    };
+
+    for (i = 0; i < ARRAY_SIZE (to_headers); i++) {
+	tohdr = notmuch_message_get_header (message, to_headers[i]);
+
+	/* Note: tohdr potentially contains a list of email addresses. */
+	addr = user_address_in_string (tohdr, config);
+	if (addr)
+	    return addr;
+    }
+
+    return NULL;
+}
+
 static GMimeMessage *
 create_reply_message(void *ctx,
 		     notmuch_config_t *config,
@@ -508,9 +541,30 @@ create_reply_message(void *ctx,
     from_addr = add_recipients_from_message (reply, config,
 					     message, reply_all);
 
+    /*
+     * Sadly, there is no standard way to find out to which email
+     * address a mail was delivered - what is in the headers depends
+     * on the MTAs used along the way.
+     *
+     * If none of the user's email addresses are in the To: or Cc:
+     * headers, we try a number of heuristics which hopefully will
+     * answer this question.
+     *
+     * First, check for Envelope-To:, X-Original-To:, and
+     * Delivered-To: headers.
+     */
+    if (from_addr == NULL)
+	from_addr = get_from_in_to_headers (config, message);
+
+    /*
+     * Check for a (for <email@add.res>) clause in Received: headers,
+     * and the domain part of known email addresses in the 'by' part
+     * of Received: headers
+     */
     if (from_addr == NULL)
-	from_addr = guess_from_received_header (config, message);
+	from_addr = guess_from_in_received_headers (config, message);
 
+    /* Default to user's primary address. */
     if (from_addr == NULL)
 	from_addr = notmuch_config_get_user_primary_email (config);
 
@@ -704,7 +758,7 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[])
     notmuch_database_t *notmuch;
     notmuch_query_t *query;
     char *query_string;
-    int opt_index, ret = 0;
+    int opt_index;
     int (*reply_format_func) (void *ctx,
 			      notmuch_config_t *config,
 			      notmuch_query_t *query,
@@ -739,10 +793,8 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[])
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-    if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
-    }
+    if (opt_index < 0)
+	return EXIT_FAILURE;
 
     if (format == FORMAT_HEADERS_ONLY) {
 	reply_format_func = notmuch_reply_format_headers_only;
@@ -761,30 +813,30 @@ notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[])
     query_string = query_string_from_args (config, argc-opt_index, argv+opt_index);
     if (query_string == NULL) {
 	fprintf (stderr, "Out of memory\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (*query_string == '\0') {
 	fprintf (stderr, "Error: notmuch reply requires at least one search term.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_ONLY, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
     query = notmuch_query_create (notmuch, query_string);
     if (query == NULL) {
 	fprintf (stderr, "Out of memory\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (reply_format_func (config, config, query, &params, reply_all, sp) != 0)
-	return 1;
+	return EXIT_FAILURE;
 
     notmuch_crypto_cleanup (&params.crypto);
     notmuch_query_destroy (query);
     notmuch_database_destroy (notmuch);
 
-    return ret;
+    return EXIT_SUCCESS;
 }
diff --git a/notmuch-restore.c b/notmuch-restore.c
index 1419621c..584d9f96 100644
--- a/notmuch-restore.c
+++ b/notmuch-restore.c
@@ -19,9 +19,10 @@
  */
 
 #include "notmuch-client.h"
-#include "dump-restore-private.h"
+#include "hex-escape.h"
 #include "tag-util.h"
 #include "string-util.h"
+#include "zlib-extra.h"
 
 static regex_t regex;
 
@@ -128,10 +129,10 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
     tag_op_list_t *tag_ops;
 
     char *input_file_name = NULL;
-    FILE *input = stdin;
+    const char *name_for_error = NULL;
+    gzFile input = NULL;
     char *line = NULL;
     void *line_ctx = NULL;
-    size_t line_size;
     ssize_t line_len;
 
     int ret = 0;
@@ -140,7 +141,7 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_WRITE, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
     if (notmuch_config_get_maildir_synchronize_flags (config))
 	flags |= TAG_FLAG_MAILDIR_SYNC;
@@ -157,44 +158,69 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-
     if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
+	ret = EXIT_FAILURE;
+	goto DONE;
     }
 
+    name_for_error = input_file_name ? input_file_name : "stdin";
+
     if (! accumulate)
 	flags |= TAG_FLAG_REMOVE_ALL;
 
-    if (input_file_name) {
-	input = fopen (input_file_name, "r");
-	if (input == NULL) {
-	    fprintf (stderr, "Error opening %s for reading: %s\n",
-		     input_file_name, strerror (errno));
-	    return 1;
+    errno = 0;
+    if (input_file_name)
+	input = gzopen (input_file_name, "r");
+    else {
+	int infd = dup (STDIN_FILENO);
+	if (infd < 0) {
+	    fprintf (stderr, "Error duping stdin: %s\n",
+		     strerror (errno));
+	    ret = EXIT_FAILURE;
+	    goto DONE;
 	}
+	input = gzdopen (infd, "r");
+	if (! input)
+	    close (infd);
+    }
+
+    if (input == NULL) {
+	fprintf (stderr, "Error opening %s for (gzip) reading: %s\n",
+		 name_for_error, strerror (errno));
+	ret = EXIT_FAILURE;
+	goto DONE;
     }
 
     if (opt_index < argc) {
-	fprintf (stderr,
-		 "Unused positional parameter: %s\n",
-		 argv[opt_index]);
-	return 1;
+	fprintf (stderr, "Unused positional parameter: %s\n", argv[opt_index]);
+	ret = EXIT_FAILURE;
+	goto DONE;
     }
 
     tag_ops = tag_op_list_create (config);
     if (tag_ops == NULL) {
 	fprintf (stderr, "Out of memory.\n");
-	return 1;
+	ret = EXIT_FAILURE;
+	goto DONE;
     }
 
     do {
-	line_len = getline (&line, &line_size, input);
+	util_status_t status;
+
+	status = gz_getline (line_ctx, &line, &line_len, input);
 
 	/* empty input file not considered an error */
-	if (line_len < 0)
-	    return 0;
+	if (status == UTIL_EOF) {
+	    ret = EXIT_SUCCESS;
+	    goto DONE;
+	}
 
+	if (status) {
+	    fprintf (stderr, "Error reading (gzipped) input: %s\n",
+		     gz_error_string(status, input));
+	    ret = EXIT_FAILURE;
+	    goto DONE;
+	}
     } while ((line_len == 0) ||
 	     (line[0] == '#') ||
 	     /* the cast is safe because we checked about for line_len < 0 */
@@ -259,21 +285,38 @@ notmuch_restore_command (notmuch_config_t *config, int argc, char *argv[])
 	if (ret)
 	    break;
 
-    }  while ((line_len = getline (&line, &line_size, input)) != -1);
+    }  while (! (ret = gz_getline (line_ctx, &line, &line_len, input)));
+    
 
-    if (line_ctx != NULL)
-	talloc_free (line_ctx);
+    /* EOF is normal loop termination condition, UTIL_SUCCESS is
+     * impossible here */
+    if (ret == UTIL_EOF) {
+	ret = EXIT_SUCCESS;
+    } else {
+	fprintf (stderr, "Error reading (gzipped) input: %s\n",
+		 gz_error_string (ret, input));
+	ret = EXIT_FAILURE;
+    }
+
+    /* currently this should not be after DONE: since we don't 
+     * know if the xregcomp was reached
+     */
 
     if (input_format == DUMP_FORMAT_SUP)
 	regfree (&regex);
 
-    if (line)
-	free (line);
+ DONE:
+    if (line_ctx != NULL)
+	talloc_free (line_ctx);
 
-    notmuch_database_destroy (notmuch);
+    if (notmuch)
+	notmuch_database_destroy (notmuch);
 
-    if (input != stdin)
-	fclose (input);
+    if (input && gzclose_r (input)) {
+	fprintf (stderr, "Error closing %s: %s\n",
+		 name_for_error, gzerror (input, NULL));
+	ret = EXIT_FAILURE;
+    }
 
-    return ret;
+    return ret ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/notmuch-search.c b/notmuch-search.c
index 7c973b3d..bc9be459 100644
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -30,23 +30,6 @@ typedef enum {
     OUTPUT_TAGS
 } output_t;
 
-static char *
-sanitize_string (const void *ctx, const char *str)
-{
-    char *out, *loop;
-
-    if (NULL == str)
-	return NULL;
-
-    loop = out = talloc_strdup (ctx, str);
-
-    for (; *loop; loop++) {
-	if ((unsigned char)(*loop) < 32)
-	    *loop = '?';
-    }
-    return out;
-}
-
 /* Return two stable query strings that identify exactly the matched
  * and unmatched messages currently in thread.  If there are no
  * matched or unmatched messages, the returned buffers will be
@@ -401,10 +384,8 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-
-    if (opt_index < 0) {
-	return 1;
-    }
+    if (opt_index < 0)
+	return EXIT_FAILURE;
 
     switch (format_sel) {
     case NOTMUCH_FORMAT_TEXT:
@@ -413,7 +394,7 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
     case NOTMUCH_FORMAT_TEXT0:
 	if (output == OUTPUT_SUMMARY) {
 	    fprintf (stderr, "Error: --format=text0 is not compatible with --output=summary.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 	format = sprinter_text0_create (config, stdout);
 	break;
@@ -432,22 +413,22 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_ONLY, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
     query_str = query_string_from_args (notmuch, argc-opt_index, argv+opt_index);
     if (query_str == NULL) {
 	fprintf (stderr, "Out of memory.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
     if (*query_str == '\0') {
 	fprintf (stderr, "Error: notmuch search requires at least one search term.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     query = notmuch_query_create (notmuch, query_str);
     if (query == NULL) {
 	fprintf (stderr, "Out of memory\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     notmuch_query_set_sort (query, sort);
@@ -491,5 +472,5 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
 
     talloc_free (format);
 
-    return ret;
+    return ret ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/notmuch-setup.c b/notmuch-setup.c
index 475248b1..36a6171a 100644
--- a/notmuch-setup.c
+++ b/notmuch-setup.c
@@ -140,7 +140,7 @@ notmuch_setup_command (notmuch_config_t *config,
 	fflush (stdout);					\
 	if (getline (&response, &response_size, stdin) < 0) {	\
 	    printf ("Exiting.\n");				\
-	    exit (1);						\
+	    exit (EXIT_FAILURE);				\
 	}							\
 	chomp_newline (response);				\
     } while (0)
@@ -223,12 +223,11 @@ notmuch_setup_command (notmuch_config_t *config,
 	g_ptr_array_free (tags, TRUE);
     }
 
+    if (notmuch_config_save (config))
+	return EXIT_FAILURE;
 
-    if (! notmuch_config_save (config)) {
-	if (notmuch_config_is_new (config))
-	  welcome_message_post_setup ();
-	return 0;
-    } else {
-	return 1;
-    }
+    if (notmuch_config_is_new (config))
+	welcome_message_post_setup ();
+
+    return EXIT_SUCCESS;
 }
diff --git a/notmuch-show.c b/notmuch-show.c
index c07f8871..d416fbd5 100644
--- a/notmuch-show.c
+++ b/notmuch-show.c
@@ -1015,9 +1015,13 @@ do_show (void *ctx,
     notmuch_messages_t *messages;
     notmuch_status_t status, res = NOTMUCH_STATUS_SUCCESS;
 
+    threads = notmuch_query_search_threads (query);
+    if (! threads)
+	return 1;
+
     sp->begin_list (sp);
 
-    for (threads = notmuch_query_search_threads (query);
+    for ( ;
 	 notmuch_threads_valid (threads);
 	 notmuch_threads_move_to_next (threads))
     {
@@ -1113,10 +1117,8 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[])
     };
 
     opt_index = parse_arguments (argc, argv, options, 1);
-    if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
-    }
+    if (opt_index < 0)
+	return EXIT_FAILURE;
 
     /* decryption implies verification */
     if (params.crypto.decrypt)
@@ -1143,7 +1145,7 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[])
     case NOTMUCH_FORMAT_MBOX:
 	if (params.part > 0) {
 	    fprintf (stderr, "Error: specifying parts is incompatible with mbox output format.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 
 	format = &format_mbox;
@@ -1193,22 +1195,22 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[])
     query_string = query_string_from_args (config, argc-opt_index, argv+opt_index);
     if (query_string == NULL) {
 	fprintf (stderr, "Out of memory\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (*query_string == '\0') {
 	fprintf (stderr, "Error: notmuch show requires at least one search term.\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_ONLY, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
     query = notmuch_query_create (notmuch, query_string);
     if (query == NULL) {
 	fprintf (stderr, "Out of memory\n");
-	return 1;
+	return EXIT_FAILURE;
     }
 
     /* Create structure printer. */
@@ -1242,5 +1244,5 @@ notmuch_show_command (notmuch_config_t *config, int argc, char *argv[])
     notmuch_query_destroy (query);
     notmuch_database_destroy (notmuch);
 
-    return ret;
+    return ret ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/notmuch-tag.c b/notmuch-tag.c
index 3b09df99..5b2f1e48 100644
--- a/notmuch-tag.c
+++ b/notmuch-tag.c
@@ -193,7 +193,7 @@ notmuch_tag_command (notmuch_config_t *config, int argc, char *argv[])
     FILE *input = stdin;
     char *input_file_name = NULL;
     int opt_index;
-    int ret = 0;
+    int ret;
 
     /* Setup our handler for SIGINT */
     memset (&action, 0, sizeof (struct sigaction));
@@ -211,7 +211,7 @@ notmuch_tag_command (notmuch_config_t *config, int argc, char *argv[])
 
     opt_index = parse_arguments (argc, argv, options, 1);
     if (opt_index < 0)
-	return 1;
+	return EXIT_FAILURE;
 
     if (input_file_name) {
 	batch = TRUE;
@@ -219,44 +219,44 @@ notmuch_tag_command (notmuch_config_t *config, int argc, char *argv[])
 	if (input == NULL) {
 	    fprintf (stderr, "Error opening %s for reading: %s\n",
 		     input_file_name, strerror (errno));
-	    return 1;
+	    return EXIT_FAILURE;
 	}
     }
 
     if (batch) {
 	if (opt_index != argc) {
 	    fprintf (stderr, "Can't specify both cmdline and stdin!\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 	if (remove_all) {
 	    fprintf (stderr, "Can't specify both --remove-all and --batch\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
     } else {
 	tag_ops = tag_op_list_create (config);
 	if (tag_ops == NULL) {
 	    fprintf (stderr, "Out of memory.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 
 	if (parse_tag_command_line (config, argc - opt_index, argv + opt_index,
 				    &query_string, tag_ops))
-	    return 1;
+	    return EXIT_FAILURE;
 
 	if (tag_op_list_size (tag_ops) == 0 && ! remove_all) {
 	    fprintf (stderr, "Error: 'notmuch tag' requires at least one tag to add or remove.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 
 	if (*query_string == '\0') {
 	    fprintf (stderr, "Error: notmuch tag requires at least one search term.\n");
-	    return 1;
+	    return EXIT_FAILURE;
 	}
     }
 
     if (notmuch_database_open (notmuch_config_get_database_path (config),
 			       NOTMUCH_DATABASE_MODE_READ_WRITE, &notmuch))
-	return 1;
+	return EXIT_FAILURE;
 
     if (notmuch_config_get_maildir_synchronize_flags (config))
 	tag_flags |= TAG_FLAG_MAILDIR_SYNC;
@@ -274,5 +274,5 @@ notmuch_tag_command (notmuch_config_t *config, int argc, char *argv[])
     if (input != stdin)
 	fclose (input);
 
-    return ret || interrupted;
+    return ret || interrupted ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/notmuch.c b/notmuch.c
index 54f46c68..dcda0392 100644
--- a/notmuch.c
+++ b/notmuch.c
@@ -22,6 +22,12 @@
 
 #include "notmuch-client.h"
 
+/*
+ * Notmuch subcommand hook.
+ *
+ * The return value will be used as notmuch exit status code,
+ * preferrably EXIT_SUCCESS or EXIT_FAILURE.
+ */
 typedef int (*command_function_t) (notmuch_config_t *config, int argc, char *argv[]);
 
 typedef struct command {
@@ -68,6 +74,18 @@ static command_t commands[] = {
       "This message, or more detailed help for the named command." }
 };
 
+typedef struct help_topic {
+    const char *name;
+    const char *summary;
+} help_topic_t;
+
+static help_topic_t help_topics[] = {
+    { "search-terms",
+      "Common search term syntax." },
+    { "hooks",
+      "Hooks that will be run before or after certain commands." },
+};
+
 static command_t *
 find_command (const char *name)
 {
@@ -87,6 +105,7 @@ static void
 usage (FILE *out)
 {
     command_t *command;
+    help_topic_t *topic;
     unsigned int i;
 
     fprintf (out,
@@ -101,13 +120,22 @@ usage (FILE *out)
 	command = &commands[i];
 
 	if (command->name)
-	    fprintf (out, "  %-11s  %s\n", command->name, command->summary);
+	    fprintf (out, "  %-12s  %s\n", command->name, command->summary);
+    }
+
+    fprintf (out, "\n");
+    fprintf (out, "Additional help topics are as follows:\n");
+    fprintf (out, "\n");
+
+    for (i = 0; i < ARRAY_SIZE (help_topics); i++) {
+	topic = &help_topics[i];
+	fprintf (out, "  %-12s  %s\n", topic->name, topic->summary);
     }
 
     fprintf (out, "\n");
     fprintf (out,
-    "Use \"notmuch help <command>\" for more details on each command\n"
-    "and \"notmuch help search-terms\" for the common search-terms syntax.\n\n");
+	     "Use \"notmuch help <command or topic>\" for more details "
+	     "on each command or topic.\n\n");
 }
 
 void
@@ -150,13 +178,15 @@ static int
 notmuch_help_command (notmuch_config_t *config, int argc, char *argv[])
 {
     command_t *command;
+    help_topic_t *topic;
+    unsigned int i;
 
     argc--; argv++; /* Ignore "help" */
 
     if (argc == 0) {
 	printf ("The notmuch mail system.\n\n");
 	usage (stdout);
-	return 0;
+	return EXIT_SUCCESS;
     }
 
     if (strcmp (argv[0], "help") == 0) {
@@ -165,7 +195,7 @@ notmuch_help_command (notmuch_config_t *config, int argc, char *argv[])
 		"\tof difficulties check that MANPATH includes the pages\n"
 		"\tinstalled by notmuch.\n\n"
 		"\tTry \"notmuch help\" for a list of topics.\n");
-	return 0;
+	return EXIT_SUCCESS;
     }
 
     command = find_command (argv[0]);
@@ -174,16 +204,18 @@ notmuch_help_command (notmuch_config_t *config, int argc, char *argv[])
 	exec_man (page);
     }
 
-    if (strcmp (argv[0], "search-terms") == 0) {
-	exec_man ("notmuch-search-terms");
-    } else if (strcmp (argv[0], "hooks") == 0) {
-	exec_man ("notmuch-hooks");
+    for (i = 0; i < ARRAY_SIZE (help_topics); i++) {
+	topic = &help_topics[i];
+	if (strcmp (argv[0], topic->name) == 0) {
+	    char *page = talloc_asprintf (config, "notmuch-%s", topic->name);
+	    exec_man (page);
+	}
     }
 
     fprintf (stderr,
 	     "\nSorry, %s is not a known command. There's not much I can do to help.\n\n",
 	     argv[0]);
-    return 1;
+    return EXIT_FAILURE;
 }
 
 /* Handle the case of "notmuch" being invoked with no command
@@ -211,7 +243,7 @@ notmuch_command (notmuch_config_t *config,
 	if (errno != ENOENT) {
 	    fprintf (stderr, "Error looking for notmuch database at %s: %s\n",
 		     db_path, strerror (errno));
-	    return 1;
+	    return EXIT_FAILURE;
 	}
 	printf ("Notmuch is configured, but there's not yet a database at\n\n\t%s\n\n",
 		db_path);
@@ -219,7 +251,7 @@ notmuch_command (notmuch_config_t *config,
 		"Note that the first run of \"notmuch new\" can take a very long time\n"
 		"and that the resulting database will use roughly the same amount of\n"
 		"storage space as the email being indexed.\n\n");
-	return 0;
+	return EXIT_SUCCESS;
     }
 
     printf ("Notmuch is configured and appears to have a database. Excellent!\n\n"
@@ -239,7 +271,7 @@ notmuch_command (notmuch_config_t *config,
 	    notmuch_config_get_user_name (config),
 	    notmuch_config_get_user_primary_email (config));
 
-    return 0;
+    return EXIT_SUCCESS;
 }
 
 int
@@ -250,10 +282,10 @@ main (int argc, char *argv[])
     const char *command_name = NULL;
     command_t *command;
     char *config_file_name = NULL;
-    notmuch_config_t *config;
+    notmuch_config_t *config = NULL;
     notmuch_bool_t print_help=FALSE, print_version=FALSE;
     int opt_index;
-    int ret = 0;
+    int ret;
 
     notmuch_opt_desc_t options[] = {
 	{ NOTMUCH_OPT_BOOLEAN, &print_help, "help", 'h', 0 },
@@ -276,16 +308,19 @@ main (int argc, char *argv[])
 
     opt_index = parse_arguments (argc, argv, options, 1);
     if (opt_index < 0) {
-	/* diagnostics already printed */
-	return 1;
+	ret = EXIT_FAILURE;
+	goto DONE;
     }
 
-    if (print_help)
-	return notmuch_help_command (NULL, argc - 1, &argv[1]);
+    if (print_help) {
+	ret = notmuch_help_command (NULL, argc - 1, &argv[1]);
+	goto DONE;
+    }
 
     if (print_version) {
 	printf ("notmuch " STRINGIFY(NOTMUCH_VERSION) "\n");
-	return 0;
+	ret = EXIT_SUCCESS;
+	goto DONE;
     }
 
     if (opt_index < argc)
@@ -295,16 +330,21 @@ main (int argc, char *argv[])
     if (!command) {
 	fprintf (stderr, "Error: Unknown command '%s' (see \"notmuch help\")\n",
 		 command_name);
-	return 1;
+	ret = EXIT_FAILURE;
+	goto DONE;
     }
 
     config = notmuch_config_open (local, config_file_name, command->create_config);
-    if (!config)
-	return 1;
+    if (!config) {
+	ret = EXIT_FAILURE;
+	goto DONE;
+    }
 
     ret = (command->function)(config, argc - opt_index, argv + opt_index);
 
-    notmuch_config_close (config);
+  DONE:
+    if (config)
+	notmuch_config_close (config);
 
     talloc_report = getenv ("NOTMUCH_TALLOC_REPORT");
     if (talloc_report && strcmp (talloc_report, "") != 0) {
diff --git a/performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc b/performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc
new file mode 100644
index 00000000..72dedd8b
--- /dev/null
+++ b/performance-test/download/notmuch-email-corpus-0.4.tar.xz.asc
@@ -0,0 +1,14 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.15 (GNU/Linux)
+
+iQGcBAABCAAGBQJSdaDkAAoJEPIClx2kp54sQ54L/ikkvF1fy88hjLitN59v6g2J
+vw85YNRifNHyp/UXI6nt2eXFzyWJiRHuvHFoBgmEsJVxauOKw61Gs2zd53x9Ear4
+MGcQWyiM1cnwX/nD7GvxRQNh33f+FEamTjg+QhG47K0A2YdLWcDC7r9GMatGT11x
+5KE24WQGOqtgQn/9qNtJvkiKIehpRiDTaW/QJ7mTCYeJFjIHJUY8dxyfiTtkJ0z7
+cJ6omehvWSw4STbEg65XJgqykxMdltNEavfvSbAT73FgmkkyXxul0s5hDZ/esd0n
+re3dyDxGt085POiAgPti05a4tJI5EQC2wLBUFri0s2JdMtazcD6yVuHNbVzZ4Do3
+nL/sgwKGUq5wRrPqPWp6HXtZ9zG+/V7hFNrr/l42qGrLqsSh0bqvEnUiwczZLBGy
+NEs4G8VjmfS2cMKePsWaekBAvFUtb47PSB6JIPwpCNvKXDrcCb28eOQVB2atgj1h
+9SktOtWYJhWIQp2YW9iae30Z6lhCcdPRRHTFMQq2nQ==
+=eSMY
+-----END PGP SIGNATURE-----
diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh
index 9ee76613..75e3d878 100644
--- a/performance-test/perf-test-lib.sh
+++ b/performance-test/perf-test-lib.sh
@@ -41,52 +41,71 @@ add_email_corpus ()
 {
     rm -rf ${MAIL_DIR}
 
-    case "$corpus_size" in
-	small)
-	    mail_subdir="mail/enron/bailey-s"
-	    check_for="${TEST_DIRECTORY}/corpus/$mail_subdir"
-	    ;;
-	medium)
-	    mail_subdir="mail/notmuch-archive"
-	    check_for="${TEST_DIRECTORY}/corpus/$mail_subdir"
-	    ;;
-	*)
-	    mail_subdir=mail
-	    check_for="${TEST_DIRECTORY}/corpus/$mail_subdir/enron/wolfe-j"
-    esac
+    CORPUS_DIR=${TEST_DIRECTORY}/corpus
+    mkdir -p "${CORPUS_DIR}"
 
-    MAIL_CORPUS="${TEST_DIRECTORY}/corpus/$mail_subdir"
-    TAG_CORPUS="${TEST_DIRECTORY}/corpus/tags"
+    MAIL_CORPUS="${CORPUS_DIR}/mail.${corpus_size}"
+    TAG_CORPUS="${CORPUS_DIR}/tags"
 
-    args=()
-    if [ ! -d "$TAG_CORPUS" ] ; then
-	args+=("notmuch-email-corpus/tags")
+    if command -v pixz > /dev/null; then
+	XZ=pixz
+    else
+	XZ=xz
+    fi
+
+    if [ ! -d "${CORPUS_DIR}/manifest" ]; then
+
+	printf "Unpacking manifests\n"
+	tar --extract --use-compress-program ${XZ} --strip-components=1 \
+	    --directory ${TEST_DIRECTORY}/corpus \
+	    --wildcards --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \
+	    'notmuch-email-corpus/manifest/*'
     fi
 
-    if [ ! -d "$check_for" ] ; then
-	args+=("notmuch-email-corpus/$mail_subdir")
+    file_list=$(mktemp file_listXXXXXX)
+    declare -a extract_dirs
+    if [ ! -d "$TAG_CORPUS" ] ; then
+	extract_dirs=("${extract_dirs[@]}" notmuch-email-corpus/tags)
     fi
 
-    if [[ ${#args[@]} > 0 ]]; then
-	if command -v pixz > /dev/null; then
-	    XZ=pixz
+    if [ ! -d "$MAIL_CORPUS" ] ; then
+	if [[ "$corpus_size" != "large" ]]; then
+	    sed s,^,notmuch-email-corpus/, < \
+		${TEST_DIRECTORY}/corpus/manifest/MANIFEST.${corpus_size} >> $file_list
 	else
-	    XZ=xz
+	    extract_dirs=("${extract_dirs[@]}" notmuch-email-corpus/mail)
 	fi
+    fi
 
-	printf "Unpacking corpus\n"
-	mkdir -p "${TEST_DIRECTORY}/corpus"
+    if [[ -s $file_list || -n "${extract_dirs[*]}" ]]; then
 
+	printf "Unpacking corpus\n"
 	tar --checkpoint=.5000 --extract --strip-components=1 \
 	    --directory ${TEST_DIRECTORY}/corpus \
 	    --use-compress-program ${XZ} \
 	    --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \
-	    "${args[@]}"
+	    --anchored --recursion \
+	    --files-from $file_list "${extract_dirs[@]}"
 
 	printf "\n"
 
+	if [[ ! -d ${MAIL_CORPUS} ]]; then
+	    printf "creating link farm\n"
+
+	    if [[ "$corpus_size" = large ]]; then
+		cp -rl ${TEST_DIRECTORY}/corpus/mail ${MAIL_CORPUS}
+	    else
+		while read -r file; do
+		    tdir=${MAIL_CORPUS}/$(dirname $file)
+		    mkdir -p $tdir
+		    ln ${TEST_DIRECTORY}/corpus/$file $tdir
+		done <${TEST_DIRECTORY}/corpus/manifest/MANIFEST.${corpus_size}
+	    fi
+	fi
+
     fi
 
+    rm $file_list
     cp -lr $TAG_CORPUS $TMP_DIRECTORY/corpus.tags
     cp -lr $MAIL_CORPUS $MAIL_DIR
 }
diff --git a/performance-test/version.sh b/performance-test/version.sh
index afafc737..f02527a7 100644
--- a/performance-test/version.sh
+++ b/performance-test/version.sh
@@ -1,3 +1,3 @@
 # this should be both a valid Makefile fragment and valid POSIX(ish) shell.
 
-PERFTEST_VERSION=0.3
+PERFTEST_VERSION=0.4
diff --git a/tag-util.c b/tag-util.c
index 3bde4097..343c161f 100644
--- a/tag-util.c
+++ b/tag-util.c
@@ -31,14 +31,7 @@ line_error (tag_parse_status_t status,
     return status;
 }
 
-/*
- * Test tags for some forbidden cases.
- *
- * return: NULL if OK,
- *	   explanatory message otherwise.
- */
-
-static const char *
+const char *
 illegal_tag (const char *tag, notmuch_bool_t remove)
 {
 
@@ -179,7 +172,7 @@ parse_tag_command_line (void *ctx, int argc, char **argv,
 
 	msg = illegal_tag (argv[i] + 1, is_remove);
 	if (msg) {
-	    fprintf (stderr, "Error: %s", msg);
+	    fprintf (stderr, "Error: %s\n", msg);
 	    return TAG_PARSE_INVALID;
 	}
 
diff --git a/tag-util.h b/tag-util.h
index 4628f163..8a4074ce 100644
--- a/tag-util.h
+++ b/tag-util.h
@@ -90,6 +90,18 @@ parse_tag_command_line (void *ctx, int argc, char **argv,
 			char **query_str, tag_op_list_t *ops);
 
 /*
+ * Test tags for some forbidden cases.
+ *
+ * Relax the checks if 'remove' is true to allow removal of previously
+ * added forbidden tags.
+ *
+ * return: NULL if OK,
+ *	   explanatory message otherwise.
+ */
+const char *
+illegal_tag (const char *tag, notmuch_bool_t remove);
+
+/*
  * Create an empty list of tag operations
  *
  * ctx is passed to talloc
diff --git a/test/Makefile.local b/test/Makefile.local
index 8870ca37..d622eafe 100644
--- a/test/Makefile.local
+++ b/test/Makefile.local
@@ -2,7 +2,9 @@
 
 dir := test
 
-extra_cflags += -I.
+# save against changes in $(dir)
+test_src_dir := $(dir)
+extra_cflags += -I$(srcdir)
 
 smtp_dummy_srcs =		\
 	$(notmuch_compat_srcs)	\
@@ -11,10 +13,10 @@ smtp_dummy_srcs =		\
 smtp_dummy_modules = $(smtp_dummy_srcs:.c=.o)
 
 $(dir)/arg-test: $(dir)/arg-test.o command-line-arguments.o util/libutil.a
-	$(call quiet,CC) -I. $^ -o $@
+	$(call quiet,CC) $^ -o $@
 
 $(dir)/hex-xcode: $(dir)/hex-xcode.o command-line-arguments.o util/libutil.a
-	$(call quiet,CC) -I. $^ -o $@ -ltalloc
+	$(call quiet,CC) $^ $(TALLOC_LDFLAGS) -o $@
 
 random_corpus_deps =  $(dir)/random-corpus.o  $(dir)/database-test.o \
 			notmuch-config.o command-line-arguments.o \
@@ -33,9 +35,25 @@ $(dir)/symbol-test: $(dir)/symbol-test.o lib/$(LINKER_NAME)
 $(dir)/parse-time: $(dir)/parse-time.o parse-time-string/parse-time-string.o
 	$(call quiet,CC) $^ -o $@
 
+$(dir)/have-compact: Makefile.config
+ifeq ($(HAVE_XAPIAN_COMPACT),1)
+	ln -sf /bin/true $@
+else
+	ln -sf /bin/false $@
+endif
+
+$(dir)/have-man: Makefile.config
+ifeq ($(HAVE_SPHINX)$(HAVE_RST2MAN),00)
+	ln -sf /bin/false $@
+else
+	ln -sf /bin/true $@
+endif
+
 .PHONY: test check
 
 TEST_BINARIES=$(dir)/arg-test \
+	      $(dir)/have-compact \
+	      $(dir)/have-man \
 	      $(dir)/hex-xcode \
 	      $(dir)/random-corpus \
 	      $(dir)/parse-time \
@@ -45,16 +63,11 @@ TEST_BINARIES=$(dir)/arg-test \
 test-binaries: $(TEST_BINARIES)
 
 test:	all test-binaries
-	@${dir}/notmuch-test $(OPTIONS)
+	@${test_src_dir}/notmuch-test $(OPTIONS)
 
 check: test
 
 SRCS := $(SRCS) $(smtp_dummy_srcs)
-CLEAN := $(CLEAN) $(dir)/smtp-dummy $(dir)/smtp-dummy.o \
-	 $(dir)/symbol-test $(dir)/symbol-test.o \
-	 $(dir)/arg-test $(dir)/arg-test.o \
-	 $(dir)/hex-xcode $(dir)/hex-xcode.o \
+CLEAN += $(TEST_BINARIES) $(addsuffix .o,$(TEST_BINARIES)) \
 	 $(dir)/database-test.o \
-	 $(dir)/random-corpus $(dir)/random-corpus.o \
-	 $(dir)/parse-time $(dir)/parse-time.o \
 	 $(dir)/corpus.mail $(dir)/test-results $(dir)/tmp.*
diff --git a/test/README b/test/README
index d12cff24..81a1c82d 100644
--- a/test/README
+++ b/test/README
@@ -64,6 +64,14 @@ The following command-line options are available when running tests:
 	Pointing this argument at a tmpfs filesystem can improve the
 	speed of the test suite for some users.
 
+Certain tests require precomputed databases to complete. You can fetch these
+databases with
+
+	make download-test-databases
+
+If you do not download the test databases, the relevant tests will be
+skipped.
+
 When invoking the test suite via "make test" any of the above options
 can be specified as follows:
 
@@ -76,6 +84,14 @@ the tests in one of the following ways.
 	TEST_EMACS=my-special-emacs TEST_EMACSCLIENT=my-emacsclient ./emacs
 	make test TEST_EMACS=my-special-emacs TEST_EMACSCLIENT=my-emacsclient
 
+Quiet Execution
+---------------
+
+Normally, when new script starts and when test PASSes you get a message
+printed on screen. This printing can be disabled by setting the
+NOTMUCH_TEST_QUIET variable to a non-null value. Message on test
+failures and skips are still printed.
+
 Skipping Tests
 --------------
 If, for any reason, you need to skip one or more tests, you can do so
diff --git a/test/basic b/test/T000-basic.sh
index 64eb7d74..ebbb6d2e 100755
--- a/test/basic
+++ b/test/T000-basic.sh
@@ -49,24 +49,6 @@ test_expect_code 2 'failure to clean up causes the test to fail' '
     test_when_finished "(exit 2)"
 '
 
-# Ensure that all tests are being run
-test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
-eval $(sed -n -e '/^TESTS="$/,/^"$/p' $TEST_DIRECTORY/notmuch-test)
-tests_in_suite=$(for i in $TESTS; do echo $i; done | sort)
-available=$(find "$TEST_DIRECTORY" -maxdepth 1 -type f \
-   '(' -perm -100 -o -perm -10 -o -perm -1 ')' \
-    ! -name aggregate-results.sh	\
-    ! -name arg-test			\
-    ! -name hex-xcode			\
-    ! -name notmuch-test		\
-    ! -name parse-time			\
-    ! -name random-corpus		\
-    ! -name smtp-dummy			\
-    ! -name symbol-test			\
-    ! -name test-verbose		\
-    | sed 's,.*/,,' | sort)
-test_expect_equal "$tests_in_suite" "$available"
-
 EXPECTED=$TEST_DIRECTORY/test.expected-output
 suppress_diff_date() {
     sed -e 's/\(.*\-\-\- test-verbose\.4\.\expected\).*/\1/' \
@@ -74,12 +56,12 @@ suppress_diff_date() {
 }
 
 test_begin_subtest "Ensure that test output is suppressed unless the test fails"
-output=$(cd $TEST_DIRECTORY; ./test-verbose 2>&1 | suppress_diff_date)
+output=$(cd $TEST_DIRECTORY; NOTMUCH_TEST_QUIET= ./test-verbose 2>&1 | suppress_diff_date)
 expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date)
 test_expect_equal "$output" "$expected"
 
 test_begin_subtest "Ensure that -v does not suppress test output"
-output=$(cd $TEST_DIRECTORY; ./test-verbose -v 2>&1 | suppress_diff_date)
+output=$(cd $TEST_DIRECTORY; NOTMUCH_TEST_QUIET= ./test-verbose -v 2>&1 | suppress_diff_date)
 expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date)
 # Do not include the results of test-verbose in totals
 rm $TEST_DIRECTORY/test-results/test-verbose
@@ -104,8 +86,9 @@ test_expect_success \
     'NOTMUCH_CONFIG is set and points to an existing file' \
     'test -f "${NOTMUCH_CONFIG}"'
 
-test_expect_success \
-    'PATH is set to this repository' \
-    'test "`echo $PATH|cut -f1 -d: | sed -e 's,/test/valgrind/bin$,,'`" = "`dirname ${TEST_DIRECTORY}`"'
+test_begin_subtest 'PATH is set to build directory'
+test_expect_equal \
+    "$(dirname ${TEST_DIRECTORY})" \
+    "$(echo $PATH|cut -f1 -d: | sed -e 's,/test/valgrind/bin$,,')"
 
 test_done
diff --git a/test/T010-help-test.sh b/test/T010-help-test.sh
new file mode 100755
index 00000000..77410bc5
--- /dev/null
+++ b/test/T010-help-test.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+test_description="online help"
+. ./test-lib.sh
+
+test_expect_success 'notmuch --help' 'notmuch --help'
+test_expect_success 'notmuch help' 'notmuch help'
+test_expect_success 'notmuch --version' 'notmuch --version'
+
+if ${TEST_DIRECTORY}/have-man; then
+    test_expect_success 'notmuch --help tag' 'notmuch --help tag'
+    test_expect_success 'notmuch help tag' 'notmuch help tag'
+else
+    test_expect_success 'notmuch --help tag (man pages not available)' \
+	'test_must_fail notmuch --help tag'
+    test_expect_success 'notmuch help tag (man pages not available)' \
+	'test_must_fail notmuch help tag'
+fi
+
+test_done
diff --git a/test/compact b/test/T020-compact.sh
index ac174cec..77bb9632 100755
--- a/test/compact
+++ b/test/T020-compact.sh
@@ -10,6 +10,17 @@ notmuch tag +tag1 \*
 notmuch tag +tag2 subject:Two
 notmuch tag -tag1 +tag3 subject:Three
 
+if ! ${TEST_DIRECTORY}/have-compact; then
+    test_begin_subtest "Compact unsupported: error message"
+    output=$(notmuch compact --quiet 2>&1)
+    test_expect_equal "$output" "notmuch was compiled against a xapian version lacking compaction support.
+Compaction failed: Unsupported operation"
+
+    test_expect_code 1 "Compact unsupported: status code" "notmuch compact"
+
+    test_done
+fi
+
 test_expect_success "Running compact" "notmuch compact --backup=${TEST_DIRECTORY}/xapian.old"
 
 test_begin_subtest "Compact preserves database"
diff --git a/test/config b/test/T030-config.sh
index ca4cf330..ca4cf330 100755
--- a/test/config
+++ b/test/T030-config.sh
diff --git a/test/setup b/test/T040-setup.sh
index 124ef1c8..124ef1c8 100755
--- a/test/setup
+++ b/test/T040-setup.sh
diff --git a/test/new b/test/T050-new.sh
index f27423da..ad46ee6d 100755
--- a/test/new
+++ b/test/T050-new.sh
@@ -248,4 +248,36 @@ ln -s i_do_not_exist "${MAIL_DIR}"/broken_link
 output=$(NOTMUCH_NEW 2>&1)
 test_expect_equal "$output" "No new mail."
 
+test_begin_subtest "Quiet: No new mail."
+output=$(NOTMUCH_NEW --quiet)
+test_expect_equal "$output" ""
+
+test_begin_subtest "Quiet: new, removed and renamed messages."
+# new
+generate_message
+# deleted
+notmuch search --format=text0 --output=files --limit=1 '*' | xargs -0 rm
+# moved
+mkdir "${MAIL_DIR}"/moved_messages
+notmuch search --format=text0 --output=files --offset=1 --limit=1 '*' | xargs -0 -I {} mv {} "${MAIL_DIR}"/moved_messages
+output=$(NOTMUCH_NEW --quiet)
+test_expect_equal "$output" ""
+
+OLDCONFIG=$(notmuch config get new.tags)
+
+test_begin_subtest "Empty tags in new.tags are forbidden"
+notmuch config set new.tags "foo;;bar"
+output=$(NOTMUCH_NEW 2>&1)
+test_expect_equal "$output" "Error: tag '' in new.tags: empty tag forbidden"
+
+test_begin_subtest "Tags starting with '-' in new.tags are forbidden"
+notmuch config set new.tags "-foo;bar"
+output=$(NOTMUCH_NEW 2>&1)
+test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
+
+test_expect_code 1 "Invalid tags set exit code" \
+    "NOTMUCH_NEW 2>&1"
+
+notmuch config set new.tags $OLDCONFIG
+
 test_done
diff --git a/test/count b/test/T060-count.sh
index da86c8cc..da86c8cc 100755
--- a/test/count
+++ b/test/T060-count.sh
diff --git a/test/insert b/test/T070-insert.sh
index 550b4132..ea9db07e 100755
--- a/test/insert
+++ b/test/T070-insert.sh
@@ -113,17 +113,27 @@ output=$(notmuch search --output=files id:$gen_msg_id)
 dirname=$(dirname "$output")
 test_expect_equal "$dirname" "$MAIL_DIR/cur"
 
+test_begin_subtest "Insert message with maildir sync off goes to new/"
+OLDCONFIG=$(notmuch config get maildir.synchronize_flags)
+notmuch config set maildir.synchronize_flags false
+gen_insert_msg
+notmuch insert +flagged < "$gen_msg_filename"
+output=$(notmuch search --output=files id:$gen_msg_id)
+dirname=$(dirname "$output")
+notmuch config set maildir.synchronize_flags $OLDCONFIG
+test_expect_equal "$dirname" "$MAIL_DIR/new"
+
 test_begin_subtest "Insert message into folder"
 gen_insert_msg
 notmuch insert --folder=Drafts < "$gen_msg_filename"
-output=$(notmuch search --output=files folder:Drafts)
+output=$(notmuch search --output=files path:Drafts/new)
 dirname=$(dirname "$output")
 test_expect_equal "$dirname" "$MAIL_DIR/Drafts/new"
 
 test_begin_subtest "Insert message into folder, add/remove tags"
 gen_insert_msg
 notmuch insert --folder=Drafts +draft -unread < "$gen_msg_filename"
-output=$(notmuch search --output=messages folder:Drafts tag:draft NOT tag:unread)
+output=$(notmuch search --output=messages path:Drafts/cur tag:draft NOT tag:unread)
 test_expect_equal "$output" "id:$gen_msg_id"
 
 gen_insert_msg
@@ -133,25 +143,44 @@ test_expect_code 1 "Insert message into non-existent folder" \
 test_begin_subtest "Insert message, create folder"
 gen_insert_msg
 notmuch insert --folder=F --create-folder +folder < "$gen_msg_filename"
-output=$(notmuch search --output=files folder:F tag:folder)
+output=$(notmuch search --output=files path:F/new tag:folder)
 basename=$(basename "$output")
 test_expect_equal_file "$gen_msg_filename" "$MAIL_DIR/F/new/${basename}"
 
 test_begin_subtest "Insert message, create subfolder"
 gen_insert_msg
 notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
-output=$(notmuch search --output=files folder:F/G/H/I/J tag:folder)
+output=$(notmuch search --output=files path:F/G/H/I/J/new tag:folder)
 basename=$(basename "$output")
 test_expect_equal_file "$gen_msg_filename" "${MAIL_DIR}/F/G/H/I/J/new/${basename}"
 
 test_begin_subtest "Insert message, create existing subfolder"
 gen_insert_msg
 notmuch insert --folder=F/G/H/I/J --create-folder +folder < "$gen_msg_filename"
-output=$(notmuch count folder:F/G/H/I/J tag:folder)
+output=$(notmuch count path:F/G/H/I/J/new tag:folder)
 test_expect_equal "$output" "2"
 
 gen_insert_msg
 test_expect_code 1 "Insert message, create invalid subfolder" \
     "notmuch insert --folder=../G --create-folder $gen_msg_filename"
 
+OLDCONFIG=$(notmuch config get new.tags)
+
+test_begin_subtest "Empty tags in new.tags are forbidden"
+notmuch config set new.tags "foo;;bar"
+gen_insert_msg
+output=$(notmuch insert $gen_msg_filename 2>&1)
+test_expect_equal "$output" "Error: tag '' in new.tags: empty tag forbidden"
+
+test_begin_subtest "Tags starting with '-' in new.tags are forbidden"
+notmuch config set new.tags "-foo;bar"
+gen_insert_msg
+output=$(notmuch insert $gen_msg_filename 2>&1)
+test_expect_equal "$output" "Error: tag '-foo' in new.tags: tag starting with '-' forbidden"
+
+test_expect_code 1 "Invalid tags set exit code" \
+    "notmuch insert $gen_msg_filename 2>&1"
+
+notmuch config set new.tags $OLDCONFIG
+
 test_done
diff --git a/test/search b/test/T080-search.sh
index a7a0b18d..05027fb0 100755
--- a/test/search
+++ b/test/T080-search.sh
@@ -59,7 +59,15 @@ test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] searchbyfrom@example.
 
 test_begin_subtest "Search by from: (name)"
 add_message '[subject]="search by from (name)"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000"' '[from]="Search By From Name <test@example.com>"'
-output=$(notmuch search from:"Search By From Name" | notmuch_search_sanitize)
+output=$(notmuch search 'from:"Search By From Name"' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Search By From Name; search by from (name) (inbox unread)"
+
+test_begin_subtest "Search by from: (name and address)"
+output=$(notmuch search 'from:"Search By From Name <test@example.com>"' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Search By From Name; search by from (name) (inbox unread)"
+
+test_begin_subtest "Search by from: without prefix (name and address)"
+output=$(notmuch search '"Search By From Name <test@example.com>"' | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Search By From Name; search by from (name) (inbox unread)"
 
 test_begin_subtest "Search by to: (address)"
@@ -69,7 +77,15 @@ test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; s
 
 test_begin_subtest "Search by to: (name)"
 add_message '[subject]="search by to (name)"' '[date]="Sat, 01 Jan 2000 12:00:00 -0000"' '[to]="Search By To Name <test@example.com>"'
-output=$(notmuch search to:"Search By To Name" | notmuch_search_sanitize)
+output=$(notmuch search 'to:"Search By To Name"' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; search by to (name) (inbox unread)"
+
+test_begin_subtest "Search by to: (name and adress)"
+output=$(notmuch search 'to:"Search By To Name <test@example.com>"' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; search by to (name) (inbox unread)"
+
+test_begin_subtest "Search by to: without prefix (name and adress)"
+output=$(notmuch search '"Search By To Name <test@example.com>"' | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; search by to (name) (inbox unread)"
 
 test_begin_subtest "Search by subject: (phrase)"
@@ -129,4 +145,42 @@ add_message '[subject]="utf8-message-body-subject"' '[date]="Sat, 01 Jan 2000 12
 output=$(notmuch search "bödý" | notmuch_search_sanitize)
 test_expect_equal "$output" "thread:XXX   2000-01-01 [1/1] Notmuch Test Suite; utf8-message-body-subject (inbox unread)"
 
+
+cat <<EOF > ${MAIL_DIR}/termpos
+From: Source <source@example.com>
+To: Dest <dest@example.com>
+Subject: part overlap test
+Date: Sat, 01 January 2000 00:00:00 +0000
+Message-ID: <termpos>
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="==-=="
+
+--==-==
+Content-Type: text/plain
+
+a b c
+
+--==-==
+Content-Type: text/plain
+
+x y z
+
+--==-==--
+EOF
+notmuch new > /dev/null
+
+test_begin_subtest "headers do not have adjacent term positions"
+# Regression test for a bug where term positions for non-prefixed
+# terms weren't updated
+output=$(notmuch search id:termpos and '"com dest"')
+test_expect_equal "$output" ""
+
+test_begin_subtest "parts have non-overlapping term positions"
+output=$(notmuch search id:termpos and '"a y c"')
+test_expect_equal "$output" ""
+
+test_begin_subtest "parts do not have adjacent term positions"
+output=$(notmuch search id:termpos and '"c x"')
+test_expect_equal "$output" ""
+
 test_done
diff --git a/test/search-output b/test/T090-search-output.sh
index 5ccfeaf9..947d572e 100755
--- a/test/search-output
+++ b/test/T090-search-output.sh
@@ -181,67 +181,71 @@ EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files"
-notmuch search --output=files '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files '*' | notmuch_search_files_sanitize | sort >OUTPUT
 cat <<EOF >EXPECTED
-MAIL_DIR/cur/52:2,
-MAIL_DIR/cur/53:2,
-MAIL_DIR/cur/50:2,
-MAIL_DIR/cur/49:2,
-MAIL_DIR/cur/48:2,
-MAIL_DIR/cur/47:2,
-MAIL_DIR/cur/46:2,
-MAIL_DIR/cur/45:2,
-MAIL_DIR/cur/44:2,
-MAIL_DIR/cur/43:2,
-MAIL_DIR/cur/42:2,
-MAIL_DIR/cur/41:2,
-MAIL_DIR/cur/40:2,
-MAIL_DIR/cur/39:2,
-MAIL_DIR/cur/38:2,
-MAIL_DIR/cur/37:2,
-MAIL_DIR/cur/36:2,
-MAIL_DIR/cur/35:2,
-MAIL_DIR/cur/34:2,
-MAIL_DIR/cur/33:2,
-MAIL_DIR/cur/32:2,
-MAIL_DIR/cur/31:2,
-MAIL_DIR/cur/30:2,
+MAIL_DIR/01:2,
+MAIL_DIR/02:2,
+MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/new/22:2,
 MAIL_DIR/cur/29:2,
-MAIL_DIR/cur/28:2,
-MAIL_DIR/cur/27:2,
-MAIL_DIR/cur/26:2,
-MAIL_DIR/cur/25:2,
-MAIL_DIR/cur/24:2,
-MAIL_DIR/cur/23:2,
-MAIL_DIR/cur/22:2,
-MAIL_DIR/cur/21:2,
-MAIL_DIR/cur/19:2,
-MAIL_DIR/cur/18:2,
+MAIL_DIR/cur/30:2,
+MAIL_DIR/cur/31:2,
+MAIL_DIR/cur/32:2,
+MAIL_DIR/cur/33:2,
+MAIL_DIR/cur/34:2,
+MAIL_DIR/cur/35:2,
+MAIL_DIR/cur/36:2,
+MAIL_DIR/cur/37:2,
+MAIL_DIR/cur/38:2,
+MAIL_DIR/cur/39:2,
+MAIL_DIR/cur/40:2,
+MAIL_DIR/cur/41:2,
+MAIL_DIR/cur/42:2,
+MAIL_DIR/cur/43:2,
+MAIL_DIR/cur/44:2,
+MAIL_DIR/cur/45:2,
+MAIL_DIR/cur/46:2,
+MAIL_DIR/cur/47:2,
+MAIL_DIR/cur/48:2,
+MAIL_DIR/cur/49:2,
+MAIL_DIR/cur/50:2,
 MAIL_DIR/cur/51:2,
-MAIL_DIR/cur/20:2,
-MAIL_DIR/cur/17:2,
-MAIL_DIR/cur/16:2,
-MAIL_DIR/cur/15:2,
-MAIL_DIR/cur/14:2,
-MAIL_DIR/cur/13:2,
-MAIL_DIR/cur/12:2,
-MAIL_DIR/cur/11:2,
-MAIL_DIR/cur/10:2,
-MAIL_DIR/cur/09:2,
-MAIL_DIR/cur/08:2,
-MAIL_DIR/cur/06:2,
-MAIL_DIR/cur/05:2,
-MAIL_DIR/cur/04:2,
-MAIL_DIR/cur/03:2,
-MAIL_DIR/cur/07:2,
-MAIL_DIR/cur/02:2,
-MAIL_DIR/cur/01:2,
+MAIL_DIR/cur/52:2,
+MAIL_DIR/cur/53:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/foo/baz/11:2,
+MAIL_DIR/foo/baz/12:2,
+MAIL_DIR/foo/baz/cur/13:2,
+MAIL_DIR/foo/baz/cur/14:2,
+MAIL_DIR/foo/baz/new/15:2,
+MAIL_DIR/foo/baz/new/16:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/new/04:2,
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
+dup1=$(notmuch search --output=files id:20091117232137.GA7669@griffis1.net | head -n 1 | sed -e "s,$MAIL_DIR,MAIL_DIR,")
+dup2=$(notmuch search --output=files id:20091117232137.GA7669@griffis1.net | tail -n 1 | sed -e "s,$MAIL_DIR,MAIL_DIR,")
+
 test_begin_subtest "--output=files --duplicate=1"
-notmuch search --output=files --duplicate=1 '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
-cat <<EOF >EXPECTED
+notmuch search --output=files --duplicate=1 '*' | notmuch_search_files_sanitize | sort >OUTPUT
+cat <<EOF | sort >EXPECTED
+$dup1
 MAIL_DIR/cur/52:2,
 MAIL_DIR/cur/53:2,
 MAIL_DIR/cur/50:2,
@@ -266,40 +270,40 @@ MAIL_DIR/cur/32:2,
 MAIL_DIR/cur/31:2,
 MAIL_DIR/cur/30:2,
 MAIL_DIR/cur/29:2,
-MAIL_DIR/cur/28:2,
-MAIL_DIR/cur/27:2,
-MAIL_DIR/cur/26:2,
-MAIL_DIR/cur/25:2,
-MAIL_DIR/cur/24:2,
-MAIL_DIR/cur/23:2,
-MAIL_DIR/cur/22:2,
-MAIL_DIR/cur/21:2,
-MAIL_DIR/cur/19:2,
-MAIL_DIR/cur/18:2,
-MAIL_DIR/cur/20:2,
-MAIL_DIR/cur/17:2,
-MAIL_DIR/cur/16:2,
-MAIL_DIR/cur/15:2,
-MAIL_DIR/cur/14:2,
-MAIL_DIR/cur/13:2,
-MAIL_DIR/cur/12:2,
-MAIL_DIR/cur/11:2,
-MAIL_DIR/cur/10:2,
-MAIL_DIR/cur/09:2,
-MAIL_DIR/cur/08:2,
-MAIL_DIR/cur/06:2,
-MAIL_DIR/cur/05:2,
-MAIL_DIR/cur/04:2,
-MAIL_DIR/cur/03:2,
-MAIL_DIR/cur/07:2,
-MAIL_DIR/cur/02:2,
-MAIL_DIR/cur/01:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/17:2,
+MAIL_DIR/foo/baz/new/16:2,
+MAIL_DIR/foo/baz/new/15:2,
+MAIL_DIR/foo/baz/cur/14:2,
+MAIL_DIR/foo/baz/cur/13:2,
+MAIL_DIR/foo/baz/12:2,
+MAIL_DIR/foo/baz/11:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/new/04:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/02:2,
+MAIL_DIR/01:2,
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files --format=json"
-notmuch search --format=json --output=files '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
-cat <<EOF >EXPECTED
+notmuch search --format=json --output=files '*' | notmuch_search_files_sanitize \
+    | test_sort_json >OUTPUT
+cat <<EOF | test_sort_json >EXPECTED
 ["MAIL_DIR/cur/52:2,",
 "MAIL_DIR/cur/53:2,",
 "MAIL_DIR/cur/50:2,",
@@ -324,42 +328,42 @@ cat <<EOF >EXPECTED
 "MAIL_DIR/cur/31:2,",
 "MAIL_DIR/cur/30:2,",
 "MAIL_DIR/cur/29:2,",
-"MAIL_DIR/cur/28:2,",
-"MAIL_DIR/cur/27:2,",
-"MAIL_DIR/cur/26:2,",
-"MAIL_DIR/cur/25:2,",
-"MAIL_DIR/cur/24:2,",
-"MAIL_DIR/cur/23:2,",
-"MAIL_DIR/cur/22:2,",
-"MAIL_DIR/cur/21:2,",
-"MAIL_DIR/cur/19:2,",
-"MAIL_DIR/cur/18:2,",
+"MAIL_DIR/bar/baz/new/28:2,",
+"MAIL_DIR/bar/baz/new/27:2,",
+"MAIL_DIR/bar/baz/cur/26:2,",
+"MAIL_DIR/bar/baz/cur/25:2,",
+"MAIL_DIR/bar/baz/24:2,",
+"MAIL_DIR/bar/baz/23:2,",
+"MAIL_DIR/bar/new/22:2,",
+"MAIL_DIR/bar/new/21:2,",
+"MAIL_DIR/bar/cur/19:2,",
+"MAIL_DIR/bar/18:2,",
 "MAIL_DIR/cur/51:2,",
-"MAIL_DIR/cur/20:2,",
-"MAIL_DIR/cur/17:2,",
-"MAIL_DIR/cur/16:2,",
-"MAIL_DIR/cur/15:2,",
-"MAIL_DIR/cur/14:2,",
-"MAIL_DIR/cur/13:2,",
-"MAIL_DIR/cur/12:2,",
-"MAIL_DIR/cur/11:2,",
-"MAIL_DIR/cur/10:2,",
-"MAIL_DIR/cur/09:2,",
-"MAIL_DIR/cur/08:2,",
-"MAIL_DIR/cur/06:2,",
-"MAIL_DIR/cur/05:2,",
-"MAIL_DIR/cur/04:2,",
-"MAIL_DIR/cur/03:2,",
-"MAIL_DIR/cur/07:2,",
-"MAIL_DIR/cur/02:2,",
-"MAIL_DIR/cur/01:2,"]
+"MAIL_DIR/bar/cur/20:2,",
+"MAIL_DIR/bar/17:2,",
+"MAIL_DIR/foo/baz/new/16:2,",
+"MAIL_DIR/foo/baz/new/15:2,",
+"MAIL_DIR/foo/baz/cur/14:2,",
+"MAIL_DIR/foo/baz/cur/13:2,",
+"MAIL_DIR/foo/baz/12:2,",
+"MAIL_DIR/foo/baz/11:2,",
+"MAIL_DIR/foo/new/10:2,",
+"MAIL_DIR/foo/new/09:2,",
+"MAIL_DIR/foo/cur/08:2,",
+"MAIL_DIR/foo/06:2,",
+"MAIL_DIR/bar/baz/05:2,",
+"MAIL_DIR/new/04:2,",
+"MAIL_DIR/foo/new/03:2,",
+"MAIL_DIR/foo/cur/07:2,",
+"MAIL_DIR/02:2,",
+"MAIL_DIR/01:2,"]
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "--output=files --format=json --duplicate=2"
-notmuch search --format=json --output=files --duplicate=2 '*' | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --format=json --output=files --duplicate=2 '*' | notmuch_search_files_sanitize >OUTPUT
 cat <<EOF >EXPECTED
-["MAIL_DIR/cur/51:2,"]
+["$dup2"]
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
@@ -388,7 +392,7 @@ add_message "[subject]='two =?ISO-8859-1?Q?line=0A_subject?=
 	headers'"
 notmuch search id:"$gen_msg_id" | notmuch_search_sanitize >OUTPUT
 cat <<EOF >EXPECTED
-thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; two line? subject headers (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; two line  subject headers (inbox unread)
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
diff --git a/test/T100-search-by-folder.sh b/test/T100-search-by-folder.sh
new file mode 100755
index 00000000..583bdf5e
--- /dev/null
+++ b/test/T100-search-by-folder.sh
@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+test_description='"notmuch search" by folder: and path: (with variations)'
+. ./test-lib.sh
+
+add_message '[dir]=bad' '[subject]="To the bone"'
+add_message '[dir]=.' '[subject]="Top level"'
+add_message '[dir]=bad/news' '[subject]="Bears"'
+mkdir -p "${MAIL_DIR}/duplicate/bad/news"
+cp "$gen_msg_filename" "${MAIL_DIR}/duplicate/bad/news"
+
+add_message '[dir]=things' '[subject]="These are a few"'
+add_message '[dir]=things/favorite' '[subject]="Raindrops, whiskers, kettles"'
+add_message '[dir]=things/bad' '[subject]="Bites, stings, sad feelings"'
+
+test_begin_subtest "Single-world folder: specification (multiple results)"
+output=$(notmuch search folder:bad folder:bad/news folder:things/bad | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)
+thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)"
+
+test_begin_subtest "Top level folder"
+output=$(notmuch search folder:'""' | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Top level (inbox unread)"
+
+test_begin_subtest "Two-word path to narrow results to one"
+output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
+
+test_begin_subtest "Folder search with --output=files"
+output=$(notmuch search --output=files folder:bad/news | notmuch_search_files_sanitize)
+test_expect_equal "$output" "MAIL_DIR/bad/news/msg-003
+MAIL_DIR/duplicate/bad/news/msg-003"
+
+test_begin_subtest "After removing duplicate instance of matching path"
+rm -r "${MAIL_DIR}/bad/news"
+notmuch new
+output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
+test_expect_equal "$output" ""
+
+test_begin_subtest "Folder search with --output=files part #2"
+output=$(notmuch search --output=files folder:duplicate/bad/news | notmuch_search_files_sanitize)
+test_expect_equal "$output" "MAIL_DIR/duplicate/bad/news/msg-003"
+
+test_begin_subtest "After removing duplicate instance of matching path part #2"
+output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
+
+test_begin_subtest "After rename, old path returns nothing"
+mv "${MAIL_DIR}/duplicate/bad/news" "${MAIL_DIR}/duplicate/bad/olds"
+notmuch new
+output=$(notmuch search folder:duplicate/bad/news | notmuch_search_sanitize)
+test_expect_equal "$output" ""
+
+test_begin_subtest "After rename, new path returns result"
+output=$(notmuch search folder:duplicate/bad/olds | notmuch_search_sanitize)
+test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
+
+# folder: and path: searches with full corpus
+rm -rf $MAIL_DIR
+add_email_corpus
+
+# add some more dupes
+cp $MAIL_DIR/foo/new/03:2, $MAIL_DIR/new
+cp $MAIL_DIR/bar/baz/05:2, $MAIL_DIR/foo
+notmuch new >/dev/null
+
+test_begin_subtest "folder: search"
+output=$(notmuch search --output=files folder:foo | notmuch_search_files_sanitize | sort)
+# bar/baz/05:2, is a duplicate of foo/05:2,
+# new/03:2, is a duplicate of foo/new/03:2,
+test_expect_equal "$output" "MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/foo/05:2,
+MAIL_DIR/foo/06:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/new/10:2,
+MAIL_DIR/new/03:2,"
+
+test_begin_subtest "top level folder: search"
+output=$(notmuch search --output=files folder:'""' | notmuch_search_files_sanitize | sort)
+# bar/18:2, is a duplicate of cur/51:2,
+# foo/new/03:2, is a duplicate of new/03:2,
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/29:2,
+MAIL_DIR/cur/30:2,
+MAIL_DIR/cur/31:2,
+MAIL_DIR/cur/32:2,
+MAIL_DIR/cur/33:2,
+MAIL_DIR/cur/34:2,
+MAIL_DIR/cur/35:2,
+MAIL_DIR/cur/36:2,
+MAIL_DIR/cur/37:2,
+MAIL_DIR/cur/38:2,
+MAIL_DIR/cur/39:2,
+MAIL_DIR/cur/40:2,
+MAIL_DIR/cur/41:2,
+MAIL_DIR/cur/42:2,
+MAIL_DIR/cur/43:2,
+MAIL_DIR/cur/44:2,
+MAIL_DIR/cur/45:2,
+MAIL_DIR/cur/46:2,
+MAIL_DIR/cur/47:2,
+MAIL_DIR/cur/48:2,
+MAIL_DIR/cur/49:2,
+MAIL_DIR/cur/50:2,
+MAIL_DIR/cur/51:2,
+MAIL_DIR/cur/52:2,
+MAIL_DIR/cur/53:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/new/03:2,
+MAIL_DIR/new/04:2,"
+
+test_begin_subtest "path: search"
+output=$(notmuch search --output=files path:"bar" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/51:2,"
+
+test_begin_subtest "top level path: search"
+output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,"
+
+test_begin_subtest "recursive path: search"
+output=$(notmuch search --output=files path:"bar/**" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+# foo/05:2, is a duplicate of bar/baz/05:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/cur/51:2,
+MAIL_DIR/foo/05:2,"
+
+test_done
diff --git a/test/search-position-overlap-bug b/test/T110-search-position-overlap-bug.sh
index 5da6ad6f..5da6ad6f 100755
--- a/test/search-position-overlap-bug
+++ b/test/T110-search-position-overlap-bug.sh
diff --git a/test/search-insufficient-from-quoting b/test/T120-search-insufficient-from-quoting.sh
index e83ea3d1..e83ea3d1 100755
--- a/test/search-insufficient-from-quoting
+++ b/test/T120-search-insufficient-from-quoting.sh
diff --git a/test/search-limiting b/test/T130-search-limiting.sh
index 303762cf..303762cf 100755
--- a/test/search-limiting
+++ b/test/T130-search-limiting.sh
diff --git a/test/excludes b/test/T140-excludes.sh
index 8bbbc2dd..8bbbc2dd 100755
--- a/test/excludes
+++ b/test/T140-excludes.sh
diff --git a/test/tagging b/test/T150-tagging.sh
index dc118f33..dc118f33 100755
--- a/test/tagging
+++ b/test/T150-tagging.sh
diff --git a/test/json b/test/T160-json.sh
index c1cf649d..c1cf649d 100755
--- a/test/json
+++ b/test/T160-json.sh
diff --git a/test/sexp b/test/T170-sexp.sh
index 667e3195..667e3195 100755
--- a/test/sexp
+++ b/test/T170-sexp.sh
diff --git a/test/text b/test/T180-text.sh
index b5ccefc9..b5ccefc9 100755
--- a/test/text
+++ b/test/T180-text.sh
diff --git a/test/multipart b/test/T190-multipart.sh
index 85cbf672..85cbf672 100755
--- a/test/multipart
+++ b/test/T190-multipart.sh
diff --git a/test/thread-naming b/test/T200-thread-naming.sh
index 1a1a48f6..1a1a48f6 100755
--- a/test/thread-naming
+++ b/test/T200-thread-naming.sh
diff --git a/test/raw b/test/T210-raw.sh
index daf5735c..daf5735c 100755
--- a/test/raw
+++ b/test/T210-raw.sh
diff --git a/test/reply b/test/T220-reply.sh
index b0d854a1..b0d854a1 100755
--- a/test/reply
+++ b/test/T220-reply.sh
diff --git a/test/reply-to-sender b/test/T230-reply-to-sender.sh
index 30e5e385..30e5e385 100755
--- a/test/reply-to-sender
+++ b/test/T230-reply-to-sender.sh
diff --git a/test/dump-restore b/test/T240-dump-restore.sh
index 0004438d..efe463ea 100755
--- a/test/dump-restore
+++ b/test/T240-dump-restore.sh
@@ -68,6 +68,32 @@ test_begin_subtest "dump --output=outfile --"
 notmuch dump --output=dump-1-arg-dash.actual --
 test_expect_equal_file dump.expected dump-1-arg-dash.actual
 
+# gzipped output
+
+test_begin_subtest "dump --gzip"
+notmuch dump --gzip > dump-gzip.gz
+gunzip dump-gzip.gz
+test_expect_equal_file dump.expected dump-gzip
+
+test_begin_subtest "dump --gzip --output=outfile"
+notmuch dump --gzip --output=dump-gzip-outfile.gz
+gunzip dump-gzip-outfile.gz
+test_expect_equal_file dump.expected dump-gzip-outfile
+
+test_begin_subtest "restoring gzipped stdin"
+notmuch dump --gzip --output=backup.gz
+notmuch tag +new_tag '*'
+notmuch restore < backup.gz
+notmuch dump --output=dump.actual
+test_expect_equal_file dump.expected dump.actual
+
+test_begin_subtest "restoring gzipped file"
+notmuch dump --gzip --output=backup.gz
+notmuch tag +new_tag '*'
+notmuch restore --input=backup.gz
+notmuch dump --output=dump.actual
+test_expect_equal_file dump.expected dump.actual
+
 # Note, we assume all messages from cworth have a message-id
 # containing cworth.org
 
@@ -98,6 +124,15 @@ notmuch dump --format=batch-tag from:cworth | sed 's/^.*-- id://' | \
     sort > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
+test_begin_subtest "format=batch-tag, missing newline"
+printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN
+notmuch restore --accumulate < IN
+notmuch dump id:20091117232137.GA7669@griffis1.net > OUT
+cat <<EOF > EXPECTED
++a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net
+EOF
+test_expect_equal_file EXPECTED OUT
+
 test_begin_subtest "format=batch-tag, # round-trip"
 notmuch dump --format=sup | sort > EXPECTED.$test_count
 notmuch dump --format=batch-tag | notmuch restore --format=batch-tag
diff --git a/test/uuencode b/test/T250-uuencode.sh
index b3e1ac19..b3e1ac19 100755
--- a/test/uuencode
+++ b/test/T250-uuencode.sh
diff --git a/test/thread-order b/test/T260-thread-order.sh
index 6c3a4b3f..6c3a4b3f 100755
--- a/test/thread-order
+++ b/test/T260-thread-order.sh
diff --git a/test/author-order b/test/T270-author-order.sh
index 6ffeffc7..6ffeffc7 100755
--- a/test/author-order
+++ b/test/T270-author-order.sh
diff --git a/test/from-guessing b/test/T280-from-guessing.sh
index 6dfaa40a..6dfaa40a 100755
--- a/test/from-guessing
+++ b/test/T280-from-guessing.sh
diff --git a/test/long-id b/test/T290-long-id.sh
index 85e620fa..85e620fa 100755
--- a/test/long-id
+++ b/test/T290-long-id.sh
diff --git a/test/encoding b/test/T300-encoding.sh
index b6c86bf0..b6c86bf0 100755
--- a/test/encoding
+++ b/test/T300-encoding.sh
diff --git a/test/emacs b/test/T310-emacs.sh
index 863219d9..ac966e52 100755
--- a/test/emacs
+++ b/test/T310-emacs.sh
@@ -7,6 +7,10 @@ EXPECTED=$TEST_DIRECTORY/emacs.expected-output
 
 add_email_corpus
 
+# syntax errors in test-lib.el cause mysterious failures
+test_expect_success 'Syntax of emacs test library' \
+    "${TEST_EMACS} -Q --batch --load $TEST_DIRECTORY/test-lib.el"
+
 test_begin_subtest "Basic notmuch-hello view in emacs"
 test_emacs '(notmuch-hello)
 	    (test-output)'
@@ -723,7 +727,7 @@ inbox,stashtest
 ${gen_msg_filename}
 http://mid.gmane.org/bought
 http://marc.info/?i=bought
-http://mail-archive.com/search?l=mid&q=bought
+http://mid.mail-archive.com/bought
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
@@ -949,4 +953,15 @@ test_emacs '(notmuch-search "subject:\"search race test\" -subject:two")
 output=$(notmuch search --output=messages 'tag:search-global-race-tag')
 test_expect_equal "$output" "id:$gen_msg_id_1"
 
+test_begin_subtest "Term escaping"
+output=$(test_emacs "(mapcar 'notmuch-escape-boolean-term (list
+	\"\"
+	\"abc\`~\!@#\$%^&*-=_+123\"
+	\"(abc\"
+	\")abc\"
+	\"\\\"abc\"
+	\"\x01xyz\"
+	\"\\x201cxyz\\x201d\"))")
+test_expect_equal "$output" '("\"\"" "abc`~!@#$%^&*-=_+123" "\"(abc\"" "\")abc\"" "\"\"\"abc\"" "\"'$'\x01''xyz\"" "\"“xyz”\"")'
+
 test_done
diff --git a/test/emacs-large-search-buffer b/test/T320-emacs-large-search-buffer.sh
index 8b1251fe..8b1251fe 100755
--- a/test/emacs-large-search-buffer
+++ b/test/T320-emacs-large-search-buffer.sh
diff --git a/test/emacs-subject-to-filename b/test/T330-emacs-subject-to-filename.sh
index 230c324d..230c324d 100755
--- a/test/emacs-subject-to-filename
+++ b/test/T330-emacs-subject-to-filename.sh
diff --git a/test/maildir-sync b/test/T340-maildir-sync.sh
index 3186e70f..3186e70f 100755
--- a/test/maildir-sync
+++ b/test/T340-maildir-sync.sh
diff --git a/test/crypto b/test/T350-crypto.sh
index 477b397e..477b397e 100755
--- a/test/crypto
+++ b/test/T350-crypto.sh
diff --git a/test/symbol-hiding b/test/T360-symbol-hiding.sh
index 636ec917..636ec917 100755
--- a/test/symbol-hiding
+++ b/test/T360-symbol-hiding.sh
diff --git a/test/search-folder-coherence b/test/T370-search-folder-coherence.sh
index 3f6ec763..5e72a6cc 100755
--- a/test/search-folder-coherence
+++ b/test/T370-search-folder-coherence.sh
@@ -27,7 +27,7 @@ cat <<EOF >EXPECTED
 MAIL_DIR/msg-001
 MAIL_DIR/spam/msg-001
 EOF
-notmuch search --output=files id:$id_x | sed -e "s,$MAIL_DIR,MAIL_DIR," >OUTPUT
+notmuch search --output=files id:$id_x | notmuch_search_files_sanitize >OUTPUT
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "Test matches folder:spam"
diff --git a/test/atomicity b/test/T380-atomicity.sh
index 1c786fa2..1c786fa2 100755
--- a/test/atomicity
+++ b/test/T380-atomicity.sh
diff --git a/test/python b/test/T390-python.sh
index 3f03a2e3..3f03a2e3 100755
--- a/test/python
+++ b/test/T390-python.sh
diff --git a/test/hooks b/test/T400-hooks.sh
index 77e8569b..77e8569b 100755
--- a/test/hooks
+++ b/test/T400-hooks.sh
diff --git a/test/argument-parsing b/test/T410-argument-parsing.sh
index 94e90874..94e90874 100755
--- a/test/argument-parsing
+++ b/test/T410-argument-parsing.sh
diff --git a/test/emacs-test-functions b/test/T420-emacs-test-functions.sh
index ca4a7988..ca4a7988 100755
--- a/test/emacs-test-functions
+++ b/test/T420-emacs-test-functions.sh
diff --git a/test/emacs-address-cleaning b/test/T430-emacs-address-cleaning.sh
index 04723467..04723467 100755
--- a/test/emacs-address-cleaning
+++ b/test/T430-emacs-address-cleaning.sh
diff --git a/test/emacs-hello b/test/T440-emacs-hello.sh
index f7296166..f7296166 100755
--- a/test/emacs-hello
+++ b/test/T440-emacs-hello.sh
diff --git a/test/emacs-show b/test/T450-emacs-show.sh
index 2a3a5356..2a3a5356 100755
--- a/test/emacs-show
+++ b/test/T450-emacs-show.sh
diff --git a/test/emacs-tree b/test/T460-emacs-tree.sh
index 4bdfddd2..8e9f37cb 100755
--- a/test/emacs-tree
+++ b/test/T460-emacs-tree.sh
@@ -8,16 +8,14 @@ EXPECTED=$TEST_DIRECTORY/tree.expected-output
 add_email_corpus
 
 test_begin_subtest "Basic notmuch-tree view in emacs"
-test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-	    (notmuch-tree "tag:inbox")
+test_emacs '(notmuch-tree "tag:inbox")
 	    (notmuch-test-wait)
 	    (test-output)
 	    (delete-other-windows)'
 test_expect_equal_file OUTPUT $EXPECTED/notmuch-tree-tag-inbox
 
 test_begin_subtest "Refreshed notmuch-tree view in emacs"
-test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-	    (notmuch-tree "tag:inbox")
+test_emacs '(notmuch-tree "tag:inbox")
 	    (notmuch-test-wait)
 	    (notmuch-tree-refresh-view)
 	    (notmuch-test-wait)
@@ -30,8 +28,7 @@ test_expect_equal_file OUTPUT $EXPECTED/notmuch-tree-tag-inbox
 # correctly.
 
 test_begin_subtest "Tag message in notmuch tree view (display)"
-test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-	    (notmuch-tree "tag:inbox")
+test_emacs '(notmuch-tree "tag:inbox")
 	    (notmuch-test-wait)
 	    (forward-line)
 	    (notmuch-tree-tag (list "+test_tag"))
@@ -44,8 +41,7 @@ output=$(notmuch search --output=messages 'tag:test_tag')
 test_expect_equal "$output" "id:877h1wv7mg.fsf@inf-8657.int-evry.fr"
 
 test_begin_subtest "Untag message in notmuch tree view"
-test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-	    (notmuch-tree "tag:inbox")
+test_emacs '(notmuch-tree "tag:inbox")
 	    (notmuch-test-wait)
 	    (forward-line)
 	    (notmuch-tree-tag (list "-test_tag"))
@@ -58,8 +54,7 @@ output=$(notmuch search --output=messages 'tag:test_tag')
 test_expect_equal "$output" ""
 
 test_begin_subtest "Tag thread in notmuch tree view"
-test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-	    (notmuch-tree "tag:inbox")
+test_emacs '(notmuch-tree "tag:inbox")
 	    (notmuch-test-wait)
 	    ;; move to a sizable thread
 	    (forward-line 26)
@@ -80,8 +75,7 @@ id:87iqd9rn3l.fsf@vertex.dottedmag
 id:20091117190054.GU3165@dottiness.seas.harvard.edu"
 
 test_begin_subtest "Untag thread in notmuch tree view"
-test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-	    (notmuch-tree "tag:inbox")
+test_emacs '(notmuch-tree "tag:inbox")
 	    (notmuch-test-wait)
 	    ;; move to the same sizable thread as above
 	    (forward-line 26)
@@ -147,26 +141,20 @@ cp OUTPUT /tmp/mjwout
 test_expect_equal_file OUTPUT $EXPECTED/notmuch-tree-show-window
 
 test_begin_subtest "Stash id"
-output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-		     (require (quote notmuch-tree))
-		     (notmuch-tree "id:1258498485-sup-142@elly")
+output=$(test_emacs '(notmuch-tree "id:1258498485-sup-142@elly")
 		     (notmuch-test-wait)
 		     (notmuch-show-stash-message-id)')
 test_expect_equal "$output" "\"Stashed: id:1258498485-sup-142@elly\""
 
 test_begin_subtest "Move to next matching message"
-output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-		     (require (quote notmuch-tree))
-		     (notmuch-tree "from:cworth")
+output=$(test_emacs '(notmuch-tree "from:cworth")
 		     (notmuch-test-wait)
 		     (notmuch-tree-next-matching-message)
 		     (notmuch-show-stash-message-id)')
 test_expect_equal "$output" "\"Stashed: id:878we4qdqf.fsf@yoom.home.cworth.org\""
 
 test_begin_subtest "Move to next thread"
-output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-		     (require (quote notmuch-tree))
-		     (notmuch-tree "tag:inbox")
+output=$(test_emacs '(notmuch-tree "tag:inbox")
 		     (notmuch-test-wait)
 		     (forward-line 26)
 		     (notmuch-tree-next-thread)
@@ -174,9 +162,7 @@ output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
 test_expect_equal "$output" "\"Stashed: id:1258471718-6781-1-git-send-email-dottedmag@dottedmag.net\""
 
 test_begin_subtest "Move to previous thread"
-output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-		     (require (quote notmuch-tree))
-		     (notmuch-tree "tag:inbox")
+output=$(test_emacs '(notmuch-tree "tag:inbox")
 		     (notmuch-test-wait)
 		     (forward-line 26)
 		     (notmuch-tree-prev-thread)
@@ -184,9 +170,7 @@ output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
 test_expect_equal "$output" "\"Stashed: id:20091117190054.GU3165@dottiness.seas.harvard.edu\""
 
 test_begin_subtest "Move to previous previous thread"
-output=$(test_emacs '(add-to-list (quote load-path) "'$PICK_DIR'")
-		     (require (quote notmuch-tree))
-		     (notmuch-tree "tag:inbox")
+output=$(test_emacs '(notmuch-tree "tag:inbox")
 		     (notmuch-test-wait)
 		     (forward-line 26)
 		     (notmuch-tree-prev-thread)
diff --git a/test/missing-headers b/test/T470-missing-headers.sh
index cb38301c..cb38301c 100755
--- a/test/missing-headers
+++ b/test/T470-missing-headers.sh
diff --git a/test/hex-escaping b/test/T480-hex-escaping.sh
index ad50e1bc..ad50e1bc 100755
--- a/test/hex-escaping
+++ b/test/T480-hex-escaping.sh
diff --git a/test/parse-time-string b/test/T490-parse-time-string.sh
index 8ae0b4c2..8ae0b4c2 100755
--- a/test/parse-time-string
+++ b/test/T490-parse-time-string.sh
diff --git a/test/search-date b/test/T500-search-date.sh
index 70bcf344..70bcf344 100755
--- a/test/search-date
+++ b/test/T500-search-date.sh
diff --git a/test/thread-replies b/test/T510-thread-replies.sh
index eeb70d06..eeb70d06 100755
--- a/test/thread-replies
+++ b/test/T510-thread-replies.sh
diff --git a/test/T520-show.sh b/test/T520-show.sh
new file mode 100755
index 00000000..0657c993
--- /dev/null
+++ b/test/T520-show.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+test_description='"notmuch show"'
+
+. ./test-lib.sh
+
+add_email_corpus
+
+test_begin_subtest "exit code for show invalid query"
+notmuch show foo..
+exit_code=$?
+test_expect_equal 1 $exit_code
+
+test_done
diff --git a/test/T530-upgrade.sh b/test/T530-upgrade.sh
new file mode 100755
index 00000000..7d5d5aa8
--- /dev/null
+++ b/test/T530-upgrade.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+test_description="database upgrade"
+
+. ./test-lib.sh
+
+dbtarball=database-v1.tar.xz
+
+# XXX: Accomplish the same with test lib helpers
+if [ ! -e ${TEST_DIRECTORY}/test-databases/${dbtarball} ]; then
+    test_subtest_missing_external_prereq_["${dbtarball} - fetch with 'make download-test-databases'"]=t
+fi
+
+test_expect_success \
+    'database checksum' \
+    '( cd $TEST_DIRECTORY/test-databases &&
+       sha256sum --quiet --check --status ${dbtarball}.sha256 )'
+
+tar xf $TEST_DIRECTORY/test-databases/${dbtarball} -C ${MAIL_DIR} --strip-components=1
+
+test_begin_subtest "folder: search does not work with old database version"
+output=$(notmuch search folder:foo)
+test_expect_equal "$output" ""
+
+test_begin_subtest "path: search does not work with old database version"
+output=$(notmuch search path:foo)
+test_expect_equal "$output" ""
+
+test_expect_success 'pre upgrade dump' 'notmuch dump | sort > pre-upgrade-dump'
+
+test_begin_subtest "database upgrade from format version 1"
+output=$(notmuch new | sed -e 's/^Backing up tags to .*$/Backing up tags to FILENAME/')
+test_expect_equal "$output" "\
+Welcome to a new version of notmuch! Your database will now be upgraded.
+This process is safe to interrupt.
+Backing up tags to FILENAME
+Your notmuch database has now been upgraded to database format version 2.
+No new mail."
+
+test_begin_subtest "tag backup matches pre-upgrade dump"
+gunzip -c ${MAIL_DIR}/.notmuch/dump-*.gz | sort > backup-dump
+test_expect_equal_file pre-upgrade-dump backup-dump
+
+test_begin_subtest "folder: no longer matches in the middle of path"
+output=$(notmuch search folder:baz)
+test_expect_equal "$output" ""
+
+test_begin_subtest "folder: search"
+output=$(notmuch search --output=files folder:foo | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/foo/06:2,
+MAIL_DIR/foo/cur/07:2,
+MAIL_DIR/foo/cur/08:2,
+MAIL_DIR/foo/new/03:2,
+MAIL_DIR/foo/new/09:2,
+MAIL_DIR/foo/new/10:2,"
+
+test_begin_subtest "top level folder: search"
+output=$(notmuch search --output=files folder:'""' | notmuch_search_files_sanitize | sort)
+# bar/18:2, is a duplicate of cur/51:2,
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/29:2,
+MAIL_DIR/cur/30:2,
+MAIL_DIR/cur/31:2,
+MAIL_DIR/cur/32:2,
+MAIL_DIR/cur/33:2,
+MAIL_DIR/cur/34:2,
+MAIL_DIR/cur/35:2,
+MAIL_DIR/cur/36:2,
+MAIL_DIR/cur/37:2,
+MAIL_DIR/cur/38:2,
+MAIL_DIR/cur/39:2,
+MAIL_DIR/cur/40:2,
+MAIL_DIR/cur/41:2,
+MAIL_DIR/cur/42:2,
+MAIL_DIR/cur/43:2,
+MAIL_DIR/cur/44:2,
+MAIL_DIR/cur/45:2,
+MAIL_DIR/cur/46:2,
+MAIL_DIR/cur/47:2,
+MAIL_DIR/cur/48:2,
+MAIL_DIR/cur/49:2,
+MAIL_DIR/cur/50:2,
+MAIL_DIR/cur/51:2,
+MAIL_DIR/cur/52:2,
+MAIL_DIR/cur/53:2,
+MAIL_DIR/new/04:2,"
+
+test_begin_subtest "path: search"
+output=$(notmuch search --output=files path:"bar" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/cur/51:2,"
+
+test_begin_subtest "top level path: search"
+output=$(notmuch search --output=files path:'""' | notmuch_search_files_sanitize | sort)
+test_expect_equal "$output" "MAIL_DIR/01:2,
+MAIL_DIR/02:2,"
+
+test_begin_subtest "recursive path: search"
+output=$(notmuch search --output=files path:"bar/**" | notmuch_search_files_sanitize | sort)
+# cur/51:2, is a duplicate of bar/18:2,
+test_expect_equal "$output" "MAIL_DIR/bar/17:2,
+MAIL_DIR/bar/18:2,
+MAIL_DIR/bar/baz/05:2,
+MAIL_DIR/bar/baz/23:2,
+MAIL_DIR/bar/baz/24:2,
+MAIL_DIR/bar/baz/cur/25:2,
+MAIL_DIR/bar/baz/cur/26:2,
+MAIL_DIR/bar/baz/new/27:2,
+MAIL_DIR/bar/baz/new/28:2,
+MAIL_DIR/bar/cur/19:2,
+MAIL_DIR/bar/cur/20:2,
+MAIL_DIR/bar/new/21:2,
+MAIL_DIR/bar/new/22:2,
+MAIL_DIR/cur/51:2,"
+
+test_done
diff --git a/test/atomicity.gdb b/test/atomicity.gdb
index fd675257..15adb16c 100644
--- a/test/atomicity.gdb
+++ b/test/atomicity.gdb
@@ -18,6 +18,10 @@ shell echo 0 > outcount
 
 shell touch inodes
 
+# work around apparent issue with lazy library loading on some
+# platforms
+set breakpoint pending on
+
 break rename
 commands
 # As an optimization, only consider snapshots after a Xapian commit.
diff --git a/test/corpus/cur/01:2, b/test/corpus/01:2,
index 7e9e3490..7e9e3490 100644
--- a/test/corpus/cur/01:2,
+++ b/test/corpus/01:2,
diff --git a/test/corpus/cur/02:2, b/test/corpus/02:2,
index dadcdaac..dadcdaac 100644
--- a/test/corpus/cur/02:2,
+++ b/test/corpus/02:2,
diff --git a/test/corpus/cur/17:2, b/test/corpus/bar/17:2,
index d3b75685..d3b75685 100644
--- a/test/corpus/cur/17:2,
+++ b/test/corpus/bar/17:2,
diff --git a/test/corpus/cur/18:2, b/test/corpus/bar/18:2,
index f522f69e..f522f69e 100644
--- a/test/corpus/cur/18:2,
+++ b/test/corpus/bar/18:2,
diff --git a/test/corpus/cur/05:2, b/test/corpus/bar/baz/05:2,
index 75b05fa4..75b05fa4 100644
--- a/test/corpus/cur/05:2,
+++ b/test/corpus/bar/baz/05:2,
diff --git a/test/corpus/cur/23:2, b/test/corpus/bar/baz/23:2,
index 9bb62d73..9bb62d73 100644
--- a/test/corpus/cur/23:2,
+++ b/test/corpus/bar/baz/23:2,
diff --git a/test/corpus/cur/24:2, b/test/corpus/bar/baz/24:2,
index c8000207..c8000207 100644
--- a/test/corpus/cur/24:2,
+++ b/test/corpus/bar/baz/24:2,
diff --git a/test/corpus/cur/25:2, b/test/corpus/bar/baz/cur/25:2,
index 7378f820..7378f820 100644
--- a/test/corpus/cur/25:2,
+++ b/test/corpus/bar/baz/cur/25:2,
diff --git a/test/corpus/cur/26:2, b/test/corpus/bar/baz/cur/26:2,
index f3c5f53d..f3c5f53d 100644
--- a/test/corpus/cur/26:2,
+++ b/test/corpus/bar/baz/cur/26:2,
diff --git a/test/corpus/cur/27:2, b/test/corpus/bar/baz/new/27:2,
index 7f0f045b..7f0f045b 100644
--- a/test/corpus/cur/27:2,
+++ b/test/corpus/bar/baz/new/27:2,
diff --git a/test/corpus/cur/28:2, b/test/corpus/bar/baz/new/28:2,
index 83ce01bd..83ce01bd 100644
--- a/test/corpus/cur/28:2,
+++ b/test/corpus/bar/baz/new/28:2,
diff --git a/test/corpus/cur/19:2, b/test/corpus/bar/cur/19:2,
index 1b7872b0..1b7872b0 100644
--- a/test/corpus/cur/19:2,
+++ b/test/corpus/bar/cur/19:2,
diff --git a/test/corpus/cur/20:2, b/test/corpus/bar/cur/20:2,
index f08a3145..f08a3145 100644
--- a/test/corpus/cur/20:2,
+++ b/test/corpus/bar/cur/20:2,
diff --git a/test/corpus/cur/21:2, b/test/corpus/bar/new/21:2,
index 7ff55ccf..7ff55ccf 100644
--- a/test/corpus/cur/21:2,
+++ b/test/corpus/bar/new/21:2,
diff --git a/test/corpus/cur/22:2, b/test/corpus/bar/new/22:2,
index 08adada6..08adada6 100644
--- a/test/corpus/cur/22:2,
+++ b/test/corpus/bar/new/22:2,
diff --git a/test/corpus/cur/06:2, b/test/corpus/foo/06:2,
index 3baad491..3baad491 100644
--- a/test/corpus/cur/06:2,
+++ b/test/corpus/foo/06:2,
diff --git a/test/corpus/cur/11:2, b/test/corpus/foo/baz/11:2,
index c0701def..c0701def 100644
--- a/test/corpus/cur/11:2,
+++ b/test/corpus/foo/baz/11:2,
diff --git a/test/corpus/cur/12:2, b/test/corpus/foo/baz/12:2,
index fbc604c0..fbc604c0 100644
--- a/test/corpus/cur/12:2,
+++ b/test/corpus/foo/baz/12:2,
diff --git a/test/corpus/cur/13:2, b/test/corpus/foo/baz/cur/13:2,
index 03cb3744..03cb3744 100644
--- a/test/corpus/cur/13:2,
+++ b/test/corpus/foo/baz/cur/13:2,
diff --git a/test/corpus/cur/14:2, b/test/corpus/foo/baz/cur/14:2,
index d3fe78db..d3fe78db 100644
--- a/test/corpus/cur/14:2,
+++ b/test/corpus/foo/baz/cur/14:2,
diff --git a/test/corpus/cur/15:2, b/test/corpus/foo/baz/new/15:2,
index 6824d5ea..6824d5ea 100644
--- a/test/corpus/cur/15:2,
+++ b/test/corpus/foo/baz/new/15:2,
diff --git a/test/corpus/cur/16:2, b/test/corpus/foo/baz/new/16:2,
index f531eb93..f531eb93 100644
--- a/test/corpus/cur/16:2,
+++ b/test/corpus/foo/baz/new/16:2,
diff --git a/test/corpus/cur/07:2, b/test/corpus/foo/cur/07:2,
index 7b1e2bbe..7b1e2bbe 100644
--- a/test/corpus/cur/07:2,
+++ b/test/corpus/foo/cur/07:2,
diff --git a/test/corpus/cur/08:2, b/test/corpus/foo/cur/08:2,
index baf34d1f..baf34d1f 100644
--- a/test/corpus/cur/08:2,
+++ b/test/corpus/foo/cur/08:2,
diff --git a/test/corpus/cur/03:2, b/test/corpus/foo/new/03:2,
index c154ac5e..c154ac5e 100644
--- a/test/corpus/cur/03:2,
+++ b/test/corpus/foo/new/03:2,
diff --git a/test/corpus/cur/09:2, b/test/corpus/foo/new/09:2,
index 26b51b1f..26b51b1f 100644
--- a/test/corpus/cur/09:2,
+++ b/test/corpus/foo/new/09:2,
diff --git a/test/corpus/cur/10:2, b/test/corpus/foo/new/10:2,
index 4211d734..4211d734 100644
--- a/test/corpus/cur/10:2,
+++ b/test/corpus/foo/new/10:2,
diff --git a/test/corpus/cur/04:2, b/test/corpus/new/04:2,
index 0ce678bf..0ce678bf 100644
--- a/test/corpus/cur/04:2,
+++ b/test/corpus/new/04:2,
diff --git a/test/help-test b/test/help-test
deleted file mode 100755
index f7df725e..00000000
--- a/test/help-test
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-test_description="online help"
-. ./test-lib.sh
-
-test_expect_success 'notmuch --help' 'notmuch --help'
-test_expect_success 'notmuch --help tag' 'notmuch --help tag'
-test_expect_success 'notmuch help' 'notmuch help'
-test_expect_success 'notmuch help tag' 'notmuch help tag'
-test_expect_success 'notmuch --version' 'notmuch --version'
-
-test_done
diff --git a/test/notmuch-test b/test/notmuch-test
index 18593f61..b8437127 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -16,61 +16,7 @@ fi
 
 cd $(dirname "$0")
 
-TESTS="
-  basic
-  help-test
-  compact
-  config
-  setup
-  new
-  count
-  insert
-  search
-  search-output
-  search-by-folder
-  search-position-overlap-bug
-  search-insufficient-from-quoting
-  search-limiting
-  excludes
-  tagging
-  json
-  sexp
-  text
-  multipart
-  thread-naming
-  raw
-  reply
-  reply-to-sender
-  dump-restore
-  uuencode
-  thread-order
-  author-order
-  from-guessing
-  long-id
-  encoding
-  emacs
-  emacs-large-search-buffer
-  emacs-subject-to-filename
-  maildir-sync
-  crypto
-  symbol-hiding
-  search-folder-coherence
-  atomicity
-  python
-  hooks
-  argument-parsing
-  emacs-test-functions
-  emacs-address-cleaning
-  emacs-hello
-  emacs-show
-  emacs-tree
-  missing-headers
-  hex-escaping
-  parse-time-string
-  search-date
-  thread-replies
-"
-TESTS=${NOTMUCH_TESTS:=$TESTS}
+TESTS=${NOTMUCH_TESTS:-`echo T[0-9][0-9][0-9]-*.sh`}
 
 # Clean up any results from a previous run
 rm -r test-results >/dev/null 2>/dev/null
@@ -98,6 +44,7 @@ done
 trap - HUP INT TERM
 
 # Report results
+echo
 ./aggregate-results.sh test-results/*
 ev=$?
 
diff --git a/test/search-by-folder b/test/search-by-folder
deleted file mode 100755
index 5cc2ca8d..00000000
--- a/test/search-by-folder
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-test_description='"notmuch search" by folder: (with variations)'
-. ./test-lib.sh
-
-add_message '[dir]=bad' '[subject]="To the bone"'
-add_message '[dir]=bad/news' '[subject]="Bears"'
-mkdir -p "${MAIL_DIR}/duplicate/bad/news"
-cp "$gen_msg_filename" "${MAIL_DIR}/duplicate/bad/news"
-
-add_message '[dir]=things' '[subject]="These are a few"'
-add_message '[dir]=things/favorite' '[subject]="Raindrops, whiskers, kettles"'
-add_message '[dir]=things/bad' '[subject]="Bites, stings, sad feelings"'
-
-test_begin_subtest "Single-world folder: specification (multiple results)"
-output=$(notmuch search folder:bad | notmuch_search_sanitize)
-test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; To the bone (inbox unread)
-thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)
-thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bites, stings, sad feelings (inbox unread)"
-
-test_begin_subtest "Two-word path to narrow results to one"
-output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
-test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
-
-test_begin_subtest "After removing duplicate instance of matching path"
-rm -r "${MAIL_DIR}/bad/news"
-notmuch new
-output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
-test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
-
-test_begin_subtest "After rename, old path returns nothing"
-mv "${MAIL_DIR}/duplicate/bad/news" "${MAIL_DIR}/duplicate/bad/olds"
-notmuch new
-output=$(notmuch search folder:bad/news | notmuch_search_sanitize)
-test_expect_equal "$output" ""
-
-test_begin_subtest "After rename, new path returns result"
-output=$(notmuch search folder:bad/olds | notmuch_search_sanitize)
-test_expect_equal "$output" "thread:XXX   2001-01-05 [1/1] Notmuch Test Suite; Bears (inbox unread)"
-
-test_done
diff --git a/test/test-databases/.gitignore b/test/test-databases/.gitignore
new file mode 100644
index 00000000..b5624b74
--- /dev/null
+++ b/test/test-databases/.gitignore
@@ -0,0 +1 @@
+*.tar.xz
diff --git a/test/test-databases/Makefile b/test/test-databases/Makefile
new file mode 100644
index 00000000..b250a8be
--- /dev/null
+++ b/test/test-databases/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C ../.. all
+
+.DEFAULT:
+	$(MAKE) -C ../.. $@
diff --git a/test/test-databases/Makefile.local b/test/test-databases/Makefile.local
new file mode 100644
index 00000000..0572e784
--- /dev/null
+++ b/test/test-databases/Makefile.local
@@ -0,0 +1,14 @@
+# -*- makefile -*-
+
+TEST_DATABASE_MIRROR=http://notmuchmail.org/releases/test-databases
+
+dir := test/test-databases
+
+test_databases := $(dir)/database-v1.tar.xz
+
+%.tar.xz:
+	wget -nv -O $@ ${TEST_DATABASE_MIRROR}/$(notdir $@);
+
+download-test-databases: ${test_databases}
+
+DISTCLEAN := $(DISTCLEAN) ${test_databases}
diff --git a/test/test-databases/database-v1.tar.xz.sha256 b/test/test-databases/database-v1.tar.xz.sha256
new file mode 100644
index 00000000..2cc4f965
--- /dev/null
+++ b/test/test-databases/database-v1.tar.xz.sha256
@@ -0,0 +1 @@
+4299e051b10e1fa7b33ea2862790a09ebfe96859681804e5251e130f800e69d2  database-v1.tar.xz
diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh
index e1eaa5a0..892991e2 100644
--- a/test/test-lib-common.sh
+++ b/test/test-lib-common.sh
@@ -98,12 +98,12 @@ then
 	PATH=$GIT_VALGRIND/bin:$PATH
 	GIT_EXEC_PATH=$GIT_VALGRIND/bin
 	export GIT_VALGRIND
-	test -n "$notmuch_path" && MANPATH="$notmuch_path/man:$MANPATH"
+	test -n "$notmuch_path" && MANPATH="$notmuch_path/doc/_build/man"
 else # normal case
 	if test -n "$notmuch_path"
 		then
 			PATH="$notmuch_path:$PATH"
-			MANPATH="$notmuch_path/man:$MANPATH"
+			MANPATH="$notmuch_path/doc/_build/man"
 		fi
 fi
 export PATH MANPATH
diff --git a/test/test-lib.el b/test/test-lib.el
index 37fcb3d0..437f83f4 100644
--- a/test/test-lib.el
+++ b/test/test-lib.el
@@ -165,3 +165,8 @@ nothing."
 
      (t
       (notmuch-test-report-unexpected output expected)))))
+
+;; For historical reasons, we hide deleted tags by default in the test
+;; suite
+(setq notmuch-tag-deleted-formats
+      '((".*" nil)))
diff --git a/test/test-lib.sh b/test/test-lib.sh
index efa9fb6f..17deaaba 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -25,6 +25,10 @@ fi
 # Make sure echo builtin does not expand backslash-escape sequences by default.
 shopt -u xpg_echo
 
+this_test=${0##*/}
+this_test=${this_test%.sh}
+this_test_bare=${this_test#T[0-9][0-9][0-9]-}
+
 # if --tee was passed, write the output not only to the terminal, but
 # additionally to the file test-results/$BASENAME.out, too.
 case "$GIT_TEST_TEE_STARTED, $* " in
@@ -33,7 +37,7 @@ done,*)
 	;;
 *' --tee '*|*' --va'*)
 	mkdir -p test-results
-	BASE=test-results/$(basename "$0" .sh)
+	BASE=test-results/$this_test
 	(GIT_TEST_TEE_STARTED=done ${SHELL-sh} "$0" "$@" 2>&1;
 	 echo $? > $BASE.exit) | tee $BASE.out
 	test "$(cat $BASE.exit)" = 0
@@ -187,7 +191,18 @@ then
 	exit 0
 fi
 
-echo $(basename "$0"): "Testing ${test_description}"
+test_description_printed=
+print_test_description ()
+{
+	test -z "$test_description_printed" || return 0
+	echo
+	echo $this_test: "Testing ${test_description}"
+	test_description_printed=1
+}
+if [ -z "$NOTMUCH_TEST_QUIET" ]
+then
+	print_test_description
+fi
 
 exec 5>&1
 
@@ -359,8 +374,12 @@ generate_message ()
 	# we use decreasing timestamps here for historical reasons;
 	# the existing test suite when we converted to unique timestamps just
 	# happened to have signicantly fewer failures with that choice.
-	template[date]=$(TZ=UTC printf "%(%a, %d %b %Y %T %z)T\n" \
-			$((978709437 - gen_msg_cnt)))
+	local date_secs=$((978709437 - gen_msg_cnt))
+	# printf %(..)T is bash 4.2+ feature. use perl fallback if needed...
+	TZ=UTC printf -v template[date] "%(%a, %d %b %Y %T %z)T" $date_secs 2>/dev/null ||
+	    template[date]=`perl -le 'use POSIX "strftime";
+				@time = gmtime '"$date_secs"';
+				print strftime "%a, %d %b %Y %T +0000", @time'`
     fi
 
     additional_headers=""
@@ -603,6 +622,12 @@ test_expect_equal_json () {
     test_expect_equal "$output" "$expected" "$@"
 }
 
+# Sort the top-level list of JSON data from stdin.
+test_sort_json () {
+    PYTHONIOENCODING=utf-8 python -c \
+        "import sys, json; json.dump(sorted(json.load(sys.stdin)),sys.stdout)"
+}
+
 test_emacs_expect_t () {
 	test "$#" = 2 && { prereq=$1; shift; } || prereq=
 	test "$#" = 1 ||
@@ -642,6 +667,11 @@ notmuch_search_sanitize ()
     perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'
 }
 
+notmuch_search_files_sanitize()
+{
+    sed -e "s,$MAIL_DIR,MAIL_DIR,"
+}
+
 NOTMUCH_SHOW_FILENAME_SQUELCH='s,filename:.*/mail,filename:/XXX/mail,'
 notmuch_show_sanitize ()
 {
@@ -748,6 +778,9 @@ test_ok_ () {
 		return
 	fi
 	test_success=$(($test_success + 1))
+	if test -n "$NOTMUCH_TEST_QUIET"; then
+		return 0
+	fi
 	say_color pass "%-6s" "PASS"
 	echo " $test_subtest_name"
 }
@@ -758,6 +791,7 @@ test_failure_ () {
 		return
 	fi
 	test_failure=$(($test_failure + 1))
+	print_test_description
 	test_failure_message_ "FAIL" "$test_subtest_name" "$@"
 	test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
 	return 1
@@ -806,6 +840,12 @@ test_skip () {
 		case $this_test.$test_count in
 		$skp)
 			to_skip=t
+			break
+		esac
+		case $this_test_bare.$test_count in
+		$skp)
+			to_skip=t
+			break
 		esac
 	done
 	if test -z "$to_skip" && test -n "$prereq" &&
@@ -1009,7 +1049,7 @@ test_done () {
 	GIT_EXIT_OK=t
 	test_results_dir="$TEST_DIRECTORY/test-results"
 	mkdir -p "$test_results_dir"
-	test_results_path="$test_results_dir/${0%.sh}"
+	test_results_path="$test_results_dir/$this_test"
 
 	echo "total $test_count" >> $test_results_path
 	echo "success $test_success" >> $test_results_path
@@ -1018,8 +1058,6 @@ test_done () {
 	echo "failed $test_failure" >> $test_results_path
 	echo "" >> $test_results_path
 
-	echo
-
 	[ -n "$EMACS_SERVER" ] && test_emacs '(kill-emacs)'
 
 	if [ "$test_failure" = "0" ]; then
@@ -1043,15 +1081,14 @@ export NOTMUCH_CONFIG=$NOTMUCH_CONFIG
 
 # Here's what we are using here:
 #
-# --no-init-file	Don't load users ~/.emacs
-#
-# --no-site-file	Don't load the site-wide startup stuff
+# --quick              Use minimal customization. This implies --no-init-file,
+#		       --no-site-file and (emacs 24) --no-site-lisp
 #
 # --directory		Ensure that the local elisp sources are found
 #
 # --load		Force loading of notmuch.el and test-lib.el
 
-exec ${TEST_EMACS} --no-init-file --no-site-file \
+exec ${TEST_EMACS} --quick \
 	--directory "$TEST_DIRECTORY/../emacs" --load notmuch.el \
 	--directory "$TEST_DIRECTORY" --load test-lib.el \
 	"\$@"
@@ -1068,7 +1105,7 @@ test_emacs () {
 	test -z "$missing_dependencies" || return
 
 	if [ -z "$EMACS_SERVER" ]; then
-		emacs_tests="$(basename $0).el"
+		emacs_tests="${this_test_bare}.el"
 		if [ -f "$TEST_DIRECTORY/$emacs_tests" ]; then
 			load_emacs_tests="--eval '(load \"$emacs_tests\")'"
 		else
@@ -1182,7 +1219,6 @@ else
 	exec 4>test.output 3>&4
 fi
 
-this_test=${0##*/}
 for skp in $NOTMUCH_SKIP_TESTS
 do
 	to_skip=
@@ -1191,6 +1227,12 @@ do
 		case "$this_test" in
 		$skp)
 			to_skip=t
+			break
+		esac
+		case "$this_test_bare" in
+		$skp)
+			to_skip=t
+			break
 		esac
 	done
 	case "$to_skip" in
diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no
index 0bca7540..1a2ff619 100644
--- a/test/test.expected-output/test-verbose-no
+++ b/test/test.expected-output/test-verbose-no
@@ -1,3 +1,4 @@
+
 test-verbose: Testing the verbosity options of the test framework itself.
  PASS   print something in test_expect_success and pass
  FAIL   print something in test_expect_success and fail
diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes
index ebe51874..d25466e9 100644
--- a/test/test.expected-output/test-verbose-yes
+++ b/test/test.expected-output/test-verbose-yes
@@ -1,3 +1,4 @@
+
 test-verbose: Testing the verbosity options of the test framework itself.
 hello stdout
 hello stderr
diff --git a/test/tree.expected-output/notmuch-tree-single-thread b/test/tree.expected-output/notmuch-tree-single-thread
index c9e5ef82..2285d10e 100644
--- a/test/tree.expected-output/notmuch-tree-single-thread
+++ b/test/tree.expected-output/notmuch-tree-single-thread
@@ -1,6 +1,6 @@
   2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox)
-  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox, unread)
-  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox, unread)
-  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Carl Worth              ╰─► ...                                             (inbox, unread)
+  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox unread)
+  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox unread)
+  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox unread)
+  2009-11-18  Carl Worth              ╰─► ...                                             (inbox unread)
 End of search results.
diff --git a/test/tree.expected-output/notmuch-tree-tag-inbox b/test/tree.expected-output/notmuch-tree-tag-inbox
index 484141ec..f28d4856 100644
--- a/test/tree.expected-output/notmuch-tree-tag-inbox
+++ b/test/tree.expected-output/notmuch-tree-tag-inbox
@@ -1,53 +1,53 @@
-  2010-12-29  François Boulogne     ─►[aur-general] Guidelines: cp, mkdir vs install      (inbox, unread)
-  2010-12-16  Olivier Berger        ─►Essai accentué                                      (inbox, unread)
-  2009-11-18  Chris Wilson          ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox, unread)
-  2009-11-18  Alex Botero-Lowry     ┬►[notmuch] [PATCH] Error out if no query is supplied to search	instead of going into an infinite loop (attachment, inbox, unread)
-  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (inbox, unread)
-  2009-11-17  Ingmar Vanhassel      ┬►[notmuch] [PATCH] Typsos                            (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Adrian Perez de Cast  ┬►[notmuch] Introducing myself                        (inbox, signed, unread)
-  2009-11-18  Keith Packard         ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Israel Herraiz        ┬►[notmuch] New to the list                           (inbox, unread)
-  2009-11-18  Keith Packard         ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Jan Janak             ┬►[notmuch] What a great idea!                        (inbox, unread)
-  2009-11-17  Jan Janak             ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Jan Janak             ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Aron Griffis          ┬►[notmuch] archive                                   (inbox, unread)
-  2009-11-18  Keith Packard         ╰┬► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth             ╰─► ...                                              (inbox, unread)
-  2009-11-17  Keith Packard         ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove	inbox (and unread) tags (inbox, unread)
-  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox, unread)
-  2009-11-17  Lars Kellogg-Stedman  ┬►[notmuch] Working with Maildir storage?             (inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       ├┬► ...                                               (inbox, signed, unread)
-  2009-11-17  Lars Kellogg-Stedman  │╰┬► ...                                              (inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       │ ├─► ...                                             (inbox, unread)
-  2009-11-17  Keith Packard         │ ╰┬► ...                                             (inbox, unread)
-  2009-11-18  Lars Kellogg-Stedman  │  ╰─► ...                                            (inbox, signed, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox, unread)
-  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox, unread)
-  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox, unread)
-  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Carl Worth              ╰─► ...                                             (inbox, unread)
-  2009-11-18  Keith Packard         ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that	highlights URLs and uses word-wrap (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ╰─►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ─►[notmuch] request for pull                          (inbox, unread)
-  2009-11-18  Jjgod Jiang           ┬►[notmuch] Mac OS X/Darwin compatibility issues      (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ╰┬► ...                                               (inbox, unread)
-  2009-11-18  Jjgod Jiang            ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Alexander Botero-Low    ╰─► ...                                             (inbox, unread)
-  2009-11-18  Rolland Santimano     ─►[notmuch] Link to mailing list archives ?           (inbox, unread)
-  2009-11-18  Jan Janak             ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool	subdirectories into tags (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] count_files: sort directory in inode order before	statting (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++	libs. (inbox, unread)
-  2009-11-18  Lars Kellogg-Stedman  ┬►[notmuch] "notmuch help" outputs to stderr?         (attachment, inbox, signed, unread)
-  2009-11-18  Lars Kellogg-Stedman  ╰─► ...                                               (attachment, inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       ─►[notmuch] [PATCH] Handle rename of message file     (inbox, unread)
-  2009-11-17  Alex Botero-Lowry     ┬►[notmuch] preliminary FreeBSD support               (attachment, inbox, unread)
-  2009-11-17  Carl Worth            ╰─► ...                                               (inbox, unread)
+  2010-12-29  François Boulogne     ─►[aur-general] Guidelines: cp, mkdir vs install      (inbox unread)
+  2010-12-16  Olivier Berger        ─►Essai accentué                                      (inbox unread)
+  2009-11-18  Chris Wilson          ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox unread)
+  2009-11-18  Alex Botero-Lowry     ┬►[notmuch] [PATCH] Error out if no query is supplied to search	instead of going into an infinite loop (attachment inbox unread)
+  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (inbox unread)
+  2009-11-17  Ingmar Vanhassel      ┬►[notmuch] [PATCH] Typsos                            (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Adrian Perez de Cast  ┬►[notmuch] Introducing myself                        (inbox signed unread)
+  2009-11-18  Keith Packard         ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Israel Herraiz        ┬►[notmuch] New to the list                           (inbox unread)
+  2009-11-18  Keith Packard         ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Jan Janak             ┬►[notmuch] What a great idea!                        (inbox unread)
+  2009-11-17  Jan Janak             ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Jan Janak             ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Aron Griffis          ┬►[notmuch] archive                                   (inbox unread)
+  2009-11-18  Keith Packard         ╰┬► ...                                               (inbox unread)
+  2009-11-18  Carl Worth             ╰─► ...                                              (inbox unread)
+  2009-11-17  Keith Packard         ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove	inbox (and unread) tags (inbox unread)
+  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox unread)
+  2009-11-17  Lars Kellogg-Stedman  ┬►[notmuch] Working with Maildir storage?             (inbox signed unread)
+  2009-11-17  Mikhail Gusarov       ├┬► ...                                               (inbox signed unread)
+  2009-11-17  Lars Kellogg-Stedman  │╰┬► ...                                              (inbox signed unread)
+  2009-11-17  Mikhail Gusarov       │ ├─► ...                                             (inbox unread)
+  2009-11-17  Keith Packard         │ ╰┬► ...                                             (inbox unread)
+  2009-11-18  Lars Kellogg-Stedman  │  ╰─► ...                                            (inbox signed unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox unread)
+  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox unread)
+  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox unread)
+  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox unread)
+  2009-11-18  Carl Worth              ╰─► ...                                             (inbox unread)
+  2009-11-18  Keith Packard         ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that	highlights URLs and uses word-wrap (inbox unread)
+  2009-11-18  Alexander Botero-Low  ╰─►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox unread)
+  2009-11-18  Alexander Botero-Low  ─►[notmuch] request for pull                          (inbox unread)
+  2009-11-18  Jjgod Jiang           ┬►[notmuch] Mac OS X/Darwin compatibility issues      (inbox unread)
+  2009-11-18  Alexander Botero-Low  ╰┬► ...                                               (inbox unread)
+  2009-11-18  Jjgod Jiang            ╰┬► ...                                              (inbox unread)
+  2009-11-18  Alexander Botero-Low    ╰─► ...                                             (inbox unread)
+  2009-11-18  Rolland Santimano     ─►[notmuch] Link to mailing list archives ?           (inbox unread)
+  2009-11-18  Jan Janak             ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool	subdirectories into tags (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] count_files: sort directory in inode order before	statting (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++	libs. (inbox unread)
+  2009-11-18  Lars Kellogg-Stedman  ┬►[notmuch] "notmuch help" outputs to stderr?         (attachment inbox signed unread)
+  2009-11-18  Lars Kellogg-Stedman  ╰─► ...                                               (attachment inbox signed unread)
+  2009-11-17  Mikhail Gusarov       ─►[notmuch] [PATCH] Handle rename of message file     (inbox unread)
+  2009-11-17  Alex Botero-Lowry     ┬►[notmuch] preliminary FreeBSD support               (attachment inbox unread)
+  2009-11-17  Carl Worth            ╰─► ...                                               (inbox unread)
 End of search results.
diff --git a/test/tree.expected-output/notmuch-tree-tag-inbox-tagged b/test/tree.expected-output/notmuch-tree-tag-inbox-tagged
index 1f75a34b..428c0ae8 100644
--- a/test/tree.expected-output/notmuch-tree-tag-inbox-tagged
+++ b/test/tree.expected-output/notmuch-tree-tag-inbox-tagged
@@ -1,53 +1,53 @@
-  2010-12-29  François Boulogne     ─►[aur-general] Guidelines: cp, mkdir vs install      (inbox, unread)
-  2010-12-16  Olivier Berger        ─►Essai accentué                                      (inbox, test_tag, unread)
-  2009-11-18  Chris Wilson          ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox, unread)
-  2009-11-18  Alex Botero-Lowry     ┬►[notmuch] [PATCH] Error out if no query is supplied to search	instead of going into an infinite loop (attachment, inbox, unread)
-  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (inbox, unread)
-  2009-11-17  Ingmar Vanhassel      ┬►[notmuch] [PATCH] Typsos                            (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Adrian Perez de Cast  ┬►[notmuch] Introducing myself                        (inbox, signed, unread)
-  2009-11-18  Keith Packard         ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Israel Herraiz        ┬►[notmuch] New to the list                           (inbox, unread)
-  2009-11-18  Keith Packard         ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Jan Janak             ┬►[notmuch] What a great idea!                        (inbox, unread)
-  2009-11-17  Jan Janak             ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Jan Janak             ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Aron Griffis          ┬►[notmuch] archive                                   (inbox, unread)
-  2009-11-18  Keith Packard         ╰┬► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth             ╰─► ...                                              (inbox, unread)
-  2009-11-17  Keith Packard         ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove	inbox (and unread) tags (inbox, unread)
-  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox, unread)
-  2009-11-17  Lars Kellogg-Stedman  ┬►[notmuch] Working with Maildir storage?             (inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       ├┬► ...                                               (inbox, signed, unread)
-  2009-11-17  Lars Kellogg-Stedman  │╰┬► ...                                              (inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       │ ├─► ...                                             (inbox, unread)
-  2009-11-17  Keith Packard         │ ╰┬► ...                                             (inbox, unread)
-  2009-11-18  Lars Kellogg-Stedman  │  ╰─► ...                                            (inbox, signed, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox, unread)
-  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox, unread)
-  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox, unread)
-  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Carl Worth              ╰─► ...                                             (inbox, unread)
-  2009-11-18  Keith Packard         ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that	highlights URLs and uses word-wrap (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ╰─►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ─►[notmuch] request for pull                          (inbox, unread)
-  2009-11-18  Jjgod Jiang           ┬►[notmuch] Mac OS X/Darwin compatibility issues      (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ╰┬► ...                                               (inbox, unread)
-  2009-11-18  Jjgod Jiang            ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Alexander Botero-Low    ╰─► ...                                             (inbox, unread)
-  2009-11-18  Rolland Santimano     ─►[notmuch] Link to mailing list archives ?           (inbox, unread)
-  2009-11-18  Jan Janak             ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool	subdirectories into tags (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] count_files: sort directory in inode order before	statting (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++	libs. (inbox, unread)
-  2009-11-18  Lars Kellogg-Stedman  ┬►[notmuch] "notmuch help" outputs to stderr?         (attachment, inbox, signed, unread)
-  2009-11-18  Lars Kellogg-Stedman  ╰─► ...                                               (attachment, inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       ─►[notmuch] [PATCH] Handle rename of message file     (inbox, unread)
-  2009-11-17  Alex Botero-Lowry     ┬►[notmuch] preliminary FreeBSD support               (attachment, inbox, unread)
-  2009-11-17  Carl Worth            ╰─► ...                                               (inbox, unread)
+  2010-12-29  François Boulogne     ─►[aur-general] Guidelines: cp, mkdir vs install      (inbox unread)
+  2010-12-16  Olivier Berger        ─►Essai accentué                                      (inbox test_tag unread)
+  2009-11-18  Chris Wilson          ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox unread)
+  2009-11-18  Alex Botero-Lowry     ┬►[notmuch] [PATCH] Error out if no query is supplied to search	instead of going into an infinite loop (attachment inbox unread)
+  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (inbox unread)
+  2009-11-17  Ingmar Vanhassel      ┬►[notmuch] [PATCH] Typsos                            (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Adrian Perez de Cast  ┬►[notmuch] Introducing myself                        (inbox signed unread)
+  2009-11-18  Keith Packard         ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Israel Herraiz        ┬►[notmuch] New to the list                           (inbox unread)
+  2009-11-18  Keith Packard         ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Jan Janak             ┬►[notmuch] What a great idea!                        (inbox unread)
+  2009-11-17  Jan Janak             ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Jan Janak             ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Aron Griffis          ┬►[notmuch] archive                                   (inbox unread)
+  2009-11-18  Keith Packard         ╰┬► ...                                               (inbox unread)
+  2009-11-18  Carl Worth             ╰─► ...                                              (inbox unread)
+  2009-11-17  Keith Packard         ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove	inbox (and unread) tags (inbox unread)
+  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox unread)
+  2009-11-17  Lars Kellogg-Stedman  ┬►[notmuch] Working with Maildir storage?             (inbox signed unread)
+  2009-11-17  Mikhail Gusarov       ├┬► ...                                               (inbox signed unread)
+  2009-11-17  Lars Kellogg-Stedman  │╰┬► ...                                              (inbox signed unread)
+  2009-11-17  Mikhail Gusarov       │ ├─► ...                                             (inbox unread)
+  2009-11-17  Keith Packard         │ ╰┬► ...                                             (inbox unread)
+  2009-11-18  Lars Kellogg-Stedman  │  ╰─► ...                                            (inbox signed unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox unread)
+  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox unread)
+  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox unread)
+  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox unread)
+  2009-11-18  Carl Worth              ╰─► ...                                             (inbox unread)
+  2009-11-18  Keith Packard         ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that	highlights URLs and uses word-wrap (inbox unread)
+  2009-11-18  Alexander Botero-Low  ╰─►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox unread)
+  2009-11-18  Alexander Botero-Low  ─►[notmuch] request for pull                          (inbox unread)
+  2009-11-18  Jjgod Jiang           ┬►[notmuch] Mac OS X/Darwin compatibility issues      (inbox unread)
+  2009-11-18  Alexander Botero-Low  ╰┬► ...                                               (inbox unread)
+  2009-11-18  Jjgod Jiang            ╰┬► ...                                              (inbox unread)
+  2009-11-18  Alexander Botero-Low    ╰─► ...                                             (inbox unread)
+  2009-11-18  Rolland Santimano     ─►[notmuch] Link to mailing list archives ?           (inbox unread)
+  2009-11-18  Jan Janak             ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool	subdirectories into tags (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] count_files: sort directory in inode order before	statting (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++	libs. (inbox unread)
+  2009-11-18  Lars Kellogg-Stedman  ┬►[notmuch] "notmuch help" outputs to stderr?         (attachment inbox signed unread)
+  2009-11-18  Lars Kellogg-Stedman  ╰─► ...                                               (attachment inbox signed unread)
+  2009-11-17  Mikhail Gusarov       ─►[notmuch] [PATCH] Handle rename of message file     (inbox unread)
+  2009-11-17  Alex Botero-Lowry     ┬►[notmuch] preliminary FreeBSD support               (attachment inbox unread)
+  2009-11-17  Carl Worth            ╰─► ...                                               (inbox unread)
 End of search results.
diff --git a/test/tree.expected-output/notmuch-tree-tag-inbox-thread-tagged b/test/tree.expected-output/notmuch-tree-tag-inbox-thread-tagged
index a7aba6e2..828c5251 100644
--- a/test/tree.expected-output/notmuch-tree-tag-inbox-thread-tagged
+++ b/test/tree.expected-output/notmuch-tree-tag-inbox-thread-tagged
@@ -1,53 +1,53 @@
-  2010-12-29  François Boulogne     ─►[aur-general] Guidelines: cp, mkdir vs install      (inbox, unread)
-  2010-12-16  Olivier Berger        ─►Essai accentué                                      (inbox, unread)
-  2009-11-18  Chris Wilson          ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox, unread)
-  2009-11-18  Alex Botero-Lowry     ┬►[notmuch] [PATCH] Error out if no query is supplied to search	instead of going into an infinite loop (attachment, inbox, unread)
-  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (inbox, unread)
-  2009-11-17  Ingmar Vanhassel      ┬►[notmuch] [PATCH] Typsos                            (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Adrian Perez de Cast  ┬►[notmuch] Introducing myself                        (inbox, signed, unread)
-  2009-11-18  Keith Packard         ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Israel Herraiz        ┬►[notmuch] New to the list                           (inbox, unread)
-  2009-11-18  Keith Packard         ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Jan Janak             ┬►[notmuch] What a great idea!                        (inbox, unread)
-  2009-11-17  Jan Janak             ├─► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Jan Janak             ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, unread)
-  2009-11-17  Aron Griffis          ┬►[notmuch] archive                                   (inbox, unread)
-  2009-11-18  Keith Packard         ╰┬► ...                                               (inbox, unread)
-  2009-11-18  Carl Worth             ╰─► ...                                              (inbox, unread)
-  2009-11-17  Keith Packard         ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove	inbox (and unread) tags (inbox, unread)
-  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox, unread)
-  2009-11-17  Lars Kellogg-Stedman  ┬►[notmuch] Working with Maildir storage?             (inbox, signed, test_thread_tag, unread)
-  2009-11-17  Mikhail Gusarov       ├┬► ...                                               (inbox, signed, test_thread_tag, unread)
-  2009-11-17  Lars Kellogg-Stedman  │╰┬► ...                                              (inbox, signed, test_thread_tag, unread)
-  2009-11-17  Mikhail Gusarov       │ ├─► ...                                             (inbox, test_thread_tag, unread)
-  2009-11-17  Keith Packard         │ ╰┬► ...                                             (inbox, test_thread_tag, unread)
-  2009-11-18  Lars Kellogg-Stedman  │  ╰─► ...                                            (inbox, signed, test_thread_tag, unread)
-  2009-11-18  Carl Worth            ╰─► ...                                               (inbox, test_thread_tag, unread)
-  2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox, unread)
-  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox, unread)
-  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox, unread)
-  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Carl Worth              ╰─► ...                                             (inbox, unread)
-  2009-11-18  Keith Packard         ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that	highlights URLs and uses word-wrap (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ╰─►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ─►[notmuch] request for pull                          (inbox, unread)
-  2009-11-18  Jjgod Jiang           ┬►[notmuch] Mac OS X/Darwin compatibility issues      (inbox, unread)
-  2009-11-18  Alexander Botero-Low  ╰┬► ...                                               (inbox, unread)
-  2009-11-18  Jjgod Jiang            ╰┬► ...                                              (inbox, unread)
-  2009-11-18  Alexander Botero-Low    ╰─► ...                                             (inbox, unread)
-  2009-11-18  Rolland Santimano     ─►[notmuch] Link to mailing list archives ?           (inbox, unread)
-  2009-11-18  Jan Janak             ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool	subdirectories into tags (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] count_files: sort directory in inode order before	statting (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox, unread)
-  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++	libs. (inbox, unread)
-  2009-11-18  Lars Kellogg-Stedman  ┬►[notmuch] "notmuch help" outputs to stderr?         (attachment, inbox, signed, unread)
-  2009-11-18  Lars Kellogg-Stedman  ╰─► ...                                               (attachment, inbox, signed, unread)
-  2009-11-17  Mikhail Gusarov       ─►[notmuch] [PATCH] Handle rename of message file     (inbox, unread)
-  2009-11-17  Alex Botero-Lowry     ┬►[notmuch] preliminary FreeBSD support               (attachment, inbox, unread)
-  2009-11-17  Carl Worth            ╰─► ...                                               (inbox, unread)
+  2010-12-29  François Boulogne     ─►[aur-general] Guidelines: cp, mkdir vs install      (inbox unread)
+  2010-12-16  Olivier Berger        ─►Essai accentué                                      (inbox unread)
+  2009-11-18  Chris Wilson          ─►[notmuch] [PATCH 1/2] Makefile: evaluate pkg-config once (inbox unread)
+  2009-11-18  Alex Botero-Lowry     ┬►[notmuch] [PATCH] Error out if no query is supplied to search	instead of going into an infinite loop (attachment inbox unread)
+  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Error out if no query is supplied to search instead of going into an infinite loop (inbox unread)
+  2009-11-17  Ingmar Vanhassel      ┬►[notmuch] [PATCH] Typsos                            (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Adrian Perez de Cast  ┬►[notmuch] Introducing myself                        (inbox signed unread)
+  2009-11-18  Keith Packard         ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Israel Herraiz        ┬►[notmuch] New to the list                           (inbox unread)
+  2009-11-18  Keith Packard         ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Jan Janak             ┬►[notmuch] What a great idea!                        (inbox unread)
+  2009-11-17  Jan Janak             ├─► ...                                               (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Jan Janak             ┬►[notmuch] [PATCH] Older versions of install do not support -C. (inbox unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox unread)
+  2009-11-17  Aron Griffis          ┬►[notmuch] archive                                   (inbox unread)
+  2009-11-18  Keith Packard         ╰┬► ...                                               (inbox unread)
+  2009-11-18  Carl Worth             ╰─► ...                                              (inbox unread)
+  2009-11-17  Keith Packard         ┬►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove	inbox (and unread) tags (inbox unread)
+  2009-11-18  Carl Worth            ╰─►[notmuch] [PATCH] Make notmuch-show 'X' (and 'x') commands remove inbox (and unread) tags (inbox unread)
+  2009-11-17  Lars Kellogg-Stedman  ┬►[notmuch] Working with Maildir storage?             (inbox signed test_thread_tag unread)
+  2009-11-17  Mikhail Gusarov       ├┬► ...                                               (inbox signed test_thread_tag unread)
+  2009-11-17  Lars Kellogg-Stedman  │╰┬► ...                                              (inbox signed test_thread_tag unread)
+  2009-11-17  Mikhail Gusarov       │ ├─► ...                                             (inbox test_thread_tag unread)
+  2009-11-17  Keith Packard         │ ╰┬► ...                                             (inbox test_thread_tag unread)
+  2009-11-18  Lars Kellogg-Stedman  │  ╰─► ...                                            (inbox signed test_thread_tag unread)
+  2009-11-18  Carl Worth            ╰─► ...                                               (inbox test_thread_tag unread)
+  2009-11-17  Mikhail Gusarov       ┬►[notmuch] [PATCH 1/2] Close message file after parsing message	headers (inbox unread)
+  2009-11-17  Mikhail Gusarov       ├─►[notmuch] [PATCH 2/2] Include <stdint.h> to get uint32_t in C++	file with gcc 4.4 (inbox unread)
+  2009-11-17  Carl Worth            ╰┬►[notmuch] [PATCH 1/2] Close message file after parsing message headers (inbox unread)
+  2009-11-17  Keith Packard          ╰┬► ...                                              (inbox unread)
+  2009-11-18  Carl Worth              ╰─► ...                                             (inbox unread)
+  2009-11-18  Keith Packard         ┬►[notmuch] [PATCH] Create a default notmuch-show-hook that	highlights URLs and uses word-wrap (inbox unread)
+  2009-11-18  Alexander Botero-Low  ╰─►[notmuch] [PATCH] Create a default notmuch-show-hook that highlights URLs and uses word-wrap (inbox unread)
+  2009-11-18  Alexander Botero-Low  ─►[notmuch] request for pull                          (inbox unread)
+  2009-11-18  Jjgod Jiang           ┬►[notmuch] Mac OS X/Darwin compatibility issues      (inbox unread)
+  2009-11-18  Alexander Botero-Low  ╰┬► ...                                               (inbox unread)
+  2009-11-18  Jjgod Jiang            ╰┬► ...                                              (inbox unread)
+  2009-11-18  Alexander Botero-Low    ╰─► ...                                             (inbox unread)
+  2009-11-18  Rolland Santimano     ─►[notmuch] Link to mailing list archives ?           (inbox unread)
+  2009-11-18  Jan Janak             ─►[notmuch] [PATCH] notmuch new: Support for conversion of spool	subdirectories into tags (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] count_files: sort directory in inode order before	statting (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH 2/2] Read mail directory in inode number order (inbox unread)
+  2009-11-18  Stewart Smith         ─►[notmuch] [PATCH] Fix linking with gcc to use g++ to link in C++	libs. (inbox unread)
+  2009-11-18  Lars Kellogg-Stedman  ┬►[notmuch] "notmuch help" outputs to stderr?         (attachment inbox signed unread)
+  2009-11-18  Lars Kellogg-Stedman  ╰─► ...                                               (attachment inbox signed unread)
+  2009-11-17  Mikhail Gusarov       ─►[notmuch] [PATCH] Handle rename of message file     (inbox unread)
+  2009-11-17  Alex Botero-Lowry     ┬►[notmuch] preliminary FreeBSD support               (attachment inbox unread)
+  2009-11-17  Carl Worth            ╰─► ...                                               (inbox unread)
 End of search results.
diff --git a/util/Makefile.local b/util/Makefile.local
index 29c0ce6e..905f2376 100644
--- a/util/Makefile.local
+++ b/util/Makefile.local
@@ -4,7 +4,8 @@ dir := util
 extra_cflags += -I$(srcdir)/$(dir)
 
 libutil_c_srcs := $(dir)/xutil.c $(dir)/error_util.c $(dir)/hex-escape.c \
-		  $(dir)/string-util.c $(dir)/talloc-extra.c
+		  $(dir)/string-util.c $(dir)/talloc-extra.c $(dir)/zlib-extra.c \
+		$(dir)/util.c
 
 libutil_modules := $(libutil_c_srcs:.c=.o)
 
diff --git a/util/string-util.c b/util/string-util.c
index a5622d7a..3e7066cd 100644
--- a/util/string-util.c
+++ b/util/string-util.c
@@ -37,6 +37,28 @@ strtok_len (char *s, const char *delim, size_t *len)
     return *len ? s : NULL;
 }
 
+char *
+sanitize_string (const void *ctx, const char *str)
+{
+    char *out, *loop;
+
+    if (! str)
+	return NULL;
+
+    out = talloc_strdup (ctx, str);
+    if (! out)
+	return NULL;
+
+    for (loop = out; *loop; loop++) {
+	if (*loop == '\t' || *loop == '\n')
+	    *loop = ' ';
+	else if ((unsigned char)(*loop) < 32)
+	    *loop = '?';
+    }
+
+    return out;
+}
+
 static int
 is_unquoted_terminator (unsigned char c)
 {
@@ -53,10 +75,12 @@ make_boolean_term (void *ctx, const char *prefix, const char *term,
     int need_quoting = 0;
 
     /* Do we need quoting?  To be paranoid, we quote anything
-     * containing a quote, even though it only matters at the
+     * containing a quote or '(', even though these only matter at the
      * beginning, and anything containing non-ASCII text. */
+    if (! term[0])
+	need_quoting = 1;
     for (in = term; *in && !need_quoting; in++)
-	if (is_unquoted_terminator (*in) || *in == '"'
+	if (is_unquoted_terminator (*in) || *in == '"' || *in == '('
 	    || (unsigned char)*in > 127)
 	    need_quoting = 1;
 
diff --git a/util/string-util.h b/util/string-util.h
index 0194607e..8a3ad19e 100644
--- a/util/string-util.h
+++ b/util/string-util.h
@@ -19,6 +19,13 @@
 
 char *strtok_len (char *s, const char *delim, size_t *len);
 
+/* Return a talloced string with str sanitized.
+ *
+ * Whitespace characters (tabs and newlines) are replaced with spaces,
+ * non-printable characters with question marks.
+ */
+char *sanitize_string (const void *ctx, const char *str);
+
 /* Construct a boolean term query with the specified prefix (e.g.,
  * "id") and search term, quoting term as necessary.  Specifically, if
  * term contains any non-printable ASCII characters, non-ASCII
diff --git a/util/util.c b/util/util.c
new file mode 100644
index 00000000..06659b35
--- /dev/null
+++ b/util/util.c
@@ -0,0 +1,24 @@
+#include "util.h"
+#include "error_util.h"
+#include <string.h>
+#include <errno.h>
+
+const char *
+util_error_string (util_status_t errnum)
+{
+    switch (errnum) {
+    case UTIL_SUCCESS:
+	return "success";
+    case UTIL_OUT_OF_MEMORY:
+	return "out of memory";
+    case UTIL_EOF:
+	return "end of file";
+    case UTIL_ERRNO:
+	return strerror (errno);
+    case UTIL_GZERROR:
+	/* we lack context to be more informative here */
+	return "zlib error";
+    default:
+	INTERNAL_ERROR("unexpected error status %d", errnum);
+    }
+}
diff --git a/util/util.h b/util/util.h
new file mode 100644
index 00000000..d12fadb4
--- /dev/null
+++ b/util/util.h
@@ -0,0 +1,29 @@
+#ifndef _UTIL_H
+#define _UTIL_H
+
+typedef enum util_status {
+    /**
+     * No error occurred.
+     */
+    UTIL_SUCCESS = 0,
+    /**
+     * Out of memory.
+     */
+    UTIL_OUT_OF_MEMORY,
+    /**
+     * End of stream reached while attempting to read.
+     */
+    UTIL_EOF,
+    /**
+     * Low level error occured, consult errno.
+     */
+    UTIL_ERRNO,
+    /**
+     * Zlib error occured, call gzerror for details.
+     */
+    UTIL_GZERROR
+} util_status_t;
+
+const char *
+util_error_string (util_status_t status);
+#endif
diff --git a/util/zlib-extra.c b/util/zlib-extra.c
new file mode 100644
index 00000000..2e704457
--- /dev/null
+++ b/util/zlib-extra.c
@@ -0,0 +1,85 @@
+/* zlib-extra.c -  Extra or enhanced routines for compressed I/O.
+ *
+ * Copyright (c) 2014 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "zlib-extra.h"
+#include <talloc.h>
+#include <stdio.h>
+#include <string.h>
+
+/* mimic POSIX/glibc getline, but on a zlib gzFile stream, and using talloc */
+util_status_t
+gz_getline (void *talloc_ctx, char **bufptr, ssize_t *bytes_read, gzFile stream)
+{
+    char *buf = *bufptr;
+    unsigned int len;
+    size_t offset = 0;
+
+    if (buf) {
+	len = talloc_array_length (buf);
+    } else {
+	/* same as getdelim from gnulib */
+	len = 120;
+	buf = talloc_array (talloc_ctx, char, len);
+	if (buf == NULL)
+	    return UTIL_OUT_OF_MEMORY;
+    }
+
+    while (1) {
+	if (! gzgets (stream, buf + offset, len - offset)) {
+	    /* Null indicates EOF or error */
+	    int zlib_status = 0;
+	    (void) gzerror (stream, &zlib_status);
+	    switch (zlib_status) {
+	    case Z_OK:
+		/* no data read before EOF */
+		if (offset == 0)
+		    return UTIL_EOF;
+		else
+		    goto SUCCESS;
+	    case Z_ERRNO:
+		return UTIL_ERRNO;
+	    default:
+		return UTIL_GZERROR;
+	    }
+	}
+
+	offset += strlen (buf + offset);
+
+	if (buf[offset - 1] == '\n')
+	    goto SUCCESS;
+
+	len *= 2;
+	buf = talloc_realloc (talloc_ctx, buf, char, len);
+	if (buf == NULL)
+	    return UTIL_OUT_OF_MEMORY;
+    }
+ SUCCESS:
+    *bufptr = buf;
+    *bytes_read = offset;
+    return UTIL_SUCCESS;
+}
+
+const char *gz_error_string (util_status_t status, gzFile file)
+{
+    if (status == UTIL_GZERROR)
+	return gzerror (file, NULL);
+    else
+	return util_error_string (status);
+}
diff --git a/util/zlib-extra.h b/util/zlib-extra.h
new file mode 100644
index 00000000..aedfd48f
--- /dev/null
+++ b/util/zlib-extra.h
@@ -0,0 +1,25 @@
+#ifndef _ZLIB_EXTRA_H
+#define _ZLIB_EXTRA_H
+
+#include "util.h"
+#include <zlib.h>
+
+/* Like getline, but read from a gzFile. Allocation is with talloc.
+ * Returns:
+ *
+ *   UTIL_SUCCESS, UTIL_OUT_OF_MEMORY, UTIL_ERRNO, UTIL_GZERROR
+ *			Consult util.h for description
+ *
+ *   UTIL_EOF		End of file encountered before
+ *			any characters read
+ */
+util_status_t
+gz_getline (void *ctx, char **lineptr, ssize_t *bytes_read, gzFile stream);
+
+/* return a suitable error string based on the return status
+ *  from gz_readline
+ */
+
+const char *
+gz_error_string (util_status_t status, gzFile stream);
+#endif
diff --git a/version b/version
index 50653ad0..249afd51 100644
--- a/version
+++ b/version
@@ -1 +1 @@
-0.17
+0.18.1
diff --git a/vim/notmuch.vim b/vim/notmuch.vim
index d7b310c8..331e9300 100644
--- a/vim/notmuch.vim
+++ b/vim/notmuch.vim
@@ -86,17 +86,22 @@ endfunction
 function! s:compose_send()
 	let b:compose_done = 1
 	let fname = expand('%')
+	let lines = getline(5, '$')
 
-	" remove headers
-	0,4d
-	write
+ruby << EOF
+	# Generate proper mail to send
+	text = VIM::evaluate('lines').join("\n")
+	fname = VIM::evaluate('fname')
+	transport = Mail.new(text)
+	transport.message_id = generate_message_id
+	transport.charset = 'utf-8'
+	File.write(fname, transport.to_s)
+EOF
 
 	let cmdtxt = g:notmuch_sendmail . ' -t -f ' . s:reply_from . ' < ' . fname
 	let out = system(cmdtxt)
 	let err = v:shell_error
 	if err
-		undo
-		write
 		echohl Error
 		echo 'Eeek! unable to send mail'
 		echo out
@@ -572,9 +577,7 @@ ruby << EOF
 			end
 			m.cc = orig[:cc]
 			m.from = $email
-			m.message_id = generate_message_id
 			m.charset = 'utf-8'
-			m.content_transfer_encoding = '7bit'
 		end
 
 		lines = []
@@ -600,7 +603,7 @@ ruby << EOF
 
 		reply.body = body_lines.join("\n")
 
-		lines += reply.to_s.lines.map { |e| e.chomp }
+		lines += reply.present.lines.map { |e| e.chomp }
 		lines << ""
 
 		cur = lines.count - 1
@@ -611,18 +614,13 @@ ruby << EOF
 	def open_compose()
 		lines = []
 
-		lines << "Date: #{Time.now().strftime('%a, %-d %b %Y %T %z')}"
 		lines << "From: #{$email}"
 		lines << "To: "
 		cur = lines.count
 
 		lines << "Cc: "
 		lines << "Bcc: "
-		lines << "Message-Id: #{generate_message_id}"
 		lines << "Subject: "
-		lines << "Mime-Version: 1.0"
-		lines << "Content-Type: text/plain; charset=utf-8"
-		lines << "Content-Transfer-Encoding: 7bit"
 		lines << ""
 		lines << ""
 		lines << ""
@@ -633,7 +631,7 @@ ruby << EOF
 	def folders_render()
 		$curbuf.render do |b|
 			folders = VIM::evaluate('g:notmuch_folders')
-			count_threads = VIM::evaluate('g:notmuch_folders_count_threads')
+			count_threads = VIM::evaluate('g:notmuch_folders_count_threads') == 1
 			$searches.clear
 			folders.each do |name, search|
 				q = $curbuf.query(search)
@@ -919,7 +917,8 @@ ruby << EOF
 				if mime_type != "text/html"
 					text = decoded
 				else
-					IO.popen("elinks --dump", "w+") do |pipe|
+					IO.popen(VIM::evaluate('exists("g:notmuch_html_converter") ? ' +
+							'g:notmuch_html_converter : "elinks --dump"'), "w+") do |pipe|
 						pipe.write(decode_body)
 						pipe.close_write
 						text = pipe.read
@@ -927,6 +926,16 @@ ruby << EOF
 				end
 				text
 			end
+
+			def present
+				buffer = ''
+				header.fields.each do |f|
+					buffer << "%s: %s\r\n" % [f.name, f.to_s]
+				end
+				buffer << "\r\n"
+				buffer << body.to_s
+				buffer
+			end
 		end
 	end