Import notmuch_0.38.2.orig.tar.xz

[dgit import orig notmuch_0.38.2.orig.tar.xz]

43 files changed, 16791 insertions, 0 deletions

diff --git a/lib/Makefile b/lib/Makefile
new file mode 100644
index 00000000..de492a7c
--- /dev/null
+++ b/lib/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C .. all
+
+.DEFAULT:
+	$(MAKE) -C .. $@
diff --git a/lib/Makefile.local b/lib/Makefile.local
new file mode 100644
index 00000000..4e766305
--- /dev/null
+++ b/lib/Makefile.local
@@ -0,0 +1,99 @@
+# -*- makefile-gmake -*-
+
+dir := lib
+
+# The (often-reused) $dir works fine within targets/prerequisites,
+# but cannot be used reliably within commands, so copy its value to a
+# variable that is not reused.
+lib := $(dir)
+
+ifeq ($(PLATFORM),MACOSX)
+LIBRARY_SUFFIX = dylib
+# On OS X, library version numbers go before suffix.
+LINKER_NAME = libnotmuch.$(LIBRARY_SUFFIX)
+SONAME = libnotmuch.$(LIBNOTMUCH_VERSION_MAJOR).$(LIBRARY_SUFFIX)
+LIBNAME = libnotmuch.$(LIBNOTMUCH_VERSION_MAJOR).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE).$(LIBRARY_SUFFIX)
+LIBRARY_LINK_FLAG = -dynamiclib -install_name $(libdir)/$(SONAME) -compatibility_version $(LIBNOTMUCH_VERSION_MAJOR).$(LIBNOTMUCH_VERSION_MINOR) -current_version $(LIBNOTMUCH_VERSION_MAJOR).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE)
+else
+LIBRARY_SUFFIX = so
+LINKER_NAME = libnotmuch.$(LIBRARY_SUFFIX)
+SONAME = $(LINKER_NAME).$(LIBNOTMUCH_VERSION_MAJOR)
+LIBNAME = $(SONAME).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE)
+LIBRARY_LINK_FLAG = -shared -Wl,--version-script=$(srcdir)/$(lib)/notmuch.sym,-soname=$(SONAME) $(NO_UNDEFINED_LDFLAGS)
+ifeq ($(PLATFORM),OPENBSD)
+LIBRARY_LINK_FLAG += -lc
+endif
+ifeq ($(LIBDIR_IN_LDCONFIG),1)
+ifeq ($(DESTDIR),)
+LIBRARY_INSTALL_POST_COMMAND=ldconfig
+endif
+endif
+endif
+
+extra_cflags += -I$(srcdir)/$(dir) -fPIC -fvisibility=hidden
+extra_cxxflags += -fvisibility-inlines-hidden
+
+libnotmuch_c_srcs =		\
+	$(notmuch_compat_srcs)	\
+	$(dir)/filenames.c	\
+	$(dir)/string-list.c	\
+	$(dir)/message-file.c	\
+	$(dir)/message-id.c	\
+	$(dir)/messages.c	\
+	$(dir)/sha1.c		\
+	$(dir)/built-with.c	\
+	$(dir)/string-map.c	\
+	$(dir)/indexopts.c	\
+	$(dir)/tags.c
+
+libnotmuch_cxx_srcs =		\
+	$(dir)/database.cc	\
+	$(dir)/parse-time-vrp.cc	\
+	$(dir)/directory.cc	\
+	$(dir)/index.cc		\
+	$(dir)/message.cc	\
+	$(dir)/add-message.cc	\
+	$(dir)/message-property.cc \
+	$(dir)/query.cc		\
+	$(dir)/query-fp.cc      \
+	$(dir)/config.cc	\
+	$(dir)/regexp-fields.cc	\
+	$(dir)/thread.cc \
+	$(dir)/thread-fp.cc     \
+	$(dir)/features.cc	\
+	$(dir)/prefix.cc	\
+	$(dir)/open.cc		\
+	$(dir)/init.cc		\
+	$(dir)/parse-sexp.cc	\
+	$(dir)/sexp-fp.cc	\
+	$(dir)/lastmod-fp.cc
+
+libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o)
+
+$(dir)/libnotmuch.a: $(libnotmuch_modules)
+	$(call quiet,AR) rcs $@ $^
+
+$(dir)/$(LIBNAME): $(libnotmuch_modules) util/libnotmuch_util.a parse-time-string/libparse-time-string.a
+	$(call quiet,CXX $(CXXFLAGS)) $(libnotmuch_modules) $(FINAL_LIBNOTMUCH_LDFLAGS) $(LIBRARY_LINK_FLAG) -o $@ util/libnotmuch_util.a parse-time-string/libparse-time-string.a
+
+$(dir)/$(SONAME): $(dir)/$(LIBNAME)
+	ln -sf $(LIBNAME) $@
+
+$(dir)/$(LINKER_NAME): $(dir)/$(SONAME)
+	ln -sf $(LIBNAME) $@
+
+install: install-$(dir)
+
+install-$(dir): $(dir)/$(LIBNAME)
+	mkdir -p "$(DESTDIR)$(libdir)/"
+	install -m0644 "$(lib)/$(LIBNAME)" "$(DESTDIR)$(libdir)/"
+	ln -sf $(LIBNAME) "$(DESTDIR)$(libdir)/$(SONAME)"
+	ln -sf $(LIBNAME) "$(DESTDIR)$(libdir)/$(LINKER_NAME)"
+	mkdir -p "$(DESTDIR)$(includedir)"
+	install -m0644 "$(srcdir)/$(lib)/notmuch.h" "$(DESTDIR)$(includedir)/"
+	$(LIBRARY_INSTALL_POST_COMMAND)
+
+SRCS  := $(SRCS) $(libnotmuch_c_srcs) $(libnotmuch_cxx_srcs)
+CLEAN += $(libnotmuch_modules) $(dir)/$(SONAME) $(dir)/$(LINKER_NAME)
+CLEAN += $(dir)/$(LIBNAME) $(dir)/libnotmuch.a
+CLEAN += $(dir)/notmuch.h.gch
diff --git a/lib/add-message.cc b/lib/add-message.cc
new file mode 100644
index 00000000..b16748fd
--- /dev/null
+++ b/lib/add-message.cc
@@ -0,0 +1,605 @@
+#include "database-private.h"
+
+/* Parse a References header value, putting a (talloc'ed under 'ctx')
+ * copy of each referenced message-id into 'hash'.
+ *
+ * We explicitly avoid including any reference identical to
+ * 'message_id' in the result (to avoid mass confusion when a single
+ * message references itself cyclically---and yes, mail messages are
+ * not infrequent in the wild that do this---don't ask me why).
+ *
+ * Return the last reference parsed, if it is not equal to message_id.
+ */
+static char *
+parse_references (void *ctx,
+		  const char *message_id,
+		  GHashTable *hash,
+		  const char *refs)
+{
+    char *ref, *last_ref = NULL;
+
+    if (refs == NULL || *refs == '\0')
+	return NULL;
+
+    while (*refs) {
+	ref = _notmuch_message_id_parse (ctx, refs, &refs);
+
+	if (ref && strcmp (ref, message_id)) {
+	    g_hash_table_add (hash, ref);
+	    last_ref = ref;
+	}
+    }
+
+    /* The return value of this function is used to add a parent
+     * reference to the database.  We should avoid making a message
+     * its own parent, thus the above check.
+     */
+    return talloc_strdup (ctx, last_ref);
+}
+
+static const char *
+_notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
+{
+
+    notmuch->last_thread_id++;
+
+    sprintf (notmuch->thread_id_str, "%016" PRIx64, notmuch->last_thread_id);
+
+    notmuch->writable_xapian_db->set_metadata ("last_thread_id", notmuch->thread_id_str);
+
+    return notmuch->thread_id_str;
+}
+
+static char *
+_get_metadata_thread_id_key (void *ctx, const char *message_id)
+{
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (ctx, message_id);
+
+    return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
+			    message_id);
+}
+
+
+static notmuch_status_t
+_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
+				      void *ctx,
+				      const char *message_id,
+				      const char **thread_id_ret);
+
+
+/* Find the thread ID to which the message with 'message_id' belongs.
+ *
+ * Note: 'thread_id_ret' must not be NULL!
+ * On success '*thread_id_ret' is set to a newly talloced string belonging to
+ * 'ctx'.
+ *
+ * Note: If there is no message in the database with the given
+ * 'message_id' then a new thread_id will be allocated for this
+ * message ID and stored in the database metadata so that the
+ * thread ID can be looked up if the message is added to the database
+ * later.
+ */
+static notmuch_status_t
+_resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
+				  void *ctx,
+				  const char *message_id,
+				  const char **thread_id_ret)
+{
+    notmuch_private_status_t status;
+    notmuch_message_t *message;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
+	return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
+						     thread_id_ret);
+
+    /* Look for this message (regular or ghost) */
+    message = _notmuch_message_create_for_message_id (
+	notmuch, message_id, &status);
+    if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+	/* Message exists */
+	*thread_id_ret = talloc_steal (
+	    ctx, notmuch_message_get_thread_id (message));
+    } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	/* Message did not exist.  Give it a fresh thread ID and
+	 * populate this message as a ghost message. */
+	*thread_id_ret = talloc_strdup (
+	    ctx, _notmuch_database_generate_thread_id (notmuch));
+	if (! *thread_id_ret) {
+	    status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	} else {
+	    status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
+	    if (status == 0)
+		/* Commit the new ghost message */
+		_notmuch_message_sync (message);
+	}
+    } else {
+	/* Create failed. Fall through. */
+    }
+
+    notmuch_message_destroy (message);
+
+    return COERCE_STATUS (status, "Error creating ghost message");
+}
+
+/* Pre-ghost messages _resolve_message_id_to_thread_id */
+static notmuch_status_t
+_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
+				      void *ctx,
+				      const char *message_id,
+				      const char **thread_id_ret)
+{
+    notmuch_status_t status;
+    notmuch_message_t *message;
+    std::string thread_id_string;
+    char *metadata_key;
+    Xapian::WritableDatabase *db;
+
+    status = notmuch_database_find_message (notmuch, message_id, &message);
+
+    if (status)
+	return status;
+
+    if (message) {
+	*thread_id_ret = talloc_steal (ctx,
+				       notmuch_message_get_thread_id (message));
+
+	notmuch_message_destroy (message);
+
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    /* Message has not been seen yet.
+     *
+     * We may have seen a reference to it already, in which case, we
+     * can return the thread ID stored in the metadata. Otherwise, we
+     * generate a new thread ID and store it there.
+     */
+    db = notmuch->writable_xapian_db;
+    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
+    thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
+
+    if (thread_id_string.empty ()) {
+	*thread_id_ret = talloc_strdup (ctx,
+					_notmuch_database_generate_thread_id (notmuch));
+	db->set_metadata (metadata_key, *thread_id_ret);
+    } else {
+	*thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str ());
+    }
+
+    talloc_free (metadata_key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_merge_threads (notmuch_database_t *notmuch,
+		const char *winner_thread_id,
+		const char *loser_thread_id)
+{
+    Xapian::PostingIterator loser, loser_end;
+    notmuch_message_t *message = NULL;
+    notmuch_private_status_t private_status;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    _notmuch_database_find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);
+
+    for (; loser != loser_end; loser++) {
+	message = _notmuch_message_create (notmuch, notmuch,
+					   *loser, &private_status);
+	if (message == NULL) {
+	    ret = COERCE_STATUS (private_status,
+				 "Cannot find document for doc_id from query");
+	    goto DONE;
+	}
+
+	_notmuch_message_remove_term (message, "thread", loser_thread_id);
+	_notmuch_message_add_term (message, "thread", winner_thread_id);
+	_notmuch_message_sync (message);
+
+	notmuch_message_destroy (message);
+	message = NULL;
+    }
+
+  DONE:
+    if (message)
+	notmuch_message_destroy (message);
+
+    return ret;
+}
+
+static void
+_my_talloc_free_for_g_hash (void *ptr)
+{
+    talloc_free (ptr);
+}
+
+notmuch_status_t
+_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
+					   notmuch_message_t *message,
+					   notmuch_message_file_t *message_file,
+					   const char **thread_id)
+{
+    GHashTable *parents = NULL;
+    const char *refs, *in_reply_to, *in_reply_to_message_id, *strict_message_id = NULL;
+    const char *last_ref_message_id, *this_message_id;
+    GList *l, *keys = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    parents = g_hash_table_new_full (g_str_hash, g_str_equal,
+				     _my_talloc_free_for_g_hash, NULL);
+    this_message_id = notmuch_message_get_message_id (message);
+
+    refs = _notmuch_message_file_get_header (message_file, "references");
+    last_ref_message_id = parse_references (message,
+					    this_message_id,
+					    parents, refs);
+
+    in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
+    if (in_reply_to)
+	strict_message_id = _notmuch_message_id_parse_strict (message,
+							      in_reply_to);
+
+    in_reply_to_message_id = parse_references (message,
+					       this_message_id,
+					       parents, in_reply_to);
+
+    /* For the parent of this message, use
+     * 1) the In-Reply-To header, if it looks sane, otherwise
+     * 2) the last message ID of the References header, if available.
+     * 3) Otherwise, fall back to the first message ID in
+     * the In-Reply-To header.
+     */
+
+    if (strict_message_id) {
+	_notmuch_message_add_term (message, "replyto", strict_message_id);
+    } else if (last_ref_message_id) {
+	_notmuch_message_add_term (message, "replyto",
+				   last_ref_message_id);
+    } else if (in_reply_to_message_id) {
+	_notmuch_message_add_term (message, "replyto",
+				   in_reply_to_message_id);
+    }
+
+    keys = g_hash_table_get_keys (parents);
+    for (l = keys; l; l = l->next) {
+	char *parent_message_id;
+	const char *parent_thread_id = NULL;
+
+	parent_message_id = (char *) l->data;
+
+	_notmuch_message_add_term (message, "reference",
+				   parent_message_id);
+
+	ret = _resolve_message_id_to_thread_id (notmuch,
+						message,
+						parent_message_id,
+						&parent_thread_id);
+	if (ret)
+	    goto DONE;
+
+	if (*thread_id == NULL) {
+	    *thread_id = talloc_strdup (message, parent_thread_id);
+	    _notmuch_message_add_term (message, "thread", *thread_id);
+	} else if (strcmp (*thread_id, parent_thread_id)) {
+	    ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
+	    if (ret)
+		goto DONE;
+	}
+    }
+
+  DONE:
+    if (keys)
+	g_list_free (keys);
+    if (parents)
+	g_hash_table_unref (parents);
+
+    return ret;
+}
+
+static notmuch_status_t
+_notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
+					    notmuch_message_t *message,
+					    const char **thread_id)
+{
+    const char *message_id = notmuch_message_get_message_id (message);
+    Xapian::PostingIterator child, children_end;
+    notmuch_message_t *child_message = NULL;
+    const char *child_thread_id;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_private_status_t private_status;
+
+    _notmuch_database_find_doc_ids (notmuch, "reference", message_id, &child, &children_end);
+
+    for (; child != children_end; child++) {
+
+	child_message = _notmuch_message_create (message, notmuch,
+						 *child, &private_status);
+	if (child_message == NULL) {
+	    ret = COERCE_STATUS (private_status,
+				 "Cannot find document for doc_id from query");
+	    goto DONE;
+	}
+
+	child_thread_id = notmuch_message_get_thread_id (child_message);
+	if (*thread_id == NULL) {
+	    *thread_id = talloc_strdup (message, child_thread_id);
+	    _notmuch_message_add_term (message, "thread", *thread_id);
+	} else if (strcmp (*thread_id, child_thread_id)) {
+	    _notmuch_message_remove_term (child_message, "reference",
+					  message_id);
+	    _notmuch_message_sync (child_message);
+	    ret = _merge_threads (notmuch, *thread_id, child_thread_id);
+	    if (ret)
+		goto DONE;
+	}
+
+	notmuch_message_destroy (child_message);
+	child_message = NULL;
+    }
+
+  DONE:
+    if (child_message)
+	notmuch_message_destroy (child_message);
+
+    return ret;
+}
+
+/* Fetch and clear the stored thread_id for message, or NULL if none. */
+static char *
+_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
+			     notmuch_message_t *message)
+{
+    const char *message_id;
+    std::string stored_id;
+    char *metadata_key;
+
+    message_id = notmuch_message_get_message_id (message);
+    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
+
+    /* Check if we have already seen related messages to this one.
+     * If we have then use the thread_id that we stored at that time.
+     */
+    stored_id = notmuch->xapian_db->get_metadata (metadata_key);
+    if (stored_id.empty ()) {
+	return NULL;
+    } else {
+	/* Clear the metadata for this message ID. We don't need it
+	 * anymore. */
+	notmuch->writable_xapian_db->set_metadata (metadata_key, "");
+
+	return talloc_strdup (ctx, stored_id.c_str ());
+    }
+}
+
+/* Given a blank or ghost 'message' and its corresponding
+ * 'message_file' link it to existing threads in the database.
+ *
+ * First, if is_ghost, this retrieves the thread ID already stored in
+ * the message (which will be the case if a message was previously
+ * added that referenced this one).  If the message is blank
+ * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
+ * later in this function).  If the database does not support ghost
+ * messages, this checks for a thread ID stored in database metadata
+ * for this message ID.
+ *
+ * Second, we look at 'message_file' and its link-relevant headers
+ * (References and In-Reply-To) for message IDs.
+ *
+ * Finally, we look in the database for existing message that
+ * reference 'message'.
+ *
+ * In all cases, we assign to the current message the first thread ID
+ * found. We will also merge any existing, distinct threads where this
+ * message belongs to both, (which is not uncommon when messages are
+ * processed out of order).
+ *
+ * Finally, if no thread ID has been found through referenced messages, we
+ * call _notmuch_message_generate_thread_id to generate a new thread
+ * ID. This should only happen for new, top-level messages, (no
+ * References or In-Reply-To header in this message, and no previously
+ * added message refers to this message).
+ */
+static notmuch_status_t
+_notmuch_database_link_message (notmuch_database_t *notmuch,
+				notmuch_message_t *message,
+				notmuch_message_file_t *message_file,
+				bool is_ghost,
+				bool is_new)
+{
+    void *local = talloc_new (NULL);
+    notmuch_status_t status;
+    const char *thread_id = NULL;
+
+    /* Check if the message already had a thread ID */
+    if (! is_new) {
+	thread_id = notmuch_message_get_thread_id (message);
+    } else if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
+	if (is_ghost)
+	    thread_id = notmuch_message_get_thread_id (message);
+    } else {
+	thread_id = _consume_metadata_thread_id (local, notmuch, message);
+	if (thread_id)
+	    _notmuch_message_add_term (message, "thread", thread_id);
+    }
+
+    status = _notmuch_database_link_message_to_parents (notmuch, message,
+							message_file,
+							&thread_id);
+    if (status)
+	goto DONE;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
+	/* In general, it shouldn't be necessary to link children,
+	 * since the earlier indexing of those children will have
+	 * stored a thread ID for the missing parent.  However, prior
+	 * to ghost messages, these stored thread IDs were NOT
+	 * rewritten during thread merging (and there was no
+	 * performant way to do so), so if indexed children were
+	 * pulled into a different thread ID by a merge, it was
+	 * necessary to pull them *back* into the stored thread ID of
+	 * the parent.  With ghost messages, we just rewrite the
+	 * stored thread IDs during merging, so this workaround isn't
+	 * necessary. */
+	status = _notmuch_database_link_message_to_children (notmuch, message,
+							     &thread_id);
+	if (status)
+	    goto DONE;
+    }
+
+    /* If not part of any existing thread, generate a new thread ID. */
+    if (thread_id == NULL) {
+	thread_id = _notmuch_database_generate_thread_id (notmuch);
+
+	_notmuch_message_add_term (message, "thread", thread_id);
+    }
+
+  DONE:
+    talloc_free (local);
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_index_file (notmuch_database_t *notmuch,
+			     const char *filename,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_t **message_ret)
+{
+    notmuch_message_file_t *message_file;
+    notmuch_message_t *message = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
+    notmuch_private_status_t private_status;
+    notmuch_bool_t is_ghost = false, is_new = false;
+    notmuch_indexopts_t *def_indexopts = NULL;
+
+    const char *date;
+    const char *from, *to, *subject;
+    char *message_id = NULL;
+
+    if (message_ret)
+	*message_ret = NULL;
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    message_file = _notmuch_message_file_open (notmuch, filename);
+    if (message_file == NULL)
+	return NOTMUCH_STATUS_FILE_ERROR;
+
+    /* Adding a message may change many documents.  Do this all
+     * atomically. */
+    ret = notmuch_database_begin_atomic (notmuch);
+    if (ret)
+	goto DONE;
+
+    ret = _notmuch_message_file_get_headers (message_file,
+					     &from, &subject, &to, &date,
+					     &message_id);
+    if (ret)
+	goto DONE;
+
+    try {
+	/* Now that we have a message ID, we get a message object,
+	 * (which may or may not reference an existing document in the
+	 * database). */
+
+	message = _notmuch_message_create_for_message_id (notmuch,
+							  message_id,
+							  &private_status);
+
+	talloc_free (message_id);
+
+	/* We cannot call notmuch_message_get_flag for a new message */
+	switch (private_status) {
+	case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+	    is_ghost = false;
+	    is_new = true;
+	    break;
+	case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+	    ret = notmuch_message_get_flag_st (message, NOTMUCH_MESSAGE_FLAG_GHOST, &is_ghost);
+	    if (ret)
+		goto DONE;
+	    is_new = false;
+	    break;
+	default:
+	    ret = COERCE_STATUS (private_status,
+				 "Unexpected status value from _notmuch_message_create_for_message_id");
+	    goto DONE;
+	}
+
+	ret = _notmuch_message_add_filename (message, filename);
+	if (ret)
+	    goto DONE;
+
+	if (is_new || is_ghost) {
+	    _notmuch_message_add_term (message, "type", "mail");
+	    if (is_ghost)
+		/* Convert ghost message to a regular message */
+		_notmuch_message_remove_term (message, "type", "ghost");
+	}
+
+	ret = _notmuch_database_link_message (notmuch, message,
+					      message_file, is_ghost, is_new);
+	if (ret)
+	    goto DONE;
+
+	if (is_new || is_ghost)
+	    _notmuch_message_set_header_values (message, date, from, subject);
+
+	if (! indexopts) {
+	    def_indexopts = notmuch_database_get_default_indexopts (notmuch);
+	    indexopts = def_indexopts;
+	}
+
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
+	if (ret)
+	    goto DONE;
+
+	if (! is_new && ! is_ghost)
+	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	_notmuch_message_sync (message);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	goto DONE;
+    }
+
+  DONE:
+    if (def_indexopts)
+	notmuch_indexopts_destroy (def_indexopts);
+
+    if (message) {
+	if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
+	    *message_ret = message;
+	else
+	    notmuch_message_destroy (message);
+    }
+
+    if (message_file)
+	_notmuch_message_file_close (message_file);
+
+    ret2 = notmuch_database_end_atomic (notmuch);
+    if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
+	ret2 != NOTMUCH_STATUS_SUCCESS)
+	ret = ret2;
+
+    return ret;
+}
+
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *notmuch,
+			      const char *filename,
+			      notmuch_message_t **message_ret)
+{
+    return notmuch_database_index_file (notmuch, filename,
+					NULL,
+					message_ret);
+
+}
diff --git a/lib/built-with.c b/lib/built-with.c
new file mode 100644
index 00000000..275e72b8
--- /dev/null
+++ b/lib/built-with.c
@@ -0,0 +1,40 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+
+notmuch_bool_t
+notmuch_built_with (const char *name)
+{
+    if (STRNCMP_LITERAL (name, "compact") == 0) {
+	return true;
+    } else if (STRNCMP_LITERAL (name, "field_processor") == 0) {
+	return true;
+    } else if (STRNCMP_LITERAL (name, "retry_lock") == 0) {
+	return HAVE_XAPIAN_DB_RETRY_LOCK;
+    } else if (STRNCMP_LITERAL (name, "session_key") == 0) {
+	return true;
+    } else if (STRNCMP_LITERAL (name, "sexp_queries") == 0) {
+	return HAVE_SFSEXP;
+    } else {
+	return false;
+    }
+}
diff --git a/lib/config.cc b/lib/config.cc
new file mode 100644
index 00000000..6cd15fab
--- /dev/null
+++ b/lib/config.cc
@@ -0,0 +1,726 @@
+/* config.cc - API for database metadata
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <pwd.h>
+#include <netdb.h>
+
+static const std::string CONFIG_PREFIX = "C";
+
+struct _notmuch_config_list {
+    notmuch_database_t *notmuch;
+    Xapian::TermIterator iterator;
+    char *current_key;
+    char *current_val;
+};
+
+struct _notmuch_config_values {
+    const char *iterator;
+    size_t tok_len;
+    const char *string;
+    void *children; /* talloc_context */
+};
+
+struct _notmuch_config_pairs {
+    notmuch_string_map_iterator_t *iter;
+};
+
+static const char *_notmuch_config_key_to_string (notmuch_config_key_t key);
+static char *_expand_path (void *ctx, const char *key, const char *val);
+
+static int
+_notmuch_config_list_destroy (notmuch_config_list_t *list)
+{
+    /* invoke destructor w/o deallocating memory */
+    list->iterator.~TermIterator();
+    return 0;
+}
+
+notmuch_status_t
+notmuch_database_set_config (notmuch_database_t *notmuch,
+			     const char *key,
+			     const char *value)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    if (! notmuch->config) {
+	if ((status = _notmuch_config_load_from_database (notmuch)))
+	    return status;
+    }
+
+    try {
+	notmuch->writable_xapian_db->set_metadata (CONFIG_PREFIX + key, value);
+    } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	notmuch->exception_reported = true;
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred setting metadata: %s\n",
+			       error.get_msg ().c_str ());
+    }
+
+    if (status)
+	return status;
+
+    _notmuch_string_map_set (notmuch->config, key, value);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_metadata_value (notmuch_database_t *notmuch,
+		 const char *key,
+		 std::string &value)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    try {
+	value = notmuch->xapian_db->get_metadata (CONFIG_PREFIX + key);
+    } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	notmuch->exception_reported = true;
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred getting metadata: %s\n",
+			       error.get_msg ().c_str ());
+    }
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_get_config (notmuch_database_t *notmuch,
+			     const char *key,
+			     char **value)
+{
+    const char *stored_val;
+    notmuch_status_t status;
+
+    if (! notmuch->config) {
+	if ((status = _notmuch_config_load_from_database (notmuch)))
+	    return status;
+    }
+
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    stored_val = _notmuch_string_map_get (notmuch->config, key);
+    if (! stored_val) {
+	/* XXX in principle this API should be fixed so empty string
+	 * is distinguished from not found */
+	*value = strdup ("");
+    } else {
+	*value = strdup (stored_val);
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_get_config_list (notmuch_database_t *notmuch,
+				  const char *prefix,
+				  notmuch_config_list_t **out)
+{
+    notmuch_config_list_t *list = NULL;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    list = talloc (notmuch, notmuch_config_list_t);
+    if (! list) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    list->notmuch = notmuch;
+    list->current_key = NULL;
+    list->current_val = NULL;
+
+    try {
+
+	new(&(list->iterator)) Xapian::TermIterator (notmuch->xapian_db->metadata_keys_begin
+							 (CONFIG_PREFIX + (prefix ? prefix : "")));
+	talloc_set_destructor (list, _notmuch_config_list_destroy);
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred getting metadata iterator: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    *out = list;
+
+  DONE:
+    if (status) {
+	if (list) {
+	    talloc_free (list);
+	    if (status != NOTMUCH_STATUS_XAPIAN_EXCEPTION)
+		_notmuch_config_list_destroy (list);
+	}
+    } else {
+	talloc_set_destructor (list, _notmuch_config_list_destroy);
+    }
+
+    return status;
+}
+
+notmuch_bool_t
+notmuch_config_list_valid (notmuch_config_list_t *metadata)
+{
+    if (metadata->iterator == metadata->notmuch->xapian_db->metadata_keys_end ())
+	return false;
+
+    return true;
+}
+
+static inline char *
+_key_from_iterator (notmuch_config_list_t *list)
+{
+    return talloc_strdup (list, (*list->iterator).c_str () + CONFIG_PREFIX.length ());
+}
+
+const char *
+notmuch_config_list_key (notmuch_config_list_t *list)
+{
+    if (list->current_key)
+	talloc_free (list->current_key);
+
+    list->current_key = _key_from_iterator (list);
+
+    return list->current_key;
+}
+
+const char *
+notmuch_config_list_value (notmuch_config_list_t *list)
+{
+    std::string strval;
+    notmuch_status_t status;
+    char *key = _key_from_iterator (list);
+
+    /* TODO: better error reporting?? */
+    status = _metadata_value (list->notmuch, key, strval);
+    if (status)
+	return NULL;
+
+    if (list->current_val)
+	talloc_free (list->current_val);
+
+    list->current_val = talloc_strdup (list, strval.c_str ());
+    talloc_free (key);
+    return list->current_val;
+}
+
+void
+notmuch_config_list_move_to_next (notmuch_config_list_t *list)
+{
+    list->iterator++;
+}
+
+void
+notmuch_config_list_destroy (notmuch_config_list_t *list)
+{
+    talloc_free (list);
+}
+
+notmuch_status_t
+_notmuch_config_load_from_database (notmuch_database_t *notmuch)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    notmuch_config_list_t *list;
+
+    if (notmuch->config == NULL)
+	notmuch->config = _notmuch_string_map_create (notmuch);
+
+    if (unlikely (notmuch->config == NULL))
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    status = notmuch_database_get_config_list (notmuch, "", &list);
+    if (status)
+	return status;
+
+    for (; notmuch_config_list_valid (list); notmuch_config_list_move_to_next (list)) {
+	const char *key = notmuch_config_list_key (list);
+	char *normalized_val = NULL;
+
+	/* If we opened from a given path, do not overwrite it */
+	if (strcmp (key, "database.path") == 0 &&
+	    (notmuch->params & NOTMUCH_PARAM_DATABASE) &&
+	    notmuch->xapian_db)
+	    continue;
+
+	normalized_val = _expand_path (list, key, notmuch_config_list_value (list));
+	_notmuch_string_map_append (notmuch->config, key, normalized_val);
+	talloc_free (normalized_val);
+    }
+
+    return status;
+}
+
+notmuch_config_values_t *
+notmuch_config_get_values (notmuch_database_t *notmuch, notmuch_config_key_t key)
+{
+    const char *key_str = _notmuch_config_key_to_string (key);
+
+    if (! key_str)
+	return NULL;
+
+    return notmuch_config_get_values_string (notmuch, key_str);
+}
+
+notmuch_config_values_t *
+notmuch_config_get_values_string (notmuch_database_t *notmuch, const char *key_str)
+{
+    notmuch_config_values_t *values = NULL;
+    bool ok = false;
+
+    values = talloc (notmuch, notmuch_config_values_t);
+    if (unlikely (! values))
+	goto DONE;
+
+    values->children = talloc_new (values);
+
+    values->string = _notmuch_string_map_get (notmuch->config, key_str);
+    if (! values->string)
+	goto DONE;
+
+    values->iterator = strsplit_len (values->string, ';', &(values->tok_len));
+    ok = true;
+
+  DONE:
+    if (! ok) {
+	if (values)
+	    talloc_free (values);
+	return NULL;
+    }
+    return values;
+}
+
+notmuch_bool_t
+notmuch_config_values_valid (notmuch_config_values_t *values)
+{
+    if (! values)
+	return false;
+
+    return (values->iterator != NULL);
+}
+
+const char *
+notmuch_config_values_get (notmuch_config_values_t *values)
+{
+    return talloc_strndup (values->children, values->iterator, values->tok_len);
+}
+
+void
+notmuch_config_values_start (notmuch_config_values_t *values)
+{
+    if (values == NULL)
+	return;
+    if (values->children) {
+	talloc_free (values->children);
+    }
+
+    values->children = talloc_new (values);
+
+    values->iterator = strsplit_len (values->string, ';', &(values->tok_len));
+}
+
+void
+notmuch_config_values_move_to_next (notmuch_config_values_t *values)
+{
+    values->iterator += values->tok_len;
+    values->iterator = strsplit_len (values->iterator, ';', &(values->tok_len));
+}
+
+void
+notmuch_config_values_destroy (notmuch_config_values_t *values)
+{
+    talloc_free (values);
+}
+
+notmuch_config_pairs_t *
+notmuch_config_get_pairs (notmuch_database_t *notmuch,
+			  const char *prefix)
+{
+    notmuch_config_pairs_t *pairs = talloc (notmuch, notmuch_config_pairs_t);
+
+    pairs->iter = _notmuch_string_map_iterator_create (notmuch->config, prefix, false);
+    return pairs;
+}
+
+notmuch_bool_t
+notmuch_config_pairs_valid (notmuch_config_pairs_t *pairs)
+{
+    return _notmuch_string_map_iterator_valid (pairs->iter);
+}
+
+void
+notmuch_config_pairs_move_to_next (notmuch_config_pairs_t *pairs)
+{
+    _notmuch_string_map_iterator_move_to_next (pairs->iter);
+}
+
+const char *
+notmuch_config_pairs_key (notmuch_config_pairs_t *pairs)
+{
+    return _notmuch_string_map_iterator_key (pairs->iter);
+}
+
+const char *
+notmuch_config_pairs_value (notmuch_config_pairs_t *pairs)
+{
+    return _notmuch_string_map_iterator_value (pairs->iter);
+}
+
+void
+notmuch_config_pairs_destroy (notmuch_config_pairs_t *pairs)
+{
+    _notmuch_string_map_iterator_destroy (pairs->iter);
+    talloc_free (pairs);
+}
+
+static char *
+_expand_path (void *ctx, const char *key, const char *val)
+{
+    char *expanded_val;
+
+    if ((strcmp (key, "database.path") == 0 ||
+	 strcmp (key, "database.mail_root") == 0 ||
+	 strcmp (key, "database.hook_dir") == 0 ||
+	 strcmp (key, "database.backup_path") == 0 ) &&
+	val[0] != '/')
+	expanded_val = talloc_asprintf (ctx, "%s/%s", getenv ("HOME"), val);
+    else
+	expanded_val = talloc_strdup (ctx, val);
+
+    return expanded_val;
+}
+
+notmuch_status_t
+_notmuch_config_load_from_file (notmuch_database_t *notmuch,
+				GKeyFile *file,
+				char **status_string)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    gchar **groups = NULL, **keys, *val;
+
+    if (notmuch->config == NULL)
+	notmuch->config = _notmuch_string_map_create (notmuch);
+
+    if (unlikely (notmuch->config == NULL)) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    groups = g_key_file_get_groups (file, NULL);
+    for (gchar **grp = groups; *grp; grp++) {
+	keys = g_key_file_get_keys (file, *grp, NULL, NULL);
+	for (gchar **keys_p = keys; *keys_p; keys_p++) {
+	    char *absolute_key = talloc_asprintf (notmuch, "%s.%s", *grp,  *keys_p);
+	    char *normalized_val;
+	    GError *gerr = NULL;
+
+	    /* If we opened from a given path, do not overwrite it */
+	    if (strcmp (absolute_key, "database.path") == 0 &&
+		(notmuch->params & NOTMUCH_PARAM_DATABASE) &&
+		notmuch->xapian_db)
+		continue;
+
+	    val = g_key_file_get_string (file, *grp, *keys_p, &gerr);
+	    if (gerr) {
+		if (status_string)
+		    IGNORE_RESULT (asprintf (status_string,
+					     "GLib: %s\n",
+					     gerr->message));
+		g_error_free (gerr);
+	    }
+	    if (! val) {
+		status = NOTMUCH_STATUS_FILE_ERROR;
+		goto DONE;
+	    }
+
+	    normalized_val = _expand_path (notmuch, absolute_key, val);
+	    _notmuch_string_map_set (notmuch->config, absolute_key, normalized_val);
+	    g_free (val);
+	    talloc_free (absolute_key);
+	    talloc_free (normalized_val);
+	    if (status)
+		goto DONE;
+	}
+	g_strfreev (keys);
+    }
+
+  DONE:
+    if (groups)
+	g_strfreev (groups);
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_config_get_bool (notmuch_database_t *notmuch, notmuch_config_key_t key, notmuch_bool_t *val)
+{
+    const char *key_string, *val_string;
+
+    key_string = _notmuch_config_key_to_string (key);
+    if (! key_string) {
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+    }
+
+    val_string = _notmuch_string_map_get (notmuch->config, key_string);
+    if (! val_string) {
+	*val = FALSE;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    if (strcase_equal (val_string, "false") || strcase_equal (val_string, "no"))
+	*val = FALSE;
+    else if (strcase_equal (val_string, "true") || strcase_equal (val_string, "yes"))
+	*val = TRUE;
+    else
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static const char *
+_get_name_from_passwd_file (void *ctx)
+{
+    long pw_buf_size;
+    char *pw_buf;
+    struct passwd passwd, *ignored;
+    const char *name;
+    int e;
+
+    pw_buf_size = sysconf (_SC_GETPW_R_SIZE_MAX);
+    if (pw_buf_size == -1) pw_buf_size = 64;
+    pw_buf = (char *) talloc_size (ctx, pw_buf_size);
+
+    while ((e = getpwuid_r (getuid (), &passwd, pw_buf,
+			    pw_buf_size, &ignored)) == ERANGE) {
+	pw_buf_size = pw_buf_size * 2;
+	pw_buf = (char *) talloc_zero_size (ctx, pw_buf_size);
+    }
+
+    if (e == 0) {
+	char *comma = strchr (passwd.pw_gecos, ',');
+	if (comma)
+	    name = talloc_strndup (ctx, passwd.pw_gecos,
+				   comma - passwd.pw_gecos);
+	else
+	    name = talloc_strdup (ctx, passwd.pw_gecos);
+    } else {
+	name = talloc_strdup (ctx, "");
+    }
+
+    talloc_free (pw_buf);
+
+    return name;
+}
+
+static char *
+_get_username_from_passwd_file (void *ctx)
+{
+    long pw_buf_size;
+    char *pw_buf;
+    struct passwd passwd, *ignored;
+    char *name;
+    int e;
+
+    pw_buf_size = sysconf (_SC_GETPW_R_SIZE_MAX);
+    if (pw_buf_size == -1) pw_buf_size = 64;
+    pw_buf = (char *) talloc_zero_size (ctx, pw_buf_size);
+
+    while ((e = getpwuid_r (getuid (), &passwd, pw_buf,
+			    pw_buf_size, &ignored)) == ERANGE) {
+	pw_buf_size = pw_buf_size * 2;
+	pw_buf = (char *) talloc_zero_size (ctx, pw_buf_size);
+    }
+
+    if (e == 0)
+	name = talloc_strdup (ctx, passwd.pw_name);
+    else
+	name = talloc_strdup (ctx, "");
+
+    talloc_free (pw_buf);
+
+    return name;
+}
+
+static const char *
+_get_email_from_passwd_file (void *ctx)
+{
+    char *email;
+
+    char *username = _get_username_from_passwd_file (ctx);
+
+    email = talloc_asprintf (ctx, "%s@localhost", username);
+
+    talloc_free (username);
+    return email;
+}
+
+static const char *
+_notmuch_config_key_to_string (notmuch_config_key_t key)
+{
+    switch (key) {
+    case NOTMUCH_CONFIG_DATABASE_PATH:
+	return "database.path";
+    case NOTMUCH_CONFIG_MAIL_ROOT:
+	return "database.mail_root";
+    case NOTMUCH_CONFIG_HOOK_DIR:
+	return "database.hook_dir";
+    case NOTMUCH_CONFIG_BACKUP_DIR:
+	return "database.backup_dir";
+    case NOTMUCH_CONFIG_EXCLUDE_TAGS:
+	return "search.exclude_tags";
+    case NOTMUCH_CONFIG_NEW_TAGS:
+	return "new.tags";
+    case NOTMUCH_CONFIG_NEW_IGNORE:
+	return "new.ignore";
+    case NOTMUCH_CONFIG_SYNC_MAILDIR_FLAGS:
+	return "maildir.synchronize_flags";
+    case NOTMUCH_CONFIG_PRIMARY_EMAIL:
+	return "user.primary_email";
+    case NOTMUCH_CONFIG_OTHER_EMAIL:
+	return "user.other_email";
+    case NOTMUCH_CONFIG_USER_NAME:
+	return "user.name";
+    case NOTMUCH_CONFIG_AUTOCOMMIT:
+	return "database.autocommit";
+    case NOTMUCH_CONFIG_EXTRA_HEADERS:
+	return "show.extra_headers";
+    case NOTMUCH_CONFIG_INDEX_AS_TEXT:
+	return "index.as_text";
+    default:
+	return NULL;
+    }
+}
+
+static const char *
+_notmuch_config_default (notmuch_database_t *notmuch, notmuch_config_key_t key)
+{
+    char *path;
+    const char *name, *email;
+
+    switch (key) {
+    case NOTMUCH_CONFIG_DATABASE_PATH:
+	path = getenv ("MAILDIR");
+	if (path)
+	    path = talloc_strdup (notmuch, path);
+	else
+	    path = talloc_asprintf (notmuch, "%s/mail",
+				    getenv ("HOME"));
+	return path;
+    case NOTMUCH_CONFIG_MAIL_ROOT:
+	/* by default, mail root is the same as database path */
+	return notmuch_database_get_path (notmuch);
+    case NOTMUCH_CONFIG_EXCLUDE_TAGS:
+	return "";
+    case NOTMUCH_CONFIG_NEW_TAGS:
+	return "unread;inbox";
+    case NOTMUCH_CONFIG_SYNC_MAILDIR_FLAGS:
+	return "true";
+    case NOTMUCH_CONFIG_USER_NAME:
+	name = getenv ("NAME");
+	if (name)
+	    name = talloc_strdup (notmuch, name);
+	else
+	    name = _get_name_from_passwd_file (notmuch);
+	return name;
+    case NOTMUCH_CONFIG_PRIMARY_EMAIL:
+	email = getenv ("EMAIL");
+	if (email)
+	    email = talloc_strdup (notmuch, email);
+	else
+	    email = _get_email_from_passwd_file (notmuch);
+	return email;
+    case NOTMUCH_CONFIG_INDEX_AS_TEXT:
+    case NOTMUCH_CONFIG_NEW_IGNORE:
+	return "";
+    case NOTMUCH_CONFIG_AUTOCOMMIT:
+	return "8000";
+    case NOTMUCH_CONFIG_EXTRA_HEADERS:
+    case NOTMUCH_CONFIG_HOOK_DIR:
+    case NOTMUCH_CONFIG_BACKUP_DIR:
+    case NOTMUCH_CONFIG_OTHER_EMAIL:
+	return NULL;
+    default:
+    case NOTMUCH_CONFIG_LAST:
+	INTERNAL_ERROR ("illegal key enum %d", key);
+    }
+}
+
+notmuch_status_t
+_notmuch_config_load_defaults (notmuch_database_t *notmuch)
+{
+    notmuch_config_key_t key;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    if (notmuch->config == NULL)
+	notmuch->config = _notmuch_string_map_create (notmuch);
+
+    for (key = NOTMUCH_CONFIG_FIRST;
+	 key < NOTMUCH_CONFIG_LAST;
+	 key = notmuch_config_key_t (key + 1)) {
+	const char *val = notmuch_config_get (notmuch, key);
+	const char *key_string = _notmuch_config_key_to_string (key);
+
+	val = _notmuch_string_map_get (notmuch->config, key_string);
+	if (! val) {
+	    if (key == NOTMUCH_CONFIG_MAIL_ROOT && (notmuch->params & NOTMUCH_PARAM_SPLIT))
+		status = NOTMUCH_STATUS_NO_MAIL_ROOT;
+
+	    _notmuch_string_map_set (notmuch->config, key_string, _notmuch_config_default (notmuch,
+											   key));
+	}
+    }
+    return status;
+}
+
+const char *
+notmuch_config_get (notmuch_database_t *notmuch, notmuch_config_key_t key)
+{
+
+    return _notmuch_string_map_get (notmuch->config, _notmuch_config_key_to_string (key));
+}
+
+const char *
+notmuch_config_path (notmuch_database_t *notmuch)
+{
+    return notmuch->config_path;
+}
+
+notmuch_status_t
+notmuch_config_set (notmuch_database_t *notmuch, notmuch_config_key_t key, const char *val)
+{
+
+    return notmuch_database_set_config (notmuch, _notmuch_config_key_to_string (key), val);
+}
+
+void
+_notmuch_config_cache (notmuch_database_t *notmuch, notmuch_config_key_t key, const char *val)
+{
+    if (notmuch->config == NULL)
+	notmuch->config = _notmuch_string_map_create (notmuch);
+
+    _notmuch_string_map_set (notmuch->config, _notmuch_config_key_to_string (key), val);
+}
diff --git a/lib/database-private.h b/lib/database-private.h
new file mode 100644
index 00000000..61232f1a
--- /dev/null
+++ b/lib/database-private.h
@@ -0,0 +1,395 @@
+/* database-private.h - For peeking into the internals of notmuch_database_t
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_DATABASE_PRIVATE_H
+#define NOTMUCH_DATABASE_PRIVATE_H
+
+/* According to WG14/N1124, a C++ implementation won't provide us a
+ * macro like PRIx64 (which gives a printf format string for
+ * formatting a uint64_t as hexadecimal) unless we define
+ * __STDC_FORMAT_MACROS before including inttypes.h. That's annoying,
+ * but there it is.
+ */
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+
+#include "notmuch-private.h"
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
+#ifdef SILENCE_XAPIAN_DEPRECATION_WARNINGS
+#define XAPIAN_DEPRECATED(D) D
+#endif
+
+#include <xapian.h>
+
+#if HAVE_SFSEXP
+#include <sexp.h>
+#endif
+
+/* Bit masks for _notmuch_database::features.  Features are named,
+ * independent aspects of the database schema.
+ *
+ * A database stores the set of features that it "uses" (implicitly
+ * before database version 3 and explicitly as of version 3).
+ *
+ * A given library version will "recognize" a particular set of
+ * features; if a database uses a feature that the library does not
+ * recognize, the library will refuse to open it.  It is assumed the
+ * set of recognized features grows monotonically over time.  A
+ * library version will "implement" some subset of the recognized
+ * features: some operations may require that the database use (or not
+ * use) some feature, while other operations may support both
+ * databases that use and that don't use some feature.
+ *
+ * On disk, the database stores string names for these features (see
+ * the feature_names array).  These enum bit values are never
+ * persisted to disk and may change freely.
+ */
+enum _notmuch_features {
+    /* If set, file names are stored in "file-direntry" terms.  If
+     * unset, file names are stored in document data.
+     *
+     * Introduced: version 1. */
+    NOTMUCH_FEATURE_FILE_TERMS			= 1 << 0,
+
+    /* If set, directory timestamps are stored in documents with
+     * XDIRECTORY terms and relative paths.  If unset, directory
+     * timestamps are stored in documents with XTIMESTAMP terms and
+     * absolute paths.
+     *
+     * Introduced: version 1. */
+    NOTMUCH_FEATURE_DIRECTORY_DOCS		= 1 << 1,
+
+    /* If set, the from, subject, and message-id headers are stored in
+     * message document values.  If unset, message documents *may*
+     * have these values, but if the value is empty, it must be
+     * retrieved from the message file.
+     *
+     * Introduced: optional in version 1, required as of version 3.
+     */
+    NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES	= 1 << 2,
+
+    /* If set, folder terms are boolean and path terms exist.  If
+     * unset, folder terms are probabilistic and stemmed and path
+     * terms do not exist.
+     *
+     * Introduced: version 2. */
+    NOTMUCH_FEATURE_BOOL_FOLDER			= 1 << 3,
+
+    /* If set, missing messages are stored in ghost mail documents.
+     * If unset, thread IDs of ghost messages are stored as database
+     * metadata instead of in ghost documents.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_GHOSTS			= 1 << 4,
+
+
+    /* If set, then the database was created after the introduction of
+     * indexed mime types. If unset, then the database may contain a
+     * mixture of messages with indexed and non-indexed mime types.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_INDEXED_MIMETYPES		= 1 << 5,
+
+    /* If set, messages store the revision number of the last
+     * modification in NOTMUCH_VALUE_LAST_MOD.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_LAST_MOD			= 1 << 6,
+
+    /* If set, unprefixed terms are stored only for the message body,
+     * not for headers.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_UNPREFIX_BODY_ONLY		= 1 << 7,
+};
+
+/* In C++, a named enum is its own type, so define bitwise operators
+ * on _notmuch_features. */
+inline _notmuch_features
+operator| (_notmuch_features a, _notmuch_features b)
+{
+    return static_cast<_notmuch_features>(
+	static_cast<unsigned>(a) | static_cast<unsigned>(b));
+}
+
+inline _notmuch_features
+operator& (_notmuch_features a, _notmuch_features b)
+{
+    return static_cast<_notmuch_features>(
+	static_cast<unsigned>(a) & static_cast<unsigned>(b));
+}
+
+inline _notmuch_features
+operator~ (_notmuch_features a)
+{
+    return static_cast<_notmuch_features>(~static_cast<unsigned>(a));
+}
+
+inline _notmuch_features&
+operator|= (_notmuch_features &a, _notmuch_features b)
+{
+    a = a | b;
+    return a;
+}
+
+inline _notmuch_features&
+operator&= (_notmuch_features &a, _notmuch_features b)
+{
+    a = a & b;
+    return a;
+}
+
+/*
+ * Configuration options for xapian database fields */
+typedef enum {
+    NOTMUCH_FIELD_NO_FLAGS	= 0,
+    NOTMUCH_FIELD_EXTERNAL	= 1 << 0,
+    NOTMUCH_FIELD_PROBABILISTIC = 1 << 1,
+    NOTMUCH_FIELD_PROCESSOR	= 1 << 2,
+    NOTMUCH_FIELD_STRIP_TRAILING_SLASH	= 1 << 3,
+} notmuch_field_flag_t;
+
+/*
+ * define bitwise operators to hide casts */
+inline notmuch_field_flag_t
+operator| (notmuch_field_flag_t a, notmuch_field_flag_t b)
+{
+    return static_cast<notmuch_field_flag_t>(
+	static_cast<unsigned>(a) | static_cast<unsigned>(b));
+}
+
+inline notmuch_field_flag_t
+operator& (notmuch_field_flag_t a, notmuch_field_flag_t b)
+{
+    return static_cast<notmuch_field_flag_t>(
+	static_cast<unsigned>(a) & static_cast<unsigned>(b));
+}
+
+#define NOTMUCH_QUERY_PARSER_FLAGS (Xapian::QueryParser::FLAG_BOOLEAN | \
+				    Xapian::QueryParser::FLAG_PHRASE | \
+				    Xapian::QueryParser::FLAG_LOVEHATE | \
+				    Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE | \
+				    Xapian::QueryParser::FLAG_WILDCARD | \
+				    Xapian::QueryParser::FLAG_PURE_NOT)
+
+/*
+ * explicit and implied parameters to open */
+typedef enum {
+    NOTMUCH_PARAM_NONE		= 0,
+    /* database passed explicitely */
+    NOTMUCH_PARAM_DATABASE	= 1 << 0,
+    /* config file passed explicitely */
+    NOTMUCH_PARAM_CONFIG	= 1 << 1,
+    /* profile name passed explicitely */
+    NOTMUCH_PARAM_PROFILE	= 1 << 2,
+    /* split (e.g. XDG) configuration */
+    NOTMUCH_PARAM_SPLIT		= 1 << 3,
+} notmuch_open_param_t;
+
+/*
+ * define bitwise operators to hide casts */
+
+inline notmuch_open_param_t
+operator| (notmuch_open_param_t a, notmuch_open_param_t b)
+{
+    return static_cast<notmuch_open_param_t>(
+	static_cast<unsigned>(a) | static_cast<unsigned>(b));
+}
+
+inline notmuch_open_param_t&
+operator|= (notmuch_open_param_t &a, notmuch_open_param_t b)
+{
+    a = a | b;
+    return a;
+}
+
+inline notmuch_open_param_t
+operator& (notmuch_open_param_t a, notmuch_open_param_t b)
+{
+    return static_cast<notmuch_open_param_t>(
+	static_cast<unsigned>(a) & static_cast<unsigned>(b));
+}
+
+struct _notmuch_database {
+    bool exception_reported;
+
+    /* Path to actual database */
+    const char *xapian_path;
+
+    /* Path to config loaded, if any */
+    const char *config_path;
+
+    int atomic_nesting;
+    /* true if changes have been made in this atomic section */
+    bool atomic_dirty;
+    Xapian::Database *xapian_db;
+    Xapian::WritableDatabase *writable_xapian_db;
+    bool open;
+    /* Bit mask of features used by this database.  This is a
+     * bitwise-OR of NOTMUCH_FEATURE_* values (above). */
+    enum _notmuch_features features;
+
+    unsigned int last_doc_id;
+
+    /* 16 bytes (+ terminator) for hexadecimal representation of
+     * a 64-bit integer. */
+    char thread_id_str[17];
+    uint64_t last_thread_id;
+
+    /* How many transactions have successfully completed since we last committed */
+    int transaction_count;
+    /* when to commit and reset the counter */
+    int transaction_threshold;
+
+    /* error reporting; this value persists only until the
+     * next library call. May be NULL */
+    char *status_string;
+
+    /* Highest committed revision number.  Modifications are recorded
+     * under a higher revision number, which can be generated with
+     * notmuch_database_new_revision. */
+    unsigned long revision;
+    const char *uuid;
+
+    /* Keep track of the number of times the database has been re-opened
+     * (or other global invalidations of notmuch's caching)
+     */
+    unsigned long view;
+    Xapian::QueryParser *query_parser;
+    Xapian::Stem *stemmer;
+    Xapian::TermGenerator *term_gen;
+    Xapian::RangeProcessor *value_range_processor;
+    Xapian::RangeProcessor *date_range_processor;
+    Xapian::RangeProcessor *last_mod_range_processor;
+
+    /* XXX it's slightly gross to use two parallel string->string maps
+     * here, but at least they are small */
+    notmuch_string_map_t *user_prefix;
+    notmuch_string_map_t *user_header;
+
+    /* Cached and possibly overridden configuration */
+    notmuch_string_map_t *config;
+
+    /* Track what parameters were specified when opening */
+    notmuch_open_param_t params;
+
+    /* list of regular expressions to check for text indexing */
+    regex_t *index_as_text;
+    size_t index_as_text_length;
+};
+
+/* Prior to database version 3, features were implied by the database
+ * version number, so hard-code them for earlier versions. */
+#define NOTMUCH_FEATURES_V0 ((enum _notmuch_features) 0)
+#define NOTMUCH_FEATURES_V1 (NOTMUCH_FEATURES_V0 | NOTMUCH_FEATURE_FILE_TERMS | \
+			     NOTMUCH_FEATURE_DIRECTORY_DOCS)
+#define NOTMUCH_FEATURES_V2 (NOTMUCH_FEATURES_V1 | NOTMUCH_FEATURE_BOOL_FOLDER)
+
+/* Current database features.  If any of these are missing from a
+ * database, request an upgrade.
+ * NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES and
+ * NOTMUCH_FEATURE_INDEXED_MIMETYPES are not included because upgrade
+ * doesn't currently introduce the features (though brand new databases
+ * will have it). */
+#define NOTMUCH_FEATURES_CURRENT \
+    (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_DIRECTORY_DOCS | \
+     NOTMUCH_FEATURE_BOOL_FOLDER | NOTMUCH_FEATURE_GHOSTS | \
+     NOTMUCH_FEATURE_LAST_MOD)
+
+/* Return the list of terms from the given iterator matching a prefix.
+ * The prefix will be stripped from the strings in the returned list.
+ * The list will be allocated using ctx as the talloc context.
+ *
+ * The function returns NULL on failure.
+ */
+notmuch_string_list_t *
+_notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
+					 Xapian::TermIterator &end,
+					 const char *prefix);
+
+void
+_notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
+				const char *prefix_name,
+				const char *value,
+				Xapian::PostingIterator *begin,
+				Xapian::PostingIterator *end);
+
+#define NOTMUCH_DATABASE_VERSION 3
+
+/* features.cc */
+
+_notmuch_features
+_notmuch_database_parse_features (const void *ctx, const char *features, unsigned int version,
+				  char mode, char **incompat_out);
+
+char *
+_notmuch_database_print_features (const void *ctx, unsigned int features);
+
+/* prefix.cc */
+notmuch_status_t
+_notmuch_database_setup_standard_query_fields (notmuch_database_t *notmuch);
+
+notmuch_status_t
+_notmuch_database_setup_user_query_fields (notmuch_database_t *notmuch);
+
+#if __cplusplus
+/* query.cc */
+notmuch_status_t
+_notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch,
+				       std::string query_string,
+				       Xapian::Query &output,
+				       std::string &msg);
+
+notmuch_status_t
+_notmuch_query_expand (notmuch_database_t *notmuch, const char *field, Xapian::Query subquery,
+		       Xapian::Query &output, std::string &msg);
+
+/* regexp-fields.cc */
+notmuch_status_t
+_notmuch_regexp_to_query (notmuch_database_t *notmuch, Xapian::valueno slot, std::string field,
+			  std::string regexp_str,
+			  Xapian::Query &output, std::string &msg);
+
+/* thread-fp.cc */
+notmuch_status_t
+_notmuch_query_name_to_query (notmuch_database_t *notmuch, const std::string name,
+			      Xapian::Query &output);
+
+#if HAVE_SFSEXP
+/* parse-sexp.cc */
+notmuch_status_t
+_notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *querystr,
+				      Xapian::Query &output);
+#endif
+
+/* parse-time-vrp.h */
+notmuch_status_t
+_notmuch_date_strings_to_query (Xapian::valueno slot, const std::string &from, const std::string &to,
+				Xapian::Query &output, std::string &msg);
+
+/* lastmod-fp.h */
+notmuch_status_t
+_notmuch_lastmod_strings_to_query (notmuch_database_t *notmuch,
+				   const std::string &from, const std::string &to,
+				   Xapian::Query &output, std::string &msg);
+#endif
+#endif
diff --git a/lib/database.cc b/lib/database.cc
new file mode 100644
index 00000000..737a3f30
--- /dev/null
+++ b/lib/database.cc
@@ -0,0 +1,1589 @@
+/* database.cc - The database interfaces of the notmuch mail library
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "database-private.h"
+#include "string-util.h"
+
+#include <iostream>
+
+#include <sys/time.h>
+#include <sys/stat.h>
+#include <signal.h>
+#include <ftw.h>
+
+#include <glib.h>               /* g_free, GPtrArray, GHashTable */
+#include <glib-object.h>        /* g_type_init */
+
+#include <gmime/gmime.h>        /* g_mime_init */
+
+using namespace std;
+
+typedef struct {
+    const char *name;
+    const char *prefix;
+    notmuch_field_flag_t flags;
+} prefix_t;
+
+#define NOTMUCH_DATABASE_VERSION 3
+
+#define STRINGIFY(s) _SUB_STRINGIFY (s)
+#define _SUB_STRINGIFY(s) #s
+
+#define LOG_XAPIAN_EXCEPTION(message, error) _log_xapian_exception (__location__, message, error)
+
+static void
+_log_xapian_exception (const char *where, notmuch_database_t *notmuch,  const Xapian::Error error)
+{
+    _notmuch_database_log (notmuch,
+			   "A Xapian exception occurred at %s: %s\n",
+			   where,
+			   error.get_msg ().c_str ());
+    notmuch->exception_reported = true;
+}
+
+notmuch_database_mode_t
+_notmuch_database_mode (notmuch_database_t *notmuch)
+{
+    if (notmuch->writable_xapian_db)
+	return NOTMUCH_DATABASE_MODE_READ_WRITE;
+    else
+	return NOTMUCH_DATABASE_MODE_READ_ONLY;
+}
+
+/* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
+ *
+ * We currently have three different types of documents (mail, ghost,
+ * and directory) and also some metadata.
+ *
+ * There are two kinds of prefixes used in notmuch. There are the
+ * human friendly 'prefix names' like "thread:", which are also used
+ * in the query parser, and the actual prefix terms in the database
+ * (e.g. "G"). The correspondence is maintained in the file scope data
+ * structure 'prefix_table'.
+ *
+ * Mail document
+ * -------------
+ * A mail document is associated with a particular email message. It
+ * is stored in one or more files on disk and is uniquely identified
+ * by its "id" field (which is generally the message ID). It is
+ * indexed with the following prefixed terms which the database uses
+ * to construct threads, etc.:
+ *
+ *    Single terms of given prefix:
+ *
+ *	type:	mail
+ *
+ *	id:	Unique ID of mail. This is from the Message-ID header
+ *		if present and not too long (see NOTMUCH_MESSAGE_ID_MAX).
+ *		If it's present and too long, then we use
+ *		"notmuch-sha1-<sha1_sum_of_message_id>".
+ *              If this header is not present, we use
+ *		"notmuch-sha1-<sha1_sum_of_entire_file>".
+ *
+ *	thread:	The ID of the thread to which the mail belongs
+ *
+ *	replyto: The ID from the In-Reply-To header of the mail (if any).
+ *
+ *    Multiple terms of given prefix:
+ *
+ *	reference: All message IDs from In-Reply-To and References
+ *		   headers in the message.
+ *
+ *	tag:	   Any tags associated with this message by the user.
+ *
+ *	file-direntry:  A colon-separated pair of values
+ *		        (INTEGER:STRING), where INTEGER is the
+ *		        document ID of a directory document, and
+ *		        STRING is the name of a file within that
+ *		        directory for this mail message.
+ *
+ *      property:       Has a property with key=value
+ *                 FIXME: if no = is present, should match on any value
+ *
+ *    A mail document also has four values:
+ *
+ *	TIMESTAMP:	The time_t value corresponding to the message's
+ *			Date header.
+ *
+ *	MESSAGE_ID:	The unique ID of the mail mess (see "id" above)
+ *
+ *	FROM:		The value of the "From" header
+ *
+ *	SUBJECT:	The value of the "Subject" header
+ *
+ *	LAST_MOD:	The revision number as of the last tag or
+ *			filename change.
+ *
+ * The prefixed terms described above are also searchable without an
+ * explicit field name, but as of notmuch 0.29 this is due to
+ * query-parser setup, not extra terms in the database.  In addition,
+ * terms from the content of the message are added without a prefix
+ * for use by the user in searching. Note that the prefix name "body"
+ * is used to refer to the empty prefix string in the database.
+ *
+ * The path of the containing folder is added with the "folder" prefix
+ * (see _notmuch_message_add_folder_terms).  Sub-paths of the the path
+ * of the mail message are added with the "path" prefix.
+ *
+ * The data portion of a mail document is empty.
+ *
+ * Ghost mail document [if NOTMUCH_FEATURE_GHOSTS]
+ * -----------------------------------------------
+ * A ghost mail document is like a mail document, but where we don't
+ * have the message content.  These are used to track thread reference
+ * information for messages we haven't received.
+ *
+ * A ghost mail document has type: ghost; id and thread fields that
+ * are identical to the mail document fields; and a MESSAGE_ID value.
+ *
+ * Directory document
+ * ------------------
+ * A directory document is used by a client of the notmuch library to
+ * maintain data necessary to allow for efficient polling of mail
+ * directories.
+ *
+ * All directory documents contain one term:
+ *
+ *	directory:	The directory path (relative to the database path)
+ *			Or the SHA1 sum of the directory path (if the
+ *			path itself is too long to fit in a Xapian
+ *			term).
+ *
+ * And all directory documents for directories other than top-level
+ * directories also contain the following term:
+ *
+ *	directory-direntry: A colon-separated pair of values
+ *		            (INTEGER:STRING), where INTEGER is the
+ *		            document ID of the parent directory
+ *		            document, and STRING is the name of this
+ *		            directory within that parent.
+ *
+ * All directory documents have a single value:
+ *
+ *	TIMESTAMP:	The mtime of the directory (at last scan)
+ *
+ * The data portion of a directory document contains the path of the
+ * directory (relative to the database path).
+ *
+ * Database metadata
+ * -----------------
+ * Xapian allows us to store arbitrary name-value pairs as
+ * "metadata". We currently use the following metadata names with the
+ * given meanings:
+ *
+ *	version		The database schema version, (which is distinct
+ *			from both the notmuch package version (see
+ *			notmuch --version) and the libnotmuch library
+ *			version. The version is stored as an base-10
+ *			ASCII integer. The initial database version
+ *			was 1, (though a schema existed before that
+ *			were no "version" database value existed at
+ *			all). Successive versions are allocated as
+ *			changes are made to the database (such as by
+ *			indexing new fields).
+ *
+ *	features	The set of features supported by this
+ *			database. This consists of a set of
+ *			'\n'-separated lines, where each is a feature
+ *			name, a '\t', and compatibility flags.  If the
+ *			compatibility flags contain 'w', then the
+ *			opener must support this feature to safely
+ *			write this database.  If the compatibility
+ *			flags contain 'r', then the opener must
+ *			support this feature to read this database.
+ *			Introduced in database version 3.
+ *
+ *	last_thread_id	The last thread ID generated. This is stored
+ *			as a 16-byte hexadecimal ASCII representation
+ *			of a 64-bit unsigned integer. The first ID
+ *			generated is 1 and the value will be
+ *			incremented for each thread ID.
+ *
+ *	C*		metadata keys starting with C indicate
+ *			configuration data. It can be managed with the
+ *			n_database_*config* API.  There is a convention
+ *			of hierarchical keys separated by '.' (e.g.
+ *			query.notmuch stores the value for the named
+ *			query 'notmuch'), but it is not enforced by the
+ *			API.
+ *
+ * Obsolete metadata
+ * -----------------
+ *
+ * If ! NOTMUCH_FEATURE_GHOSTS, there are no ghost mail documents.
+ * Instead, the database has the following additional database
+ * metadata:
+ *
+ *	thread_id_*	A pre-allocated thread ID for a particular
+ *			message. This is actually an arbitrarily large
+ *			family of metadata name. Any particular name is
+ *			formed by concatenating "thread_id_" with a message
+ *			ID (or the SHA1 sum of a message ID if it is very
+ *			long---see description of 'id' in the mail
+ *			document). The value stored is a thread ID.
+ *
+ *			These thread ID metadata values are stored
+ *			whenever a message references a parent message
+ *			that does not yet exist in the database. A
+ *			thread ID will be allocated and stored, and if
+ *			the message is later added, the stored thread
+ *			ID will be used (and the metadata value will
+ *			be cleared).
+ *
+ *			Even before a message is added, it's
+ *			pre-allocated thread ID is useful so that all
+ *			descendant messages that reference this common
+ *			parent can be recognized as belonging to the
+ *			same thread.
+ */
+
+
+notmuch_string_map_iterator_t *
+_notmuch_database_user_headers (notmuch_database_t *notmuch)
+{
+    return _notmuch_string_map_iterator_create (notmuch->user_header, "", false);
+}
+
+const char *
+notmuch_status_to_string (notmuch_status_t status)
+{
+    switch (status) {
+    case NOTMUCH_STATUS_SUCCESS:
+	return "No error occurred";
+    case NOTMUCH_STATUS_OUT_OF_MEMORY:
+	return "Out of memory";
+    case NOTMUCH_STATUS_READ_ONLY_DATABASE:
+	return "Attempt to write to a read-only database";
+    case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
+	return "A Xapian exception occurred";
+    case NOTMUCH_STATUS_FILE_ERROR:
+	return "Something went wrong trying to read or write a file";
+    case NOTMUCH_STATUS_FILE_NOT_EMAIL:
+	return "File is not an email";
+    case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
+	return "Message ID is identical to a message in database";
+    case NOTMUCH_STATUS_NULL_POINTER:
+	return "Erroneous NULL pointer";
+    case NOTMUCH_STATUS_TAG_TOO_LONG:
+	return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
+    case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
+	return "Unbalanced number of calls to notmuch_message_freeze/thaw";
+    case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
+	return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic";
+    case NOTMUCH_STATUS_UNSUPPORTED_OPERATION:
+	return "Unsupported operation";
+    case NOTMUCH_STATUS_UPGRADE_REQUIRED:
+	return "Operation requires a database upgrade";
+    case NOTMUCH_STATUS_PATH_ERROR:
+	return "Path supplied is illegal for this function";
+    case NOTMUCH_STATUS_IGNORED:
+	return "Argument was ignored";
+    case NOTMUCH_STATUS_ILLEGAL_ARGUMENT:
+	return "Illegal argument for function";
+    case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
+	return "Crypto protocol missing, malformed, or unintelligible";
+    case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
+	return "Crypto engine initialization failure";
+    case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
+	return "Unknown crypto protocol";
+    case NOTMUCH_STATUS_NO_CONFIG:
+	return "No configuration file found";
+    case NOTMUCH_STATUS_NO_DATABASE:
+	return "No database found";
+    case NOTMUCH_STATUS_DATABASE_EXISTS:
+	return "Database exists, not recreated";
+    case NOTMUCH_STATUS_BAD_QUERY_SYNTAX:
+	return "Syntax error in query";
+    case NOTMUCH_STATUS_NO_MAIL_ROOT:
+	return "No mail root found";
+    default:
+    case NOTMUCH_STATUS_LAST_STATUS:
+	return "Unknown error status value";
+    }
+}
+
+void
+_notmuch_database_log (notmuch_database_t *notmuch,
+		       const char *format,
+		       ...)
+{
+    va_list va_args;
+
+    va_start (va_args, format);
+
+    if (notmuch->status_string)
+	talloc_free (notmuch->status_string);
+
+    notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
+    va_end (va_args);
+}
+
+void
+_notmuch_database_log_append (notmuch_database_t *notmuch,
+			      const char *format,
+			      ...)
+{
+    va_list va_args;
+
+    va_start (va_args, format);
+
+    if (notmuch->status_string)
+	notmuch->status_string = talloc_vasprintf_append (notmuch->status_string, format, va_args);
+    else
+	notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
+
+    va_end (va_args);
+}
+
+static void
+find_doc_ids_for_term (notmuch_database_t *notmuch,
+		       const char *term,
+		       Xapian::PostingIterator *begin,
+		       Xapian::PostingIterator *end)
+{
+    *begin = notmuch->xapian_db->postlist_begin (term);
+
+    *end = notmuch->xapian_db->postlist_end (term);
+}
+
+void
+_notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
+				const char *prefix_name,
+				const char *value,
+				Xapian::PostingIterator *begin,
+				Xapian::PostingIterator *end)
+{
+    char *term;
+
+    term = talloc_asprintf (notmuch, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    find_doc_ids_for_term (notmuch, term, begin, end);
+
+    talloc_free (term);
+}
+
+notmuch_private_status_t
+_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
+				      const char *prefix_name,
+				      const char *value,
+				      unsigned int *doc_id)
+{
+    Xapian::PostingIterator i, end;
+
+    _notmuch_database_find_doc_ids (notmuch, prefix_name, value, &i, &end);
+
+    if (i == end) {
+	*doc_id = 0;
+	return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+    }
+
+    *doc_id = *i;
+
+#if DEBUG_DATABASE_SANITY
+    i++;
+
+    if (i != end)
+	INTERNAL_ERROR ("Term %s:%s is not unique as expected.\n",
+			prefix_name, value);
+#endif
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+static Xapian::Document
+find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
+{
+    return notmuch->xapian_db->get_document (doc_id);
+}
+
+/* Generate a compressed version of 'message_id' of the form:
+ *
+ *	notmuch-sha1-<sha1_sum_of_message_id>
+ */
+char *
+_notmuch_message_id_compressed (void *ctx, const char *message_id)
+{
+    char *sha1, *compressed;
+
+    sha1 = _notmuch_sha1_of_string (message_id);
+
+    compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1);
+    free (sha1);
+
+    return compressed;
+}
+
+notmuch_status_t
+notmuch_database_find_message (notmuch_database_t *notmuch,
+			       const char *message_id,
+			       notmuch_message_t **message_ret)
+{
+    notmuch_private_status_t status;
+    unsigned int doc_id;
+
+    if (message_ret == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (notmuch, message_id);
+
+    try {
+	status = _notmuch_database_find_unique_doc_id (notmuch, "id",
+						       message_id, &doc_id);
+
+	if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
+	    *message_ret = NULL;
+	else {
+	    *message_ret = _notmuch_message_create (notmuch, notmuch, doc_id,
+						    NULL);
+	    if (*message_ret == NULL)
+		return NOTMUCH_STATUS_OUT_OF_MEMORY;
+	}
+
+	return NOTMUCH_STATUS_SUCCESS;
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred finding message: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	*message_ret = NULL;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+}
+
+notmuch_status_t
+_notmuch_database_ensure_writable (notmuch_database_t *notmuch)
+{
+    if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY) {
+	_notmuch_database_log (notmuch, "Cannot write to a read-only database.\n");
+	return NOTMUCH_STATUS_READ_ONLY_DATABASE;
+    }
+
+    if (! notmuch->open) {
+	_notmuch_database_log (notmuch, "Cannot write to a closed database.\n");
+	return NOTMUCH_STATUS_CLOSED_DATABASE;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Allocate a revision number for the next change. */
+unsigned long
+_notmuch_database_new_revision (notmuch_database_t *notmuch)
+{
+    unsigned long new_revision = notmuch->revision + 1;
+
+    /* If we're in an atomic section, hold off on updating the
+     * committed revision number until we commit the atomic section.
+     */
+    if (notmuch->atomic_nesting)
+	notmuch->atomic_dirty = true;
+    else
+	notmuch->revision = new_revision;
+
+    return new_revision;
+}
+
+notmuch_status_t
+notmuch_database_close (notmuch_database_t *notmuch)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    /* Many Xapian objects (and thus notmuch objects) hold references to
+     * the database, so merely deleting the database may not suffice to
+     * close it.  Thus, we explicitly close it here. */
+    if (notmuch->open) {
+	try {
+	    /* Close the database.  This implicitly flushes
+	     * outstanding changes. If there is an open (non-flushed)
+	     * transaction, ALL pending changes will be discarded */
+	    notmuch->xapian_db->close ();
+	} catch (const Xapian::Error &error) {
+	    status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	    if (! notmuch->exception_reported) {
+		_notmuch_database_log (notmuch,
+				       "Error: A Xapian exception occurred closing database: %s\n",
+				       error.get_msg ().c_str ());
+	    }
+	}
+    }
+    notmuch->open = false;
+    return status;
+}
+
+static int
+unlink_cb (const char *path,
+	   unused (const struct stat *sb),
+	   unused (int type),
+	   unused (struct FTW *ftw))
+{
+    return remove (path);
+}
+
+static int
+rmtree (const char *path)
+{
+    return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
+}
+
+class NotmuchCompactor : public Xapian::Compactor
+{
+    notmuch_compact_status_cb_t status_cb;
+    void *status_closure;
+
+public:
+    NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) :
+	status_cb (cb), status_closure (closure)
+    {
+    }
+
+    virtual void
+    set_status (const std::string &table, const std::string &status)
+    {
+	char *msg;
+
+	if (status_cb == NULL)
+	    return;
+
+	if (status.length () == 0)
+	    msg = talloc_asprintf (NULL, "compacting table %s", table.c_str ());
+	else
+	    msg = talloc_asprintf (NULL, "     %s", status.c_str ());
+
+	if (msg == NULL) {
+	    return;
+	}
+
+	status_cb (msg, status_closure);
+	talloc_free (msg);
+    }
+};
+
+/* Compacts the given database, optionally saving the original database
+ * in backup_path. Additionally, a callback function can be provided to
+ * give the user feedback on the progress of the (likely long-lived)
+ * compaction process.
+ *
+ * The backup path must point to a directory on the same volume as the
+ * original database. Passing a NULL backup_path will result in the
+ * uncompacted database being deleted after compaction has finished.
+ * Note that the database write lock will be held during the
+ * compaction process to protect data integrity.
+ */
+notmuch_status_t
+notmuch_database_compact (const char *path,
+			  const char *backup_path,
+			  notmuch_compact_status_cb_t status_cb,
+			  void *closure)
+{
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_database_t *notmuch = NULL;
+    char *message = NULL;
+
+    ret = notmuch_database_open_with_config (path,
+					     NOTMUCH_DATABASE_MODE_READ_WRITE,
+					     "",
+					     NULL,
+					     &notmuch,
+					     &message);
+    if (ret) {
+	if (status_cb) status_cb (message, closure);
+	return ret;
+    }
+
+    _notmuch_config_cache (notmuch, NOTMUCH_CONFIG_DATABASE_PATH, path);
+
+    return notmuch_database_compact_db (notmuch,
+					backup_path,
+					status_cb,
+					closure);
+}
+
+notmuch_status_t
+notmuch_database_compact_db (notmuch_database_t *notmuch,
+			     const char *backup_path,
+			     notmuch_compact_status_cb_t status_cb,
+			     void *closure)
+{
+    void *local;
+    const char *xapian_path, *compact_xapian_path;
+    const char *path;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    struct stat statbuf;
+    bool keep_backup;
+    char *message;
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    path = notmuch_config_get (notmuch, NOTMUCH_CONFIG_DATABASE_PATH);
+    if (! path)
+	return NOTMUCH_STATUS_PATH_ERROR;
+
+    local = talloc_new (NULL);
+    if (! local)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    ret = _notmuch_choose_xapian_path (local, path, &xapian_path, &message);
+    if (ret)
+	goto DONE;
+
+    if (! (compact_xapian_path = talloc_asprintf (local, "%s.compact", xapian_path))) {
+	ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    if (backup_path == NULL) {
+	if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) {
+	    ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	    goto DONE;
+	}
+	keep_backup = false;
+    } else {
+	keep_backup = true;
+    }
+
+    if (stat (backup_path, &statbuf) != -1) {
+	_notmuch_database_log (notmuch, "Path already exists: %s\n", backup_path);
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+    if (errno != ENOENT) {
+	_notmuch_database_log (notmuch, "Unknown error while stat()ing path: %s\n",
+			       strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    /* Unconditionally attempt to remove old work-in-progress database (if
+     * any). This is "protected" by database lock. If this fails due to write
+     * errors (etc), the following code will fail and provide error message.
+     */
+    (void) rmtree (compact_xapian_path);
+
+    try {
+	NotmuchCompactor compactor (status_cb, closure);
+	notmuch->xapian_db->compact (compact_xapian_path, Xapian::DBCOMPACT_NO_RENUMBER, 0,
+				     compactor);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "Error while compacting: %s\n", error.get_msg ().c_str ());
+	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	goto DONE;
+    }
+
+    if (rename (xapian_path, backup_path)) {
+	_notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
+			       xapian_path, backup_path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (rename (compact_xapian_path, xapian_path)) {
+	_notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
+			       compact_xapian_path, xapian_path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (! keep_backup) {
+	if (rmtree (backup_path)) {
+	    _notmuch_database_log (notmuch, "Error removing old database %s: %s\n",
+				   backup_path, strerror (errno));
+	    ret = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+    }
+
+  DONE:
+    if (notmuch) {
+	notmuch_status_t ret2;
+
+	const char *str = notmuch_database_status_string (notmuch);
+	if (status_cb && str)
+	    status_cb (str, closure);
+
+	ret2 = notmuch_database_destroy (notmuch);
+
+	/* don't clobber previous error status */
+	if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS)
+	    ret = ret2;
+    }
+
+    talloc_free (local);
+
+    return ret;
+}
+
+notmuch_status_t
+notmuch_database_destroy (notmuch_database_t *notmuch)
+{
+    notmuch_status_t status;
+    const char *talloc_report;
+
+    talloc_report = getenv ("NOTMUCH_TALLOC_REPORT");
+    if (talloc_report && strcmp (talloc_report, "") != 0) {
+	FILE *report = fopen (talloc_report, "a");
+	if (report) {
+	    talloc_report_full (notmuch, report);
+	}
+    }
+
+    status = notmuch_database_close (notmuch);
+
+    delete notmuch->term_gen;
+    notmuch->term_gen = NULL;
+    delete notmuch->query_parser;
+    notmuch->query_parser = NULL;
+    delete notmuch->xapian_db;
+    notmuch->xapian_db = NULL;
+    delete notmuch->value_range_processor;
+    notmuch->value_range_processor = NULL;
+    delete notmuch->date_range_processor;
+    notmuch->date_range_processor = NULL;
+    delete notmuch->last_mod_range_processor;
+    notmuch->last_mod_range_processor = NULL;
+    delete notmuch->stemmer;
+    notmuch->stemmer = NULL;
+
+    talloc_free (notmuch);
+
+    return status;
+}
+
+const char *
+notmuch_database_get_path (notmuch_database_t *notmuch)
+{
+    return notmuch_config_get (notmuch, NOTMUCH_CONFIG_DATABASE_PATH);
+}
+
+unsigned int
+notmuch_database_get_version (notmuch_database_t *notmuch)
+{
+    unsigned int version;
+    string version_string;
+    const char *str;
+    char *end;
+
+    try {
+	version_string = notmuch->xapian_db->get_metadata ("version");
+    } catch (const Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (notmuch, error);
+	return 0;
+    }
+
+    if (version_string.empty ())
+	return 0;
+
+    str = version_string.c_str ();
+    if (str == NULL || *str == '\0')
+	return 0;
+
+    version = strtoul (str, &end, 10);
+    if (*end != '\0')
+	INTERNAL_ERROR ("Malformed database version: %s", str);
+
+    return version;
+}
+
+notmuch_bool_t
+notmuch_database_needs_upgrade (notmuch_database_t *notmuch)
+{
+    unsigned int version;
+
+    if (_notmuch_database_mode (notmuch) != NOTMUCH_DATABASE_MODE_READ_WRITE)
+	return FALSE;
+
+    if (NOTMUCH_FEATURES_CURRENT & ~notmuch->features)
+	return TRUE;
+
+    version = notmuch_database_get_version (notmuch);
+
+    return (version > 0 && version < NOTMUCH_DATABASE_VERSION);
+}
+
+static volatile sig_atomic_t do_progress_notify = 0;
+
+static void
+handle_sigalrm (unused (int signal))
+{
+    do_progress_notify = 1;
+}
+
+/* Upgrade the current database.
+ *
+ * After opening a database in read-write mode, the client should
+ * check if an upgrade is needed (notmuch_database_needs_upgrade) and
+ * if so, upgrade with this function before making any modifications.
+ *
+ * The optional progress_notify callback can be used by the caller to
+ * provide progress indication to the user. If non-NULL it will be
+ * called periodically with 'count' as the number of messages upgraded
+ * so far and 'total' the overall number of messages that will be
+ * converted.
+ */
+notmuch_status_t
+notmuch_database_upgrade (notmuch_database_t *notmuch,
+			  void (*progress_notify)(void *closure,
+						  double progress),
+			  void *closure)
+{
+    void *local = talloc_new (NULL);
+    Xapian::TermIterator t, t_end;
+    Xapian::WritableDatabase *db;
+    struct sigaction action;
+    struct itimerval timerval;
+    bool timer_is_active = false;
+    enum _notmuch_features target_features, new_features;
+    notmuch_status_t status;
+    notmuch_private_status_t private_status;
+    notmuch_query_t *query = NULL;
+    unsigned int count = 0, total = 0;
+
+    if (_notmuch_database_mode (notmuch) != NOTMUCH_DATABASE_MODE_READ_WRITE)
+	return NOTMUCH_STATUS_READ_ONLY_DATABASE;
+
+    db = notmuch->writable_xapian_db;
+
+    target_features = notmuch->features | NOTMUCH_FEATURES_CURRENT;
+    new_features = NOTMUCH_FEATURES_CURRENT & ~notmuch->features;
+
+    if (! notmuch_database_needs_upgrade (notmuch))
+	return NOTMUCH_STATUS_SUCCESS;
+
+    if (progress_notify) {
+	/* Set up our handler for SIGALRM */
+	memset (&action, 0, sizeof (struct sigaction));
+	action.sa_handler = handle_sigalrm;
+	sigemptyset (&action.sa_mask);
+	action.sa_flags = SA_RESTART;
+	sigaction (SIGALRM, &action, NULL);
+
+	/* Then start a timer to send SIGALRM once per second. */
+	timerval.it_interval.tv_sec = 1;
+	timerval.it_interval.tv_usec = 0;
+	timerval.it_value.tv_sec = 1;
+	timerval.it_value.tv_usec = 0;
+	setitimer (ITIMER_REAL, &timerval, NULL);
+
+	timer_is_active = true;
+    }
+
+    /* Figure out how much total work we need to do. */
+    if (new_features &
+	(NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
+	 NOTMUCH_FEATURE_LAST_MOD)) {
+	query = notmuch_query_create (notmuch, "");
+	unsigned msg_count;
+
+	status = notmuch_query_count_messages (query, &msg_count);
+	if (status)
+	    goto DONE;
+
+	total += msg_count;
+	notmuch_query_destroy (query);
+	query = NULL;
+    }
+    if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
+	t_end = db->allterms_end ("XTIMESTAMP");
+	for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++)
+	    ++total;
+    }
+    if (new_features & NOTMUCH_FEATURE_GHOSTS) {
+	/* The ghost message upgrade converts all thread_id_*
+	 * metadata values into ghost message documents. */
+	t_end = db->metadata_keys_end ("thread_id_");
+	for (t = db->metadata_keys_begin ("thread_id_"); t != t_end; ++t)
+	    ++total;
+    }
+
+    /* Perform the upgrade in a transaction. */
+    db->begin_transaction (true);
+
+    /* Set the target features so we write out changes in the desired
+     * format. */
+    notmuch->features = target_features;
+
+    /* Perform per-message upgrades. */
+    if (new_features &
+	(NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
+	 NOTMUCH_FEATURE_LAST_MOD)) {
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+	char *filename;
+
+	query = notmuch_query_create (notmuch, "");
+
+	status = notmuch_query_search_messages (query, &messages);
+	if (status)
+	    goto DONE;
+	for (;
+	     notmuch_messages_valid (messages);
+	     notmuch_messages_move_to_next (messages)) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    /* Before version 1, each message document had its
+	     * filename in the data field. Copy that into the new
+	     * format by calling notmuch_message_add_filename.
+	     */
+	    if (new_features & NOTMUCH_FEATURE_FILE_TERMS) {
+		filename = _notmuch_message_talloc_copy_data (message);
+		if (filename && *filename != '\0') {
+		    _notmuch_message_add_filename (message, filename);
+		    _notmuch_message_clear_data (message);
+		}
+		talloc_free (filename);
+	    }
+
+	    /* Prior to version 2, the "folder:" prefix was
+	     * probabilistic and stemmed. Change it to the current
+	     * boolean prefix. Add "path:" prefixes while at it.
+	     */
+	    if (new_features & NOTMUCH_FEATURE_BOOL_FOLDER)
+		_notmuch_message_upgrade_folder (message);
+
+	    /* Prior to NOTMUCH_FEATURE_LAST_MOD, messages did not
+	     * track modification revisions.  Give all messages the
+	     * next available revision; since we just started tracking
+	     * revisions for this database, that will be 1.
+	     */
+	    if (new_features & NOTMUCH_FEATURE_LAST_MOD)
+		_notmuch_message_upgrade_last_mod (message);
+
+	    _notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+	query = NULL;
+    }
+
+    /* Perform per-directory upgrades. */
+
+    /* Before version 1 we stored directory timestamps in
+     * XTIMESTAMP documents instead of the current XDIRECTORY
+     * documents. So copy those as well. */
+    if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
+	t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
+
+	for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
+	     t != t_end;
+	     t++) {
+	    Xapian::PostingIterator p, p_end;
+	    std::string term = *t;
+
+	    p_end = notmuch->xapian_db->postlist_end (term);
+
+	    for (p = notmuch->xapian_db->postlist_begin (term);
+		 p != p_end;
+		 p++) {
+		Xapian::Document document;
+		time_t mtime;
+		notmuch_directory_t *directory;
+
+		if (do_progress_notify) {
+		    progress_notify (closure, (double) count / total);
+		    do_progress_notify = 0;
+		}
+
+		document = find_document_for_doc_id (notmuch, *p);
+		mtime = Xapian::sortable_unserialise (
+		    document.get_value (NOTMUCH_VALUE_TIMESTAMP));
+
+		directory = _notmuch_directory_find_or_create (notmuch, term.c_str () + 10,
+							       NOTMUCH_FIND_CREATE, &status);
+		notmuch_directory_set_mtime (directory, mtime);
+		notmuch_directory_destroy (directory);
+
+		db->delete_document (*p);
+	    }
+
+	    ++count;
+	}
+    }
+
+    /* Perform metadata upgrades. */
+
+    /* Prior to NOTMUCH_FEATURE_GHOSTS, thread IDs for missing
+     * messages were stored as database metadata. Change these to
+     * ghost messages.
+     */
+    if (new_features & NOTMUCH_FEATURE_GHOSTS) {
+	notmuch_message_t *message;
+	std::string message_id, thread_id;
+
+	t_end = db->metadata_keys_end (NOTMUCH_METADATA_THREAD_ID_PREFIX);
+	for (t = db->metadata_keys_begin (NOTMUCH_METADATA_THREAD_ID_PREFIX);
+	     t != t_end; ++t) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message_id = (*t).substr (
+		strlen (NOTMUCH_METADATA_THREAD_ID_PREFIX));
+	    thread_id = db->get_metadata (*t);
+
+	    /* Create ghost message */
+	    message = _notmuch_message_create_for_message_id (
+		notmuch, message_id.c_str (), &private_status);
+	    if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+		/* Document already exists; ignore the stored thread ID */
+	    } else if (private_status ==
+		       NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+		private_status = _notmuch_message_initialize_ghost (
+		    message, thread_id.c_str ());
+		if (! private_status)
+		    _notmuch_message_sync (message);
+	    }
+
+	    if (private_status) {
+		_notmuch_database_log (notmuch,
+				       "Upgrade failed while creating ghost messages.\n");
+		status = COERCE_STATUS (private_status,
+					"Unexpected status from _notmuch_message_initialize_ghost");
+		goto DONE;
+	    }
+
+	    /* Clear saved metadata thread ID */
+	    db->set_metadata (*t, "");
+
+	    ++count;
+	}
+    }
+
+    status = NOTMUCH_STATUS_SUCCESS;
+    db->set_metadata ("features", _notmuch_database_print_features (local, notmuch->features));
+    db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
+
+  DONE:
+    if (status == NOTMUCH_STATUS_SUCCESS)
+	db->commit_transaction ();
+    else
+	db->cancel_transaction ();
+
+    if (timer_is_active) {
+	/* Now stop the timer. */
+	timerval.it_interval.tv_sec = 0;
+	timerval.it_interval.tv_usec = 0;
+	timerval.it_value.tv_sec = 0;
+	timerval.it_value.tv_usec = 0;
+	setitimer (ITIMER_REAL, &timerval, NULL);
+
+	/* And disable the signal handler. */
+	action.sa_handler = SIG_IGN;
+	sigaction (SIGALRM, &action, NULL);
+    }
+
+    if (query)
+	notmuch_query_destroy (query);
+
+    talloc_free (local);
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_begin_atomic (notmuch_database_t *notmuch)
+{
+    if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY ||
+	notmuch->atomic_nesting > 0)
+	goto DONE;
+
+    if (notmuch_database_needs_upgrade (notmuch))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    try {
+	notmuch->writable_xapian_db->begin_transaction (false);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred beginning transaction: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    notmuch->atomic_nesting++;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_end_atomic (notmuch_database_t *notmuch)
+{
+    Xapian::WritableDatabase *db;
+
+    if (notmuch->atomic_nesting == 0)
+	return NOTMUCH_STATUS_UNBALANCED_ATOMIC;
+
+    if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY ||
+	notmuch->atomic_nesting > 1)
+	goto DONE;
+
+    db = notmuch->writable_xapian_db;
+    try {
+	db->commit_transaction ();
+	notmuch->transaction_count++;
+
+	/* Xapian never flushes on a non-flushed commit, even if the
+	 * flush threshold is 1.  However, we rely on flushing to test
+	 * atomicity. On the other hand, we can't straight replace
+	 * XAPIAN_FLUSH_THRESHOLD with our autocommit counter, because
+	 * the former also applies outside notmuch atomic
+	 * commits. Hence the follow complicated  test */
+	const char *thresh = getenv ("XAPIAN_FLUSH_THRESHOLD");
+	if ((notmuch->transaction_threshold > 0 &&
+	     notmuch->transaction_count >= notmuch->transaction_threshold) ||
+	    (thresh && atoi (thresh) == 1)) {
+	    db->commit ();
+	    notmuch->transaction_count = 0;
+	}
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred committing transaction: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    if (notmuch->atomic_dirty) {
+	++notmuch->revision;
+	notmuch->atomic_dirty = false;
+    }
+
+  DONE:
+    notmuch->atomic_nesting--;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+unsigned long
+notmuch_database_get_revision (notmuch_database_t *notmuch,
+			       const char **uuid)
+{
+    if (uuid)
+	*uuid = notmuch->uuid;
+    return notmuch->revision;
+}
+
+/* We allow the user to use arbitrarily long paths for directories. But
+ * we have a term-length limit. So if we exceed that, we'll use the
+ * SHA-1 of the path for the database term.
+ *
+ * Note: This function may return the original value of 'path'. If it
+ * does not, then the caller is responsible to free() the returned
+ * value.
+ */
+const char *
+_notmuch_database_get_directory_db_path (const char *path)
+{
+    int term_len = strlen (_find_prefix ("directory")) + strlen (path);
+
+    if (term_len > NOTMUCH_TERM_MAX)
+	return _notmuch_sha1_of_string (path);
+    else
+	return path;
+}
+
+/* Given a path, split it into two parts: the directory part is all
+ * components except for the last, and the basename is that last
+ * component. Getting the return-value for either part is optional
+ * (the caller can pass NULL).
+ *
+ * The original 'path' can represent either a regular file or a
+ * directory---the splitting will be carried out in the same way in
+ * either case. Trailing slashes on 'path' will be ignored, and any
+ * cases of multiple '/' characters appearing in series will be
+ * treated as a single '/'.
+ *
+ * Allocation (if any) will have 'ctx' as the talloc owner. But
+ * pointers will be returned within the original path string whenever
+ * possible.
+ *
+ * Note: If 'path' is non-empty and contains no non-trailing slash,
+ * (that is, consists of a filename with no parent directory), then
+ * the directory returned will be an empty string. However, if 'path'
+ * is an empty string, then both directory and basename will be
+ * returned as NULL.
+ */
+notmuch_status_t
+_notmuch_database_split_path (void *ctx,
+			      const char *path,
+			      const char **directory,
+			      const char **basename)
+{
+    const char *slash;
+
+    if (path == NULL || *path == '\0') {
+	if (directory)
+	    *directory = NULL;
+	if (basename)
+	    *basename = NULL;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    /* Find the last slash (not counting a trailing slash), if any. */
+
+    slash = path + strlen (path) - 1;
+
+    /* First, skip trailing slashes. */
+    while (slash != path && *slash == '/')
+	--slash;
+
+    /* Then, find a slash. */
+    while (slash != path && *slash != '/') {
+	if (basename)
+	    *basename = slash;
+
+	--slash;
+    }
+
+    /* Finally, skip multiple slashes. */
+    while (slash != path && *(slash - 1) == '/')
+	--slash;
+
+    if (slash == path) {
+	if (directory)
+	    *directory = talloc_strdup (ctx, "");
+	if (basename)
+	    *basename = path;
+    } else {
+	if (directory)
+	    *directory = talloc_strndup (ctx, path, slash - path);
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Find the document ID of the specified directory.
+ *
+ * If (flags & NOTMUCH_FIND_CREATE), a new directory document will be
+ * created if one does not exist for 'path'.  Otherwise, if the
+ * directory document does not exist, this sets *directory_id to
+ * ((unsigned int)-1) and returns NOTMUCH_STATUS_SUCCESS.
+ */
+notmuch_status_t
+_notmuch_database_find_directory_id (notmuch_database_t *notmuch,
+				     const char *path,
+				     notmuch_find_flags_t flags,
+				     unsigned int *directory_id)
+{
+    notmuch_directory_t *directory;
+    notmuch_status_t status;
+
+    if (path == NULL) {
+	*directory_id = 0;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    directory = _notmuch_directory_find_or_create (notmuch, path, flags, &status);
+    if (status || ! directory) {
+	*directory_id = -1;
+	return status;
+    }
+
+    *directory_id = _notmuch_directory_get_document_id (directory);
+
+    notmuch_directory_destroy (directory);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+const char *
+_notmuch_database_get_directory_path (void *ctx,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id)
+{
+    Xapian::Document document;
+
+    document = find_document_for_doc_id (notmuch, doc_id);
+
+    return talloc_strdup (ctx, document.get_data ().c_str ());
+}
+
+/* Given a legal 'filename' for the database, (either relative to
+ * database path or absolute with initial components identical to
+ * database path), return a new string (with 'ctx' as the talloc
+ * owner) suitable for use as a direntry term value.
+ *
+ * If (flags & NOTMUCH_FIND_CREATE), the necessary directory documents
+ * will be created in the database as needed.  Otherwise, if the
+ * necessary directory documents do not exist, this sets
+ * *direntry to NULL and returns NOTMUCH_STATUS_SUCCESS.
+ */
+notmuch_status_t
+_notmuch_database_filename_to_direntry (void *ctx,
+					notmuch_database_t *notmuch,
+					const char *filename,
+					notmuch_find_flags_t flags,
+					char **direntry)
+{
+    const char *relative, *directory, *basename;
+    Xapian::docid directory_id;
+    notmuch_status_t status;
+
+    relative = _notmuch_database_relative_path (notmuch, filename);
+
+    status = _notmuch_database_split_path (ctx, relative,
+					   &directory, &basename);
+    if (status)
+	return status;
+
+    status = _notmuch_database_find_directory_id (notmuch, directory, flags,
+						  &directory_id);
+    if (status || directory_id == (unsigned int) -1) {
+	*direntry = NULL;
+	return status;
+    }
+
+    *direntry = talloc_asprintf (ctx, "%u:%s", directory_id, basename);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Given a legal 'path' for the database, return the relative path.
+ *
+ * The return value will be a pointer to the original path contents,
+ * and will be either the original string (if 'path' was relative) or
+ * a portion of the string (if path was absolute and begins with the
+ * database path).
+ */
+const char *
+_notmuch_database_relative_path (notmuch_database_t *notmuch,
+				 const char *path)
+{
+    const char *db_path, *relative;
+    unsigned int db_path_len;
+
+    db_path = notmuch_config_get (notmuch, NOTMUCH_CONFIG_MAIL_ROOT);
+    db_path_len = strlen (db_path);
+
+    relative = path;
+
+    if (*relative == '/') {
+	while (*relative == '/' && *(relative + 1) == '/')
+	    relative++;
+
+	if (strncmp (relative, db_path, db_path_len) == 0) {
+	    relative += db_path_len;
+	    while (*relative == '/')
+		relative++;
+	}
+    }
+
+    return relative;
+}
+
+notmuch_status_t
+notmuch_database_get_directory (notmuch_database_t *notmuch,
+				const char *path,
+				notmuch_directory_t **directory)
+{
+    notmuch_status_t status;
+
+    if (directory == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    *directory = NULL;
+
+    try {
+	*directory = _notmuch_directory_find_or_create (notmuch, path,
+							NOTMUCH_FIND_LOOKUP, &status);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred getting directory: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return status;
+}
+
+/* Allocate a document ID that satisfies the following criteria:
+ *
+ * 1. The ID does not exist for any document in the Xapian database
+ *
+ * 2. The ID was not previously returned from this function
+ *
+ * 3. The ID is the smallest integer satisfying (1) and (2)
+ *
+ * This function will trigger an internal error if these constraints
+ * cannot all be satisfied, (that is, the pool of available document
+ * IDs has been exhausted).
+ */
+unsigned int
+_notmuch_database_generate_doc_id (notmuch_database_t *notmuch)
+{
+    assert (notmuch->last_doc_id >= notmuch->xapian_db->get_lastdocid ());
+
+    notmuch->last_doc_id++;
+
+    if (notmuch->last_doc_id == 0)
+	INTERNAL_ERROR ("Xapian document IDs are exhausted.\n");
+
+    return notmuch->last_doc_id;
+}
+
+notmuch_status_t
+notmuch_database_remove_message (notmuch_database_t *notmuch,
+				 const char *filename)
+{
+    notmuch_status_t status;
+    notmuch_message_t *message;
+
+    status = notmuch_database_find_message_by_filename (notmuch, filename,
+							&message);
+
+    if (status == NOTMUCH_STATUS_SUCCESS && message) {
+	if (notmuch_message_count_files (message) > 1) {
+	    status = _notmuch_message_remove_filename (message, filename);
+	}
+	if (status == NOTMUCH_STATUS_SUCCESS)
+	    status = _notmuch_message_delete (message);
+	else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
+	    _notmuch_message_sync (message);
+
+	notmuch_message_destroy (message);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
+					   const char *filename,
+					   notmuch_message_t **message_ret)
+{
+    void *local;
+    const char *prefix = _find_prefix ("file-direntry");
+    char *direntry, *term;
+    Xapian::PostingIterator i, end;
+    notmuch_status_t status;
+
+    if (message_ret == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_FILE_TERMS))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    /* return NULL on any failure */
+    *message_ret = NULL;
+
+    local = talloc_new (notmuch);
+
+    try {
+	status = _notmuch_database_filename_to_direntry (
+	    local, notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
+	if (status || ! direntry)
+	    goto DONE;
+
+	term = talloc_asprintf (local, "%s%s", prefix, direntry);
+
+	find_doc_ids_for_term (notmuch, term, &i, &end);
+
+	if (i != end) {
+	    notmuch_private_status_t private_status;
+
+	    *message_ret = _notmuch_message_create (notmuch, notmuch, *i,
+						    &private_status);
+	    if (*message_ret == NULL)
+		status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	}
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "Error: A Xapian exception occurred finding message by filename: %s\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    talloc_free (local);
+
+    if (status && *message_ret) {
+	notmuch_message_destroy (*message_ret);
+	*message_ret = NULL;
+    }
+    return status;
+}
+
+notmuch_string_list_t *
+_notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
+					 Xapian::TermIterator &end,
+					 const char *prefix)
+{
+    int prefix_len = strlen (prefix);
+    notmuch_string_list_t *list;
+
+    list = _notmuch_string_list_create (ctx);
+    if (unlikely (list == NULL))
+	return NULL;
+
+    for (i.skip_to (prefix); i != end; i++) {
+	/* Terminate loop at first term without desired prefix. */
+	if (strncmp ((*i).c_str (), prefix, prefix_len))
+	    break;
+
+	_notmuch_string_list_append (list, (*i).c_str () + prefix_len);
+    }
+
+    return list;
+}
+
+notmuch_tags_t *
+notmuch_database_get_all_tags (notmuch_database_t *db)
+{
+    Xapian::TermIterator i, end;
+    notmuch_string_list_t *tags;
+
+    try {
+	i = db->xapian_db->allterms_begin ();
+	end = db->xapian_db->allterms_end ();
+	tags = _notmuch_database_get_terms_with_prefix (db, i, end,
+							_find_prefix ("tag"));
+	_notmuch_string_list_sort (tags);
+	return _notmuch_tags_create (db, tags);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (db, "A Xapian exception occurred getting tags: %s.\n",
+			       error.get_msg ().c_str ());
+	db->exception_reported = true;
+	return NULL;
+    }
+}
+
+const char *
+notmuch_database_status_string (const notmuch_database_t *notmuch)
+{
+    return notmuch->status_string;
+}
+
+bool
+_notmuch_database_indexable_as_text (notmuch_database_t *notmuch, const char *mime_string)
+{
+    for (size_t i = 0; i < notmuch->index_as_text_length; i++) {
+	if (regexec (&notmuch->index_as_text[i], mime_string, 0, NULL, 0) == 0) {
+	    return true;
+	}
+    }
+
+    return false;
+}
diff --git a/lib/directory.cc b/lib/directory.cc
new file mode 100644
index 00000000..5cf64d7f
--- /dev/null
+++ b/lib/directory.cc
@@ -0,0 +1,335 @@
+/* directory.cc - Results of directory-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+/* Create an iterator to iterate over the basenames of files (or
+ * directories) that all share a common parent directory.
+ */
+static notmuch_filenames_t *
+_create_filenames_for_terms_with_prefix (void *ctx,
+					 notmuch_database_t *notmuch,
+					 const char *prefix)
+{
+    notmuch_string_list_t *filename_list;
+    Xapian::TermIterator i, end;
+
+    i = notmuch->xapian_db->allterms_begin ();
+    end = notmuch->xapian_db->allterms_end ();
+    filename_list = _notmuch_database_get_terms_with_prefix (ctx, i, end,
+							     prefix);
+    if (unlikely (filename_list == NULL))
+	return NULL;
+
+    return _notmuch_filenames_create (ctx, filename_list);
+}
+
+struct _notmuch_directory {
+    notmuch_database_t *notmuch;
+    Xapian::docid document_id;
+    Xapian::Document doc;
+    time_t mtime;
+};
+
+#define LOG_XAPIAN_EXCEPTION(directory, error) _log_xapian_exception (__location__, directory, error)
+
+static void
+_log_xapian_exception (const char *where, notmuch_directory_t *dir,  const Xapian::Error error)
+{
+    notmuch_database_t *notmuch = dir->notmuch;
+
+    _notmuch_database_log (notmuch,
+			   "A Xapian exception occurred at %s: %s\n",
+			   where,
+			   error.get_msg ().c_str ());
+    notmuch->exception_reported = true;
+}
+
+
+/* We end up having to call the destructor explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_directory_destructor (notmuch_directory_t *directory)
+{
+    directory->doc.~Document ();
+
+    return 0;
+}
+
+static notmuch_private_status_t
+find_directory_document (notmuch_database_t *notmuch,
+			 const char *db_path,
+			 Xapian::Document *document)
+{
+    notmuch_private_status_t status;
+    Xapian::docid doc_id;
+
+    status = _notmuch_database_find_unique_doc_id (notmuch, "directory",
+						   db_path, &doc_id);
+    if (status) {
+	*document = Xapian::Document ();
+	return status;
+    }
+
+    *document = notmuch->xapian_db->get_document (doc_id);
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Find or create a directory document.
+ *
+ * 'path' should be a path relative to the path of 'database', or else
+ * should be an absolute path with initial components that match the
+ * path of 'database'.
+ *
+ * If (flags & NOTMUCH_FIND_CREATE), then the directory document will
+ * be created if it does not exist.  Otherwise, if the directory
+ * document does not exist, *status_ret is set to
+ * NOTMUCH_STATUS_SUCCESS and this returns NULL.
+ */
+notmuch_directory_t *
+_notmuch_directory_find_or_create (notmuch_database_t *notmuch,
+				   const char *path,
+				   notmuch_find_flags_t flags,
+				   notmuch_status_t *status_ret)
+{
+    notmuch_directory_t *directory;
+    notmuch_private_status_t private_status;
+    const char *db_path;
+    bool create = (flags & NOTMUCH_FIND_CREATE);
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_DIRECTORY_DOCS)) {
+	*status_ret = NOTMUCH_STATUS_UPGRADE_REQUIRED;
+	return NULL;
+    }
+
+    *status_ret = NOTMUCH_STATUS_SUCCESS;
+
+    path = _notmuch_database_relative_path (notmuch, path);
+
+    if (create && _notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	INTERNAL_ERROR ("Failure to ensure database is writable");
+
+    directory = talloc (notmuch, notmuch_directory_t);
+    if (unlikely (directory == NULL)) {
+	*status_ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    directory->notmuch = notmuch;
+
+    /* "placement new"---not actually allocating memory */
+    new (&directory->doc) Xapian::Document;
+
+    talloc_set_destructor (directory, _notmuch_directory_destructor);
+
+    db_path = _notmuch_database_get_directory_db_path (path);
+
+    try {
+	Xapian::TermIterator i, end;
+
+	private_status = find_directory_document (notmuch, db_path,
+						  &directory->doc);
+	directory->document_id = directory->doc.get_docid ();
+
+	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	    if (! create) {
+		notmuch_directory_destroy (directory);
+		directory = NULL;
+		*status_ret = NOTMUCH_STATUS_SUCCESS;
+		goto DONE;
+	    }
+
+	    void *local = talloc_new (directory);
+	    const char *parent, *basename;
+	    Xapian::docid parent_id;
+	    char *term = talloc_asprintf (local, "%s%s",
+					  _find_prefix ("directory"), db_path);
+	    directory->doc.add_term (term, 0);
+
+	    directory->doc.set_data (path);
+
+	    _notmuch_database_split_path (local, path, &parent, &basename);
+
+	    *status_ret = _notmuch_database_find_directory_id (
+		notmuch, parent, NOTMUCH_FIND_CREATE, &parent_id);
+	    if (*status_ret) {
+		notmuch_directory_destroy (directory);
+		directory = NULL;
+		goto DONE;
+	    }
+
+	    if (basename) {
+		term = talloc_asprintf (local, "%s%u:%s",
+					_find_prefix ("directory-direntry"),
+					parent_id, basename);
+		directory->doc.add_term (term, 0);
+	    }
+
+	    directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+				      Xapian::sortable_serialise (0));
+
+	    directory->document_id = _notmuch_database_generate_doc_id (notmuch);
+	    directory->notmuch->
+		writable_xapian_db
+		-> replace_document (directory->document_id, directory->doc);
+	    talloc_free (local);
+	}
+
+	directory->mtime = Xapian::sortable_unserialise (
+	    directory->doc.get_value (NOTMUCH_VALUE_TIMESTAMP));
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred finding/creating a directory: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	notmuch_directory_destroy (directory);
+	directory = NULL;
+	*status_ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    if (db_path != path)
+	free ((char *) db_path);
+
+    return directory;
+}
+
+unsigned int
+_notmuch_directory_get_document_id (notmuch_directory_t *directory)
+{
+    return directory->document_id;
+}
+
+notmuch_status_t
+notmuch_directory_set_mtime (notmuch_directory_t *directory,
+			     time_t mtime)
+{
+    notmuch_database_t *notmuch = directory->notmuch;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    try {
+	directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+				  Xapian::sortable_serialise (mtime));
+
+	directory->notmuch
+	    ->writable_xapian_db->replace_document (directory->document_id, directory->doc);
+
+	directory->mtime = mtime;
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred setting directory mtime: %s.\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+time_t
+notmuch_directory_get_mtime (notmuch_directory_t *directory)
+{
+    return directory->mtime;
+}
+
+notmuch_filenames_t *
+notmuch_directory_get_child_files (notmuch_directory_t *directory)
+{
+    char *term;
+    notmuch_filenames_t *child_files = NULL;
+
+    term = talloc_asprintf (directory, "%s%u:",
+			    _find_prefix ("file-direntry"),
+			    directory->document_id);
+
+    try {
+	child_files = _create_filenames_for_terms_with_prefix (directory,
+							       directory->notmuch,
+							       term);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (directory, error);
+    }
+
+    talloc_free (term);
+
+    return child_files;
+}
+
+notmuch_filenames_t *
+notmuch_directory_get_child_directories (notmuch_directory_t *directory)
+{
+    char *term;
+    notmuch_filenames_t *child_directories = NULL;
+
+    term = talloc_asprintf (directory, "%s%u:",
+			    _find_prefix ("directory-direntry"),
+			    directory->document_id);
+
+    try {
+	child_directories = _create_filenames_for_terms_with_prefix (directory,
+								     directory->notmuch, term);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (directory, error);
+    }
+
+    talloc_free (term);
+
+    return child_directories;
+}
+
+notmuch_status_t
+notmuch_directory_delete (notmuch_directory_t *directory)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (directory->notmuch);
+    if (status)
+	return status;
+
+    try {
+	directory->notmuch->
+	    writable_xapian_db->delete_document (directory->document_id);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (directory->notmuch,
+			       "A Xapian exception occurred deleting directory entry: %s.\n",
+			       error.get_msg ().c_str ());
+	directory->notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    notmuch_directory_destroy (directory);
+
+    return status;
+}
+
+void
+notmuch_directory_destroy (notmuch_directory_t *directory)
+{
+    talloc_free (directory);
+}
diff --git a/lib/features.cc b/lib/features.cc
new file mode 100644
index 00000000..cf0196c8
--- /dev/null
+++ b/lib/features.cc
@@ -0,0 +1,114 @@
+#include "database-private.h"
+
+static const struct {
+    /* NOTMUCH_FEATURE_* value. */
+    _notmuch_features value;
+    /* Feature name as it appears in the database.  This name should
+     * be appropriate for displaying to the user if an older version
+     * of notmuch doesn't support this feature. */
+    const char *name;
+    /* Compatibility flags when this feature is declared. */
+    const char *flags;
+} feature_names[] = {
+    { NOTMUCH_FEATURE_FILE_TERMS,
+      "multiple paths per message", "rw" },
+    { NOTMUCH_FEATURE_DIRECTORY_DOCS,
+      "relative directory paths", "rw" },
+    /* Header values are not required for reading a database because a
+     * reader can just refer to the message file. */
+    { NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES,
+      "from/subject/message-ID in database", "w" },
+    { NOTMUCH_FEATURE_BOOL_FOLDER,
+      "exact folder:/path: search", "rw" },
+    { NOTMUCH_FEATURE_GHOSTS,
+      "mail documents for missing messages", "w" },
+    /* Knowledge of the index mime-types are not required for reading
+     * a database because a reader will just be unable to query
+     * them. */
+    { NOTMUCH_FEATURE_INDEXED_MIMETYPES,
+      "indexed MIME types", "w" },
+    { NOTMUCH_FEATURE_LAST_MOD,
+      "modification tracking", "w" },
+    /* Existing databases will work fine for all queries not involving
+     * 'body:' */
+    { NOTMUCH_FEATURE_UNPREFIX_BODY_ONLY,
+      "index body and headers separately", "w" },
+};
+
+char *
+_notmuch_database_print_features (const void *ctx, unsigned int features)
+{
+    unsigned int i;
+    char *res = talloc_strdup (ctx, "");
+
+    for (i = 0; i < ARRAY_SIZE (feature_names); ++i)
+	if (features & feature_names[i].value)
+	    res = talloc_asprintf_append_buffer (
+		res, "%s\t%s\n", feature_names[i].name, feature_names[i].flags);
+
+    return res;
+}
+
+
+/* Parse a database features string from the given database version.
+ * Returns the feature bit set.
+ *
+ * For version < 3, this ignores the features string and returns a
+ * hard-coded set of features.
+ *
+ * If there are unrecognized features that are required to open the
+ * database in mode (which should be 'r' or 'w'), return a
+ * comma-separated list of unrecognized but required features in
+ * *incompat_out suitable for presenting to the user.  *incompat_out
+ * will be allocated from ctx.
+ */
+_notmuch_features
+_notmuch_database_parse_features (const void *ctx, const char *features, unsigned int version,
+				  char mode, char **incompat_out)
+{
+    _notmuch_features res = static_cast<_notmuch_features>(0);
+    unsigned int namelen, i;
+    size_t llen = 0;
+    const char *flags;
+
+    /* Prior to database version 3, features were implied by the
+     * version number. */
+    if (version == 0)
+	return NOTMUCH_FEATURES_V0;
+    else if (version == 1)
+	return NOTMUCH_FEATURES_V1;
+    else if (version == 2)
+	return NOTMUCH_FEATURES_V2;
+
+    /* Parse the features string */
+    while ((features = strtok_len_c (features + llen, "\n", &llen)) != NULL) {
+	flags = strchr (features, '\t');
+	if (! flags || flags > features + llen)
+	    continue;
+	namelen = flags - features;
+
+	for (i = 0; i < ARRAY_SIZE (feature_names); ++i) {
+	    if (strlen (feature_names[i].name) == namelen &&
+		strncmp (feature_names[i].name, features, namelen) == 0) {
+		res |= feature_names[i].value;
+		break;
+	    }
+	}
+
+	if (i == ARRAY_SIZE (feature_names) && incompat_out) {
+	    /* Unrecognized feature */
+	    const char *have = strchr (flags, mode);
+	    if (have && have < features + llen) {
+		/* This feature is required to access this database in
+		 * 'mode', but we don't understand it. */
+		if (! *incompat_out)
+		    *incompat_out = talloc_strdup (ctx, "");
+		*incompat_out = talloc_asprintf_append_buffer (
+		    *incompat_out, "%s%.*s", **incompat_out ? ", " : "",
+		    namelen, features);
+	    }
+	}
+    }
+
+    return res;
+}
diff --git a/lib/filenames.c b/lib/filenames.c
new file mode 100644
index 00000000..37d631d6
--- /dev/null
+++ b/lib/filenames.c
@@ -0,0 +1,76 @@
+/* filenames.c - Iterator for a list of filenames
+ *
+ * Copyright © 2010 Intel Corporation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+struct _notmuch_filenames {
+    notmuch_string_node_t *iterator;
+};
+
+/* The notmuch_filenames_t iterates over a notmuch_string_list_t of
+ * file names */
+notmuch_filenames_t *
+_notmuch_filenames_create (const void *ctx,
+			   notmuch_string_list_t *list)
+{
+    notmuch_filenames_t *filenames;
+
+    filenames = talloc (ctx, notmuch_filenames_t);
+    if (unlikely (filenames == NULL))
+	return NULL;
+
+    filenames->iterator = list->head;
+    (void) talloc_reference (filenames, list);
+
+    return filenames;
+}
+
+notmuch_bool_t
+notmuch_filenames_valid (notmuch_filenames_t *filenames)
+{
+    if (filenames == NULL)
+	return false;
+
+    return (filenames->iterator != NULL);
+}
+
+const char *
+notmuch_filenames_get (notmuch_filenames_t *filenames)
+{
+    if ((filenames == NULL) || (filenames->iterator == NULL))
+	return NULL;
+
+    return filenames->iterator->string;
+}
+
+void
+notmuch_filenames_move_to_next (notmuch_filenames_t *filenames)
+{
+    if ((filenames == NULL) || (filenames->iterator == NULL))
+	return;
+
+    filenames->iterator = filenames->iterator->next;
+}
+
+void
+notmuch_filenames_destroy (notmuch_filenames_t *filenames)
+{
+    talloc_free (filenames);
+}
diff --git a/lib/index.cc b/lib/index.cc
new file mode 100644
index 00000000..629dcb22
--- /dev/null
+++ b/lib/index.cc
@@ -0,0 +1,770 @@
+/*
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <gmime/gmime.h>
+#include <gmime/gmime-filter.h>
+
+#include <xapian.h>
+
+
+typedef struct {
+    int state;
+    int a;
+    int b;
+    int next_if_match;
+    int next_if_not_match;
+} scanner_state_t;
+
+/* Simple, linear state-transition diagram for the uuencode filter.
+ *
+ * If the character being processed is within the range of [a, b]
+ * for the current state then we transition next_if_match
+ * state. If not, we transition to the next_if_not_match state.
+ *
+ * The final two states are special in that they are the states in
+ * which we discard data. */
+static const int first_uuencode_skipping_state = 11;
+static const scanner_state_t uuencode_states[] = {
+    { 0,  'b',  'b',  1,  0 },
+    { 1,  'e',  'e',  2,  0 },
+    { 2,  'g',  'g',  3,  0 },
+    { 3,  'i',  'i',  4,  0 },
+    { 4,  'n',  'n',  5,  0 },
+    { 5,  ' ',  ' ',  6,  0 },
+    { 6,  '0',  '7',  7,  0 },
+    { 7,  '0',  '7',  8,  0 },
+    { 8,  '0',  '7',  9,  0 },
+    { 9,  ' ',  ' ',  10, 0 },
+    { 10, '\n', '\n', 11, 10 },
+    { 11, 'M',  'M',  12, 0 },
+    { 12, ' ',  '`',  12, 11 }
+};
+
+/* The following table is intended to implement this DFA (in 'dot'
+ * format). Note that 2 and 3 are "hidden" states used to step through
+ * the possible out edges of state 1.
+ *
+ * digraph html_filter {
+ *     0 -> 1  [label="<"];
+ *     0 -> 0;
+ *     1 -> 4 [label="'"];
+ *     1 -> 5 [label="\""];
+ *     1 -> 0 [label=">"];
+ *     1 -> 1;
+ *     4 -> 1 [label="'"];
+ *     4 -> 4;
+ *     5 -> 1 [label="\""];
+ *     5 -> 5;
+ * }
+ */
+static const int first_html_skipping_state = 1;
+static const scanner_state_t html_states[] = {
+    { 0,  '<',  '<',  1,  0 },
+    { 1,  '\'', '\'', 4,  2 },  /* scanning for quote or > */
+    { 1,  '"',  '"',  5,  3 },
+    { 1,  '>',  '>',  0,  1 },
+    { 4,  '\'', '\'', 1,  4 },  /* inside single quotes */
+    { 5,  '"', '"',   1,  5 },  /* inside double quotes */
+};
+
+/* Oh, how I wish that gobject didn't require so much noisy boilerplate!
+ * (Though I have at least eliminated some of the stock set...) */
+typedef struct _NotmuchFilterDiscardNonTerm NotmuchFilterDiscardNonTerm;
+typedef struct _NotmuchFilterDiscardNonTermClass NotmuchFilterDiscardNonTermClass;
+
+/**
+ * NotmuchFilterDiscardNonTerm:
+ *
+ * @parent_object: parent #GMimeFilter
+ * @encode: encoding vs decoding
+ * @state: State of the parser
+ *
+ * A filter to discard uuencoded portions of an email.
+ *
+ * A uuencoded portion is identified as beginning with a line
+ * matching:
+ *
+ *	begin [0-7][0-7][0-7] .*
+ *
+ * After that detection, and beginning with the following line,
+ * characters will be discarded as long as the first character of each
+ * line begins with M and subsequent characters on the line are within
+ * the range of ASCII characters from ' ' to '`'.
+ *
+ * This is not a perfect UUencode filter. It's possible to have a
+ * message that will legitimately match that pattern, (so that some
+ * legitimate content is discarded). And for most UUencoded files, the
+ * final line of encoded data (the line not starting with M) will be
+ * indexed.
+ **/
+struct _NotmuchFilterDiscardNonTerm {
+    GMimeFilter parent_object;
+    GMimeContentType *content_type;
+    int state;
+    int first_skipping_state;
+    const scanner_state_t *states;
+};
+
+struct _NotmuchFilterDiscardNonTermClass {
+    GMimeFilterClass parent_class;
+};
+
+static GMimeFilter *notmuch_filter_discard_non_term_new (GMimeContentType *content);
+
+static void notmuch_filter_discard_non_term_finalize (GObject *object);
+
+static GMimeFilter *filter_copy (GMimeFilter *filter);
+static void filter_filter (GMimeFilter *filter, char *in, size_t len, size_t prespace,
+			   char **out, size_t *outlen, size_t *outprespace);
+static void filter_complete (GMimeFilter *filter, char *in, size_t len, size_t prespace,
+			     char **out, size_t *outlen, size_t *outprespace);
+static void filter_reset (GMimeFilter *filter);
+
+
+static GMimeFilterClass *parent_class = NULL;
+
+static void
+notmuch_filter_discard_non_term_class_init (NotmuchFilterDiscardNonTermClass *klass,
+					    unused (void *class_data))
+{
+    GObjectClass *object_class = G_OBJECT_CLASS (klass);
+    GMimeFilterClass *filter_class = GMIME_FILTER_CLASS (klass);
+
+    object_class->finalize = notmuch_filter_discard_non_term_finalize;
+
+    filter_class->copy = filter_copy;
+    filter_class->filter = filter_filter;
+    filter_class->complete = filter_complete;
+    filter_class->reset = filter_reset;
+}
+
+static void
+notmuch_filter_discard_non_term_finalize (GObject *object)
+{
+    G_OBJECT_CLASS (parent_class)->finalize (object);
+}
+
+static GMimeFilter *
+filter_copy (GMimeFilter *gmime_filter)
+{
+    NotmuchFilterDiscardNonTerm *filter = (NotmuchFilterDiscardNonTerm *) gmime_filter;
+
+    return notmuch_filter_discard_non_term_new (filter->content_type);
+}
+
+static void
+filter_filter (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t prespace,
+	       char **outbuf, size_t *outlen, size_t *outprespace)
+{
+    NotmuchFilterDiscardNonTerm *filter = (NotmuchFilterDiscardNonTerm *) gmime_filter;
+    const scanner_state_t *states = filter->states;
+    const char *inptr = inbuf;
+    const char *inend = inbuf + inlen;
+    char *outptr;
+
+    (void) prespace;
+
+    int next;
+
+    g_mime_filter_set_size (gmime_filter, inlen, false);
+    outptr = gmime_filter->outbuf;
+
+    next = filter->state;
+    while (inptr < inend) {
+	/* Each state is defined by a contiguous set of rows of the
+	 * state table marked by a common value for '.state'. The
+	 * state numbers must be equal to the index of the first row
+	 * in a given state; thus the loop condition here looks for a
+	 * jump to a first row of a state, which is a real transition
+	 * in the underlying DFA.
+	 */
+	do {
+	    if (*inptr >= states[next].a && *inptr <= states[next].b) {
+		next = states[next].next_if_match;
+	    } else {
+		next = states[next].next_if_not_match;
+	    }
+
+	} while (next != states[next].state);
+
+	if (filter->state < filter->first_skipping_state)
+	    *outptr++ = *inptr;
+
+	filter->state = next;
+	inptr++;
+    }
+
+    *outlen = outptr - gmime_filter->outbuf;
+    *outprespace = gmime_filter->outpre;
+    *outbuf = gmime_filter->outbuf;
+}
+
+static void
+filter_complete (GMimeFilter *filter, char *inbuf, size_t inlen, size_t prespace,
+		 char **outbuf, size_t *outlen, size_t *outprespace)
+{
+    if (inbuf && inlen)
+	filter_filter (filter, inbuf, inlen, prespace, outbuf, outlen, outprespace);
+}
+
+static void
+filter_reset (GMimeFilter *gmime_filter)
+{
+    NotmuchFilterDiscardNonTerm *filter = (NotmuchFilterDiscardNonTerm *) gmime_filter;
+
+    filter->state = 0;
+}
+
+/**
+ * notmuch_filter_discard_non_term_new:
+ *
+ * Returns: a new #NotmuchFilterDiscardNonTerm filter.
+ **/
+static GType type = 0;
+
+static const GTypeInfo info = {
+    .class_size = sizeof (NotmuchFilterDiscardNonTermClass),
+    .base_init = NULL,
+    .base_finalize = NULL,
+    .class_init = (GClassInitFunc) notmuch_filter_discard_non_term_class_init,
+    .class_finalize = NULL,
+    .class_data = NULL,
+    .instance_size = sizeof (NotmuchFilterDiscardNonTerm),
+    .n_preallocs = 0,
+    .instance_init = NULL,
+    .value_table = NULL,
+};
+
+void
+_notmuch_filter_init () {
+    type = g_type_register_static (GMIME_TYPE_FILTER, "NotmuchFilterDiscardNonTerm", &info,
+				   (GTypeFlags) 0);
+    parent_class = (GMimeFilterClass *) g_type_class_ref (GMIME_TYPE_FILTER);
+}
+
+static GMimeFilter *
+notmuch_filter_discard_non_term_new (GMimeContentType *content_type)
+{
+    NotmuchFilterDiscardNonTerm *filter;
+
+    filter = (NotmuchFilterDiscardNonTerm *) g_object_new (type, NULL);
+    filter->content_type = content_type;
+    filter->state = 0;
+    if (g_mime_content_type_is_type (content_type, "text", "html")) {
+	filter->states = html_states;
+	filter->first_skipping_state = first_html_skipping_state;
+    } else {
+	filter->states = uuencode_states;
+	filter->first_skipping_state = first_uuencode_skipping_state;
+    }
+
+    return (GMimeFilter *) filter;
+}
+
+/* We're finally down to a single (NAME + address) email "mailbox". */
+static void
+_index_address_mailbox (notmuch_message_t *message,
+			const char *prefix_name,
+			InternetAddress *address)
+{
+    InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
+    const char *name, *addr, *combined;
+    void *local = talloc_new (message);
+
+    name = internet_address_get_name (address);
+    addr = internet_address_mailbox_get_addr (mailbox);
+
+    /* Combine the name and address and index them as a phrase. */
+    if (name && addr)
+	combined = talloc_asprintf (local, "%s %s", name, addr);
+    else if (name)
+	combined = name;
+    else
+	combined = addr;
+
+    if (combined)
+	_notmuch_message_gen_terms (message, prefix_name, combined);
+
+    talloc_free (local);
+}
+
+static void
+_index_address_list (notmuch_message_t *message,
+		     const char *prefix_name,
+		     InternetAddressList *addresses);
+
+/* The outer loop over the InternetAddressList wasn't quite enough.
+ * There can actually be a tree here where a single member of the list
+ * is a "group" containing another list. Recurse please.
+ */
+static void
+_index_address_group (notmuch_message_t *message,
+		      const char *prefix_name,
+		      InternetAddress *address)
+{
+    InternetAddressGroup *group;
+    InternetAddressList *list;
+
+    group = INTERNET_ADDRESS_GROUP (address);
+    list = internet_address_group_get_members (group);
+
+    if (! list)
+	return;
+
+    _index_address_list (message, prefix_name, list);
+}
+
+static void
+_index_address_list (notmuch_message_t *message,
+		     const char *prefix_name,
+		     InternetAddressList *addresses)
+{
+    int i;
+    InternetAddress *address;
+
+    if (addresses == NULL)
+	return;
+
+    for (i = 0; i < internet_address_list_length (addresses); i++) {
+	address = internet_address_list_get_address (addresses, i);
+	if (INTERNET_ADDRESS_IS_MAILBOX (address)) {
+	    _index_address_mailbox (message, prefix_name, address);
+	} else if (INTERNET_ADDRESS_IS_GROUP (address)) {
+	    _index_address_group (message, prefix_name, address);
+	} else {
+	    INTERNAL_ERROR ("GMime InternetAddress is neither a mailbox nor a group.\n");
+	}
+    }
+}
+
+static void
+_index_content_type (notmuch_message_t *message, GMimeObject *part)
+{
+    GMimeContentType *content_type = g_mime_object_get_content_type (part);
+
+    if (content_type) {
+	char *mime_string = g_mime_content_type_get_mime_type (content_type);
+	if (mime_string) {
+	    _notmuch_message_gen_terms (message, "mimetype", mime_string);
+	    g_free (mime_string);
+	}
+    }
+}
+
+static void
+_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts,
+			    GMimeObject *part,
+			    _notmuch_message_crypto_t *msg_crypto);
+
+static void
+_index_pkcs7_part (notmuch_message_t *message,
+		   notmuch_indexopts_t *indexopts,
+		   GMimeObject *part,
+		   _notmuch_message_crypto_t *msg_crypto);
+
+static bool
+_indexable_as_text (notmuch_message_t *message, GMimeObject *part)
+{
+    GMimeContentType *content_type = g_mime_object_get_content_type (part);
+    notmuch_database_t *notmuch = notmuch_message_get_database (message);
+
+    if (content_type) {
+	char *mime_string = g_mime_content_type_get_mime_type (content_type);
+	if (mime_string) {
+	    bool ret = _notmuch_database_indexable_as_text (notmuch, mime_string);
+	    g_free (mime_string);
+	    return ret;
+	}
+    }
+    return false;
+}
+
+/* Callback to generate terms for each mime part of a message. */
+static void
+_index_mime_part (notmuch_message_t *message,
+		  notmuch_indexopts_t *indexopts,
+		  GMimeObject *part,
+		  _notmuch_message_crypto_t *msg_crypto)
+{
+    GMimeStream *stream, *filter;
+    GMimeFilter *discard_non_term_filter;
+    GMimeDataWrapper *wrapper;
+    GByteArray *byte_array;
+    GMimeContentDisposition *disposition;
+    GMimeContentType *content_type;
+    char *body;
+    const char *charset;
+    GMimeObject *repaired_part = NULL;
+
+    if (! part) {
+	_notmuch_database_log (notmuch_message_get_database (message),
+			       "Warning: Not indexing empty mime part.\n");
+	goto DONE;
+    }
+
+    repaired_part = _notmuch_repair_mixed_up_mangled (part);
+    if (repaired_part) {
+	/* This was likely "Mixed Up" in transit!  We will instead use
+	 * the more likely-to-be-correct variant. */
+	notmuch_message_add_property (message, "index.repaired", "mixedup");
+	part = repaired_part;
+    }
+
+    _index_content_type (message, part);
+
+    if (GMIME_IS_MULTIPART (part)) {
+	GMimeMultipart *multipart = GMIME_MULTIPART (part);
+	int i;
+
+	if (GMIME_IS_MULTIPART_SIGNED (multipart))
+	    _notmuch_message_add_term (message, "tag", "signed");
+
+	if (GMIME_IS_MULTIPART_ENCRYPTED (multipart))
+	    _notmuch_message_add_term (message, "tag", "encrypted");
+
+	for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
+	    GMimeObject *child;
+	    if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
+		/* Don't index the signature, but index its content type. */
+		if (i == GMIME_MULTIPART_SIGNED_SIGNATURE) {
+		    _index_content_type (message,
+					 g_mime_multipart_get_part (multipart, i));
+		    continue;
+		} else if (i != GMIME_MULTIPART_SIGNED_CONTENT) {
+		    _notmuch_database_log (notmuch_message_get_database (message),
+					   "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+		}
+	    }
+	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
+		_index_content_type (message,
+				     g_mime_multipart_get_part (multipart, i));
+		if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) {
+		    _index_encrypted_mime_part (message, indexopts,
+						part,
+						msg_crypto);
+		} else {
+		    if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) {
+			_notmuch_database_log (notmuch_message_get_database (message),
+					       "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		    }
+		}
+		continue;
+	    }
+	    child = g_mime_multipart_get_part (multipart, i);
+	    GMimeObject *toindex = child;
+	    if (_notmuch_message_crypto_potential_payload (msg_crypto, child, part, i) &&
+		msg_crypto->decryption_status == NOTMUCH_MESSAGE_DECRYPTED_FULL) {
+		toindex = _notmuch_repair_crypto_payload_skip_legacy_display (child);
+		if (toindex != child)
+		    notmuch_message_add_property (message, "index.repaired",
+						  "skip-protected-headers-legacy-display");
+	    }
+	    _index_mime_part (message, indexopts, toindex, msg_crypto);
+	}
+	goto DONE;
+    }
+
+    if (GMIME_IS_MESSAGE_PART (part)) {
+	GMimeMessage *mime_message;
+
+	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
+
+	_index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message),
+			  msg_crypto);
+
+	goto DONE;
+    }
+
+    if (GMIME_IS_APPLICATION_PKCS7_MIME (part)) {
+	_index_pkcs7_part (message, indexopts, part, msg_crypto);
+	goto DONE;
+    }
+
+    if (! (GMIME_IS_PART (part))) {
+	_notmuch_database_log (notmuch_message_get_database (message),
+			       "Warning: Not indexing unknown mime part: %s.\n",
+			       g_type_name (G_OBJECT_TYPE (part)));
+	goto DONE;
+    }
+
+    disposition = g_mime_object_get_content_disposition (part);
+    if (disposition &&
+	strcasecmp (g_mime_content_disposition_get_disposition (disposition),
+		    GMIME_DISPOSITION_ATTACHMENT) == 0) {
+	const char *filename = g_mime_part_get_filename (GMIME_PART (part));
+
+	_notmuch_message_add_term (message, "tag", "attachment");
+	_notmuch_message_gen_terms (message, "attachment", filename);
+
+	if (! _indexable_as_text (message, part)) {
+	    /* XXX: Would be nice to call out to something here to parse
+	     * the attachment into text and then index that. */
+	    goto DONE;
+	}
+    }
+
+    byte_array = g_byte_array_new ();
+
+    stream = g_mime_stream_mem_new_with_byte_array (byte_array);
+    g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), false);
+
+    filter = g_mime_stream_filter_new (stream);
+
+    content_type = g_mime_object_get_content_type (part);
+    discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
+
+    g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
+			      discard_non_term_filter);
+
+    charset = g_mime_object_get_content_type_parameter (part, "charset");
+    if (charset) {
+	GMimeFilter *charset_filter;
+	charset_filter = g_mime_filter_charset_new (charset, "UTF-8");
+	/* This result can be NULL for things like "unknown-8bit".
+	 * Don't set a NULL filter as that makes GMime print
+	 * annoying assertion-failure messages on stderr. */
+	if (charset_filter) {
+	    g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
+				      charset_filter);
+	    g_object_unref (charset_filter);
+	}
+    }
+
+    wrapper = g_mime_part_get_content (GMIME_PART (part));
+    if (wrapper)
+	g_mime_data_wrapper_write_to_stream (wrapper, filter);
+
+    g_object_unref (stream);
+    g_object_unref (filter);
+    g_object_unref (discard_non_term_filter);
+
+    g_byte_array_append (byte_array, (guint8 *) "\0", 1);
+    body = (char *) g_byte_array_free (byte_array, false);
+
+    if (body) {
+	_notmuch_message_gen_terms (message, NULL, body);
+
+	free (body);
+    }
+  DONE:
+    if (repaired_part)
+	g_object_unref (repaired_part);
+}
+
+/* descend (if desired) into the cleartext part of an encrypted MIME
+ * part while indexing. */
+static void
+_index_encrypted_mime_part (notmuch_message_t *message,
+			    notmuch_indexopts_t *indexopts,
+			    GMimeObject *encrypted_data,
+			    _notmuch_message_crypto_t *msg_crypto)
+{
+    notmuch_status_t status;
+    GError *err = NULL;
+    notmuch_database_t *notmuch = NULL;
+    GMimeObject *clear = NULL;
+
+    if (! indexopts || (notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_FALSE))
+	return;
+
+    notmuch = notmuch_message_get_database (message);
+
+    bool attempted = false;
+    GMimeDecryptResult *decrypt_result = NULL;
+    bool get_sk = (notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_TRUE);
+
+    clear = _notmuch_crypto_decrypt (&attempted, notmuch_indexopts_get_decrypt_policy (indexopts),
+				     message, encrypted_data, get_sk ? &decrypt_result : NULL, &err);
+    if (! attempted)
+	return;
+    if (err || ! clear) {
+	if (decrypt_result)
+	    g_object_unref (decrypt_result);
+	if (err) {
+	    _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n",
+				   err->domain, err->code, err->message);
+	    g_error_free (err);
+	} else {
+	    _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (unknown error)\n");
+	}
+	/* Indicate that we failed to decrypt during indexing */
+	status = notmuch_message_add_property (message, "index.decryption", "failure");
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					  "property (%d)\n", status);
+	return;
+    }
+    if (decrypt_result) {
+	status = _notmuch_message_crypto_successful_decryption (msg_crypto);
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to mark the message as decrypted (%s)\n",
+					  notmuch_status_to_string (status));
+	if (get_sk) {
+	    status = notmuch_message_add_property (message, "session-key",
+						   g_mime_decrypt_result_get_session_key (
+						       decrypt_result));
+	    if (status)
+		_notmuch_database_log (notmuch, "failed to add session-key "
+				       "property (%d)\n", status);
+	}
+	g_object_unref (decrypt_result);
+    }
+    GMimeObject *toindex = clear;
+
+    if (_notmuch_message_crypto_potential_payload (msg_crypto, clear, encrypted_data,
+						   GMIME_MULTIPART_ENCRYPTED_CONTENT) &&
+	msg_crypto->decryption_status == NOTMUCH_MESSAGE_DECRYPTED_FULL) {
+	toindex = _notmuch_repair_crypto_payload_skip_legacy_display (clear);
+	if (toindex != clear)
+	    notmuch_message_add_property (message, "index.repaired",
+					  "skip-protected-headers-legacy-display");
+    }
+    _index_mime_part (message, indexopts, toindex, msg_crypto);
+    g_object_unref (clear);
+
+    status = notmuch_message_add_property (message, "index.decryption", "success");
+    if (status)
+	_notmuch_database_log (notmuch, "failed to add index.decryption "
+			       "property (%d)\n", status);
+
+}
+
+static void
+_index_pkcs7_part (notmuch_message_t *message,
+		   notmuch_indexopts_t *indexopts,
+		   GMimeObject *part,
+		   _notmuch_message_crypto_t *msg_crypto)
+{
+    GMimeApplicationPkcs7Mime *pkcs7;
+    GMimeSecureMimeType p7type;
+    GMimeObject *mimeobj = NULL;
+    GMimeSignatureList *sigs = NULL;
+    GError *err = NULL;
+    notmuch_database_t *notmuch = NULL;
+
+    pkcs7 = GMIME_APPLICATION_PKCS7_MIME (part);
+    p7type = g_mime_application_pkcs7_mime_get_smime_type (pkcs7);
+    notmuch = notmuch_message_get_database (message);
+    _index_content_type (message, part);
+
+    if (p7type == GMIME_SECURE_MIME_TYPE_SIGNED_DATA) {
+	sigs = g_mime_application_pkcs7_mime_verify (pkcs7, GMIME_VERIFY_NONE, &mimeobj, &err);
+	if (sigs == NULL) {
+	    _notmuch_database_log (notmuch,
+				   "Failed to verify PKCS#7 SignedData during indexing. (%d:%d) [%s]\n",
+				   err->domain, err->code, err->message);
+	    g_error_free (err);
+	    goto DONE;
+	}
+	_notmuch_message_add_term (message, "tag", "signed");
+	GMimeObject *toindex = mimeobj;
+	if (_notmuch_message_crypto_potential_payload (msg_crypto, mimeobj, part, 0) &&
+	    msg_crypto->decryption_status == NOTMUCH_MESSAGE_DECRYPTED_FULL) {
+	    toindex = _notmuch_repair_crypto_payload_skip_legacy_display (mimeobj);
+	    if (toindex != mimeobj)
+		notmuch_message_add_property (message, "index.repaired",
+					      "skip-protected-headers-legacy-display");
+	}
+	_index_mime_part (message, indexopts, toindex, msg_crypto);
+    } else if (p7type == GMIME_SECURE_MIME_TYPE_ENVELOPED_DATA) {
+	_notmuch_message_add_term (message, "tag", "encrypted");
+	_index_encrypted_mime_part (message, indexopts,
+				    part,
+				    msg_crypto);
+    } else {
+	_notmuch_database_log (notmuch, "Cannot currently handle PKCS#7 smime-type '%s'\n",
+			       g_mime_object_get_content_type_parameter (part, "smime-type"));
+    }
+  DONE:
+    if (mimeobj)
+	g_object_unref (mimeobj);
+    if (sigs)
+	g_object_unref (sigs);
+}
+
+static notmuch_status_t
+_notmuch_message_index_user_headers (notmuch_message_t *message, GMimeMessage *mime_message)
+{
+
+    notmuch_database_t *notmuch = notmuch_message_get_database (message);
+    notmuch_string_map_iterator_t *iter = _notmuch_database_user_headers (notmuch);
+
+    for (; _notmuch_string_map_iterator_valid (iter);
+	 _notmuch_string_map_iterator_move_to_next (iter)) {
+
+	const char *prefix_name = _notmuch_string_map_iterator_key (iter);
+
+	const char *header_name = _notmuch_string_map_iterator_value (iter);
+
+	const char *header = g_mime_object_get_header (GMIME_OBJECT (mime_message), header_name);
+	if (header)
+	    _notmuch_message_gen_terms (message, prefix_name, header);
+    }
+
+    if (iter)
+	_notmuch_string_map_iterator_destroy (iter);
+    return NOTMUCH_STATUS_SUCCESS;
+
+}
+
+notmuch_status_t
+_notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_file_t *message_file)
+{
+    GMimeMessage *mime_message;
+    InternetAddressList *addresses;
+    const char *subject;
+    notmuch_status_t status;
+    _notmuch_message_crypto_t *msg_crypto;
+
+    status = _notmuch_message_file_get_mime_message (message_file,
+						     &mime_message);
+    if (status)
+	return status;
+
+    addresses = g_mime_message_get_from (mime_message);
+    if (addresses) {
+	_index_address_list (message, "from", addresses);
+    }
+
+    addresses = g_mime_message_get_all_recipients (mime_message);
+    if (addresses) {
+	_index_address_list (message, "to", addresses);
+	g_object_unref (addresses);
+    }
+
+    subject = g_mime_message_get_subject (mime_message);
+    _notmuch_message_gen_terms (message, "subject", subject);
+
+    status = _notmuch_message_index_user_headers (message, mime_message);
+
+    msg_crypto = _notmuch_message_crypto_new (NULL);
+    _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message), msg_crypto);
+    if (msg_crypto && msg_crypto->payload_subject) {
+	_notmuch_message_gen_terms (message, "subject", msg_crypto->payload_subject);
+	_notmuch_message_update_subject (message, msg_crypto->payload_subject);
+    }
+
+    talloc_free (msg_crypto);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/indexopts.c b/lib/indexopts.c
new file mode 100644
index 00000000..2ffd1942
--- /dev/null
+++ b/lib/indexopts.c
@@ -0,0 +1,81 @@
+/* indexopts.c - options for indexing messages (currently a stub)
+ *
+ * Copyright © 2017 Daniel Kahn Gillmor
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
+ */
+
+#include "notmuch-private.h"
+
+struct _notmuch_indexopts {
+    _notmuch_crypto_t crypto;
+};
+
+notmuch_indexopts_t *
+notmuch_database_get_default_indexopts (notmuch_database_t *db)
+{
+    notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t);
+
+    if (! ret)
+	return ret;
+    ret->crypto.decrypt = NOTMUCH_DECRYPT_AUTO;
+
+    char *decrypt_policy;
+    notmuch_status_t err = notmuch_database_get_config (db, "index.decrypt", &decrypt_policy);
+
+    if (err)
+	return NULL;
+
+    if (decrypt_policy) {
+	if ((! (strcasecmp (decrypt_policy, "true"))) ||
+	    (! (strcasecmp (decrypt_policy, "yes"))) ||
+	    (! (strcasecmp (decrypt_policy, "1"))))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_TRUE);
+	else if ((! (strcasecmp (decrypt_policy, "false"))) ||
+		 (! (strcasecmp (decrypt_policy, "no"))) ||
+		 (! (strcasecmp (decrypt_policy, "0"))))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_FALSE);
+	else if (! strcasecmp (decrypt_policy, "nostash"))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_NOSTASH);
+    }
+
+    free (decrypt_policy);
+    return ret;
+}
+
+notmuch_status_t
+notmuch_indexopts_set_decrypt_policy (notmuch_indexopts_t *indexopts,
+				      notmuch_decryption_policy_t decrypt_policy)
+{
+    if (! indexopts)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    indexopts->crypto.decrypt = decrypt_policy;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_decryption_policy_t
+notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts)
+{
+    if (! indexopts)
+	return false;
+    return indexopts->crypto.decrypt;
+}
+
+void
+notmuch_indexopts_destroy (notmuch_indexopts_t *indexopts)
+{
+    talloc_free (indexopts);
+}
diff --git a/lib/init.cc b/lib/init.cc
new file mode 100644
index 00000000..cf29200f
--- /dev/null
+++ b/lib/init.cc
@@ -0,0 +1,21 @@
+#include "notmuch-private.h"
+
+#include <mutex>
+
+static void do_init ()
+{
+    /* Initialize the GLib type system and threads */
+#if ! GLIB_CHECK_VERSION (2, 35, 1)
+    g_type_init ();
+#endif
+
+    g_mime_init ();
+    _notmuch_filter_init ();
+}
+
+void
+_notmuch_init ()
+{
+    static std::once_flag initialized;
+    std::call_once (initialized, do_init);
+}
diff --git a/lib/lastmod-fp.cc b/lib/lastmod-fp.cc
new file mode 100644
index 00000000..f85efd28
--- /dev/null
+++ b/lib/lastmod-fp.cc
@@ -0,0 +1,83 @@
+/* lastmod-fp.cc - lastmod range query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2022 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "lastmod-fp.h"
+
+notmuch_status_t
+_notmuch_lastmod_strings_to_query (notmuch_database_t *notmuch,
+				   const std::string &from, const std::string &to,
+				   Xapian::Query &output, std::string &msg)
+{
+    long from_idx = 0L, to_idx = LONG_MAX;
+    long current;
+    std::string str;
+
+    /* revision should not change, but for the avoidance of doubt,
+     * grab for both ends of range, if needed*/
+    current = notmuch_database_get_revision (notmuch, NULL);
+
+    try {
+	if (from.empty ())
+	    from_idx = 0L;
+	else
+	    from_idx = std::stol (from);
+    } catch (std::logic_error &e) {
+	msg = "bad 'from' revision: '" + from + "'";
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    if (from_idx < 0)
+	from_idx += current;
+
+    try {
+	if (EMPTY_STRING (to))
+	    to_idx = LONG_MAX;
+	else
+	    to_idx = std::stol (to);
+    } catch (std::logic_error &e) {
+	msg = "bad 'to' revision: '" + to + "'";
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    if (to_idx < 0)
+	to_idx += current;
+
+    output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, NOTMUCH_VALUE_LAST_MOD,
+			    Xapian::sortable_serialise (from_idx),
+			    Xapian::sortable_serialise (to_idx));
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+Xapian::Query
+LastModRangeProcessor::operator() (const std::string &begin, const std::string &end)
+{
+
+    Xapian::Query output;
+    std::string msg;
+
+    if (_notmuch_lastmod_strings_to_query (notmuch, begin, end, output, msg))
+	throw Xapian::QueryParserError (msg);
+
+    return output;
+}
+
diff --git a/lib/lastmod-fp.h b/lib/lastmod-fp.h
new file mode 100644
index 00000000..8168fe7b
--- /dev/null
+++ b/lib/lastmod-fp.h
@@ -0,0 +1,41 @@
+/* lastmod-fp.h - database revision query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2022 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_LASTMOD_FP_H
+#define NOTMUCH_LASTMOD_FP_H
+
+#include <xapian.h>
+
+class LastModRangeProcessor : public Xapian::RangeProcessor {
+protected:
+    notmuch_database_t *notmuch;
+
+public:
+    LastModRangeProcessor (notmuch_database_t *notmuch_, const std::string prefix_)
+	:  Xapian::RangeProcessor (NOTMUCH_VALUE_LAST_MOD, prefix_, 0), notmuch (notmuch_)
+    {
+    }
+
+    Xapian::Query operator() (const std::string &begin, const std::string &end);
+};
+
+#endif /* NOTMUCH_LASTMOD_FP_H */
diff --git a/lib/message-file.c b/lib/message-file.c
new file mode 100644
index 00000000..68f646a4
--- /dev/null
+++ b/lib/message-file.c
@@ -0,0 +1,402 @@
+/* message.c - Utility functions for parsing an email message for notmuch.
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include <stdarg.h>
+
+#include "notmuch-private.h"
+
+#include <gmime/gmime.h>
+
+#include <glib.h> /* GHashTable */
+
+struct _notmuch_message_file {
+    /* open stream to (possibly gzipped) file */
+    GMimeStream *stream;
+    char *filename;
+
+    /* Cache for decoded headers */
+    GHashTable *headers;
+
+    GMimeMessage *message;
+};
+
+static int
+_notmuch_message_file_destructor (notmuch_message_file_t *message)
+{
+    if (message->headers)
+	g_hash_table_destroy (message->headers);
+
+    if (message->message)
+	g_object_unref (message->message);
+
+    if (message->stream)
+	g_object_unref (message->stream);
+
+    return 0;
+}
+
+/* Create a new notmuch_message_file_t for 'filename' with 'ctx' as
+ * the talloc owner. */
+notmuch_message_file_t *
+_notmuch_message_file_open_ctx (notmuch_database_t *notmuch,
+				void *ctx, const char *filename)
+{
+    notmuch_message_file_t *message;
+
+    message = talloc_zero (ctx, notmuch_message_file_t);
+    if (unlikely (message == NULL))
+	return NULL;
+
+    const char *prefix = notmuch_config_get (notmuch, NOTMUCH_CONFIG_MAIL_ROOT);
+
+    if (prefix == NULL)
+	goto FAIL;
+
+    if (*filename == '/') {
+	if (strncmp (filename, prefix, strlen (prefix)) != 0) {
+	    _notmuch_database_log (notmuch, "Error opening %s: path outside mail root\n",
+				   filename);
+	    errno = 0;
+	    goto FAIL;
+	}
+	message->filename = talloc_strdup (message, filename);
+    } else {
+	message->filename = talloc_asprintf (message, "%s/%s", prefix, filename);
+    }
+
+    if (message->filename == NULL)
+	goto FAIL;
+
+    talloc_set_destructor (message, _notmuch_message_file_destructor);
+
+    message->stream = g_mime_stream_gzfile_open (message->filename);
+    if (message->stream == NULL)
+	goto FAIL;
+
+    return message;
+
+  FAIL:
+    if (errno)
+	_notmuch_database_log (notmuch, "Error opening %s: %s\n",
+			       filename, strerror (errno));
+    _notmuch_message_file_close (message);
+
+    return NULL;
+}
+
+notmuch_message_file_t *
+_notmuch_message_file_open (notmuch_database_t *notmuch,
+			    const char *filename)
+{
+    return _notmuch_message_file_open_ctx (notmuch, NULL, filename);
+}
+
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message_file)
+{
+    return message_file->filename;
+}
+
+void
+_notmuch_message_file_close (notmuch_message_file_t *message)
+{
+    talloc_free (message);
+}
+
+static bool
+_is_mbox (GMimeStream *stream)
+{
+    char from_buf[5];
+    bool ret = false;
+
+    /* Is this mbox? */
+    if (g_mime_stream_read (stream, from_buf, sizeof (from_buf)) == sizeof (from_buf) &&
+	strncmp (from_buf, "From ", 5) == 0)
+	ret = true;
+
+    g_mime_stream_reset (stream);
+
+    return ret;
+}
+
+notmuch_status_t
+_notmuch_message_file_parse (notmuch_message_file_t *message)
+{
+    GMimeParser *parser;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    bool is_mbox;
+
+    if (message->message)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    is_mbox = _is_mbox (message->stream);
+
+    _notmuch_init ();
+
+    message->headers = g_hash_table_new_full (strcase_hash, strcase_equal,
+					      free, g_free);
+    if (! message->headers)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    parser = g_mime_parser_new_with_stream (message->stream);
+    g_mime_parser_set_scan_from (parser, is_mbox);
+
+    message->message = g_mime_parser_construct_message (parser, NULL);
+    if (! message->message) {
+	status = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	goto DONE;
+    }
+
+    if (is_mbox && ! g_mime_parser_eos (parser)) {
+	/*
+	 * This is a multi-message mbox. (For historical reasons, we
+	 * do support single-message mboxes.)
+	 */
+	status = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+    }
+
+  DONE:
+    g_mime_stream_reset (message->stream);
+    g_object_unref (parser);
+
+    if (status) {
+	g_hash_table_destroy (message->headers);
+	message->headers = NULL;
+
+	if (message->message) {
+	    g_object_unref (message->message);
+	    message->message = NULL;
+	}
+
+    }
+
+    return status;
+}
+
+notmuch_status_t
+_notmuch_message_file_get_mime_message (notmuch_message_file_t *message,
+					GMimeMessage **mime_message)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_message_file_parse (message);
+    if (status)
+	return status;
+
+    *mime_message = message->message;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/*
+ * Get all instances of a header decoded and concatenated.
+ *
+ * The result must be freed using g_free().
+ *
+ * Return NULL on errors, empty string for non-existing headers.
+ */
+
+static char *
+_extend_header (char *combined, const char *value)
+{
+    char *decoded;
+
+    decoded = g_mime_utils_header_decode_text (NULL, value);
+    if (! decoded) {
+	if (combined) {
+	    g_free (combined);
+	    combined = NULL;
+	}
+	goto DONE;
+    }
+
+    if (combined) {
+	char *tmp = g_strdup_printf ("%s %s", combined, decoded);
+	g_free (decoded);
+	g_free (combined);
+	if (! tmp) {
+	    combined = NULL;
+	    goto DONE;
+	}
+
+	combined = tmp;
+    } else {
+	combined = decoded;
+    }
+  DONE:
+    return combined;
+}
+
+static char *
+_notmuch_message_file_get_combined_header (notmuch_message_file_t *message,
+					   const char *header)
+{
+    char *combined = NULL;
+    GMimeHeaderList *headers;
+
+    headers = g_mime_object_get_header_list (GMIME_OBJECT (message->message));
+    if (! headers)
+	return NULL;
+
+
+    for (int i = 0; i < g_mime_header_list_get_count (headers); i++) {
+	const char *value;
+	GMimeHeader *g_header = g_mime_header_list_get_header_at (headers, i);
+
+	if (strcasecmp (g_mime_header_get_name (g_header), header) != 0)
+	    continue;
+
+	/* GMime retains ownership of value, we hope */
+	value = g_mime_header_get_value (g_header);
+
+	combined = _extend_header (combined, value);
+    }
+
+    /* Return empty string for non-existing headers. */
+    if (! combined)
+	combined = g_strdup ("");
+
+    return combined;
+}
+
+const char *
+_notmuch_message_file_get_header (notmuch_message_file_t *message,
+				  const char *header)
+{
+    const char *value;
+    char *decoded;
+
+    if (_notmuch_message_file_parse (message))
+	return NULL;
+
+    /* If we have a cached decoded value, use it. */
+    value = g_hash_table_lookup (message->headers, header);
+    if (value)
+	return value;
+
+    if (strcasecmp (header, "received") == 0 ||
+	strcasecmp (header, "delivered-to") == 0) {
+	/*
+	 * The Received: header is special. We concatenate all instances of the
+	 * header as we use this when analyzing the path the mail has taken
+	 * from sender to recipient.
+	 *
+	 * Similarly, multiple instances of Delivered-To may be present. We
+	 * concatenate them so the one with highest priority may be picked (eg.
+	 * primary_email before other_email).
+	 */
+	decoded = _notmuch_message_file_get_combined_header (message, header);
+    } else {
+	value = g_mime_object_get_header (GMIME_OBJECT (message->message),
+					  header);
+	if (value)
+	    decoded = g_mime_utils_header_decode_text (NULL, value);
+	else
+	    decoded = g_strdup ("");
+    }
+
+    if (! decoded)
+	return NULL;
+
+    /* Cache the decoded value. We also own the strings. */
+    g_hash_table_insert (message->headers, xstrdup (header), decoded);
+
+    return decoded;
+}
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+				   const char **from_out,
+				   const char **subject_out,
+				   const char **to_out,
+				   const char **date_out,
+				   char **message_id_out)
+{
+    notmuch_status_t ret;
+    const char *header;
+    const char *from, *to, *subject, *date;
+    char *message_id = NULL;
+
+    /* Parse message up front to get better error status. */
+    ret = _notmuch_message_file_parse (message_file);
+    if (ret)
+	goto DONE;
+
+    /* Before we do any real work, (especially before doing a
+     * potential SHA-1 computation on the entire file's contents),
+     * let's make sure that what we're looking at looks like an
+     * actual email message.
+     */
+    from = _notmuch_message_file_get_header (message_file, "from");
+    subject = _notmuch_message_file_get_header (message_file, "subject");
+    to = _notmuch_message_file_get_header (message_file, "to");
+    date = _notmuch_message_file_get_header (message_file, "date");
+
+    if ((from == NULL || *from == '\0') &&
+	(subject == NULL || *subject == '\0') &&
+	(to == NULL || *to == '\0')) {
+	ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	goto DONE;
+    }
+
+    /* Now that we're sure it's mail, the first order of business
+     * is to find a message ID (or else create one ourselves).
+     */
+    header = _notmuch_message_file_get_header (message_file, "message-id");
+    if (header && *header != '\0') {
+	message_id = _notmuch_message_id_parse (message_file, header, NULL);
+
+	/* So the header value isn't RFC-compliant, but it's
+	 * better than no message-id at all.
+	 */
+	if (message_id == NULL)
+	    message_id = talloc_strdup (message_file, header);
+    }
+
+    if (message_id == NULL ) {
+	/* No message-id at all, let's generate one by taking a
+	 * hash over the file's contents.
+	 */
+	char *sha1 = _notmuch_sha1_of_file (_notmuch_message_file_get_filename (message_file));
+
+	/* If that failed too, something is really wrong. Give up. */
+	if (sha1 == NULL) {
+	    ret = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
+	free (sha1);
+    }
+  DONE:
+    if (ret == NOTMUCH_STATUS_SUCCESS) {
+	if (from_out)
+	    *from_out = from;
+	if (subject_out)
+	    *subject_out = subject;
+	if (to_out)
+	    *to_out = to;
+	if (date_out)
+	    *date_out = date;
+	if (message_id_out)
+	    *message_id_out = message_id;
+    }
+    return ret;
+}
diff --git a/lib/message-id.c b/lib/message-id.c
new file mode 100644
index 00000000..54012354
--- /dev/null
+++ b/lib/message-id.c
@@ -0,0 +1,126 @@
+#include "notmuch-private.h"
+#include "string-util.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+    const char *s;
+
+    s = *str;
+    while (*s && (isspace (*s) || *s == '(')) {
+	while (*s && isspace (*s))
+	    s++;
+	if (*s == '(') {
+	    int nesting = 1;
+	    s++;
+	    while (*s && nesting) {
+		if (*s == '(') {
+		    nesting++;
+		} else if (*s == ')') {
+		    nesting--;
+		} else if (*s == '\\') {
+		    if (*(s + 1))
+			s++;
+		}
+		s++;
+	    }
+	}
+    }
+
+    *str = s;
+}
+
+char *
+_notmuch_message_id_parse (void *ctx, const char *message_id, const char **next)
+{
+    const char *s, *end;
+    char *result;
+
+    if (message_id == NULL || *message_id == '\0')
+	return NULL;
+
+    s = message_id;
+
+    skip_space_and_comments (&s);
+
+    /* Skip any unstructured text as well. */
+    while (*s && *s != '<')
+	s++;
+
+    if (*s == '<') {
+	s++;
+    } else {
+	if (next)
+	    *next = s;
+	return NULL;
+    }
+
+    skip_space_and_comments (&s);
+
+    end = s;
+    while (*end && *end != '>')
+	end++;
+    if (next) {
+	if (*end)
+	    *next = end + 1;
+	else
+	    *next = end;
+    }
+
+    if (end > s && *end == '>')
+	end--;
+    if (end <= s)
+	return NULL;
+
+    result = talloc_strndup (ctx, s, end - s + 1);
+
+    /* Finally, collapse any whitespace that is within the message-id
+     * itself. */
+    {
+	char *r;
+	int len;
+
+	for (r = result, len = strlen (r); *r; r++, len--)
+	    if (*r == ' ' || *r == '\t')
+		memmove (r, r + 1, len);
+    }
+
+    return result;
+}
+
+char *
+_notmuch_message_id_parse_strict (void *ctx, const char *message_id)
+{
+    const char *s, *end;
+
+    if (message_id == NULL || *message_id == '\0')
+	return NULL;
+
+    s = skip_space (message_id);
+    if (*s == '<')
+	s++;
+    else
+	return NULL;
+
+    for (end = s; *end && *end != '>'; end++)
+	if (isspace (*end))
+	    return NULL;
+
+    if (*end != '>')
+	return NULL;
+    else {
+	const char *last = skip_space (end + 1);
+	if (*last != '\0')
+	    return NULL;
+    }
+
+    return talloc_strndup (ctx, s, end - s);
+}
diff --git a/lib/message-private.h b/lib/message-private.h
new file mode 100644
index 00000000..73b080e4
--- /dev/null
+++ b/lib/message-private.h
@@ -0,0 +1,16 @@
+#ifndef MESSAGE_PRIVATE_H
+#define MESSAGE_PRIVATE_H
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message);
+
+bool
+_notmuch_message_frozen (notmuch_message_t *message);
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix);
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message,  const char *prefix_name);
+
+#endif
diff --git a/lib/message-property.cc b/lib/message-property.cc
new file mode 100644
index 00000000..7f520340
--- /dev/null
+++ b/lib/message-property.cc
@@ -0,0 +1,214 @@
+/* message-property.cc - Properties are like tags, but (key,value) pairs.
+ * keys are allowed to repeat.
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+#define LOG_XAPIAN_EXCEPTION(message, error) _log_xapian_exception (__location__, message, error)
+
+static void
+_log_xapian_exception (const char *where, notmuch_message_t *message,  const Xapian::Error error)
+{
+    notmuch_database_t *notmuch = notmuch_message_get_database (message);
+
+    _notmuch_database_log (notmuch,
+			   "A Xapian exception occurred at %s: %s\n",
+			   where,
+			   error.get_msg ().c_str ());
+    notmuch->exception_reported = true;
+}
+
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value)
+{
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    *value = _notmuch_string_map_get (_notmuch_message_property_map (message), key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_count_properties (notmuch_message_t *message, const char *key, unsigned int *count)
+{
+    if (! count || ! key || ! message)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    notmuch_string_map_t *map;
+
+    map = _notmuch_message_property_map (message);
+    if (! map)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    notmuch_string_map_iterator_t *matcher = _notmuch_string_map_iterator_create (map, key, true);
+
+    if (! matcher)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    *count = 0;
+    while (_notmuch_string_map_iterator_valid (matcher)) {
+	(*count)++;
+	_notmuch_string_map_iterator_move_to_next (matcher);
+    }
+
+    _notmuch_string_map_iterator_destroy (matcher);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_notmuch_message_modify_property (notmuch_message_t *message, const char *key, const char *value,
+				  bool delete_it)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+    char *term = NULL;
+
+    status = _notmuch_database_ensure_writable (notmuch_message_get_database (message));
+    if (status)
+	return status;
+
+    if (key == NULL || value == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (strchr (key, '='))
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+
+    term = talloc_asprintf (message, "%s=%s", key, value);
+
+    try {
+	if (delete_it)
+	    private_status = _notmuch_message_remove_term (message, "property", term);
+	else
+	    private_status = _notmuch_message_add_term (message, "property", term);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    if (private_status)
+	return COERCE_STATUS (private_status,
+			      "Unhandled error modifying message property");
+    if (! _notmuch_message_frozen (message))
+	_notmuch_message_sync (message);
+
+    if (term)
+	talloc_free (term);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, true);
+}
+
+static
+notmuch_status_t
+_notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key, bool prefix)
+{
+    notmuch_status_t status;
+    const char *term_prefix;
+
+    status = _notmuch_database_ensure_writable (notmuch_message_get_database (message));
+    if (status)
+	return status;
+
+    if (key)
+	term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key,
+				       prefix ? "" : "=");
+    else
+	term_prefix = _find_prefix ("property");
+
+    try {
+	/* XXX better error reporting ? */
+	_notmuch_message_remove_terms (message, term_prefix);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    if (! _notmuch_message_frozen (message))
+	_notmuch_message_sync (message);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    return _notmuch_message_remove_all_properties (message, key, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix)
+{
+    return _notmuch_message_remove_all_properties (message, prefix, true);
+}
+
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
+{
+    notmuch_string_map_t *map;
+
+    map = _notmuch_message_property_map (message);
+    return _notmuch_string_map_iterator_create (map, key, exact);
+}
+
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_valid (properties);
+}
+
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_move_to_next (properties);
+}
+
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_key (properties);
+}
+
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_value (properties);
+}
+
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties)
+{
+    _notmuch_string_map_iterator_destroy (properties);
+}
diff --git a/lib/message.cc b/lib/message.cc
new file mode 100644
index 00000000..46638f80
--- /dev/null
+++ b/lib/message.cc
@@ -0,0 +1,2358 @@
+/* message.cc - Results of message-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+#include <stdint.h>
+
+#include <gmime/gmime.h>
+
+struct _notmuch_message {
+    notmuch_database_t *notmuch;
+    Xapian::docid doc_id;
+    int frozen;
+    char *message_id;
+    char *thread_id;
+    size_t thread_depth;
+    char *in_reply_to;
+    notmuch_string_list_t *tag_list;
+    notmuch_string_list_t *filename_term_list;
+    notmuch_string_list_t *filename_list;
+    char *maildir_flags;
+    char *author;
+    notmuch_message_file_t *message_file;
+    notmuch_string_list_t *property_term_list;
+    notmuch_string_map_t *property_map;
+    notmuch_string_list_t *reference_list;
+    notmuch_message_list_t *replies;
+    unsigned long flags;
+    /* For flags that are initialized on-demand, lazy_flags indicates
+     * if each flag has been initialized. */
+    unsigned long lazy_flags;
+
+    /* Message document modified since last sync */
+    bool modified;
+
+    /* last view of database the struct is synced with */
+    unsigned long last_view;
+
+    Xapian::Document doc;
+    Xapian::termcount termpos;
+};
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
+struct maildir_flag_tag {
+    char flag;
+    const char *tag;
+    bool inverse;
+};
+
+/* ASCII ordered table of Maildir flags and associated tags */
+static const struct maildir_flag_tag flag2tag[] = {
+    { 'D', "draft",   false },
+    { 'F', "flagged", false },
+    { 'P', "passed",  false },
+    { 'R', "replied", false },
+    { 'S', "unread",  true }
+};
+
+/* We end up having to call the destructor explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_message_destructor (notmuch_message_t *message)
+{
+    message->doc.~Document ();
+
+    return 0;
+}
+
+#define LOG_XAPIAN_EXCEPTION(message, error) _log_xapian_exception (__location__, message, error)
+
+static void
+_log_xapian_exception (const char *where, notmuch_message_t *message,  const Xapian::Error error)
+{
+    notmuch_database_t *notmuch = notmuch_message_get_database (message);
+
+    _notmuch_database_log (notmuch,
+			   "A Xapian exception occurred at %s: %s\n",
+			   where,
+			   error.get_msg ().c_str ());
+    notmuch->exception_reported = true;
+}
+
+static notmuch_message_t *
+_notmuch_message_create_for_document (const void *talloc_owner,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id,
+				      Xapian::Document doc,
+				      notmuch_private_status_t *status)
+{
+    notmuch_message_t *message;
+
+    if (status)
+	*status = NOTMUCH_PRIVATE_STATUS_SUCCESS;
+
+    message = talloc (talloc_owner, notmuch_message_t);
+    if (unlikely (message == NULL)) {
+	if (status)
+	    *status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    message->notmuch = notmuch;
+    message->doc_id = doc_id;
+
+    message->frozen = 0;
+    message->flags = 0;
+    message->lazy_flags = 0;
+
+    /* the message is initially not synchronized with Xapian */
+    message->last_view = 0;
+
+    /* Calculated after the thread structure is computed */
+    message->thread_depth = 0;
+
+    /* Each of these will be lazily created as needed. */
+    message->message_id = NULL;
+    message->thread_id = NULL;
+    message->in_reply_to = NULL;
+    message->tag_list = NULL;
+    message->filename_term_list = NULL;
+    message->filename_list = NULL;
+    message->maildir_flags = NULL;
+    message->message_file = NULL;
+    message->author = NULL;
+    message->property_term_list = NULL;
+    message->property_map = NULL;
+    message->reference_list = NULL;
+
+    message->replies = _notmuch_message_list_create (message);
+    if (unlikely (message->replies == NULL)) {
+	if (status)
+	    *status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    /* This is C++'s creepy "placement new", which is really just an
+     * ugly way to call a constructor for a pre-allocated object. So
+     * it's really not an error to not be checking for OUT_OF_MEMORY
+     * here, since this "new" isn't actually allocating memory. This
+     * is language-design comedy of the wrong kind. */
+
+    new (&message->doc) Xapian::Document;
+
+    talloc_set_destructor (message, _notmuch_message_destructor);
+
+    message->doc = doc;
+    message->termpos = 0;
+    message->modified = false;
+
+    return message;
+}
+
+/* Create a new notmuch_message_t object for an existing document in
+ * the database.
+ *
+ * Here, 'talloc owner' is an optional talloc context to which the new
+ * message will belong. This allows for the caller to not bother
+ * calling notmuch_message_destroy on the message, and know that all
+ * memory will be reclaimed when 'talloc_owner' is freed. The caller
+ * still can call notmuch_message_destroy when finished with the
+ * message if desired.
+ *
+ * The 'talloc_owner' argument can also be NULL, in which case the
+ * caller *is* responsible for calling notmuch_message_destroy.
+ *
+ * If no document exists in the database with document ID of 'doc_id'
+ * then this function returns NULL and optionally sets *status to
+ * NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND.
+ *
+ * This function can also fail to due lack of available memory,
+ * returning NULL and optionally setting *status to
+ * NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY.
+ *
+ * The caller can pass NULL for status if uninterested in
+ * distinguishing these two cases.
+ */
+notmuch_message_t *
+_notmuch_message_create (const void *talloc_owner,
+			 notmuch_database_t *notmuch,
+			 unsigned int doc_id,
+			 notmuch_private_status_t *status)
+{
+    Xapian::Document doc;
+
+    try {
+	doc = notmuch->xapian_db->get_document (doc_id);
+    } catch (const Xapian::DocNotFoundError &error) {
+	if (status)
+	    *status = NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+	return NULL;
+    }
+
+    return _notmuch_message_create_for_document (talloc_owner, notmuch,
+						 doc_id, doc, status);
+}
+
+/* Create a new notmuch_message_t object for a specific message ID,
+ * (which may or may not already exist in the database).
+ *
+ * The 'notmuch' database will be the talloc owner of the returned
+ * message.
+ *
+ * This function returns a valid notmuch_message_t whether or not
+ * there is already a document in the database with the given message
+ * ID. These two cases can be distinguished by the value of *status:
+ *
+ *
+ *   NOTMUCH_PRIVATE_STATUS_SUCCESS:
+ *
+ *     There is already a document with message ID 'message_id' in the
+ *     database. The returned message can be used to query/modify the
+ *     document. The message may be a ghost message.
+ *
+ *   NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+ *
+ *     No document with 'message_id' exists in the database. The
+ *     returned message contains a newly created document (not yet
+ *     added to the database) and a document ID that is known not to
+ *     exist in the database.  This message is "blank"; that is, it
+ *     contains only a message ID and no other metadata. The caller
+ *     can modify the message, and a call to _notmuch_message_sync
+ *     will add the document to the database.
+ *
+ * If an error occurs, this function will return NULL and *status
+ * will be set as appropriate. (The status pointer argument must
+ * not be NULL.)
+ */
+notmuch_message_t *
+_notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
+					const char *message_id,
+					notmuch_private_status_t *status_ret)
+{
+    notmuch_message_t *message;
+    Xapian::Document doc;
+    unsigned int doc_id;
+    char *term;
+
+    *status_ret = (notmuch_private_status_t) notmuch_database_find_message (notmuch,
+									    message_id,
+									    &message);
+    if (message)
+	return talloc_steal (notmuch, message);
+    else if (*status_ret)
+	return NULL;
+
+    /* If the message ID is too long, substitute its sha1 instead. */
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (message, message_id);
+
+    term = talloc_asprintf (NULL, "%s%s",
+			    _find_prefix ("id"), message_id);
+    if (term == NULL) {
+	*status_ret = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	INTERNAL_ERROR ("Failure to ensure database is writable.");
+
+    try {
+	doc.add_term (term, 0);
+	talloc_free (term);
+
+	doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
+
+	doc_id = _notmuch_database_generate_doc_id (notmuch);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred creating message: %s\n",
+			       error.get_msg ().c_str ());
+	notmuch->exception_reported = true;
+	*status_ret = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+	return NULL;
+    }
+
+    message = _notmuch_message_create_for_document (notmuch, notmuch,
+						    doc_id, doc, status_ret);
+
+    /* We want to inform the caller that we had to create a new
+     * document. */
+    if (*status_ret == NOTMUCH_PRIVATE_STATUS_SUCCESS)
+	*status_ret = NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+
+    return message;
+}
+
+static char *
+_notmuch_message_get_term (notmuch_message_t *message,
+			   Xapian::TermIterator &i, Xapian::TermIterator &end,
+			   const char *prefix)
+{
+    int prefix_len = strlen (prefix);
+    char *value;
+
+    i.skip_to (prefix);
+
+    if (i == end)
+	return NULL;
+
+    const std::string &term = *i;
+
+    if (strncmp (term.c_str (), prefix, prefix_len))
+	return NULL;
+
+    value = talloc_strdup (message, term.c_str () + prefix_len);
+
+#if DEBUG_DATABASE_SANITY
+    i++;
+
+    if (i != end && strncmp ((*i).c_str (), prefix, prefix_len) == 0) {
+	INTERNAL_ERROR ("Mail (doc_id: %d) has duplicate %s terms: %s and %s\n",
+			message->doc_id, prefix, value,
+			(*i).c_str () + prefix_len);
+    }
+#endif
+
+    return value;
+}
+
+static void
+_notmuch_message_ensure_metadata (notmuch_message_t *message, void *field)
+{
+    Xapian::TermIterator i, end;
+
+    if (field && (message->last_view >= message->notmuch->view))
+	return;
+
+    const char *thread_prefix = _find_prefix ("thread"),
+	       *tag_prefix = _find_prefix ("tag"),
+	       *id_prefix = _find_prefix ("id"),
+	       *type_prefix = _find_prefix ("type"),
+	       *filename_prefix = _find_prefix ("file-direntry"),
+	       *property_prefix = _find_prefix ("property"),
+	       *reference_prefix = _find_prefix ("reference"),
+	       *replyto_prefix = _find_prefix ("replyto");
+
+    /* We do this all in a single pass because Xapian decompresses the
+     * term list every time you iterate over it.  Thus, while this is
+     * slightly more costly than looking up individual fields if only
+     * one field of the message object is actually used, it's a huge
+     * win as more fields are used. */
+    for (int count = 0; count < 3; count++) {
+	try {
+	    i = message->doc.termlist_begin ();
+	    end = message->doc.termlist_end ();
+
+	    /* Get thread */
+	    if (! message->thread_id)
+		message->thread_id =
+		    _notmuch_message_get_term (message, i, end, thread_prefix);
+
+	    /* Get tags */
+	    assert (strcmp (thread_prefix, tag_prefix) < 0);
+	    if (! message->tag_list) {
+		message->tag_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     tag_prefix);
+		_notmuch_string_list_sort (message->tag_list);
+	    }
+
+	    /* Get id */
+	    assert (strcmp (tag_prefix, id_prefix) < 0);
+	    if (! message->message_id)
+		message->message_id =
+		    _notmuch_message_get_term (message, i, end, id_prefix);
+
+	    /* Get document type */
+	    assert (strcmp (id_prefix, type_prefix) < 0);
+	    if (! NOTMUCH_TEST_BIT (message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST)) {
+		i.skip_to (type_prefix);
+		/* "T" is the prefix "type" fields.  See
+		 * BOOLEAN_PREFIX_INTERNAL. */
+		if (*i == "Tmail")
+		    NOTMUCH_CLEAR_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+		else if (*i == "Tghost")
+		    NOTMUCH_SET_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+		else
+		    INTERNAL_ERROR ("Message without type term");
+		NOTMUCH_SET_BIT (&message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+	    }
+
+	    /* Get filename list.  Here we get only the terms.  We lazily
+	     * expand them to full file names when needed in
+	     * _notmuch_message_ensure_filename_list. */
+	    assert (strcmp (type_prefix, filename_prefix) < 0);
+	    if (! message->filename_term_list && ! message->filename_list)
+		message->filename_term_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     filename_prefix);
+
+
+	    /* Get property terms. Mimic the setup with filenames above */
+	    assert (strcmp (filename_prefix, property_prefix) < 0);
+	    if (! message->property_map && ! message->property_term_list)
+		message->property_term_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     property_prefix);
+
+	    /* get references */
+	    assert (strcmp (property_prefix, reference_prefix) < 0);
+	    if (! message->reference_list) {
+		message->reference_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     reference_prefix);
+	    }
+
+	    /* Get reply to */
+	    assert (strcmp (property_prefix, replyto_prefix) < 0);
+	    if (! message->in_reply_to)
+		message->in_reply_to =
+		    _notmuch_message_get_term (message, i, end, replyto_prefix);
+
+
+	    /* It's perfectly valid for a message to have no In-Reply-To
+	     * header. For these cases, we return an empty string. */
+	    if (! message->in_reply_to)
+		message->in_reply_to = talloc_strdup (message, "");
+
+	    /* all the way without an exception */
+	    break;
+	} catch (const Xapian::DatabaseModifiedError &error) {
+	    notmuch_status_t status = notmuch_database_reopen (message->notmuch,
+							       NOTMUCH_DATABASE_MODE_READ_ONLY);
+	    if (status != NOTMUCH_STATUS_SUCCESS)
+		INTERNAL_ERROR ("unhandled error from notmuch_database_reopen: %s\n",
+				notmuch_status_to_string (status));
+	}
+    }
+    message->last_view = message->notmuch->view;
+}
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message,
+				      const char *prefix_name)
+{
+    if (strcmp ("thread", prefix_name) == 0) {
+	talloc_free (message->thread_id);
+	message->thread_id = NULL;
+    }
+
+    if (strcmp ("tag", prefix_name) == 0) {
+	talloc_unlink (message, message->tag_list);
+	message->tag_list = NULL;
+    }
+
+    if (strcmp ("type", prefix_name) == 0) {
+	NOTMUCH_CLEAR_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+	NOTMUCH_CLEAR_BIT (&message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+    }
+
+    if (strcmp ("file-direntry", prefix_name) == 0) {
+	talloc_free (message->filename_term_list);
+	talloc_free (message->filename_list);
+	message->filename_term_list = message->filename_list = NULL;
+    }
+
+    if (strcmp ("property", prefix_name) == 0) {
+
+	if (message->property_term_list)
+	    talloc_free (message->property_term_list);
+	message->property_term_list = NULL;
+
+	if (message->property_map)
+	    talloc_unlink (message, message->property_map);
+
+	message->property_map = NULL;
+    }
+
+    if (strcmp ("replyto", prefix_name) == 0) {
+	talloc_free (message->in_reply_to);
+	message->in_reply_to = NULL;
+    }
+}
+
+unsigned int
+_notmuch_message_get_doc_id (notmuch_message_t *message)
+{
+    return message->doc_id;
+}
+
+const char *
+notmuch_message_get_message_id (notmuch_message_t *message)
+{
+    try {
+	_notmuch_message_ensure_metadata (message, message->message_id);
+    } catch (const Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NULL;
+    }
+
+    if (! message->message_id)
+	INTERNAL_ERROR ("Message with document ID of %u has no message ID.\n",
+			message->doc_id);
+    return message->message_id;
+}
+
+static void
+_notmuch_message_ensure_message_file (notmuch_message_t *message)
+{
+    const char *filename;
+
+    if (message->message_file)
+	return;
+
+    filename = notmuch_message_get_filename (message);
+    if (unlikely (filename == NULL))
+	return;
+
+    message->message_file = _notmuch_message_file_open_ctx (
+	notmuch_message_get_database (message), message, filename);
+}
+
+const char *
+notmuch_message_get_header (notmuch_message_t *message, const char *header)
+{
+    Xapian::valueno slot = Xapian::BAD_VALUENO;
+
+    /* Fetch header from the appropriate xapian value field if
+     * available */
+    if (strcasecmp (header, "from") == 0)
+	slot = NOTMUCH_VALUE_FROM;
+    else if (strcasecmp (header, "subject") == 0)
+	slot = NOTMUCH_VALUE_SUBJECT;
+    else if (strcasecmp (header, "message-id") == 0)
+	slot = NOTMUCH_VALUE_MESSAGE_ID;
+
+    if (slot != Xapian::BAD_VALUENO) {
+	try {
+	    std::string value = message->doc.get_value (slot);
+
+	    /* If we have NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES, then
+	     * empty values indicate empty headers.  If we don't, then
+	     * it could just mean we didn't record the header. */
+	    if ((message->notmuch->features &
+		 NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES) ||
+		! value.empty ())
+		return talloc_strdup (message, value.c_str ());
+
+	} catch (Xapian::Error &error) {
+	    LOG_XAPIAN_EXCEPTION (message, error);
+	    return NULL;
+	}
+    }
+
+    /* Otherwise fall back to parsing the file */
+    _notmuch_message_ensure_message_file (message);
+    if (message->message_file == NULL)
+	return NULL;
+
+    return _notmuch_message_file_get_header (message->message_file, header);
+}
+
+/* Return the message ID from the In-Reply-To header of 'message'.
+ *
+ * Returns an empty string ("") if 'message' has no In-Reply-To
+ * header.
+ *
+ * Returns NULL if any error occurs.
+ */
+const char *
+_notmuch_message_get_in_reply_to (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_metadata (message, message->in_reply_to);
+    return message->in_reply_to;
+}
+
+const char *
+notmuch_message_get_thread_id (notmuch_message_t *message)
+{
+    try {
+	_notmuch_message_ensure_metadata (message, message->thread_id);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NULL;
+    }
+    if (! message->thread_id)
+	INTERNAL_ERROR ("Message with document ID of %u has no thread ID.\n",
+			message->doc_id);
+    return message->thread_id;
+}
+
+void
+_notmuch_message_add_reply (notmuch_message_t *message,
+			    notmuch_message_t *reply)
+{
+    _notmuch_message_list_add_message (message->replies, reply);
+}
+
+size_t
+_notmuch_message_get_thread_depth (notmuch_message_t *message)
+{
+    return message->thread_depth;
+}
+
+void
+_notmuch_message_label_depths (notmuch_message_t *message,
+			       size_t depth)
+{
+    message->thread_depth = depth;
+
+    for (notmuch_messages_t *messages = _notmuch_messages_create (message->replies);
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	notmuch_message_t *child = notmuch_messages_get (messages);
+	_notmuch_message_label_depths (child, depth + 1);
+    }
+}
+
+const notmuch_string_list_t *
+_notmuch_message_get_references (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_metadata (message, message->reference_list);
+    return message->reference_list;
+}
+
+static int
+_cmpmsg (const void *pa, const void *pb)
+{
+    notmuch_message_t **a = (notmuch_message_t **) pa;
+    notmuch_message_t **b = (notmuch_message_t **) pb;
+    time_t time_a = notmuch_message_get_date (*a);
+    time_t time_b = notmuch_message_get_date (*b);
+
+    if (time_a == time_b)
+	return 0;
+    else if (time_a < time_b)
+	return -1;
+    else
+	return 1;
+}
+
+notmuch_message_list_t *
+_notmuch_message_sort_subtrees (void *ctx, notmuch_message_list_t *list)
+{
+
+    size_t count = 0;
+    size_t capacity = 16;
+
+    if (! list)
+	return list;
+
+    void *local = talloc_new (NULL);
+    notmuch_message_list_t *new_list = _notmuch_message_list_create (ctx);
+    notmuch_message_t **message_array = talloc_zero_array (local, notmuch_message_t *, capacity);
+
+    for (notmuch_messages_t *messages = _notmuch_messages_create (list);
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	notmuch_message_t *root = notmuch_messages_get (messages);
+	if (count >= capacity) {
+	    capacity *= 2;
+	    message_array = talloc_realloc (local, message_array, notmuch_message_t *, capacity);
+	}
+	message_array[count++] = root;
+	root->replies = _notmuch_message_sort_subtrees (root, root->replies);
+    }
+
+    qsort (message_array, count, sizeof (notmuch_message_t *), _cmpmsg);
+    for (size_t i = 0; i < count; i++) {
+	_notmuch_message_list_add_message (new_list, message_array[i]);
+    }
+
+    talloc_free (local);
+    talloc_free (list);
+    return new_list;
+}
+
+notmuch_messages_t *
+notmuch_message_get_replies (notmuch_message_t *message)
+{
+    return _notmuch_messages_create (message->replies);
+}
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
+{
+    Xapian::TermIterator i;
+    size_t prefix_len = 0;
+
+    prefix_len = strlen (prefix);
+
+    while (1) {
+	i = message->doc.termlist_begin ();
+	i.skip_to (prefix);
+
+	/* Terminate loop when no terms remain with desired prefix. */
+	if (i == message->doc.termlist_end () ||
+	    strncmp ((*i).c_str (), prefix, prefix_len))
+	    break;
+
+	try {
+	    message->doc.remove_term ((*i));
+	    message->modified = true;
+	} catch (const Xapian::InvalidArgumentError) {
+	    /* Ignore failure to remove non-existent term. */
+	}
+    }
+
+    _notmuch_message_invalidate_metadata (message, "property");
+}
+
+
+/* Remove all terms generated by indexing, i.e. not tags or
+ * properties, along with any automatic tags*/
+/* According to Xapian API docs, none of these calls throw
+ * exceptions */
+static notmuch_private_status_t
+_notmuch_message_remove_indexed_terms (notmuch_message_t *message)
+{
+    Xapian::TermIterator i;
+
+    const std::string
+	id_prefix = _find_prefix ("id"),
+	property_prefix = _find_prefix ("property"),
+	tag_prefix = _find_prefix ("tag"),
+	type_prefix = _find_prefix ("type");
+
+    /* Make sure we have the data to restore to Xapian*/
+    _notmuch_message_ensure_metadata (message, NULL);
+
+    /* Empirically, it turns out to be faster to remove all the terms,
+     * and add back the ones we want. */
+    message->doc.clear_terms ();
+    message->modified = true;
+
+    /* still a mail message */
+    message->doc.add_term (type_prefix + "mail");
+
+    /* Put back message-id */
+    message->doc.add_term (id_prefix + message->message_id);
+
+    /* Put back non-automatic tags */
+    for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags)) {
+
+	const char *tag = notmuch_tags_get (tags);
+
+	if (strcmp (tag, "encrypted") != 0 &&
+	    strcmp (tag, "signed") != 0 &&
+	    strcmp (tag, "attachment") != 0) {
+	    std::string term = tag_prefix + tag;
+	    message->doc.add_term (term);
+	}
+    }
+
+    /* Put back properties */
+    notmuch_message_properties_t *list;
+
+    for (list = notmuch_message_get_properties (message, "", false);
+	 notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+	std::string term = property_prefix +
+			   notmuch_message_properties_key (list) + "=" +
+			   notmuch_message_properties_value (list);
+
+	message->doc.add_term (term);
+    }
+
+    notmuch_message_properties_destroy (list);
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+
+/* Return true if p points at "new" or "cur". */
+static bool
+is_maildir (const char *p)
+{
+    return strcmp (p, "cur") == 0 || strcmp (p, "new") == 0;
+}
+
+/* Add "folder:" term for directory. */
+NODISCARD static notmuch_status_t
+_notmuch_message_add_folder_terms (notmuch_message_t *message,
+				   const char *directory)
+{
+    char *folder, *last;
+
+    folder = talloc_strdup (NULL, directory);
+    if (! folder)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    /*
+     * If the message file is in a leaf directory named "new" or
+     * "cur", presume maildir and index the parent directory. Thus a
+     * "folder:" prefix search matches messages in the specified
+     * maildir folder, i.e. in the specified directory and its "new"
+     * and "cur" subdirectories.
+     *
+     * Note that this means the "folder:" prefix can't be used for
+     * distinguishing between message files in "new" or "cur". The
+     * "path:" prefix needs to be used for that.
+     *
+     * Note the deliberate difference to _filename_is_in_maildir(). We
+     * don't want to index different things depending on the existence
+     * or non-existence of all maildir sibling directories "new",
+     * "cur", and "tmp". Doing so would be surprising, and difficult
+     * for the user to fix in case all subdirectories were not in
+     * place during indexing.
+     */
+    last = strrchr (folder, '/');
+    if (last) {
+	if (is_maildir (last + 1))
+	    *last = '\0';
+    } else if (is_maildir (folder)) {
+	*folder = '\0';
+    }
+
+    if (notmuch_status_t status = COERCE_STATUS (_notmuch_message_add_term (message, "folder",
+									    folder),
+						 "adding folder term"))
+	return status;
+
+    talloc_free (folder);
+
+    message->modified = true;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+#define RECURSIVE_SUFFIX "/**"
+
+/* Add "path:" terms for directory. */
+NODISCARD static notmuch_status_t
+_notmuch_message_add_path_terms (notmuch_message_t *message,
+				 const char *directory)
+{
+    notmuch_status_t status;
+
+    /* Add exact "path:" term. */
+    status = COERCE_STATUS (_notmuch_message_add_term (message, "path", directory),
+			    "adding path term");
+    if (status)
+	return status;
+
+    if (strlen (directory)) {
+	char *path, *p;
+
+	path = talloc_asprintf (NULL, "%s%s", directory, RECURSIVE_SUFFIX);
+	if (! path)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+	/* Add recursive "path:" terms for directory and all parents. */
+	for (p = path + strlen (path) - 1; p > path; p--) {
+	    if (*p == '/') {
+		strcpy (p, RECURSIVE_SUFFIX);
+		status = COERCE_STATUS (_notmuch_message_add_term (message, "path", path),
+					"adding path term");
+		if (status)
+		    return status;
+	    }
+	}
+
+	talloc_free (path);
+    }
+
+    /* Recursive all-matching path:** for consistency. */
+    status = COERCE_STATUS (_notmuch_message_add_term (message, "path", "**"),
+			    "adding path term");
+    if (status)
+	return status;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Add directory based terms for all filenames of the message. */
+static notmuch_status_t
+_notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
+{
+    const char *direntry_prefix = _find_prefix ("file-direntry");
+    int direntry_prefix_len = strlen (direntry_prefix);
+    Xapian::TermIterator i = message->doc.termlist_begin ();
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    for (i.skip_to (direntry_prefix); i != message->doc.termlist_end (); i++) {
+	unsigned int directory_id;
+	const char *direntry, *directory;
+	char *colon;
+	const std::string &term = *i;
+	notmuch_status_t term_status;
+
+	/* Terminate loop at first term without desired prefix. */
+	if (strncmp (term.c_str (), direntry_prefix, direntry_prefix_len))
+	    break;
+
+	/* Indicate that there are filenames remaining. */
+	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	direntry = term.c_str ();
+	direntry += direntry_prefix_len;
+
+	directory_id = strtol (direntry, &colon, 10);
+
+	if (colon == NULL || *colon != ':')
+	    INTERNAL_ERROR ("malformed direntry");
+
+	directory = _notmuch_database_get_directory_path (ctx,
+							  message->notmuch,
+							  directory_id);
+
+	term_status = _notmuch_message_add_folder_terms (message, directory);
+	if (term_status)
+	    return term_status;
+
+	term_status = _notmuch_message_add_path_terms (message, directory);
+	if (term_status)
+	    return term_status;
+    }
+
+    return status;
+}
+
+/* Add an additional 'filename' for 'message'.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync. */
+notmuch_status_t
+_notmuch_message_add_filename (notmuch_message_t *message,
+			       const char *filename)
+{
+    const char *relative, *directory;
+    notmuch_status_t status;
+    notmuch_private_status_t private_status;
+    void *local = talloc_new (message);
+    char *direntry;
+
+    if (filename == NULL)
+	INTERNAL_ERROR ("Message filename cannot be NULL.");
+
+    if (! (message->notmuch->features & NOTMUCH_FEATURE_FILE_TERMS) ||
+	! (message->notmuch->features & NOTMUCH_FEATURE_BOOL_FOLDER))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    relative = _notmuch_database_relative_path (message->notmuch, filename);
+
+    status = _notmuch_database_split_path (local, relative, &directory, NULL);
+    if (status)
+	return status;
+
+    status = _notmuch_database_filename_to_direntry (
+	local, message->notmuch, filename, NOTMUCH_FIND_CREATE, &direntry);
+    if (status)
+	return status;
+
+    /* New file-direntry allows navigating to this message with
+     * notmuch_directory_get_child_files() . */
+    private_status = _notmuch_message_add_term (message, "file-direntry", direntry);
+    switch (private_status) {
+    case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+	break;
+    case NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG:
+	_notmuch_database_log (message->notmuch, "filename too long for file-direntry term: %s\n",
+			       filename);
+	return NOTMUCH_STATUS_PATH_ERROR;
+    default:
+	return COERCE_STATUS (private_status, "adding file-direntry term");
+    }
+
+    status = _notmuch_message_add_folder_terms (message, directory);
+    if (status)
+	return status;
+
+    status = _notmuch_message_add_path_terms (message, directory);
+    if (status)
+	return status;
+
+    talloc_free (local);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Remove a particular 'filename' from 'message'.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync.
+ *
+ * If this message still has other filenames, returns
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID.
+ *
+ * Note: This function does not remove a document from the database,
+ * even if the specified filename is the only filename for this
+ * message. For that functionality, see
+ * notmuch_database_remove_message. */
+notmuch_status_t
+_notmuch_message_remove_filename (notmuch_message_t *message,
+				  const char *filename)
+{
+    void *local = talloc_new (message);
+    char *direntry;
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    if (! (message->notmuch->features & NOTMUCH_FEATURE_FILE_TERMS) ||
+	! (message->notmuch->features & NOTMUCH_FEATURE_BOOL_FOLDER))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    status = _notmuch_database_filename_to_direntry (
+	local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
+    if (status || ! direntry)
+	return status;
+
+    /* Unlink this file from its parent directory. */
+    private_status = _notmuch_message_remove_term (message,
+						   "file-direntry", direntry);
+    status = COERCE_STATUS (private_status,
+			    "Unexpected error from _notmuch_message_remove_term");
+    if (status)
+	return status;
+
+    /* Re-synchronize "folder:" and "path:" terms for this message. */
+
+    /* Remove all "folder:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("folder"));
+
+    /* Remove all "path:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("path"));
+
+    /* Add back terms for all remaining filenames of the message. */
+    status = _notmuch_message_add_directory_terms (local, message);
+
+    talloc_free (local);
+
+    return status;
+}
+
+/* Upgrade the "folder:" prefix from V1 to V2. */
+#define FOLDER_PREFIX_V1       "XFOLDER"
+#define ZFOLDER_PREFIX_V1      "Z" FOLDER_PREFIX_V1
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message)
+{
+    /* Remove all old "folder:" terms. */
+    _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1);
+
+    /* Remove all old "folder:" stemmed terms. */
+    _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1);
+
+    /* Add new boolean "folder:" and "path:" terms. */
+    _notmuch_message_add_directory_terms (message, message);
+}
+
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message)
+{
+    return talloc_strdup (message, message->doc.get_data ().c_str ());
+}
+
+void
+_notmuch_message_clear_data (notmuch_message_t *message)
+{
+    message->doc.set_data ("");
+    message->modified = true;
+}
+
+static void
+_notmuch_message_ensure_filename_list (notmuch_message_t *message)
+{
+    notmuch_string_node_t *node;
+
+    if (message->filename_list)
+	return;
+
+    _notmuch_message_ensure_metadata (message, message->filename_term_list);
+
+    message->filename_list = _notmuch_string_list_create (message);
+    node = message->filename_term_list->head;
+
+    if (! node) {
+	/* A message document created by an old version of notmuch
+	 * (prior to rename support) will have the filename in the
+	 * data of the document rather than as a file-direntry term.
+	 *
+	 * It would be nice to do the upgrade of the document directly
+	 * here, but the database is likely open in read-only mode. */
+
+	std::string datastr = message->doc.get_data ();
+	const char *data = datastr.c_str ();
+
+	if (data == NULL)
+	    INTERNAL_ERROR ("message with no filename");
+
+	_notmuch_string_list_append (message->filename_list, data);
+
+	return;
+    }
+
+    for (; node; node = node->next) {
+	void *local = talloc_new (message);
+	const char *db_path, *directory, *basename, *filename;
+	char *colon, *direntry = NULL;
+	unsigned int directory_id;
+
+	direntry = node->string;
+
+	directory_id = strtol (direntry, &colon, 10);
+
+	if (colon == NULL || *colon != ':')
+	    INTERNAL_ERROR ("malformed direntry");
+
+	basename = colon + 1;
+
+	*colon = '\0';
+
+	db_path = notmuch_config_get (message->notmuch, NOTMUCH_CONFIG_MAIL_ROOT);
+
+	directory = _notmuch_database_get_directory_path (local,
+							  message->notmuch,
+							  directory_id);
+
+	if (strlen (directory))
+	    filename = talloc_asprintf (message, "%s/%s/%s",
+					db_path, directory, basename);
+	else
+	    filename = talloc_asprintf (message, "%s/%s",
+					db_path, basename);
+
+	_notmuch_string_list_append (message->filename_list, filename);
+
+	talloc_free (local);
+    }
+
+    talloc_free (message->filename_term_list);
+    message->filename_term_list = NULL;
+}
+
+const char *
+notmuch_message_get_filename (notmuch_message_t *message)
+{
+    try {
+	_notmuch_message_ensure_filename_list (message);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NULL;
+    }
+
+    if (message->filename_list == NULL)
+	return NULL;
+
+    if (message->filename_list->head == NULL ||
+	message->filename_list->head->string == NULL) {
+	INTERNAL_ERROR ("message with no filename");
+    }
+
+    return message->filename_list->head->string;
+}
+
+notmuch_filenames_t *
+notmuch_message_get_filenames (notmuch_message_t *message)
+{
+    try {
+	_notmuch_message_ensure_filename_list (message);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NULL;
+    }
+
+    return _notmuch_filenames_create (message, message->filename_list);
+}
+
+int
+notmuch_message_count_files (notmuch_message_t *message)
+{
+    try {
+	_notmuch_message_ensure_filename_list (message);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return -1;
+    }
+
+    return _notmuch_string_list_length (message->filename_list);
+}
+
+notmuch_status_t
+notmuch_message_get_flag_st (notmuch_message_t *message,
+			     notmuch_message_flag_t flag,
+			     notmuch_bool_t *is_set)
+{
+    if (! is_set)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    try {
+	if (flag == NOTMUCH_MESSAGE_FLAG_GHOST &&
+	    ! NOTMUCH_TEST_BIT (message->lazy_flags, flag))
+	    _notmuch_message_ensure_metadata (message, NULL);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    *is_set = NOTMUCH_TEST_BIT (message->flags, flag);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_bool_t
+notmuch_message_get_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag)
+{
+    notmuch_bool_t is_set;
+    notmuch_status_t status;
+
+    status = notmuch_message_get_flag_st (message, flag, &is_set);
+
+    if (status)
+	return FALSE;
+    else
+	return is_set;
+}
+
+void
+notmuch_message_set_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag, notmuch_bool_t enable)
+{
+    if (enable)
+	NOTMUCH_SET_BIT (&message->flags, flag);
+    else
+	NOTMUCH_CLEAR_BIT (&message->flags, flag);
+    NOTMUCH_SET_BIT (&message->lazy_flags, flag);
+}
+
+time_t
+notmuch_message_get_date (notmuch_message_t *message)
+{
+    std::string value;
+
+    try {
+	value = message->doc.get_value (NOTMUCH_VALUE_TIMESTAMP);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return 0;
+    }
+
+    if (value.empty ())
+	/* sortable_unserialise is undefined on empty string */
+	return 0;
+    return Xapian::sortable_unserialise (value);
+}
+
+notmuch_tags_t *
+notmuch_message_get_tags (notmuch_message_t *message)
+{
+    notmuch_tags_t *tags;
+
+    try {
+	_notmuch_message_ensure_metadata (message, message->tag_list);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NULL;
+    }
+
+    tags = _notmuch_tags_create (message, message->tag_list);
+    /* _notmuch_tags_create steals the reference to the tag_list, but
+     * in this case it's still used by the message, so we add an
+     * *additional* talloc reference to the list.  As a result, it's
+     * possible to modify the message tags (which talloc_unlink's the
+     * current list from the message) while still iterating because
+     * the iterator will keep the current list alive. */
+    if (! talloc_reference (message, message->tag_list))
+	return NULL;
+
+    return tags;
+}
+
+const char *
+_notmuch_message_get_author (notmuch_message_t *message)
+{
+    return message->author;
+}
+
+void
+_notmuch_message_set_author (notmuch_message_t *message,
+			     const char *author)
+{
+    if (message->author)
+	talloc_free (message->author);
+    message->author = talloc_strdup (message, author);
+    return;
+}
+
+void
+_notmuch_message_set_header_values (notmuch_message_t *message,
+				    const char *date,
+				    const char *from,
+				    const char *subject)
+{
+    time_t time_value;
+
+    /* GMime really doesn't want to see a NULL date, so protect its
+     * sensibilities. */
+    if (date == NULL || *date == '\0') {
+	time_value = 0;
+    } else {
+	time_value = g_mime_utils_header_decode_date_unix (date);
+	/*
+	 * Workaround for https://bugzilla.gnome.org/show_bug.cgi?id=779923
+	 */
+	if (time_value < 0)
+	    time_value = 0;
+    }
+
+    message->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+			    Xapian::sortable_serialise (time_value));
+    message->doc.add_value (NOTMUCH_VALUE_FROM, from);
+    message->doc.add_value (NOTMUCH_VALUE_SUBJECT, subject);
+    message->modified = true;
+}
+
+void
+_notmuch_message_update_subject (notmuch_message_t *message,
+				 const char *subject)
+{
+    message->doc.add_value (NOTMUCH_VALUE_SUBJECT, subject);
+    message->modified = true;
+}
+
+/* Upgrade a message to support NOTMUCH_FEATURE_LAST_MOD.  The caller
+ * must call _notmuch_message_sync. */
+void
+_notmuch_message_upgrade_last_mod (notmuch_message_t *message)
+{
+    /* _notmuch_message_sync will update the last modification
+     * revision; we just have to ask it to. */
+    message->modified = true;
+}
+
+/* Synchronize changes made to message->doc out into the database. */
+void
+_notmuch_message_sync (notmuch_message_t *message)
+{
+    if (_notmuch_database_mode (message->notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	return;
+
+    if (! message->modified)
+	return;
+
+    /* Update the last modification of this message. */
+    if (message->notmuch->features & NOTMUCH_FEATURE_LAST_MOD)
+	/* sortable_serialise gives a reasonably compact encoding,
+	 * which directly translates to reduced IO when scanning the
+	 * value stream.  Since it's built for doubles, we only get 53
+	 * effective bits, but that's still enough for the database to
+	 * last a few centuries at 1 million revisions per second. */
+	message->doc.add_value (NOTMUCH_VALUE_LAST_MOD,
+				Xapian::sortable_serialise (
+				    _notmuch_database_new_revision (
+					message->notmuch)));
+
+    message->notmuch->writable_xapian_db->
+	replace_document (message->doc_id, message->doc);
+    message->modified = false;
+}
+
+/* Delete a message document from the database, leaving a ghost
+ * message in its place */
+notmuch_status_t
+_notmuch_message_delete (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+    const char *mid, *tid;
+    notmuch_message_t *ghost;
+    notmuch_private_status_t private_status;
+    notmuch_database_t *notmuch;
+    unsigned int count = 0;
+    bool is_ghost;
+
+    mid = notmuch_message_get_message_id (message);
+    tid = notmuch_message_get_thread_id (message);
+    notmuch = message->notmuch;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    try {
+	Xapian::PostingIterator thread_doc, thread_doc_end;
+	Xapian::PostingIterator mail_doc, mail_doc_end;
+
+	/* look for a non-ghost message in the same thread */
+	/* if this was a ghost to begin with, we are done */
+	private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost);
+	if (private_status)
+	    return COERCE_STATUS (private_status,
+				  "Error trying to determine whether message was a ghost");
+
+	message->notmuch->writable_xapian_db->delete_document (message->doc_id);
+
+	if (is_ghost)
+	    return NOTMUCH_STATUS_SUCCESS;
+
+	_notmuch_database_find_doc_ids (message->notmuch, "thread", tid, &thread_doc,
+					&thread_doc_end);
+	_notmuch_database_find_doc_ids (message->notmuch, "type", "mail", &mail_doc, &mail_doc_end);
+
+	while (count == 0 &&
+	       thread_doc != thread_doc_end &&
+	       mail_doc != mail_doc_end) {
+	    thread_doc.skip_to (*mail_doc);
+	    if (thread_doc != thread_doc_end) {
+		if (*thread_doc == *mail_doc) {
+		    count++;
+		} else {
+		    mail_doc.skip_to (*thread_doc);
+		    if (mail_doc != mail_doc_end && *thread_doc == *mail_doc)
+			count++;
+		}
+	    }
+	}
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    if (count > 0) {
+	/* reintroduce a ghost in its place because there are still
+	 * other active messages in this thread: */
+	ghost = _notmuch_message_create_for_message_id (notmuch, mid, &private_status);
+	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	    private_status = _notmuch_message_initialize_ghost (ghost, tid);
+	    if (! private_status)
+		_notmuch_message_sync (ghost);
+	} else if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+	    /* this is deeply weird, and we should not have gotten
+	     * into this state.  is there a better error message to
+	     * return here? */
+	    status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+	}
+
+	notmuch_message_destroy (ghost);
+	status = COERCE_STATUS (private_status, "Error converting to ghost message");
+    } else {
+	/* the thread now contains only ghosts: delete them */
+	try {
+	    Xapian::PostingIterator doc, doc_end;
+
+	    _notmuch_database_find_doc_ids (message->notmuch, "thread", tid, &doc, &doc_end);
+
+	    for (; doc != doc_end; doc++) {
+		message->notmuch->writable_xapian_db->delete_document (*doc);
+	    }
+	} catch (Xapian::Error &error) {
+	    LOG_XAPIAN_EXCEPTION (message, error);
+	    return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	}
+
+    }
+    return status;
+}
+
+/* Transform a blank message into a ghost message.  The caller must
+ * _notmuch_message_sync the message. */
+notmuch_private_status_t
+_notmuch_message_initialize_ghost (notmuch_message_t *message,
+				   const char *thread_id)
+{
+    notmuch_private_status_t status;
+
+    status = _notmuch_message_add_term (message, "type", "ghost");
+    if (status)
+	return status;
+    status = _notmuch_message_add_term (message, "thread", thread_id);
+    if (status)
+	return status;
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Ensure that 'message' is not holding any file object open. Future
+ * calls to various functions will still automatically open the
+ * message file as needed.
+ */
+void
+_notmuch_message_close (notmuch_message_t *message)
+{
+    if (message->message_file) {
+	_notmuch_message_file_close (message->message_file);
+	message->message_file = NULL;
+    }
+}
+
+/* Add a name:value term to 'message', (the actual term will be
+ * encoded by prefixing the value with a short prefix). See
+ * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term
+ * names to prefix values.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync. */
+NODISCARD notmuch_private_status_t
+_notmuch_message_add_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value)
+{
+
+    char *term;
+    notmuch_private_status_t status = NOTMUCH_PRIVATE_STATUS_SUCCESS;
+
+    if (value == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term = talloc_asprintf (message, "%s%s",
+			    _find_prefix (prefix_name), value);
+    if (strlen (term) > NOTMUCH_TERM_MAX) {
+	status = NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
+	goto DONE;
+    }
+
+    try {
+	message->doc.add_term (term, 0);
+	message->modified = true;
+	_notmuch_message_invalidate_metadata (message, prefix_name);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	status = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    talloc_free (term);
+    return status;
+}
+
+/* Parse 'text' and add a term to 'message' for each parsed word. Each
+ * term will be added with the appropriate prefix if prefix_name is
+ * non-NULL.
+ */
+notmuch_private_status_t
+_notmuch_message_gen_terms (notmuch_message_t *message,
+			    const char *prefix_name,
+			    const char *text)
+{
+    Xapian::TermGenerator *term_gen = message->notmuch->term_gen;
+
+    if (text == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term_gen->set_document (message->doc);
+    term_gen->set_termpos (message->termpos);
+
+    if (prefix_name) {
+	const char *prefix = _notmuch_database_prefix (message->notmuch, prefix_name);
+	if (prefix == NULL)
+	    return NOTMUCH_PRIVATE_STATUS_BAD_PREFIX;
+
+	_notmuch_message_invalidate_metadata (message, prefix_name);
+	term_gen->index_text (text, 1, prefix);
+    } else {
+	term_gen->index_text (text);
+    }
+
+    /* Create a gap between this an the next terms so they don't
+     * appear to be a phrase. */
+    message->termpos = term_gen->get_termpos () + 100;
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Remove a name:value term from 'message', (the actual term will be
+ * encoded by prefixing the value with a short prefix). See
+ * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term
+ * names to prefix values.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync. */
+NODISCARD notmuch_private_status_t
+_notmuch_message_remove_term (notmuch_message_t *message,
+			      const char *prefix_name,
+			      const char *value)
+{
+    char *term;
+
+    if (value == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term = talloc_asprintf (message, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    if (strlen (term) > NOTMUCH_TERM_MAX)
+	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
+
+    try {
+	message->doc.remove_term (term);
+	message->modified = true;
+    } catch (const Xapian::InvalidArgumentError &error) {
+	/* We'll let the philosophers try to wrestle with the
+	 * question of whether failing to remove that which was not
+	 * there in the first place is failure. For us, we'll silently
+	 * consider it all good. */
+	LOG_XAPIAN_EXCEPTION (message, error);
+    }
+
+    talloc_free (term);
+
+    _notmuch_message_invalidate_metadata (message, prefix_name);
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+notmuch_private_status_t
+_notmuch_message_has_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value,
+			   bool *result)
+{
+    char *term;
+    bool out = false;
+    notmuch_private_status_t status = NOTMUCH_PRIVATE_STATUS_SUCCESS;
+
+    if (value == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term = talloc_asprintf (message, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    if (strlen (term) > NOTMUCH_TERM_MAX)
+	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
+
+    try {
+	/* Look for the exact term */
+	Xapian::TermIterator i = message->doc.termlist_begin ();
+	i.skip_to (term);
+	if (i != message->doc.termlist_end () &&
+	    ! strcmp ((*i).c_str (), term))
+	    out = true;
+    } catch (Xapian::Error &error) {
+	status = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+    }
+    talloc_free (term);
+
+    *result = out;
+    return status;
+}
+
+notmuch_status_t
+notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    try {
+	status = _notmuch_database_ensure_writable (message->notmuch);
+	if (status)
+	    return status;
+
+	if (tag == NULL)
+	    return NOTMUCH_STATUS_NULL_POINTER;
+
+	if (strlen (tag) > NOTMUCH_TAG_MAX)
+	    return NOTMUCH_STATUS_TAG_TOO_LONG;
+
+	private_status = _notmuch_message_add_term (message, "tag", tag);
+	if (private_status) {
+	    return COERCE_STATUS (private_status,
+				  "_notmuch_message_remove_term return unexpected value: %d\n",
+				  private_status);
+	}
+
+	if (! message->frozen)
+	    _notmuch_message_sync (message);
+
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    try {
+	status = _notmuch_database_ensure_writable (message->notmuch);
+	if (status)
+	    return status;
+
+	if (tag == NULL)
+	    return NOTMUCH_STATUS_NULL_POINTER;
+
+	if (strlen (tag) > NOTMUCH_TAG_MAX)
+	    return NOTMUCH_STATUS_TAG_TOO_LONG;
+
+	private_status = _notmuch_message_remove_term (message, "tag", tag);
+	if (private_status) {
+	    return COERCE_STATUS (private_status,
+				  "_notmuch_message_remove_term return unexpected value: %d\n",
+				  private_status);
+	}
+
+	if (! message->frozen)
+	    _notmuch_message_sync (message);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Is the given filename within a maildir directory?
+ *
+ * Specifically, is the final directory component of 'filename' either
+ * "cur" or "new". If so, return a pointer to that final directory
+ * component within 'filename'. If not, return NULL.
+ *
+ * A non-NULL return value is guaranteed to be a valid string pointer
+ * pointing to the characters "new/" or "cur/", (but not
+ * NUL-terminated).
+ */
+static const char *
+_filename_is_in_maildir (const char *filename)
+{
+    const char *slash, *dir = NULL;
+
+    /* Find the last '/' separating directory from filename. */
+    slash = strrchr (filename, '/');
+    if (slash == NULL)
+	return NULL;
+
+    /* Jump back 4 characters to where the previous '/' will be if the
+     * directory is named "cur" or "new". */
+    if (slash - filename < 4)
+	return NULL;
+
+    slash -= 4;
+
+    if (*slash != '/')
+	return NULL;
+
+    dir = slash + 1;
+
+    if (STRNCMP_LITERAL (dir, "cur/") == 0 ||
+	STRNCMP_LITERAL (dir, "new/") == 0) {
+	return dir;
+    }
+
+    return NULL;
+}
+
+static notmuch_status_t
+_ensure_maildir_flags (notmuch_message_t *message, bool force)
+{
+    const char *flags;
+    notmuch_filenames_t *filenames;
+    const char *filename, *dir;
+    char *combined_flags = talloc_strdup (message, "");
+    int seen_maildir_info = 0;
+
+    if (message->maildir_flags) {
+	if (force) {
+	    talloc_free (message->maildir_flags);
+	    message->maildir_flags = NULL;
+	}
+    }
+    filenames = notmuch_message_get_filenames (message);
+    if (! filenames)
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    for (;
+	 notmuch_filenames_valid (filenames);
+	 notmuch_filenames_move_to_next (filenames)) {
+	filename = notmuch_filenames_get (filenames);
+	dir = _filename_is_in_maildir (filename);
+
+	if (! dir)
+	    continue;
+
+	flags = strstr (filename, ":2,");
+	if (flags) {
+	    seen_maildir_info = 1;
+	    flags += 3;
+	    combined_flags = talloc_strdup_append (combined_flags, flags);
+	} else if (STRNCMP_LITERAL (dir, "new/") == 0) {
+	    /* Messages are delivered to new/ with no "info" part, but
+	     * they effectively have default maildir flags.  According
+	     * to the spec, we should ignore the info part for
+	     * messages in new/, but some MUAs (mutt) can set maildir
+	     * flags on messages in new/, so we're liberal in what we
+	     * accept. */
+	    seen_maildir_info = 1;
+	}
+    }
+    if (seen_maildir_info)
+	message->maildir_flags = combined_flags;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_bool_t
+notmuch_message_has_maildir_flag (notmuch_message_t *message, char flag)
+{
+    notmuch_status_t status;
+    notmuch_bool_t ret;
+
+    status = notmuch_message_has_maildir_flag_st (message, flag, &ret);
+    if (status)
+	return FALSE;
+
+    return ret;
+}
+
+notmuch_status_t
+notmuch_message_has_maildir_flag_st (notmuch_message_t *message,
+				     char flag,
+				     notmuch_bool_t *is_set)
+{
+    notmuch_status_t status;
+
+    if (! is_set)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    status = _ensure_maildir_flags (message, false);
+    if (status)
+	return status;
+
+    *is_set =  message->maildir_flags && (strchr (message->maildir_flags, flag) != NULL);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+    unsigned i;
+
+    status = _ensure_maildir_flags (message, true);
+    if (status)
+	return status;
+    /* If none of the filenames have any maildir info field (not even
+     * an empty info with no flags set) then there's no information to
+     * go on, so do nothing. */
+    if (! message->maildir_flags)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    status = notmuch_message_freeze (message);
+    if (status)
+	return status;
+
+    for (i = 0; i < ARRAY_SIZE (flag2tag); i++) {
+	if ((strchr (message->maildir_flags, flag2tag[i].flag) != NULL)
+	    ^
+	    flag2tag[i].inverse) {
+	    status = notmuch_message_add_tag (message, flag2tag[i].tag);
+	} else {
+	    status = notmuch_message_remove_tag (message, flag2tag[i].tag);
+	}
+	if (status)
+	    return status;
+    }
+    status = notmuch_message_thaw (message);
+
+    return status;
+}
+
+/* From the set of tags on 'message' and the flag2tag table, compute a
+ * set of maildir-flag actions to be taken, (flags that should be
+ * either set or cleared).
+ *
+ * The result is returned as two talloced strings: to_set, and to_clear
+ */
+static void
+_get_maildir_flag_actions (notmuch_message_t *message,
+			   char **to_set_ret,
+			   char **to_clear_ret)
+{
+    char *to_set, *to_clear;
+    notmuch_tags_t *tags;
+    const char *tag;
+    unsigned i;
+
+    to_set = talloc_strdup (message, "");
+    to_clear = talloc_strdup (message, "");
+
+    /* First, find flags for all set tags. */
+    for (tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags)) {
+	tag = notmuch_tags_get (tags);
+
+	for (i = 0; i < ARRAY_SIZE (flag2tag); i++) {
+	    if (strcmp (tag, flag2tag[i].tag) == 0) {
+		if (flag2tag[i].inverse)
+		    to_clear = talloc_asprintf_append (to_clear,
+						       "%c",
+						       flag2tag[i].flag);
+		else
+		    to_set = talloc_asprintf_append (to_set,
+						     "%c",
+						     flag2tag[i].flag);
+	    }
+	}
+    }
+
+    /* Then, find the flags for all tags not present. */
+    for (i = 0; i < ARRAY_SIZE (flag2tag); i++) {
+	if (flag2tag[i].inverse) {
+	    if (strchr (to_clear, flag2tag[i].flag) == NULL)
+		to_set = talloc_asprintf_append (to_set, "%c", flag2tag[i].flag);
+	} else {
+	    if (strchr (to_set, flag2tag[i].flag) == NULL)
+		to_clear = talloc_asprintf_append (to_clear, "%c", flag2tag[i].flag);
+	}
+    }
+
+    *to_set_ret = to_set;
+    *to_clear_ret = to_clear;
+}
+
+/* Given 'filename' and a set of maildir flags to set and to clear,
+ * compute the new maildir filename.
+ *
+ * If the existing filename is in the directory "new", the new
+ * filename will be in the directory "cur", except for the case when
+ * no flags are changed and the existing filename does not contain
+ * maildir info (starting with ",2:").
+ *
+ * After a sequence of ":2," in the filename, any subsequent
+ * single-character flags will be added or removed according to the
+ * characters in flags_to_set and flags_to_clear. Any existing flags
+ * not mentioned in either string will remain. The final list of flags
+ * will be in ASCII order.
+ *
+ * If the original flags seem invalid, (repeated characters or
+ * non-ASCII ordering of flags), this function will return NULL
+ * (meaning that renaming would not be safe and should not occur).
+ */
+static char *
+_new_maildir_filename (void *ctx,
+		       const char *filename,
+		       const char *flags_to_set,
+		       const char *flags_to_clear)
+{
+    const char *info, *flags;
+    unsigned int flag, last_flag;
+    char *filename_new, *dir;
+    char flag_map[128];
+    int flags_in_map = 0;
+    bool flags_changed = false;
+    unsigned int i;
+    char *s;
+
+    memset (flag_map, 0, sizeof (flag_map));
+
+    info = strstr (filename, ":2,");
+
+    if (info == NULL) {
+	info = filename + strlen (filename);
+    } else {
+	/* Loop through existing flags in filename. */
+	for (flags = info + 3, last_flag = 0;
+	     *flags;
+	     last_flag = flag, flags++) {
+	    flag = *flags;
+
+	    /* Original flags not in ASCII order. Abort. */
+	    if (flag < last_flag)
+		return NULL;
+
+	    /* Non-ASCII flag. Abort. */
+	    if (flag > sizeof (flag_map) - 1)
+		return NULL;
+
+	    /* Repeated flag value. Abort. */
+	    if (flag_map[flag])
+		return NULL;
+
+	    flag_map[flag] = 1;
+	    flags_in_map++;
+	}
+    }
+
+    /* Then set and clear our flags from tags. */
+    for (flags = flags_to_set; *flags; flags++) {
+	flag = *flags;
+	if (flag_map[flag] == 0) {
+	    flag_map[flag] = 1;
+	    flags_in_map++;
+	    flags_changed = true;
+	}
+    }
+
+    for (flags = flags_to_clear; *flags; flags++) {
+	flag = *flags;
+	if (flag_map[flag]) {
+	    flag_map[flag] = 0;
+	    flags_in_map--;
+	    flags_changed = true;
+	}
+    }
+
+    /* Messages in new/ without maildir info can be kept in new/ if no
+     * flags have changed. */
+    dir = (char *) _filename_is_in_maildir (filename);
+    if (dir && STRNCMP_LITERAL (dir, "new/") == 0 && ! *info && ! flags_changed)
+	return talloc_strdup (ctx, filename);
+
+    filename_new = (char *) talloc_size (ctx,
+					 info - filename +
+					 strlen (":2,") + flags_in_map + 1);
+    if (unlikely (filename_new == NULL))
+	return NULL;
+
+    strncpy (filename_new, filename, info - filename);
+    filename_new[info - filename] = '\0';
+
+    strcat (filename_new, ":2,");
+
+    s = filename_new + strlen (filename_new);
+    for (i = 0; i < sizeof (flag_map); i++) {
+	if (flag_map[i]) {
+	    *s = i;
+	    s++;
+	}
+    }
+    *s = '\0';
+
+    /* If message is in new/ move it under cur/. */
+    dir = (char *) _filename_is_in_maildir (filename_new);
+    if (dir && STRNCMP_LITERAL (dir, "new/") == 0)
+	memcpy (dir, "cur/", 4);
+
+    return filename_new;
+}
+
+notmuch_status_t
+notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
+{
+    notmuch_filenames_t *filenames;
+    const char *filename;
+    char *filename_new;
+    char *to_set, *to_clear;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    _get_maildir_flag_actions (message, &to_set, &to_clear);
+
+    for (filenames = notmuch_message_get_filenames (message);
+	 notmuch_filenames_valid (filenames);
+	 notmuch_filenames_move_to_next (filenames)) {
+	filename = notmuch_filenames_get (filenames);
+
+	if (! _filename_is_in_maildir (filename))
+	    continue;
+
+	filename_new = _new_maildir_filename (message, filename,
+					      to_set, to_clear);
+	if (filename_new == NULL)
+	    continue;
+
+	if (strcmp (filename, filename_new)) {
+	    int err;
+	    notmuch_status_t new_status;
+
+	    err = rename (filename, filename_new);
+	    if (err)
+		continue;
+
+	    new_status = _notmuch_message_remove_filename (message,
+							   filename);
+	    /* Hold on to only the first error. */
+	    if (! status && new_status
+		&& new_status != NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) {
+		status = new_status;
+		continue;
+	    }
+
+	    new_status = _notmuch_message_add_filename (message,
+							filename_new);
+	    /* Hold on to only the first error. */
+	    if (! status && new_status) {
+		status = new_status;
+		continue;
+	    }
+
+	    _notmuch_message_sync (message);
+	}
+
+	talloc_free (filename_new);
+    }
+
+    talloc_free (to_set);
+    talloc_free (to_clear);
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_message_remove_all_tags (notmuch_message_t *message)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+    notmuch_tags_t *tags;
+    const char *tag;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+    tags = notmuch_message_get_tags (message);
+    if (! tags)
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+
+    for (;
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags)) {
+	tag = notmuch_tags_get (tags);
+
+	private_status = _notmuch_message_remove_term (message, "tag", tag);
+	if (private_status) {
+	    return COERCE_STATUS (private_status,
+				  "_notmuch_message_remove_term return unexpected value: %d\n",
+				  private_status);
+	}
+    }
+
+    if (! message->frozen)
+	_notmuch_message_sync (message);
+
+    talloc_free (tags);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_freeze (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    message->frozen++;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_thaw (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    if (message->frozen > 0) {
+	message->frozen--;
+	if (message->frozen == 0)
+	    _notmuch_message_sync (message);
+	return NOTMUCH_STATUS_SUCCESS;
+    } else {
+	return NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW;
+    }
+}
+
+void
+notmuch_message_destroy (notmuch_message_t *message)
+{
+    talloc_free (message);
+}
+
+notmuch_database_t *
+notmuch_message_get_database (const notmuch_message_t *message)
+{
+    return message->notmuch;
+}
+
+static void
+_notmuch_message_ensure_property_map (notmuch_message_t *message)
+{
+    notmuch_string_node_t *node;
+
+    if (message->property_map)
+	return;
+
+    _notmuch_message_ensure_metadata (message, message->property_term_list);
+
+    message->property_map = _notmuch_string_map_create (message);
+
+    for (node = message->property_term_list->head; node; node = node->next) {
+	const char *key;
+	char *value;
+
+	value = strchr (node->string, '=');
+	if (! value)
+	    INTERNAL_ERROR ("malformed property term");
+
+	*value = '\0';
+	value++;
+	key = node->string;
+
+	_notmuch_string_map_append (message->property_map, key, value);
+
+    }
+
+    talloc_free (message->property_term_list);
+    message->property_term_list = NULL;
+}
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_property_map (message);
+
+    return message->property_map;
+}
+
+bool
+_notmuch_message_frozen (notmuch_message_t *message)
+{
+    return message->frozen;
+}
+
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+			 notmuch_indexopts_t *indexopts)
+{
+    notmuch_database_t *notmuch = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_private_status_t private_status;
+    notmuch_filenames_t *orig_filenames = NULL;
+    const char *orig_thread_id = NULL;
+    notmuch_message_file_t *message_file = NULL;
+
+    int found = 0;
+
+    if (message == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    /* Save in case we need to delete message */
+    orig_thread_id = notmuch_message_get_thread_id (message);
+    if (! orig_thread_id) {
+	/* the following is correct as long as there is only one reason
+	 * n_m_get_thread_id returns NULL
+	 */
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    /* strdup it because the metadata may be invalidated */
+    orig_thread_id = talloc_strdup (message, orig_thread_id);
+
+    notmuch = notmuch_message_get_database (message);
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    orig_filenames = notmuch_message_get_filenames (message);
+
+    private_status = _notmuch_message_remove_indexed_terms (message);
+    if (private_status) {
+	ret = COERCE_STATUS (private_status, "error removing terms");
+	goto DONE;
+    }
+
+    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
+    if (ret)
+	goto DONE; /* XXX TODO: distinguish from other error returns above? */
+    if (indexopts && notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_FALSE) {
+	ret = notmuch_message_remove_all_properties (message, "session-key");
+	if (ret)
+	    goto DONE;
+    }
+
+    /* re-add the filenames with the associated indexopts */
+    for (; notmuch_filenames_valid (orig_filenames);
+	 notmuch_filenames_move_to_next (orig_filenames)) {
+
+	const char *date;
+	const char *from, *to, *subject;
+	char *message_id = NULL;
+	const char *thread_id = NULL;
+
+	const char *filename = notmuch_filenames_get (orig_filenames);
+
+	message_file = _notmuch_message_file_open (notmuch, filename);
+	if (message_file == NULL)
+	    continue;
+
+	ret = _notmuch_message_file_get_headers (message_file,
+						 &from, &subject, &to, &date,
+						 &message_id);
+	if (ret)
+	    goto DONE;
+
+	/* XXX TODO: deal with changing message id? */
+
+	_notmuch_message_add_filename (message, filename);
+
+	ret = _notmuch_database_link_message_to_parents (notmuch, message,
+							 message_file,
+							 &thread_id);
+	if (ret)
+	    goto DONE;
+
+	if (thread_id == NULL)
+	    thread_id = orig_thread_id;
+
+	ret = COERCE_STATUS (_notmuch_message_add_term (message, "thread", thread_id),
+			     "adding thread term");
+	if (ret)
+	    goto DONE;
+
+	/* Take header values only from first filename */
+	if (found == 0)
+	    _notmuch_message_set_header_values (message, date, from, subject);
+
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
+
+	if (ret == NOTMUCH_STATUS_FILE_ERROR)
+	    continue;
+	if (ret)
+	    goto DONE;
+
+	found++;
+	_notmuch_message_file_close (message_file);
+	message_file = NULL;
+    }
+    if (found == 0) {
+	/* put back thread id to help cleanup */
+	ret = COERCE_STATUS (_notmuch_message_add_term (message, "thread", orig_thread_id),
+			     "adding thread term");
+	if (ret)
+	    goto DONE;
+
+	ret = _notmuch_message_delete (message);
+    } else {
+	_notmuch_message_sync (message);
+    }
+
+  DONE:
+    if (message_file)
+	_notmuch_message_file_close (message_file);
+
+    /* XXX TODO destroy orig_filenames? */
+    return ret;
+}
diff --git a/lib/messages.c b/lib/messages.c
new file mode 100644
index 00000000..eec0a162
--- /dev/null
+++ b/lib/messages.c
@@ -0,0 +1,194 @@
+/* messages.c - Iterator for a set of messages
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <glib.h>
+
+/* Create a new notmuch_message_list_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+notmuch_message_list_t *
+_notmuch_message_list_create (const void *ctx)
+{
+    notmuch_message_list_t *list;
+
+    list = talloc (ctx, notmuch_message_list_t);
+    if (unlikely (list == NULL))
+	return NULL;
+
+    list->head = NULL;
+    list->tail = &list->head;
+
+    return list;
+}
+
+/* Append 'message' to the end of 'list'. */
+void
+_notmuch_message_list_add_message (notmuch_message_list_t *list,
+				   notmuch_message_t *message)
+{
+    notmuch_message_node_t *node = talloc (list, notmuch_message_node_t);
+
+    node->message = message;
+    node->next = NULL;
+
+    *(list->tail) = node;
+    list->tail = &node->next;
+}
+
+bool
+_notmuch_message_list_empty (notmuch_message_list_t *list)
+{
+    if (list == NULL)
+	return TRUE;
+
+    return (list->head == NULL);
+}
+
+notmuch_messages_t *
+_notmuch_messages_create (notmuch_message_list_t *list)
+{
+    notmuch_messages_t *messages;
+
+    if (list->head == NULL)
+	return NULL;
+
+    messages = talloc (list, notmuch_messages_t);
+    if (unlikely (messages == NULL))
+	return NULL;
+
+    messages->is_of_list_type = true;
+    messages->iterator = list->head;
+
+    return messages;
+}
+
+/* We're using the "is_of_type_list" to conditionally defer to the
+ * notmuch_mset_messages_t implementation of notmuch_messages_t in
+ * query.cc. It's ugly that that's over in query.cc, and it's ugly
+ * that we're not using a union here. Both of those uglies are due to
+ * C++:
+ *
+ *	1. I didn't want to force a C++ header file onto
+ *	   notmuch-private.h and suddenly subject all our code to a
+ *	   C++ compiler and its rules.
+ *
+ *	2. C++ won't allow me to put C++ objects, (with non-trivial
+ *	   constructors) into a union anyway. Even though I'd
+ *	   carefully control object construction with placement new
+ *	   anyway. *sigh*
+ */
+notmuch_bool_t
+notmuch_messages_valid (notmuch_messages_t *messages)
+{
+    if (messages == NULL)
+	return false;
+
+    if (! messages->is_of_list_type)
+	return _notmuch_mset_messages_valid (messages);
+
+    return (messages->iterator != NULL);
+}
+
+bool
+_notmuch_messages_has_next (notmuch_messages_t *messages)
+{
+    if (! notmuch_messages_valid (messages))
+	return false;
+
+    if (! messages->is_of_list_type)
+	INTERNAL_ERROR ("_notmuch_messages_has_next not implemented for msets");
+
+    return (messages->iterator->next != NULL);
+}
+
+notmuch_message_t *
+notmuch_messages_get (notmuch_messages_t *messages)
+{
+    if (! messages->is_of_list_type)
+	return _notmuch_mset_messages_get (messages);
+
+    if (messages->iterator == NULL)
+	return NULL;
+
+    return messages->iterator->message;
+}
+
+void
+notmuch_messages_move_to_next (notmuch_messages_t *messages)
+{
+    if (! messages->is_of_list_type) {
+	_notmuch_mset_messages_move_to_next (messages);
+	return;
+    }
+
+    if (messages->iterator == NULL)
+	return;
+
+    messages->iterator = messages->iterator->next;
+}
+
+void
+notmuch_messages_destroy (notmuch_messages_t *messages)
+{
+    talloc_free (messages);
+}
+
+
+notmuch_tags_t *
+notmuch_messages_collect_tags (notmuch_messages_t *messages)
+{
+    notmuch_string_list_t *tags;
+    notmuch_tags_t *msg_tags;
+    notmuch_message_t *msg;
+    GHashTable *htable;
+    GList *keys, *l;
+    const char *tag;
+
+    tags = _notmuch_string_list_create (messages);
+    if (tags == NULL) return NULL;
+
+    htable = g_hash_table_new_full (g_str_hash, g_str_equal, free, NULL);
+
+    while ((msg = notmuch_messages_get (messages))) {
+	msg_tags = notmuch_message_get_tags (msg);
+	while ((tag = notmuch_tags_get (msg_tags))) {
+	    g_hash_table_insert (htable, xstrdup (tag), NULL);
+	    notmuch_tags_move_to_next (msg_tags);
+	}
+	notmuch_tags_destroy (msg_tags);
+	notmuch_message_destroy (msg);
+	notmuch_messages_move_to_next (messages);
+    }
+
+    keys = g_hash_table_get_keys (htable);
+    for (l = keys; l; l = l->next) {
+	_notmuch_string_list_append (tags, (char *) l->data);
+    }
+
+    g_list_free (keys);
+    g_hash_table_destroy (htable);
+
+    _notmuch_string_list_sort (tags);
+    return _notmuch_tags_create (messages, tags);
+}
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
new file mode 100644
index 00000000..367e23e6
--- /dev/null
+++ b/lib/notmuch-private.h
@@ -0,0 +1,768 @@
+/* notmuch-private.h - Internal interfaces for notmuch.
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_PRIVATE_H
+#define NOTMUCH_PRIVATE_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE /* For getline and asprintf */
+#endif
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "compat.h"
+
+#include "notmuch.h"
+
+#include "xutil.h"
+#include "error_util.h"
+#include "string-util.h"
+#include "crypto.h"
+#include "repair.h"
+
+NOTMUCH_BEGIN_DECLS
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include <talloc.h>
+
+#ifdef DEBUG
+# define DEBUG_DATABASE_SANITY 1
+# define DEBUG_THREADING 1
+# define DEBUG_QUERY 1
+#endif
+
+#define COMPILE_TIME_ASSERT(pred) ((void) sizeof (char[1 - 2 * ! (pred)]))
+
+#define STRNCMP_LITERAL(var, literal) \
+    strncmp ((var), (literal), sizeof (literal) - 1)
+
+/* Robust bit test/set/reset macros */
+#define _NOTMUCH_VALID_BIT(bit) \
+    ((bit) >= 0 && ((unsigned long) bit) < CHAR_BIT * sizeof (unsigned long long))
+#define NOTMUCH_TEST_BIT(val, bit) \
+    (_NOTMUCH_VALID_BIT (bit) ? ! ! ((val) & (1ull << (bit))) : 0)
+#define NOTMUCH_SET_BIT(valp, bit) \
+    (_NOTMUCH_VALID_BIT (bit) ? (*(valp) |= (1ull << (bit))) : *(valp))
+#define NOTMUCH_CLEAR_BIT(valp,  bit) \
+    (_NOTMUCH_VALID_BIT (bit) ? (*(valp) &= ~(1ull << (bit))) : *(valp))
+
+#define unused(x) x ## _unused __attribute__ ((unused))
+
+/* Thanks to Andrew Tridgell's (SAMBA's) talloc for this definition of
+ * unlikely. The talloc source code comes to us via the GNU LGPL v. 3.
+ */
+/* these macros gain us a few percent of speed on gcc */
+#if (__GNUC__ >= 3)
+/* the strange !! is to ensure that __builtin_expect() takes either 0 or 1
+ * as its first argument */
+#ifndef likely
+#define likely(x)   __builtin_expect (! ! (x), 1)
+#endif
+#ifndef unlikely
+#define unlikely(x) __builtin_expect (! ! (x), 0)
+#endif
+#else
+#ifndef likely
+#define likely(x) (x)
+#endif
+#ifndef unlikely
+#define unlikely(x) (x)
+#endif
+#endif
+
+typedef enum {
+    NOTMUCH_VALUE_TIMESTAMP = 0,
+    NOTMUCH_VALUE_MESSAGE_ID,
+    NOTMUCH_VALUE_FROM,
+    NOTMUCH_VALUE_SUBJECT,
+    NOTMUCH_VALUE_LAST_MOD,
+} notmuch_value_t;
+
+/* Xapian (with flint backend) complains if we provide a term longer
+ * than this, but I haven't yet found a way to query the limit
+ * programmatically. */
+#define NOTMUCH_TERM_MAX 245
+
+#define NOTMUCH_METADATA_THREAD_ID_PREFIX "thread_id_"
+
+/* For message IDs we have to be even more restrictive. Beyond fitting
+ * into the term limit, we also use message IDs to construct
+ * metadata-key values. And the documentation says that these should
+ * be restricted to about 200 characters. (The actual limit for the
+ * chert backend at least is 252.)
+ */
+#define NOTMUCH_MESSAGE_ID_MAX (200 - sizeof (NOTMUCH_METADATA_THREAD_ID_PREFIX))
+
+typedef enum {
+    /* First, copy all the public status values. */
+    NOTMUCH_PRIVATE_STATUS_SUCCESS			= NOTMUCH_STATUS_SUCCESS,
+    NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY		= NOTMUCH_STATUS_OUT_OF_MEMORY,
+    NOTMUCH_PRIVATE_STATUS_READ_ONLY_DATABASE		= NOTMUCH_STATUS_READ_ONLY_DATABASE,
+    NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION		= NOTMUCH_STATUS_XAPIAN_EXCEPTION,
+    NOTMUCH_PRIVATE_STATUS_FILE_ERROR			= NOTMUCH_STATUS_FILE_ERROR,
+    NOTMUCH_PRIVATE_STATUS_FILE_NOT_EMAIL		= NOTMUCH_STATUS_FILE_NOT_EMAIL,
+    NOTMUCH_PRIVATE_STATUS_NULL_POINTER			= NOTMUCH_STATUS_NULL_POINTER,
+    NOTMUCH_PRIVATE_STATUS_TAG_TOO_LONG			= NOTMUCH_STATUS_TAG_TOO_LONG,
+    NOTMUCH_PRIVATE_STATUS_UNBALANCED_FREEZE_THAW	= NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
+    NOTMUCH_PRIVATE_STATUS_UNBALANCED_ATOMIC		= NOTMUCH_STATUS_UNBALANCED_ATOMIC,
+    NOTMUCH_PRIVATE_STATUS_UNSUPPORTED_OPERATION	= NOTMUCH_STATUS_UNSUPPORTED_OPERATION,
+    NOTMUCH_PRIVATE_STATUS_UPGRADE_REQUIRED		= NOTMUCH_STATUS_UPGRADE_REQUIRED,
+    NOTMUCH_PRIVATE_STATUS_PATH_ERROR			= NOTMUCH_STATUS_PATH_ERROR,
+    NOTMUCH_PRIVATE_STATUS_IGNORED			= NOTMUCH_STATUS_IGNORED,
+    NOTMUCH_PRIVATE_STATUS_ILLEGAL_ARGUMENT		= NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    NOTMUCH_PRIVATE_STATUS_MALFORMED_CRYPTO_PROTOCOL		= NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL,
+    NOTMUCH_PRIVATE_STATUS_FAILED_CRYPTO_CONTEXT_CREATION	= NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION,
+    NOTMUCH_PRIVATE_STATUS_UNKNOWN_CRYPTO_PROTOCOL		= NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL,
+    NOTMUCH_PRIVATE_STATUS_NO_CONFIG				= NOTMUCH_STATUS_NO_CONFIG,
+    NOTMUCH_PRIVATE_STATUS_NO_DATABASE				= NOTMUCH_STATUS_NO_DATABASE,
+    NOTMUCH_PRIVATE_STATUS_DATABASE_EXISTS			= NOTMUCH_STATUS_DATABASE_EXISTS,
+    NOTMUCH_PRIVATE_STATUS_NO_MAIL_ROOT				= NOTMUCH_STATUS_NO_MAIL_ROOT,
+    NOTMUCH_PRIVATE_STATUS_BAD_QUERY_SYNTAX			= NOTMUCH_STATUS_BAD_QUERY_SYNTAX,
+    NOTMUCH_PRIVATE_STATUS_CLOSED_DATABASE			= NOTMUCH_STATUS_CLOSED_DATABASE,
+
+    /* Then add our own private values. */
+    NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG		= NOTMUCH_STATUS_LAST_STATUS,
+    NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND,
+    NOTMUCH_PRIVATE_STATUS_BAD_PREFIX,
+
+    NOTMUCH_PRIVATE_STATUS_LAST_STATUS
+} notmuch_private_status_t;
+
+/* Coerce a notmuch_private_status_t value to a notmuch_status_t
+ * value, generating an internal error if the private value is equal
+ * to or greater than NOTMUCH_STATUS_LAST_STATUS. (The idea here is
+ * that the caller has previously handled any expected
+ * notmuch_private_status_t values.)
+ *
+ * Note that the function _internal_error does not return. Evaluating
+ * to NOTMUCH_STATUS_SUCCESS is done purely to appease the compiler.
+ */
+#define COERCE_STATUS(private_status, format, ...)                      \
+    ((private_status >= (notmuch_private_status_t) NOTMUCH_STATUS_LAST_STATUS) \
+     ?                                                                  \
+     _internal_error (format " (%s).\n",                                \
+		      ##__VA_ARGS__,                                    \
+		      __location__),                                    \
+     (notmuch_status_t) NOTMUCH_PRIVATE_STATUS_SUCCESS                  \
+     :                                                                  \
+     (notmuch_status_t) private_status)
+
+/* Flags shared by various lookup functions. */
+typedef enum {
+    /* Lookup without creating any documents.  This is the default
+     * behavior. */
+    NOTMUCH_FIND_LOOKUP = 0,
+    /* If set, create the necessary document (or documents) if they
+     * are missing.  Requires a read/write database. */
+    NOTMUCH_FIND_CREATE = 1 << 0,
+} notmuch_find_flags_t;
+
+typedef struct _notmuch_doc_id_set notmuch_doc_id_set_t;
+
+/* database.cc */
+
+/* Lookup a prefix value by name.
+ *
+ * XXX: This should really be static inside of message.cc, and we can
+ * do that once we convert database.cc to use the
+ * _notmuch_message_add/remove_term functions. */
+const char *
+_find_prefix (const char *name);
+
+/* Lookup a prefix value by name, including possibly user defined prefixes
+ */
+const char *
+_notmuch_database_prefix (notmuch_database_t *notmuch, const char *name);
+
+char *
+_notmuch_message_id_compressed (void *ctx, const char *message_id);
+
+notmuch_status_t
+_notmuch_database_ensure_writable (notmuch_database_t *notmuch);
+
+void
+_notmuch_database_log (notmuch_database_t *notmuch,
+		       const char *format, ...);
+
+void
+_notmuch_database_log_append (notmuch_database_t *notmuch,
+			      const char *format, ...);
+
+unsigned long
+_notmuch_database_new_revision (notmuch_database_t *notmuch);
+
+const char *
+_notmuch_database_relative_path (notmuch_database_t *notmuch,
+				 const char *path);
+
+notmuch_status_t
+_notmuch_database_split_path (void *ctx,
+			      const char *path,
+			      const char **directory,
+			      const char **basename);
+
+const char *
+_notmuch_database_get_directory_db_path (const char *path);
+
+unsigned int
+_notmuch_database_generate_doc_id (notmuch_database_t *notmuch);
+
+notmuch_private_status_t
+_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
+				      const char *prefix_name,
+				      const char *value,
+				      unsigned int *doc_id);
+
+notmuch_status_t
+_notmuch_database_find_directory_id (notmuch_database_t *database,
+				     const char *path,
+				     notmuch_find_flags_t flags,
+				     unsigned int *directory_id);
+
+const char *
+_notmuch_database_get_directory_path (void *ctx,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id);
+
+notmuch_status_t
+_notmuch_database_filename_to_direntry (void *ctx,
+					notmuch_database_t *notmuch,
+					const char *filename,
+					notmuch_find_flags_t flags,
+					char **direntry);
+
+bool
+_notmuch_database_indexable_as_text (notmuch_database_t *notmuch,
+				     const char *mime_string);
+
+/* directory.cc */
+
+notmuch_directory_t *
+_notmuch_directory_find_or_create (notmuch_database_t *notmuch,
+				   const char *path,
+				   notmuch_find_flags_t flags,
+				   notmuch_status_t *status_ret);
+
+unsigned int
+_notmuch_directory_get_document_id (notmuch_directory_t *directory);
+
+notmuch_database_mode_t
+_notmuch_database_mode (notmuch_database_t *notmuch);
+
+/* message.cc */
+
+notmuch_message_t *
+_notmuch_message_create (const void *talloc_owner,
+			 notmuch_database_t *notmuch,
+			 unsigned int doc_id,
+			 notmuch_private_status_t *status);
+
+notmuch_message_t *
+_notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
+					const char *message_id,
+					notmuch_private_status_t *status);
+
+unsigned int
+_notmuch_message_get_doc_id (notmuch_message_t *message);
+
+const char *
+_notmuch_message_get_in_reply_to (notmuch_message_t *message);
+
+notmuch_private_status_t
+_notmuch_message_add_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value);
+
+notmuch_private_status_t
+_notmuch_message_remove_term (notmuch_message_t *message,
+			      const char *prefix_name,
+			      const char *value);
+
+notmuch_private_status_t
+_notmuch_message_has_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value,
+			   bool *result);
+
+notmuch_private_status_t
+_notmuch_message_gen_terms (notmuch_message_t *message,
+			    const char *prefix_name,
+			    const char *text);
+
+void
+_notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
+
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message);
+
+notmuch_status_t
+_notmuch_message_add_filename (notmuch_message_t *message,
+			       const char *filename);
+
+notmuch_status_t
+_notmuch_message_remove_filename (notmuch_message_t *message,
+				  const char *filename);
+
+notmuch_status_t
+_notmuch_message_rename (notmuch_message_t *message,
+			 const char *new_filename);
+
+void
+_notmuch_message_ensure_thread_id (notmuch_message_t *message);
+
+void
+_notmuch_message_set_header_values (notmuch_message_t *message,
+				    const char *date,
+				    const char *from,
+				    const char *subject);
+
+void
+_notmuch_message_update_subject (notmuch_message_t *message,
+				 const char *subject);
+
+void
+_notmuch_message_upgrade_last_mod (notmuch_message_t *message);
+
+void
+_notmuch_message_sync (notmuch_message_t *message);
+
+notmuch_status_t
+_notmuch_message_delete (notmuch_message_t *message);
+
+notmuch_private_status_t
+_notmuch_message_initialize_ghost (notmuch_message_t *message,
+				   const char *thread_id);
+
+void
+_notmuch_message_close (notmuch_message_t *message);
+
+/* Get a copy of the data in this message document.
+ *
+ * Caller should talloc_free the result when done.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message);
+
+/* Clear the data in this message document.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+void
+_notmuch_message_clear_data (notmuch_message_t *message);
+
+/* Set the author member of 'message' - this is the representation used
+ * when displaying the message */
+void
+_notmuch_message_set_author (notmuch_message_t *message, const char *author);
+
+/* Get the author member of 'message' */
+const char *
+_notmuch_message_get_author (notmuch_message_t *message);
+
+/* message-file.c */
+
+/* XXX: I haven't decided yet whether these will actually get exported
+ * into the public interface in notmuch.h
+ */
+
+typedef struct _notmuch_message_file notmuch_message_file_t;
+
+/* Open a file containing a single email message.
+ *
+ * The caller should call notmuch_message_close when done with this.
+ *
+ * Returns NULL if any error occurs.
+ */
+notmuch_message_file_t *
+_notmuch_message_file_open (notmuch_database_t *notmuch, const char *filename);
+
+/* Like notmuch_message_file_open but with 'ctx' as the talloc owner. */
+notmuch_message_file_t *
+_notmuch_message_file_open_ctx (notmuch_database_t *notmuch,
+				void *ctx, const char *filename);
+
+/* Close a notmuch message previously opened with notmuch_message_open. */
+void
+_notmuch_message_file_close (notmuch_message_file_t *message);
+
+/* Parse the message.
+ *
+ * This will be done automatically as necessary on other calls
+ * depending on it, but an explicit call allows for better error
+ * status reporting.
+ */
+notmuch_status_t
+_notmuch_message_file_parse (notmuch_message_file_t *message);
+
+/* Get the gmime message of a message file.
+ *
+ * The message file is parsed as necessary.
+ *
+ * The GMimeMessage* is set to *mime_message on success (which the
+ * caller must not unref).
+ *
+ * XXX: Would be nice to not have to expose GMimeMessage here.
+ */
+notmuch_status_t
+_notmuch_message_file_get_mime_message (notmuch_message_file_t *message,
+					GMimeMessage **mime_message);
+
+/* Get the value of the specified header from the message as a UTF-8 string.
+ *
+ * The message file is parsed as necessary.
+ *
+ * The header name is case insensitive.
+ *
+ * The Received: header is special - for it all Received: headers in
+ * the message are concatenated
+ *
+ * The returned value is owned by the notmuch message and is valid
+ * only until the message is closed. The caller should copy it if
+ * needing to modify the value or to hold onto it for longer.
+ *
+ * Returns NULL on errors, empty string if the message does not
+ * contain a header line matching 'header'.
+ */
+const char *
+_notmuch_message_file_get_header (notmuch_message_file_t *message,
+				  const char *header);
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+				   const char **from_out,
+				   const char **subject_out,
+				   const char **to_out,
+				   const char **date_out,
+				   char **message_id_out);
+
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message);
+
+/* add-message.cc */
+notmuch_status_t
+_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
+					   notmuch_message_t *message,
+					   notmuch_message_file_t *message_file,
+					   const char **thread_id);
+/* index.cc */
+
+void
+_notmuch_filter_init ();
+
+notmuch_status_t
+_notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_file_t *message_file);
+
+/* init.cc */
+void
+_notmuch_init ();
+
+/* messages.c */
+
+typedef struct _notmuch_message_node {
+    notmuch_message_t *message;
+    struct _notmuch_message_node *next;
+} notmuch_message_node_t;
+
+typedef struct _notmuch_message_list {
+    notmuch_message_node_t *head;
+    notmuch_message_node_t **tail;
+} notmuch_message_list_t;
+
+/* There's a rumor that there's an alternate struct _notmuch_messages
+ * somewhere with some nasty C++ objects in it. We'll try to maintain
+ * ignorance of that here. (See notmuch_mset_messages_t in query.cc)
+ */
+struct _notmuch_messages {
+    bool is_of_list_type;
+    notmuch_doc_id_set_t *excluded_doc_ids;
+    notmuch_message_node_t *iterator;
+};
+
+notmuch_message_list_t *
+_notmuch_message_list_create (const void *ctx);
+
+bool
+_notmuch_message_list_empty (notmuch_message_list_t *list);
+
+void
+_notmuch_message_list_add_message (notmuch_message_list_t *list,
+				   notmuch_message_t *message);
+
+notmuch_messages_t *
+_notmuch_messages_create (notmuch_message_list_t *list);
+
+bool
+_notmuch_messages_has_next (notmuch_messages_t *messages);
+
+/* query.cc */
+
+bool
+_notmuch_mset_messages_valid (notmuch_messages_t *messages);
+
+notmuch_message_t *
+_notmuch_mset_messages_get (notmuch_messages_t *messages);
+
+void
+_notmuch_mset_messages_move_to_next (notmuch_messages_t *messages);
+
+bool
+_notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
+			      unsigned int doc_id);
+
+void
+_notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
+			    unsigned int doc_id);
+
+/* querying xapian documents by type (e.g. "mail" or "ghost"): */
+notmuch_status_t
+_notmuch_query_search_documents (notmuch_query_t *query,
+				 const char *type,
+				 notmuch_messages_t **out);
+
+notmuch_status_t
+_notmuch_query_count_documents (notmuch_query_t *query,
+				const char *type,
+				unsigned *count_out);
+/* message-id.c */
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimiters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+char *
+_notmuch_message_id_parse (void *ctx, const char *message_id, const char **next);
+
+/* Parse a message-id, discarding leading and trailing whitespace, and
+ * '<' and '>' delimiters.
+ *
+ * Apply a probably-stricter-than RFC definition of what is allowed in
+ * a message-id. In particular, forbid whitespace.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id.
+ */
+
+char *
+_notmuch_message_id_parse_strict (void *ctx, const char *message_id);
+
+
+/* message.cc */
+
+void
+_notmuch_message_add_reply (notmuch_message_t *message,
+			    notmuch_message_t *reply);
+
+void
+_notmuch_message_remove_unprefixed_terms (notmuch_message_t *message);
+
+size_t _notmuch_message_get_thread_depth (notmuch_message_t *message);
+
+void
+_notmuch_message_label_depths (notmuch_message_t *message,
+			       size_t depth);
+
+notmuch_message_list_t *
+_notmuch_message_sort_subtrees (void *ctx, notmuch_message_list_t *list);
+
+/* sha1.c */
+
+char *
+_notmuch_sha1_of_string (const char *str);
+
+char *
+_notmuch_sha1_of_file (const char *filename);
+
+/* string-list.c */
+
+typedef struct _notmuch_string_node {
+    char *string;
+    struct _notmuch_string_node *next;
+} notmuch_string_node_t;
+
+typedef struct _notmuch_string_list {
+    int length;
+    notmuch_string_node_t *head;
+    notmuch_string_node_t **tail;
+} notmuch_string_list_t;
+
+notmuch_string_list_t *
+_notmuch_string_list_create (const void *ctx);
+
+/*
+ * return the number of strings in 'list'
+ */
+int
+_notmuch_string_list_length (notmuch_string_list_t *list);
+
+/* Add 'string' to 'list'.
+ *
+ * The list will create its own talloced copy of 'string'.
+ */
+void
+_notmuch_string_list_append (notmuch_string_list_t *list,
+			     const char *string);
+
+void
+_notmuch_string_list_sort (notmuch_string_list_t *list);
+
+const notmuch_string_list_t *
+_notmuch_message_get_references (notmuch_message_t *message);
+
+/* string-map.c */
+typedef struct _notmuch_string_map notmuch_string_map_t;
+typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx);
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value);
+
+void
+_notmuch_string_map_set (notmuch_string_map_t *map,
+			 const char *key,
+			 const char *value);
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     bool exact);
+
+bool
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+
+/* Create an iterator for user headers. Destroy with
+ * _notmuch_string_map_iterator_destroy. Actually in database.cc*/
+notmuch_string_map_iterator_t *
+_notmuch_database_user_headers (notmuch_database_t *notmuch);
+
+/* tags.c */
+
+notmuch_tags_t *
+_notmuch_tags_create (const void *ctx, notmuch_string_list_t *list);
+
+/* filenames.c */
+
+/* The notmuch_filenames_t iterates over a notmuch_string_list_t of
+ * file names */
+notmuch_filenames_t *
+_notmuch_filenames_create (const void *ctx,
+			   notmuch_string_list_t *list);
+
+/* thread.cc */
+
+notmuch_thread_t *
+_notmuch_thread_create (void *ctx,
+			notmuch_database_t *notmuch,
+			unsigned int seed_doc_id,
+			notmuch_doc_id_set_t *match_set,
+			notmuch_string_list_t *excluded_terms,
+			notmuch_exclude_t omit_exclude,
+			notmuch_sort_t sort);
+
+/* indexopts.c */
+
+struct _notmuch_indexopts;
+
+#define CONFIG_HEADER_PREFIX "index.header."
+
+#define EMPTY_STRING(s) ((s)[0] == '\0')
+
+/* config.cc */
+notmuch_status_t
+_notmuch_config_load_from_database (notmuch_database_t *db);
+
+notmuch_status_t
+_notmuch_config_load_from_file (notmuch_database_t *db, GKeyFile *file, char **status_string);
+
+notmuch_status_t
+_notmuch_config_load_defaults (notmuch_database_t *db);
+
+void
+_notmuch_config_cache (notmuch_database_t *db, notmuch_config_key_t key, const char *val);
+
+/* open.cc */
+notmuch_status_t
+_notmuch_choose_xapian_path (void *ctx, const char *database_path, const char **xapian_path,
+			     char **message);
+
+NOTMUCH_END_DECLS
+
+#ifdef __cplusplus
+/* Implicit typecast from 'void *' to 'T *' is okay in C, but not in
+ * C++. In talloc_steal, an explicit cast is provided for type safety
+ * in some GCC versions. Otherwise, a cast is required. Provide a
+ * template function for this to maintain type safety, and redefine
+ * talloc_steal to use it.
+ */
+#if ! (__GNUC__ >= 3)
+template <class T> T *
+_notmuch_talloc_steal (const void *new_ctx, const T *ptr)
+{
+    return static_cast<T *> (talloc_steal (new_ctx, ptr));
+}
+#undef talloc_steal
+#define talloc_steal _notmuch_talloc_steal
+#endif
+
+#if __cplusplus >= 201703L || __cppcheck__
+#define NODISCARD [[nodiscard]]
+#else
+#define NODISCARD /**/
+#endif
+#endif
+
+#endif
diff --git a/lib/notmuch.h b/lib/notmuch.h
new file mode 100644
index 00000000..4e2b0fa4
--- /dev/null
+++ b/lib/notmuch.h
@@ -0,0 +1,2872 @@
+/* notmuch - Not much of an email library, (just index and search)
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+/**
+ * @defgroup notmuch The notmuch API
+ *
+ * Not much of an email library, (just index and search)
+ *
+ * @{
+ */
+
+#ifndef NOTMUCH_H
+#define NOTMUCH_H
+
+#ifndef __DOXYGEN__
+
+#ifdef  __cplusplus
+# define NOTMUCH_BEGIN_DECLS  extern "C" {
+# define NOTMUCH_END_DECLS    }
+#else
+# define NOTMUCH_BEGIN_DECLS
+# define NOTMUCH_END_DECLS
+#endif
+
+NOTMUCH_BEGIN_DECLS
+
+#include <time.h>
+
+#pragma GCC visibility push(default)
+
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#ifndef TRUE
+#define TRUE 1
+#endif
+
+/*
+ * The library version number.  This must agree with the soname
+ * version in Makefile.local.
+ */
+#define LIBNOTMUCH_MAJOR_VERSION        5
+#define LIBNOTMUCH_MINOR_VERSION        6
+#define LIBNOTMUCH_MICRO_VERSION        0
+
+
+#if defined (__clang_major__) && __clang_major__ >= 3 \
+    || defined (__GNUC__) && __GNUC__ >= 5 \
+    || defined (__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ >= 5
+#define NOTMUCH_DEPRECATED(major, minor) \
+    __attribute__ ((deprecated ("function deprecated as of libnotmuch " #major "." #minor)))
+#else
+#define NOTMUCH_DEPRECATED(major, minor) __attribute__ ((deprecated))
+#endif
+
+
+#endif /* __DOXYGEN__ */
+
+/**
+ * Check the version of the notmuch library being compiled against.
+ *
+ * Return true if the library being compiled against is of the
+ * specified version or above. For example:
+ *
+ * @code
+ * #if LIBNOTMUCH_CHECK_VERSION(3, 1, 0)
+ *     (code requiring libnotmuch 3.1.0 or above)
+ * #endif
+ * @endcode
+ *
+ * LIBNOTMUCH_CHECK_VERSION has been defined since version 3.1.0; to
+ * check for versions prior to that, use:
+ *
+ * @code
+ * #if !defined(NOTMUCH_CHECK_VERSION)
+ *     (code requiring libnotmuch prior to 3.1.0)
+ * #endif
+ * @endcode
+ */
+#define LIBNOTMUCH_CHECK_VERSION(major, minor, micro)                   \
+    (LIBNOTMUCH_MAJOR_VERSION > (major) ||                                      \
+     (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION > (minor)) || \
+     (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION == (minor) && \
+      LIBNOTMUCH_MICRO_VERSION >= (micro)))
+
+/**
+ * Notmuch boolean type.
+ */
+typedef int notmuch_bool_t;
+
+/**
+ * Status codes used for the return values of most functions.
+ *
+ * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function
+ * completed without error. Any other value indicates an error.
+ */
+typedef enum {
+    /**
+     * No error occurred.
+     */
+    NOTMUCH_STATUS_SUCCESS = 0,
+    /**
+     * Out of memory.
+     */
+    NOTMUCH_STATUS_OUT_OF_MEMORY,
+    /**
+     * An attempt was made to write to a database opened in read-only
+     * mode.
+     */
+    NOTMUCH_STATUS_READ_ONLY_DATABASE,
+    /**
+     * A Xapian exception occurred.
+     *
+     * @todo We don't really want to expose this lame XAPIAN_EXCEPTION
+     * value. Instead we should map to things like DATABASE_LOCKED or
+     * whatever.
+     */
+    NOTMUCH_STATUS_XAPIAN_EXCEPTION,
+    /**
+     * An error occurred trying to read or write to a file (this could
+     * be file not found, permission denied, etc.)
+     */
+    NOTMUCH_STATUS_FILE_ERROR,
+    /**
+     * A file was presented that doesn't appear to be an email
+     * message.
+     */
+    NOTMUCH_STATUS_FILE_NOT_EMAIL,
+    /**
+     * A file contains a message ID that is identical to a message
+     * already in the database.
+     */
+    NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID,
+    /**
+     * The user erroneously passed a NULL pointer to a notmuch
+     * function.
+     */
+    NOTMUCH_STATUS_NULL_POINTER,
+    /**
+     * A tag value is too long (exceeds NOTMUCH_TAG_MAX).
+     */
+    NOTMUCH_STATUS_TAG_TOO_LONG,
+    /**
+     * The notmuch_message_thaw function has been called more times
+     * than notmuch_message_freeze.
+     */
+    NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
+    /**
+     * notmuch_database_end_atomic has been called more times than
+     * notmuch_database_begin_atomic.
+     */
+    NOTMUCH_STATUS_UNBALANCED_ATOMIC,
+    /**
+     * The operation is not supported.
+     */
+    NOTMUCH_STATUS_UNSUPPORTED_OPERATION,
+    /**
+     * The operation requires a database upgrade.
+     */
+    NOTMUCH_STATUS_UPGRADE_REQUIRED,
+    /**
+     * There is a problem with the proposed path, e.g. a relative path
+     * passed to a function expecting an absolute path.
+     */
+    NOTMUCH_STATUS_PATH_ERROR,
+    /**
+     * The requested operation was ignored. Depending on the function,
+     * this may not be an actual error.
+     */
+    NOTMUCH_STATUS_IGNORED,
+    /**
+     * One of the arguments violates the preconditions for the
+     * function, in a way not covered by a more specific argument.
+     */
+    NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    /**
+     * A MIME object claimed to have cryptographic protection which
+     * notmuch tried to handle, but the protocol was not specified in
+     * an intelligible way.
+     */
+    NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL,
+    /**
+     * Notmuch attempted to do crypto processing, but could not
+     * initialize the engine needed to do so.
+     */
+    NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION,
+    /**
+     * A MIME object claimed to have cryptographic protection, and
+     * notmuch attempted to process it, but the specific protocol was
+     * something that notmuch doesn't know how to handle.
+     */
+    NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL,
+    /**
+     * Unable to load a config file
+     */
+    NOTMUCH_STATUS_NO_CONFIG,
+    /**
+     * Unable to load a database
+     */
+    NOTMUCH_STATUS_NO_DATABASE,
+    /**
+     * Database exists, so not (re)-created
+     */
+    NOTMUCH_STATUS_DATABASE_EXISTS,
+    /**
+     * Syntax error in query
+     */
+    NOTMUCH_STATUS_BAD_QUERY_SYNTAX,
+    /**
+     * No mail root could be deduced from parameters and environment
+     */
+    NOTMUCH_STATUS_NO_MAIL_ROOT,
+    /**
+     * Database is not fully opened, or has been closed
+     */
+    NOTMUCH_STATUS_CLOSED_DATABASE,
+    /**
+     * Not an actual status value. Just a way to find out how many
+     * valid status values there are.
+     */
+    NOTMUCH_STATUS_LAST_STATUS
+} notmuch_status_t;
+
+/**
+ * Get a string representation of a notmuch_status_t value.
+ *
+ * The result is read-only.
+ */
+const char *
+notmuch_status_to_string (notmuch_status_t status);
+
+/* Various opaque data types. For each notmuch_<foo>_t see the various
+ * notmuch_<foo> functions below. */
+#ifndef __DOXYGEN__
+typedef struct _notmuch_database notmuch_database_t;
+typedef struct _notmuch_query notmuch_query_t;
+typedef struct _notmuch_threads notmuch_threads_t;
+typedef struct _notmuch_thread notmuch_thread_t;
+typedef struct _notmuch_messages notmuch_messages_t;
+typedef struct _notmuch_message notmuch_message_t;
+typedef struct _notmuch_tags notmuch_tags_t;
+typedef struct _notmuch_directory notmuch_directory_t;
+typedef struct _notmuch_filenames notmuch_filenames_t;
+typedef struct _notmuch_config_list notmuch_config_list_t;
+typedef struct _notmuch_config_values notmuch_config_values_t;
+typedef struct _notmuch_config_pairs notmuch_config_pairs_t;
+typedef struct _notmuch_indexopts notmuch_indexopts_t;
+#endif /* __DOXYGEN__ */
+
+/**
+ * Create a new, empty notmuch database located at 'path'.
+ *
+ * The path should be a top-level directory to a collection of
+ * plain-text email messages (one message per file). This call will
+ * create a new ".notmuch" directory within 'path' where notmuch will
+ * store its data.
+ *
+ * After a successful call to notmuch_database_create, the returned
+ * database will be open so the caller should call
+ * notmuch_database_destroy when finished with it.
+ *
+ * The database will not yet have any data in it
+ * (notmuch_database_create itself is a very cheap function). Messages
+ * contained within 'path' can be added to the database by calling
+ * notmuch_database_index_file.
+ *
+ * In case of any failure, this function returns an error status and
+ * sets *database to NULL (after printing an error message on stderr).
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully created the database.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'path' argument is NULL.
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
+ *
+ * NOTMUCH_STATUS_PATH_ERROR: filename is too long
+ *
+ * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to create the
+ *	database file (such as permission denied, or file not found,
+ *	etc.), or the database already exists.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred.
+ */
+notmuch_status_t
+notmuch_database_create (const char *path, notmuch_database_t **database);
+
+/**
+ * Like notmuch_database_create, except optionally return an error
+ * message. This message is allocated by malloc and should be freed by
+ * the caller.
+ */
+notmuch_status_t
+notmuch_database_create_verbose (const char *path,
+				 notmuch_database_t **database,
+				 char **error_message);
+
+/**
+ * Database open mode for notmuch_database_open.
+ */
+typedef enum {
+    /**
+     * Open database for reading only.
+     */
+    NOTMUCH_DATABASE_MODE_READ_ONLY = 0,
+    /**
+     * Open database for reading and writing.
+     */
+    NOTMUCH_DATABASE_MODE_READ_WRITE
+} notmuch_database_mode_t;
+
+/**
+ * Deprecated alias for notmuch_database_open_with_config with
+ * config_path="" and error_message=NULL
+ * @deprecated Deprecated as of libnotmuch 5.4 (notmuch 0.32)
+ */
+NOTMUCH_DEPRECATED(5, 4)
+notmuch_status_t
+notmuch_database_open (const char *path,
+		       notmuch_database_mode_t mode,
+		       notmuch_database_t **database);
+/**
+ * Deprecated alias for notmuch_database_open_with_config with
+ * config_path=""
+ *
+ * @deprecated Deprecated as of libnotmuch 5.4 (notmuch 0.32)
+ *
+ */
+NOTMUCH_DEPRECATED(5, 4)
+notmuch_status_t
+notmuch_database_open_verbose (const char *path,
+			       notmuch_database_mode_t mode,
+			       notmuch_database_t **database,
+			       char **error_message);
+
+/**
+ * Open an existing notmuch database located at 'database_path', using
+ * configuration in 'config_path'.
+ *
+ * @param[in]	database_path
+ * @parblock
+ * Path to existing database.
+ *
+ * A notmuch database is a Xapian database containing appropriate
+ * metadata.
+ *
+ * The database should have been created at some time in the past,
+ * (not necessarily by this process), by calling
+ * notmuch_database_create.
+ *
+ * If 'database_path' is NULL, use the location specified
+ *
+ * - in the environment variable NOTMUCH_DATABASE, if non-empty
+ *
+ * - in a configuration file, located as described under 'config_path'
+ *
+ * - by $XDG_DATA_HOME/notmuch/$PROFILE where XDG_DATA_HOME defaults
+ *   to "$HOME/.local/share" and PROFILE as as discussed in
+ *   'profile'
+ *
+ * If 'database_path' is non-NULL, but does not appear to be a Xapian
+ * database, check for a directory '.notmuch/xapian' below
+ * 'database_path' (this is the behavior of
+ * notmuch_database_open_verbose pre-0.32).
+ *
+ * @endparblock
+ * @param[in]	mode
+ * @parblock
+ * Mode to open database. Use one of #NOTMUCH_DATABASE_MODE_READ_WRITE
+ * or #NOTMUCH_DATABASE_MODE_READ_ONLY
+ *
+ * @endparblock
+ * @param[in]  config_path
+ * @parblock
+ * Path to config file.
+ *
+ * Config file is key-value, with mandatory sections. See
+ * <em>notmuch-config(5)</em> for more information. The key-value pair
+ * overrides the corresponding configuration data stored in the
+ * database (see <em>notmuch_database_get_config</em>)
+ *
+ * If <em>config_path</em> is NULL use the path specified
+ *
+ * - in environment variable <em>NOTMUCH_CONFIG</em>, if non-empty
+ *
+ * - by  <em>XDG_CONFIG_HOME</em>/notmuch/ where
+ *   XDG_CONFIG_HOME defaults to "$HOME/.config".
+ *
+ * - by $HOME/.notmuch-config
+ *
+ * If <em>config_path</em> is "" (empty string) then do not
+ * open any configuration file.
+ * @endparblock
+ * @param[in] profile:
+ * @parblock
+ * Name of profile (configuration/database variant).
+ *
+ * If non-NULL, append to the directory / file path determined for
+ * <em>config_path</em> and <em>database_path</em>.
+ *
+ * If NULL then use
+ * - environment variable NOTMUCH_PROFILE if defined,
+ * - otherwise "default" for directories and "" (empty string) for paths.
+ *
+ * @endparblock
+ * @param[out] database
+ * @parblock
+ * Pointer to database object. May not be NULL.
+ *
+ * The caller should call notmuch_database_destroy when finished with
+ * this database.
+ *
+ * In case of any failure, this function returns an error status and
+ * sets *database to NULL.
+ *
+ * @endparblock
+ * @param[out] error_message
+ * If non-NULL, store error message from opening the database.
+ * Any such message is allocated by \a malloc(3) and should be freed
+ * by the caller.
+ *
+ * @retval NOTMUCH_STATUS_SUCCESS: Successfully opened the database.
+ *
+ * @retval NOTMUCH_STATUS_NULL_POINTER: The given \a database
+ * argument is NULL.
+ *
+ * @retval NOTMUCH_STATUS_NO_CONFIG: No config file was found. Fatal.
+ *
+ * @retval NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
+ *
+ * @retval NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to open the
+ *	database or config file (such as permission denied, or file not found,
+ *	etc.), or the database version is unknown.
+ *
+ * @retval NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred.
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ */
+
+notmuch_status_t
+notmuch_database_open_with_config (const char *database_path,
+				   notmuch_database_mode_t mode,
+				   const char *config_path,
+				   const char *profile,
+				   notmuch_database_t **database,
+				   char **error_message);
+
+
+/**
+ * Loads configuration from config file, database, and/or defaults
+ *
+ * For description of arguments, @see notmuch_database_open_with_config
+ *
+ * For errors other then NO_DATABASE and NO_CONFIG, *database is set to
+ * NULL.
+ *
+ * @retval NOTMUCH_STATUS_SUCCESS: Successfully loaded configuration.
+ *
+ * @retval NOTMUCH_STATUS_NO_CONFIG: No config file was loaded. Not fatal.
+ *
+ * @retval NOTMUCH_STATUS_NO_DATABASE: No config information was
+ *	loaded from a database. Not fatal.
+ *
+ * @retval NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
+ *
+ * @retval NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to open the
+ *	database or config file (such as permission denied, or file not found,
+ *	etc.)
+ *
+ * @retval NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred.
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ */
+
+notmuch_status_t
+notmuch_database_load_config (const char *database_path,
+			      const char *config_path,
+			      const char *profile,
+			      notmuch_database_t **database,
+			      char **error_message);
+
+/**
+ * Create a new notmuch database located at 'database_path', using
+ * configuration in 'config_path'.
+ *
+ * For description of arguments, @see notmuch_database_open_with_config
+ *
+ * In case of any failure, this function returns an error status and
+ * sets *database to NULL.
+ *
+ * @retval NOTMUCH_STATUS_SUCCESS: Successfully created the database.
+ *
+ * @retval NOTMUCH_STATUS_DATABASE_EXISTS: Database already exists, not created
+ *
+ * @retval NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
+ *
+ * @retval NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to open the
+ *	database or config file (such as permission denied, or file not found,
+ *	etc.)
+ *
+ * @retval NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred.
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ */
+
+notmuch_status_t
+notmuch_database_create_with_config (const char *database_path,
+				     const char *config_path,
+				     const char *profile,
+				     notmuch_database_t **database,
+				     char **error_message);
+
+/**
+ * Retrieve last status string for given database.
+ *
+ */
+const char *
+notmuch_database_status_string (const notmuch_database_t *notmuch);
+
+/**
+ * Commit changes and close the given notmuch database.
+ *
+ * After notmuch_database_close has been called, calls to other
+ * functions on objects derived from this database may either behave
+ * as if the database had not been closed (e.g., if the required data
+ * has been cached) or may fail with a
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION. The only further operation
+ * permitted on the database itself is to call
+ * notmuch_database_destroy.
+ *
+ * notmuch_database_close can be called multiple times.  Later calls
+ * have no effect.
+ *
+ * For writable databases, notmuch_database_close commits all changes
+ * to disk before closing the database, unless the caller is currently
+ * in an atomic section (there was a notmuch_database_begin_atomic
+ * without a matching notmuch_database_end_atomic). In this case
+ * changes since the last commit are discarded. @see
+ * notmuch_database_end_atomic for more information.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully closed the database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred; the
+ *	database has been closed but there are no guarantees the
+ *	changes to the database, if any, have been flushed to disk.
+ */
+notmuch_status_t
+notmuch_database_close (notmuch_database_t *database);
+
+/**
+ * A callback invoked by notmuch_database_compact to notify the user
+ * of the progress of the compaction process.
+ */
+typedef void (*notmuch_compact_status_cb_t)(const char *message, void *closure);
+
+/**
+ * Compact a notmuch database, backing up the original database to the
+ * given path.
+ *
+ * The database will be opened with NOTMUCH_DATABASE_MODE_READ_WRITE
+ * during the compaction process to ensure no writes are made.
+ *
+ * If the optional callback function 'status_cb' is non-NULL, it will
+ * be called with diagnostic and informational messages. The argument
+ * 'closure' is passed verbatim to any callback invoked.
+ */
+notmuch_status_t
+notmuch_database_compact (const char *path,
+			  const char *backup_path,
+			  notmuch_compact_status_cb_t status_cb,
+			  void *closure);
+
+/**
+ * Like notmuch_database_compact, but take an open database as a
+ * parameter.
+ *
+ * @since libnnotmuch 5.4 (notmuch 0.32)
+ */
+notmuch_status_t
+notmuch_database_compact_db (notmuch_database_t *database,
+			     const char *backup_path,
+			     notmuch_compact_status_cb_t status_cb,
+			     void *closure);
+
+/**
+ * Destroy the notmuch database, closing it if necessary and freeing
+ * all associated resources.
+ *
+ * Return value as in notmuch_database_close if the database was open;
+ * notmuch_database_destroy itself has no failure modes.
+ */
+notmuch_status_t
+notmuch_database_destroy (notmuch_database_t *database);
+
+/**
+ * Return the database path of the given database.
+ *
+ * The return value is a string owned by notmuch so should not be
+ * modified nor freed by the caller.
+ */
+const char *
+notmuch_database_get_path (notmuch_database_t *database);
+
+/**
+ * Return the database format version of the given database.
+ *
+ * @retval 0 on error
+ */
+unsigned int
+notmuch_database_get_version (notmuch_database_t *database);
+
+/**
+ * Can the database be upgraded to a newer database version?
+ *
+ * If this function returns TRUE, then the caller may call
+ * notmuch_database_upgrade to upgrade the database.  If the caller
+ * does not upgrade an out-of-date database, then some functions may
+ * fail with NOTMUCH_STATUS_UPGRADE_REQUIRED.  This always returns
+ * FALSE for a read-only database because there's no way to upgrade a
+ * read-only database.
+ *
+ * Also returns FALSE if an error occurs accessing the database.
+ *
+ */
+notmuch_bool_t
+notmuch_database_needs_upgrade (notmuch_database_t *database);
+
+/**
+ * Upgrade the current database to the latest supported version.
+ *
+ * This ensures that all current notmuch functionality will be
+ * available on the database.  After opening a database in read-write
+ * mode, it is recommended that clients check if an upgrade is needed
+ * (notmuch_database_needs_upgrade) and if so, upgrade with this
+ * function before making any modifications.  If
+ * notmuch_database_needs_upgrade returns FALSE, this will be a no-op.
+ *
+ * The optional progress_notify callback can be used by the caller to
+ * provide progress indication to the user. If non-NULL it will be
+ * called periodically with 'progress' as a floating-point value in
+ * the range of [0.0 .. 1.0] indicating the progress made so far in
+ * the upgrade process.  The argument 'closure' is passed verbatim to
+ * any callback invoked.
+ */
+notmuch_status_t
+notmuch_database_upgrade (notmuch_database_t *database,
+			  void (*progress_notify)(void *closure,
+						  double progress),
+			  void *closure);
+
+/**
+ * Begin an atomic database operation.
+ *
+ * Any modifications performed between a successful begin and a
+ * notmuch_database_end_atomic will be applied to the database
+ * atomically.  Note that, unlike a typical database transaction, this
+ * only ensures atomicity, not durability; neither begin nor end
+ * necessarily flush modifications to disk.
+ *
+ * Atomic sections may be nested.  begin_atomic and end_atomic must
+ * always be called in pairs.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully entered atomic section.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	atomic section not entered.
+ */
+notmuch_status_t
+notmuch_database_begin_atomic (notmuch_database_t *notmuch);
+
+/**
+ * Indicate the end of an atomic database operation.  If repeated
+ * (with matching notmuch_database_begin_atomic) "database.autocommit"
+ * times, commit the the transaction and all previous (non-cancelled)
+ * transactions to the database.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully completed atomic section.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	atomic section not ended.
+ *
+ * NOTMUCH_STATUS_UNBALANCED_ATOMIC: The database is not currently in
+ *	an atomic section.
+ */
+notmuch_status_t
+notmuch_database_end_atomic (notmuch_database_t *notmuch);
+
+/**
+ * Return the committed database revision and UUID.
+ *
+ * The database revision number increases monotonically with each
+ * commit to the database.  Hence, all messages and message changes
+ * committed to the database (that is, visible to readers) have a last
+ * modification revision <= the committed database revision.  Any
+ * messages committed in the future will be assigned a modification
+ * revision > the committed database revision.
+ *
+ * The UUID is a NUL-terminated opaque string that uniquely identifies
+ * this database.  Two revision numbers are only comparable if they
+ * have the same database UUID. The string 'uuid' is owned by notmuch
+ * and should not be freed or modified by the user.
+ */
+unsigned long
+notmuch_database_get_revision (notmuch_database_t *notmuch,
+			       const char **uuid);
+
+/**
+ * Retrieve a directory object from the database for 'path'.
+ *
+ * Here, 'path' should be a path relative to the path of 'database'
+ * (see notmuch_database_get_path), or else should be an absolute path
+ * with initial components that match the path of 'database'.
+ *
+ * If this directory object does not exist in the database, this
+ * returns NOTMUCH_STATUS_SUCCESS and sets *directory to NULL.
+ *
+ * Otherwise the returned directory object is owned by the database
+ * and as such, will only be valid until notmuch_database_destroy is
+ * called.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully retrieved directory.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'directory' argument is NULL.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	directory not retrieved.
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ *      database to use this function.
+ */
+notmuch_status_t
+notmuch_database_get_directory (notmuch_database_t *database,
+				const char *path,
+				notmuch_directory_t **directory);
+
+/**
+ * Add a message file to a database, indexing it for retrieval by
+ * future searches.  If a message already exists with the same message
+ * ID as the specified file, their indexes will be merged, and this
+ * new filename will also be associated with the existing message.
+ *
+ * Here, 'filename' should be a path relative to the path of
+ * 'database' (see notmuch_database_get_path), or else should be an
+ * absolute filename with initial components that match the path of
+ * 'database'.
+ *
+ * The file should be a single mail message (not a multi-message mbox)
+ * that is expected to remain at its current location, (since the
+ * notmuch database will reference the filename, and will not copy the
+ * entire contents of the file.
+ *
+ * If another message with the same message ID already exists in the
+ * database, rather than creating a new message, this adds the search
+ * terms from the identified file to the existing message's index, and
+ * adds 'filename' to the list of filenames known for the message.
+ *
+ * The 'indexopts' parameter can be NULL (meaning, use the indexing
+ * defaults from the database), or can be an explicit choice of
+ * indexing options that should govern the indexing of this specific
+ * 'filename'.
+ *
+ * If 'message' is not NULL, then, on successful return
+ * (NOTMUCH_STATUS_SUCCESS or NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) '*message'
+ * will be initialized to a message object that can be used for things
+ * such as adding tags to the just-added message. The user should call
+ * notmuch_message_destroy when done with the message. On any failure
+ * '*message' will be set to NULL.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully added to database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred,
+ *	message not added.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
+ *	ID as another message already in the database. The new
+ *	filename was successfully added to the message in the database
+ *	(if not already present) and the existing message is returned.
+ *
+ * NOTMUCH_STATUS_FILE_ERROR: an error occurred trying to open the
+ *	file, (such as permission denied, or file not found,
+ *	etc.). Nothing added to the database.
+ *
+ * NOTMUCH_STATUS_FILE_NOT_EMAIL: the contents of filename don't look
+ *	like an email message. Nothing added to the database.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so no message can be added.
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ *      database to use this function.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_database_index_file (notmuch_database_t *database,
+			     const char *filename,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_t **message);
+
+/**
+ * Deprecated alias for notmuch_database_index_file called with
+ * NULL indexopts.
+ *
+ * @deprecated Deprecated as of libnotmuch 5.1 (notmuch 0.26). Please
+ * use notmuch_database_index_file instead.
+ *
+ */
+NOTMUCH_DEPRECATED (5, 1)
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *database,
+			      const char *filename,
+			      notmuch_message_t **message);
+
+/**
+ * Remove a message filename from the given notmuch database. If the
+ * message has no more filenames, remove the message.
+ *
+ * If the same message (as determined by the message ID) is still
+ * available via other filenames, then the message will persist in the
+ * database for those filenames. When the last filename is removed for
+ * a particular message, the database content for that message will be
+ * entirely removed.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: The last filename was removed and the
+ *	message was removed from the database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred,
+ *	message not removed.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: This filename was removed but
+ *	the message persists in the database with at least one other
+ *	filename.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so no message can be removed.
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ *      database to use this function.
+ */
+notmuch_status_t
+notmuch_database_remove_message (notmuch_database_t *database,
+				 const char *filename);
+
+/**
+ * Find a message with the given message_id.
+ *
+ * If a message with the given message_id is found then, on successful return
+ * (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to a message
+ * object.  The caller should call notmuch_message_destroy when done with the
+ * message.
+ *
+ * On any failure or when the message is not found, this function initializes
+ * '*message' to NULL. This means, when NOTMUCH_STATUS_SUCCESS is returned, the
+ * caller is supposed to check '*message' for NULL to find out whether the
+ * message with the given message_id was found.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successful return, check '*message'.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'message' argument is NULL
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory, creating message object
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
+ */
+notmuch_status_t
+notmuch_database_find_message (notmuch_database_t *database,
+			       const char *message_id,
+			       notmuch_message_t **message);
+
+/**
+ * Find a message with the given filename.
+ *
+ * If the database contains a message with the given filename then, on
+ * successful return (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to
+ * a message object. The caller should call notmuch_message_destroy when done
+ * with the message.
+ *
+ * On any failure or when the message is not found, this function initializes
+ * '*message' to NULL. This means, when NOTMUCH_STATUS_SUCCESS is returned, the
+ * caller is supposed to check '*message' for NULL to find out whether the
+ * message with the given filename is found.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successful return, check '*message'
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'message' argument is NULL
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory, creating the message object
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ *      database to use this function.
+ */
+notmuch_status_t
+notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
+					   const char *filename,
+					   notmuch_message_t **message);
+
+/**
+ * Return a list of all tags found in the database.
+ *
+ * This function creates a list of all tags found in the database. The
+ * resulting list contains all tags from all messages found in the database.
+ *
+ * On error this function returns NULL.
+ */
+notmuch_tags_t *
+notmuch_database_get_all_tags (notmuch_database_t *db);
+
+/**
+ * Reopen an open notmuch database.
+ *
+ * @param [in] db	open notmuch database
+ * @param [in] mode	mode (read only or read-write) for reopened database.
+ *
+ * @retval #NOTMUCH_STATUS_SUCCESS
+ * @retval #NOTMUCH_STATUS_ILLEGAL_ARGUMENT	The passed database was not open.
+ * @retval #NOTMUCH_STATUS_XAPIAN_EXCEPTION	A Xapian exception occured
+ */
+notmuch_status_t
+notmuch_database_reopen (notmuch_database_t *db, notmuch_database_mode_t mode);
+
+/**
+ * Create a new query for 'database'.
+ *
+ * Here, 'database' should be an open database, (see
+ * notmuch_database_open and notmuch_database_create).
+ *
+ * For the query string, we'll document the syntax here more
+ * completely in the future, but it's likely to be a specialized
+ * version of the general Xapian query syntax:
+ *
+ * https://xapian.org/docs/queryparser.html
+ *
+ * As a special case, passing either a length-zero string, (that is ""),
+ * or a string consisting of a single asterisk (that is "*"), will
+ * result in a query that returns all messages in the database.
+ *
+ * See notmuch_query_set_sort for controlling the order of results.
+ * See notmuch_query_search_messages and notmuch_query_search_threads
+ * to actually execute the query.
+ *
+ * User should call notmuch_query_destroy when finished with this
+ * query.
+ *
+ * Will return NULL if insufficient memory is available.
+ */
+notmuch_query_t *
+notmuch_query_create (notmuch_database_t *database,
+		      const char *query_string);
+
+typedef enum {
+    NOTMUCH_QUERY_SYNTAX_XAPIAN,
+    NOTMUCH_QUERY_SYNTAX_SEXP
+} notmuch_query_syntax_t;
+
+notmuch_status_t
+notmuch_query_create_with_syntax (notmuch_database_t *database,
+				  const char *query_string,
+				  notmuch_query_syntax_t syntax,
+				  notmuch_query_t **output);
+/**
+ * Sort values for notmuch_query_set_sort.
+ */
+typedef enum {
+    /**
+     * Oldest first.
+     */
+    NOTMUCH_SORT_OLDEST_FIRST,
+    /**
+     * Newest first.
+     */
+    NOTMUCH_SORT_NEWEST_FIRST,
+    /**
+     * Sort by message-id.
+     */
+    NOTMUCH_SORT_MESSAGE_ID,
+    /**
+     * Do not sort.
+     */
+    NOTMUCH_SORT_UNSORTED
+} notmuch_sort_t;
+
+/**
+ * Return the query_string of this query. See notmuch_query_create.
+ */
+const char *
+notmuch_query_get_query_string (const notmuch_query_t *query);
+
+/**
+ * Return the notmuch database of this query. See notmuch_query_create.
+ */
+notmuch_database_t *
+notmuch_query_get_database (const notmuch_query_t *query);
+
+/**
+ * Exclude values for notmuch_query_set_omit_excluded. The strange
+ * order is to maintain backward compatibility: the old FALSE/TRUE
+ * options correspond to the new
+ * NOTMUCH_EXCLUDE_FLAG/NOTMUCH_EXCLUDE_TRUE options.
+ */
+typedef enum {
+    NOTMUCH_EXCLUDE_FLAG,
+    NOTMUCH_EXCLUDE_TRUE,
+    NOTMUCH_EXCLUDE_FALSE,
+    NOTMUCH_EXCLUDE_ALL
+} notmuch_exclude_t;
+
+/**
+ * Specify whether to omit excluded results or simply flag them.  By
+ * default, this is set to TRUE.
+ *
+ * If set to TRUE or ALL, notmuch_query_search_messages will omit excluded
+ * messages from the results, and notmuch_query_search_threads will omit
+ * threads that match only in excluded messages.  If set to TRUE,
+ * notmuch_query_search_threads will include all messages in threads that
+ * match in at least one non-excluded message.  Otherwise, if set to ALL,
+ * notmuch_query_search_threads will omit excluded messages from all threads.
+ *
+ * If set to FALSE or FLAG then both notmuch_query_search_messages and
+ * notmuch_query_search_threads will return all matching
+ * messages/threads regardless of exclude status. If set to FLAG then
+ * the exclude flag will be set for any excluded message that is
+ * returned by notmuch_query_search_messages, and the thread counts
+ * for threads returned by notmuch_query_search_threads will be the
+ * number of non-excluded messages/matches. Otherwise, if set to
+ * FALSE, then the exclude status is completely ignored.
+ *
+ * The performance difference when calling
+ * notmuch_query_search_messages should be relatively small (and both
+ * should be very fast).  However, in some cases,
+ * notmuch_query_search_threads is very much faster when omitting
+ * excluded messages as it does not need to construct the threads that
+ * only match in excluded messages.
+ */
+void
+notmuch_query_set_omit_excluded (notmuch_query_t *query,
+				 notmuch_exclude_t omit_excluded);
+
+/**
+ * Specify the sorting desired for this query.
+ */
+void
+notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort);
+
+/**
+ * Return the sort specified for this query. See
+ * notmuch_query_set_sort.
+ */
+notmuch_sort_t
+notmuch_query_get_sort (const notmuch_query_t *query);
+
+/**
+ * Add a tag that will be excluded from the query results by default.
+ * This exclusion will be ignored if this tag appears explicitly in
+ * the query.
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_SUCCESS: excluded was added successfully.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: a Xapian exception occurred.
+ *      Most likely a problem lazily parsing the query string.
+ *
+ * NOTMUCH_STATUS_IGNORED: tag is explicitly present in the query, so
+ *		not excluded.
+ */
+notmuch_status_t
+notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag);
+
+/**
+ * Execute a query for threads, returning a notmuch_threads_t object
+ * which can be used to iterate over the results. The returned threads
+ * object is owned by the query and as such, will only be valid until
+ * notmuch_query_destroy.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_query_t *query;
+ *     notmuch_threads_t *threads;
+ *     notmuch_thread_t *thread;
+ *     notmuch_status_t stat;
+ *
+ *     query = notmuch_query_create (database, query_string);
+ *
+ *     for (stat = notmuch_query_search_threads (query, &threads);
+ *	    stat == NOTMUCH_STATUS_SUCCESS &&
+ *          notmuch_threads_valid (threads);
+ *          notmuch_threads_move_to_next (threads))
+ *     {
+ *         thread = notmuch_threads_get (threads);
+ *         ....
+ *         notmuch_thread_destroy (thread);
+ *     }
+ *
+ *     notmuch_query_destroy (query);
+ *
+ * Note: If you are finished with a thread before its containing
+ * query, you can call notmuch_thread_destroy to clean up some memory
+ * sooner (as in the above example). Otherwise, if your thread objects
+ * are long-lived, then you don't need to call notmuch_thread_destroy
+ * and all the memory will still be reclaimed when the query is
+ * destroyed.
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_threads_t object. (For consistency, we do provide a
+ * notmuch_threads_destroy function, but there's no good reason
+ * to call it if the query is about to be destroyed).
+ *
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_search_threads (notmuch_query_t *query,
+			      notmuch_threads_t **out);
+
+/**
+ * Deprecated alias for notmuch_query_search_threads.
+ *
+ * @deprecated Deprecated as of libnotmuch 5 (notmuch 0.25). Please
+ * use notmuch_query_search_threads instead.
+ *
+ */
+NOTMUCH_DEPRECATED (5, 0)
+notmuch_status_t
+notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out);
+
+/**
+ * Execute a query for messages, returning a notmuch_messages_t object
+ * which can be used to iterate over the results. The returned
+ * messages object is owned by the query and as such, will only be
+ * valid until notmuch_query_destroy.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_query_t *query;
+ *     notmuch_messages_t *messages;
+ *     notmuch_message_t *message;
+ *
+ *     query = notmuch_query_create (database, query_string);
+ *
+ *     if (notmuch_query_search_messages (query, &messages) != NOTMUCH_STATUS_SUCCESS)
+ *         return EXIT_FAILURE;
+ *
+ *     for (;
+ *          notmuch_messages_valid (messages);
+ *          notmuch_messages_move_to_next (messages))
+ *     {
+ *         message = notmuch_messages_get (messages);
+ *         ....
+ *         notmuch_message_destroy (message);
+ *     }
+ *
+ *     notmuch_query_destroy (query);
+ *
+ * Note: If you are finished with a message before its containing
+ * query, you can call notmuch_message_destroy to clean up some memory
+ * sooner (as in the above example). Otherwise, if your message
+ * objects are long-lived, then you don't need to call
+ * notmuch_message_destroy and all the memory will still be reclaimed
+ * when the query is destroyed.
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_messages_t object. (For consistency, we do provide a
+ * notmuch_messages_destroy function, but there's no good
+ * reason to call it if the query is about to be destroyed).
+ *
+ * If a Xapian exception occurs this function will return NULL.
+ *
+ * @since libnotmuch 5 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_search_messages (notmuch_query_t *query,
+			       notmuch_messages_t **out);
+/**
+ * Deprecated alias for notmuch_query_search_messages
+ *
+ * @deprecated Deprecated as of libnotmuch 5 (notmuch 0.25). Please use
+ * notmuch_query_search_messages instead.
+ *
+ */
+
+NOTMUCH_DEPRECATED (5, 0)
+notmuch_status_t
+notmuch_query_search_messages_st (notmuch_query_t *query,
+				  notmuch_messages_t **out);
+
+/**
+ * Destroy a notmuch_query_t along with any associated resources.
+ *
+ * This will in turn destroy any notmuch_threads_t and
+ * notmuch_messages_t objects generated by this query, (and in
+ * turn any notmuch_thread_t and notmuch_message_t objects generated
+ * from those results, etc.), if such objects haven't already been
+ * destroyed.
+ */
+void
+notmuch_query_destroy (notmuch_query_t *query);
+
+/**
+ * Is the given 'threads' iterator pointing at a valid thread.
+ *
+ * When this function returns TRUE, notmuch_threads_get will return a
+ * valid object. Whereas when this function returns FALSE,
+ * notmuch_threads_get will return NULL.
+ *
+ * If passed a NULL pointer, this function returns FALSE
+ *
+ * See the documentation of notmuch_query_search_threads for example
+ * code showing how to iterate over a notmuch_threads_t object.
+ */
+notmuch_bool_t
+notmuch_threads_valid (notmuch_threads_t *threads);
+
+/**
+ * Get the current thread from 'threads' as a notmuch_thread_t.
+ *
+ * Note: The returned thread belongs to 'threads' and has a lifetime
+ * identical to it (and the query to which it belongs).
+ *
+ * See the documentation of notmuch_query_search_threads for example
+ * code showing how to iterate over a notmuch_threads_t object.
+ *
+ * If an out-of-memory situation occurs, this function will return
+ * NULL.
+ */
+notmuch_thread_t *
+notmuch_threads_get (notmuch_threads_t *threads);
+
+/**
+ * Move the 'threads' iterator to the next thread.
+ *
+ * If 'threads' is already pointing at the last thread then the
+ * iterator will be moved to a point just beyond that last thread,
+ * (where notmuch_threads_valid will return FALSE and
+ * notmuch_threads_get will return NULL).
+ *
+ * See the documentation of notmuch_query_search_threads for example
+ * code showing how to iterate over a notmuch_threads_t object.
+ */
+void
+notmuch_threads_move_to_next (notmuch_threads_t *threads);
+
+/**
+ * Destroy a notmuch_threads_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_threads_t object will be reclaimed when the
+ * containing query object is destroyed.
+ */
+void
+notmuch_threads_destroy (notmuch_threads_t *threads);
+
+/**
+ * Return the number of messages matching a search.
+ *
+ * This function performs a search and returns the number of matching
+ * messages.
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_SUCCESS: query completed successfully.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: a Xapian exception occurred. The
+ *      value of *count is not defined.
+ *
+ * @since libnotmuch 5 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_count_messages (notmuch_query_t *query, unsigned int *count);
+
+/**
+ * Deprecated alias for notmuch_query_count_messages
+ *
+ *
+ * @deprecated Deprecated since libnotmuch 5.0 (notmuch 0.25). Please
+ * use notmuch_query_count_messages instead.
+ */
+NOTMUCH_DEPRECATED (5, 0)
+notmuch_status_t
+notmuch_query_count_messages_st (notmuch_query_t *query, unsigned int *count);
+
+/**
+ * Return the number of threads matching a search.
+ *
+ * This function performs a search and returns the number of unique thread IDs
+ * in the matching messages. This is the same as number of threads matching a
+ * search.
+ *
+ * Note that this is a significantly heavier operation than
+ * notmuch_query_count_messages{_st}().
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Memory allocation failed. The value
+ *      of *count is not defined
+ *
+ * NOTMUCH_STATUS_SUCCESS: query completed successfully.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: a Xapian exception occurred. The
+ *      value of *count is not defined.
+ *
+ * @since libnotmuch 5 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_count_threads (notmuch_query_t *query, unsigned *count);
+
+/**
+ * Deprecated alias for notmuch_query_count_threads
+ *
+ * @deprecated Deprecated as of libnotmuch 5.0 (notmuch 0.25). Please
+ * use notmuch_query_count_threads_st instead.
+ */
+NOTMUCH_DEPRECATED (5, 0)
+notmuch_status_t
+notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count);
+
+/**
+ * Get the thread ID of 'thread'.
+ *
+ * The returned string belongs to 'thread' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * thread is valid, (which is until notmuch_thread_destroy or until
+ * the query from which it derived is destroyed).
+ */
+const char *
+notmuch_thread_get_thread_id (notmuch_thread_t *thread);
+
+/**
+ * Get the total number of messages in 'thread'.
+ *
+ * This count consists of all messages in the database belonging to
+ * this thread. Contrast with notmuch_thread_get_matched_messages() .
+ */
+int
+notmuch_thread_get_total_messages (notmuch_thread_t *thread);
+
+/**
+ * Get the total number of files in 'thread'.
+ *
+ * This sums notmuch_message_count_files over all messages in the
+ * thread
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread);
+
+/**
+ * Get a notmuch_messages_t iterator for the top-level messages in
+ * 'thread' in oldest-first order.
+ *
+ * This iterator will not necessarily iterate over all of the messages
+ * in the thread. It will only iterate over the messages in the thread
+ * which are not replies to other messages in the thread.
+ *
+ * The returned list will be destroyed when the thread is destroyed.
+ */
+notmuch_messages_t *
+notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread);
+
+/**
+ * Get a notmuch_thread_t iterator for all messages in 'thread' in
+ * oldest-first order.
+ *
+ * The returned list will be destroyed when the thread is destroyed.
+ */
+notmuch_messages_t *
+notmuch_thread_get_messages (notmuch_thread_t *thread);
+
+/**
+ * Get the number of messages in 'thread' that matched the search.
+ *
+ * This count includes only the messages in this thread that were
+ * matched by the search from which the thread was created and were
+ * not excluded by any exclude tags passed in with the query (see
+ * notmuch_query_add_tag_exclude). Contrast with
+ * notmuch_thread_get_total_messages() .
+ */
+int
+notmuch_thread_get_matched_messages (notmuch_thread_t *thread);
+
+/**
+ * Get the authors of 'thread' as a UTF-8 string.
+ *
+ * The returned string is a comma-separated list of the names of the
+ * authors of mail messages in the query results that belong to this
+ * thread.
+ *
+ * The string contains authors of messages matching the query first, then
+ * non-matched authors (with the two groups separated by '|'). Within
+ * each group, authors are ordered by date.
+ *
+ * The returned string belongs to 'thread' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * thread is valid, (which is until notmuch_thread_destroy or until
+ * the query from which it derived is destroyed).
+ */
+const char *
+notmuch_thread_get_authors (notmuch_thread_t *thread);
+
+/**
+ * Get the subject of 'thread' as a UTF-8 string.
+ *
+ * The subject is taken from the first message (according to the query
+ * order---see notmuch_query_set_sort) in the query results that
+ * belongs to this thread.
+ *
+ * The returned string belongs to 'thread' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * thread is valid, (which is until notmuch_thread_destroy or until
+ * the query from which it derived is destroyed).
+ */
+const char *
+notmuch_thread_get_subject (notmuch_thread_t *thread);
+
+/**
+ * Get the date of the oldest message in 'thread' as a time_t value.
+ */
+time_t
+notmuch_thread_get_oldest_date (notmuch_thread_t *thread);
+
+/**
+ * Get the date of the newest message in 'thread' as a time_t value.
+ */
+time_t
+notmuch_thread_get_newest_date (notmuch_thread_t *thread);
+
+/**
+ * Get the tags for 'thread', returning a notmuch_tags_t object which
+ * can be used to iterate over all tags.
+ *
+ * Note: In the Notmuch database, tags are stored on individual
+ * messages, not on threads. So the tags returned here will be all
+ * tags of the messages which matched the search and which belong to
+ * this thread.
+ *
+ * The tags object is owned by the thread and as such, will only be
+ * valid for as long as the thread is valid, (for example, until
+ * notmuch_thread_destroy or until the query from which it derived is
+ * destroyed).
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_thread_t *thread;
+ *     notmuch_tags_t *tags;
+ *     const char *tag;
+ *
+ *     thread = notmuch_threads_get (threads);
+ *
+ *     for (tags = notmuch_thread_get_tags (thread);
+ *          notmuch_tags_valid (tags);
+ *          notmuch_tags_move_to_next (tags))
+ *     {
+ *         tag = notmuch_tags_get (tags);
+ *         ....
+ *     }
+ *
+ *     notmuch_thread_destroy (thread);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_tags_t object. (For consistency, we do provide a
+ * notmuch_tags_destroy function, but there's no good reason to call
+ * it if the message is about to be destroyed).
+ */
+notmuch_tags_t *
+notmuch_thread_get_tags (notmuch_thread_t *thread);
+
+/**
+ * Destroy a notmuch_thread_t object.
+ */
+void
+notmuch_thread_destroy (notmuch_thread_t *thread);
+
+/**
+ * Is the given 'messages' iterator pointing at a valid message.
+ *
+ * When this function returns TRUE, notmuch_messages_get will return a
+ * valid object. Whereas when this function returns FALSE,
+ * notmuch_messages_get will return NULL.
+ *
+ * See the documentation of notmuch_query_search_messages for example
+ * code showing how to iterate over a notmuch_messages_t object.
+ */
+notmuch_bool_t
+notmuch_messages_valid (notmuch_messages_t *messages);
+
+/**
+ * Get the current message from 'messages' as a notmuch_message_t.
+ *
+ * Note: The returned message belongs to 'messages' and has a lifetime
+ * identical to it (and the query to which it belongs).
+ *
+ * See the documentation of notmuch_query_search_messages for example
+ * code showing how to iterate over a notmuch_messages_t object.
+ *
+ * If an out-of-memory situation occurs, this function will return
+ * NULL.
+ */
+notmuch_message_t *
+notmuch_messages_get (notmuch_messages_t *messages);
+
+/**
+ * Move the 'messages' iterator to the next message.
+ *
+ * If 'messages' is already pointing at the last message then the
+ * iterator will be moved to a point just beyond that last message,
+ * (where notmuch_messages_valid will return FALSE and
+ * notmuch_messages_get will return NULL).
+ *
+ * See the documentation of notmuch_query_search_messages for example
+ * code showing how to iterate over a notmuch_messages_t object.
+ */
+void
+notmuch_messages_move_to_next (notmuch_messages_t *messages);
+
+/**
+ * Destroy a notmuch_messages_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_messages_t object will be reclaimed when the containing
+ * query object is destroyed.
+ */
+void
+notmuch_messages_destroy (notmuch_messages_t *messages);
+
+/**
+ * Return a list of tags from all messages.
+ *
+ * The resulting list is guaranteed not to contain duplicated tags.
+ *
+ * WARNING: You can no longer iterate over messages after calling this
+ * function, because the iterator will point at the end of the list.
+ * We do not have a function to reset the iterator yet and the only
+ * way how you can iterate over the list again is to recreate the
+ * message list.
+ *
+ * The function returns NULL on error.
+ */
+notmuch_tags_t *
+notmuch_messages_collect_tags (notmuch_messages_t *messages);
+
+/**
+ * Get the database associated with this message.
+ *
+ * @since libnotmuch 5.2 (notmuch 0.27)
+ */
+notmuch_database_t *
+notmuch_message_get_database (const notmuch_message_t *message);
+
+/**
+ * Get the message ID of 'message'.
+ *
+ * The returned string belongs to 'message' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * message is valid, (which is until the query from which it derived
+ * is destroyed).
+ *
+ * This function will return NULL if triggers an unhandled Xapian
+ * exception.
+ */
+const char *
+notmuch_message_get_message_id (notmuch_message_t *message);
+
+/**
+ * Get the thread ID of 'message'.
+ *
+ * The returned string belongs to 'message' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * message is valid, (for example, until the user calls
+ * notmuch_message_destroy on 'message' or until a query from which it
+ * derived is destroyed).
+ *
+ * This function will return NULL if triggers an unhandled Xapian
+ * exception.
+ */
+const char *
+notmuch_message_get_thread_id (notmuch_message_t *message);
+
+/**
+ * Get a notmuch_messages_t iterator for all of the replies to
+ * 'message'.
+ *
+ * Note: This call only makes sense if 'message' was ultimately
+ * obtained from a notmuch_thread_t object, (such as by coming
+ * directly from the result of calling notmuch_thread_get_
+ * toplevel_messages or by any number of subsequent
+ * calls to notmuch_message_get_replies).
+ *
+ * If 'message' was obtained through some non-thread means, (such as
+ * by a call to notmuch_query_search_messages), then this function
+ * will return NULL.
+ *
+ * If there are no replies to 'message', this function will return
+ * NULL. (Note that notmuch_messages_valid will accept that NULL
+ * value as legitimate, and simply return FALSE for it.)
+ *
+ * This function also returns NULL if it triggers a Xapian exception.
+ *
+ * The returned list will be destroyed when the thread is
+ * destroyed.
+ */
+notmuch_messages_t *
+notmuch_message_get_replies (notmuch_message_t *message);
+
+/**
+ * Get the total number of files associated with a message.
+ * @returns Non-negative integer for file count.
+ * @returns Negative integer for error.
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+int
+notmuch_message_count_files (notmuch_message_t *message);
+
+/**
+ * Get a filename for the email corresponding to 'message'.
+ *
+ * The returned filename is an absolute filename, (the initial
+ * component will match notmuch_database_get_path() ).
+ *
+ * The returned string belongs to the message so should not be
+ * modified or freed by the caller (nor should it be referenced after
+ * the message is destroyed).
+ *
+ * Note: If this message corresponds to multiple files in the mail
+ * store, (that is, multiple files contain identical message IDs),
+ * this function will arbitrarily return a single one of those
+ * filenames. See notmuch_message_get_filenames for returning the
+ * complete list of filenames.
+ *
+ * This function returns NULL if it triggers a Xapian exception.
+ */
+const char *
+notmuch_message_get_filename (notmuch_message_t *message);
+
+/**
+ * Get all filenames for the email corresponding to 'message'.
+ *
+ * Returns a notmuch_filenames_t iterator listing all the filenames
+ * associated with 'message'. These files may not have identical
+ * content, but each will have the identical Message-ID.
+ *
+ * Each filename in the iterator is an absolute filename, (the initial
+ * component will match notmuch_database_get_path() ).
+ *
+ * This function returns NULL if it triggers a Xapian exception.
+ */
+notmuch_filenames_t *
+notmuch_message_get_filenames (notmuch_message_t *message);
+
+/**
+ * Re-index the e-mail corresponding to 'message' using the supplied index options
+ *
+ * Returns the status of the re-index operation.  (see the return
+ * codes documented in notmuch_database_index_file)
+ *
+ * After reindexing, the user should discard the message object passed
+ * in here by calling notmuch_message_destroy, since it refers to the
+ * original message, not to the reindexed message.
+ */
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+			 notmuch_indexopts_t *indexopts);
+
+/**
+ * Message flags.
+ */
+typedef enum {
+    NOTMUCH_MESSAGE_FLAG_MATCH,
+    NOTMUCH_MESSAGE_FLAG_EXCLUDED,
+
+    /* This message is a "ghost message", meaning it has no filenames
+     * or content, but we know it exists because it was referenced by
+     * some other message.  A ghost message has only a message ID and
+     * thread ID.
+     */
+    NOTMUCH_MESSAGE_FLAG_GHOST,
+} notmuch_message_flag_t;
+
+/**
+ * Get a value of a flag for the email corresponding to 'message'.
+ *
+ * returns FALSE in case of errors.
+ *
+ * @deprecated Deprecated as of libnotmuch 5.3 (notmuch 0.31). Please
+ * use notmuch_message_get_flag_st instead.
+ */
+NOTMUCH_DEPRECATED (5, 3)
+notmuch_bool_t
+notmuch_message_get_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag);
+
+/**
+ * Get a value of a flag for the email corresponding to 'message'.
+ *
+ * @param message a message object
+ * @param flag flag to check
+ * @param is_set pointer to boolean to store flag value.
+ *
+ * @retval #NOTMUCH_STATUS_SUCCESS
+ * @retval #NOTMUCH_STATUS_NULL_POINTER is_set is NULL
+ * @retval #NOTMUCH_STATUS_XAPIAN_EXCEPTION Accessing the database
+ * triggered an exception.
+ *
+ * @since libnotmuch 5.3 (notmuch 0.31)
+ */
+notmuch_status_t
+notmuch_message_get_flag_st (notmuch_message_t *message,
+			     notmuch_message_flag_t flag,
+			     notmuch_bool_t *is_set);
+
+/**
+ * Set a value of a flag for the email corresponding to 'message'.
+ */
+void
+notmuch_message_set_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag, notmuch_bool_t value);
+
+/**
+ * Get the date of 'message' as a time_t value.
+ *
+ * For the original textual representation of the Date header from the
+ * message call notmuch_message_get_header() with a header value of
+ * "date".
+ *
+ * Returns 0 in case of error.
+ */
+time_t
+notmuch_message_get_date (notmuch_message_t *message);
+
+/**
+ * Get the value of the specified header from 'message' as a UTF-8 string.
+ *
+ * Common headers are stored in the database when the message is
+ * indexed and will be returned from the database.  Other headers will
+ * be read from the actual message file.
+ *
+ * The header name is case insensitive.
+ *
+ * The returned string belongs to the message so should not be
+ * modified or freed by the caller (nor should it be referenced after
+ * the message is destroyed).
+ *
+ * Returns an empty string ("") if the message does not contain a
+ * header line matching 'header'. Returns NULL if any error occurs.
+ */
+const char *
+notmuch_message_get_header (notmuch_message_t *message, const char *header);
+
+/**
+ * Get the tags for 'message', returning a notmuch_tags_t object which
+ * can be used to iterate over all tags.
+ *
+ * The tags object is owned by the message and as such, will only be
+ * valid for as long as the message is valid, (which is until the
+ * query from which it derived is destroyed).
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_t *message;
+ *     notmuch_tags_t *tags;
+ *     const char *tag;
+ *
+ *     message = notmuch_database_find_message (database, message_id);
+ *
+ *     for (tags = notmuch_message_get_tags (message);
+ *          notmuch_tags_valid (tags);
+ *          notmuch_tags_move_to_next (tags))
+ *     {
+ *         tag = notmuch_tags_get (tags);
+ *         ....
+ *     }
+ *
+ *     notmuch_message_destroy (message);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_tags_t object. (For consistency, we do provide a
+ * notmuch_tags_destroy function, but there's no good reason to call
+ * it if the message is about to be destroyed).
+ */
+notmuch_tags_t *
+notmuch_message_get_tags (notmuch_message_t *message);
+
+/**
+ * The longest possible tag value.
+ */
+#define NOTMUCH_TAG_MAX 200
+
+/**
+ * Add a tag to the given message.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Tag successfully added to message
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The 'tag' argument is NULL
+ *
+ * NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
+ *	(exceeds NOTMUCH_TAG_MAX)
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
+
+/**
+ * Remove a tag from the given message.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Tag successfully removed from message
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The 'tag' argument is NULL
+ *
+ * NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
+ *	(exceeds NOTMUCH_TAG_MAX)
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
+
+/**
+ * Remove all tags from the given message.
+ *
+ * See notmuch_message_freeze for an example showing how to safely
+ * replace tag values.
+ *
+ * @retval #NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *	read-only mode so message cannot be modified.
+ * @retval #NOTMUCH_STATUS_XAPIAN_EXCEPTION: an exception was thrown
+ *      accessing the database.
+ */
+notmuch_status_t
+notmuch_message_remove_all_tags (notmuch_message_t *message);
+
+/**
+ * Add/remove tags according to maildir flags in the message filename(s).
+ *
+ * This function examines the filenames of 'message' for maildir
+ * flags, and adds or removes tags on 'message' as follows when these
+ * flags are present:
+ *
+ *	Flag	Action if present
+ *	----	-----------------
+ *	'D'	Adds the "draft" tag to the message
+ *	'F'	Adds the "flagged" tag to the message
+ *	'P'	Adds the "passed" tag to the message
+ *	'R'	Adds the "replied" tag to the message
+ *	'S'	Removes the "unread" tag from the message
+ *
+ * For each flag that is not present, the opposite action (add/remove)
+ * is performed for the corresponding tags.
+ *
+ * Flags are identified as trailing components of the filename after a
+ * sequence of ":2,".
+ *
+ * If there are multiple filenames associated with this message, the
+ * flag is considered present if it appears in one or more
+ * filenames. (That is, the flags from the multiple filenames are
+ * combined with the logical OR operator.)
+ *
+ * A client can ensure that notmuch database tags remain synchronized
+ * with maildir flags by calling this function after each call to
+ * notmuch_database_index_file. See also
+ * notmuch_message_tags_to_maildir_flags for synchronizing tag changes
+ * back to maildir flags.
+ */
+notmuch_status_t
+notmuch_message_maildir_flags_to_tags (notmuch_message_t *message);
+
+/**
+ * return TRUE if any filename of 'message' has maildir flag 'flag',
+ * FALSE otherwise.
+ *
+ * Deprecated wrapper for notmuch_message_has_maildir_flag_st
+ *
+ * @returns FALSE in case of error
+ * @deprecated libnotmuch 5.3 (notmuch 0.31)
+ */
+NOTMUCH_DEPRECATED (5, 3)
+notmuch_bool_t
+notmuch_message_has_maildir_flag (notmuch_message_t *message, char flag);
+
+/**
+ * check message for maildir flag
+ *
+ * @param [in,out]	message message to check
+ * @param [in] flag	flag to check for
+ * @param [out] is_set  pointer to boolean
+ *
+ * @retval #NOTMUCH_STATUS_SUCCESS
+ * @retval #NOTMUCH_STATUS_NULL_POINTER is_set is NULL
+ * @retval #NOTMUCH_STATUS_XAPIAN_EXCEPTION Accessing the database
+ * triggered an exception.
+ */
+notmuch_status_t
+notmuch_message_has_maildir_flag_st (notmuch_message_t *message,
+				     char flag,
+				     notmuch_bool_t *is_set);
+
+/**
+ * Rename message filename(s) to encode tags as maildir flags.
+ *
+ * Specifically, for each filename corresponding to this message:
+ *
+ * If the filename is not in a maildir directory, do nothing.  (A
+ * maildir directory is determined as a directory named "new" or
+ * "cur".) Similarly, if the filename has invalid maildir info,
+ * (repeated or outof-ASCII-order flag characters after ":2,"), then
+ * do nothing.
+ *
+ * If the filename is in a maildir directory, rename the file so that
+ * its filename ends with the sequence ":2," followed by zero or more
+ * of the following single-character flags (in ASCII order):
+ *
+ *   * flag 'D' iff the message has the "draft" tag
+ *   * flag 'F' iff the message has the "flagged" tag
+ *   * flag 'P' iff the message has the "passed" tag
+ *   * flag 'R' iff the message has the "replied" tag
+ *   * flag 'S' iff the message does not have the "unread" tag
+ *
+ * Any existing flags unmentioned in the list above will be preserved
+ * in the renaming.
+ *
+ * Also, if this filename is in a directory named "new", rename it to
+ * be within the neighboring directory named "cur".
+ *
+ * A client can ensure that maildir filename flags remain synchronized
+ * with notmuch database tags by calling this function after changing
+ * tags, (after calls to notmuch_message_add_tag,
+ * notmuch_message_remove_tag, or notmuch_message_freeze/
+ * notmuch_message_thaw). See also notmuch_message_maildir_flags_to_tags
+ * for synchronizing maildir flag changes back to tags.
+ */
+notmuch_status_t
+notmuch_message_tags_to_maildir_flags (notmuch_message_t *message);
+
+/**
+ * Freeze the current state of 'message' within the database.
+ *
+ * This means that changes to the message state, (via
+ * notmuch_message_add_tag, notmuch_message_remove_tag, and
+ * notmuch_message_remove_all_tags), will not be committed to the
+ * database until the message is thawed with notmuch_message_thaw.
+ *
+ * Multiple calls to freeze/thaw are valid and these calls will
+ * "stack". That is there must be as many calls to thaw as to freeze
+ * before a message is actually thawed.
+ *
+ * The ability to do freeze/thaw allows for safe transactions to
+ * change tag values. For example, explicitly setting a message to
+ * have a given set of tags might look like this:
+ *
+ *    notmuch_message_freeze (message);
+ *
+ *    notmuch_message_remove_all_tags (message);
+ *
+ *    for (i = 0; i < NUM_TAGS; i++)
+ *        notmuch_message_add_tag (message, tags[i]);
+ *
+ *    notmuch_message_thaw (message);
+ *
+ * With freeze/thaw used like this, the message in the database is
+ * guaranteed to have either the full set of original tag values, or
+ * the full set of new tag values, but nothing in between.
+ *
+ * Imagine the example above without freeze/thaw and the operation
+ * somehow getting interrupted. This could result in the message being
+ * left with no tags if the interruption happened after
+ * notmuch_message_remove_all_tags but before notmuch_message_add_tag.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully frozen.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_freeze (notmuch_message_t *message);
+
+/**
+ * Thaw the current 'message', synchronizing any changes that may have
+ * occurred while 'message' was frozen into the notmuch database.
+ *
+ * See notmuch_message_freeze for an example of how to use this
+ * function to safely provide tag changes.
+ *
+ * Multiple calls to freeze/thaw are valid and these calls with
+ * "stack". That is there must be as many calls to thaw as to freeze
+ * before a message is actually thawed.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully thawed, (or at least
+ *	its frozen count has successfully been reduced by 1).
+ *
+ * NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: An attempt was made to thaw
+ *	an unfrozen message. That is, there have been an unbalanced
+ *	number of calls to notmuch_message_freeze and
+ *	notmuch_message_thaw.
+ */
+notmuch_status_t
+notmuch_message_thaw (notmuch_message_t *message);
+
+/**
+ * Destroy a notmuch_message_t object.
+ *
+ * It can be useful to call this function in the case of a single
+ * query object with many messages in the result, (such as iterating
+ * over the entire database). Otherwise, it's fine to never call this
+ * function and there will still be no memory leaks. (The memory from
+ * the messages get reclaimed when the containing query is destroyed.)
+ */
+void
+notmuch_message_destroy (notmuch_message_t *message);
+
+/**
+ * @name Message Properties
+ *
+ * This interface provides the ability to attach arbitrary (key,value)
+ * string pairs to a message, to remove such pairs, and to iterate
+ * over them.  The caller should take some care as to what keys they
+ * add or delete values for, as other subsystems or extensions may
+ * depend on these properties.
+ *
+ * Please see notmuch-properties(7) for more details about specific
+ * properties and conventions around their use.
+ *
+ */
+/**@{*/
+/**
+ * Retrieve the value for a single property key
+ *
+ * *value* is set to a string owned by the message or NULL if there is
+ * no such key. In the case of multiple values for the given key, the
+ * first one is retrieved.
+ *
+ * @returns
+ * - NOTMUCH_STATUS_NULL_POINTER: *value* may not be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value);
+
+/**
+ * Add a (key,value) pair to a message
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove a (key,value) pair from a message.
+ *
+ * It is not an error to remove a non-existent (key,value) pair
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove all (key,value) pairs from the given message.
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     key      key to delete properties for. If NULL, delete
+ *			   properties for all keys
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
+
+/**
+ * Remove all (prefix*,value) pairs from the given message
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     prefix   delete properties with keys that start with prefix.
+ *			   If NULL, delete all properties
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix);
+
+/**
+ * Opaque message property iterator
+ */
+typedef struct _notmuch_string_map_iterator notmuch_message_properties_t;
+
+/**
+ * Get the properties for *message*, returning a
+ * notmuch_message_properties_t object which can be used to iterate
+ * over all properties.
+ *
+ * The notmuch_message_properties_t object is owned by the message and
+ * as such, will only be valid for as long as the message is valid,
+ * (which is until the query from which it derived is destroyed).
+ *
+ * @param[in] message  The message to examine
+ * @param[in] key      key or key prefix
+ * @param[in] exact    if TRUE, require exact match with key. Otherwise
+ *		       treat as prefix.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_properties_t *list;
+ *
+ *     for (list = notmuch_message_get_properties (message, "testkey1", TRUE);
+ *          notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ *        printf("%s\n", notmuch_message_properties_value(list));
+ *     }
+ *
+ *     notmuch_message_properties_destroy (list);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_message_properties_t object. (For consistency, we do
+ * provide a notmuch_message_properities_destroy function, but there's
+ * no good reason to call it if the message is about to be destroyed).
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact);
+
+/**
+ * Return the number of properties named "key" belonging to the specific message.
+ *
+ * @param[in] message  The message to examine
+ * @param[in] key      key to count
+ * @param[out] count   The number of matching properties associated with this message.
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_SUCCESS: successful count, possibly some other error.
+ *
+ * @since libnotmuch 5.2 (notmuch 0.27)
+ */
+notmuch_status_t
+notmuch_message_count_properties (notmuch_message_t *message, const char *key, unsigned int *count);
+
+/**
+ * Is the given *properties* iterator pointing at a valid (key,value)
+ * pair.
+ *
+ * When this function returns TRUE,
+ * notmuch_message_properties_{key,value} will return a valid string,
+ * and notmuch_message_properties_move_to_next will do what it
+ * says. Whereas when this function returns FALSE, calling any of
+ * these functions results in undefined behaviour.
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t
+ * object.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties);
+
+/**
+ * Move the *properties* iterator to the next (key,value) pair
+ *
+ * If *properties* is already pointing at the last pair then the iterator
+ * will be moved to a point just beyond that last pair, (where
+ * notmuch_message_properties_valid will return FALSE).
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t object.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * this could be useful if iterating for a prefix
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties);
+
+/**
+ * Return the value from the current (key,value) pair.
+ *
+ * This could be useful if iterating for a prefix.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties);
+
+
+/**
+ * Destroy a notmuch_message_properties_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_message_properties_t object will be reclaimed when the
+ * containing message object is destroyed.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties);
+
+/**@}*/
+
+/**
+ * Is the given 'tags' iterator pointing at a valid tag.
+ *
+ * When this function returns TRUE, notmuch_tags_get will return a
+ * valid string. Whereas when this function returns FALSE,
+ * notmuch_tags_get will return NULL.
+ *
+ * It is acceptable to pass NULL for 'tags', in which case this
+ * function will always return FALSE.
+
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+notmuch_bool_t
+notmuch_tags_valid (notmuch_tags_t *tags);
+
+/**
+ * Get the current tag from 'tags' as a string.
+ *
+ * Note: The returned string belongs to 'tags' and has a lifetime
+ * identical to it (and the query to which it ultimately belongs).
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+const char *
+notmuch_tags_get (notmuch_tags_t *tags);
+
+/**
+ * Move the 'tags' iterator to the next tag.
+ *
+ * If 'tags' is already pointing at the last tag then the iterator
+ * will be moved to a point just beyond that last tag, (where
+ * notmuch_tags_valid will return FALSE and notmuch_tags_get will
+ * return NULL).
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+void
+notmuch_tags_move_to_next (notmuch_tags_t *tags);
+
+/**
+ * Destroy a notmuch_tags_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_tags_t object will be reclaimed when the containing
+ * message or query objects are destroyed.
+ */
+void
+notmuch_tags_destroy (notmuch_tags_t *tags);
+
+/**
+ * Store an mtime within the database for 'directory'.
+ *
+ * The 'directory' should be an object retrieved from the database
+ * with notmuch_database_get_directory for a particular path.
+ *
+ * The intention is for the caller to use the mtime to allow efficient
+ * identification of new messages to be added to the database. The
+ * recommended usage is as follows:
+ *
+ *   o Read the mtime of a directory from the filesystem
+ *
+ *   o Call index_file for all mail files in the directory
+ *
+ *   o Call notmuch_directory_set_mtime with the mtime read from the
+ *     filesystem.
+ *
+ * Then, when wanting to check for updates to the directory in the
+ * future, the client can call notmuch_directory_get_mtime and know
+ * that it only needs to add files if the mtime of the directory and
+ * files are newer than the stored timestamp.
+ *
+ * Note: The notmuch_directory_get_mtime function does not allow the
+ * caller to distinguish a timestamp of 0 from a non-existent
+ * timestamp. So don't store a timestamp of 0 unless you are
+ * comfortable with that.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: mtime successfully stored in database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception
+ *	occurred, mtime not stored.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so directory mtime cannot be modified.
+ */
+notmuch_status_t
+notmuch_directory_set_mtime (notmuch_directory_t *directory,
+			     time_t mtime);
+
+/**
+ * Get the mtime of a directory, (as previously stored with
+ * notmuch_directory_set_mtime).
+ *
+ * Returns 0 if no mtime has previously been stored for this
+ * directory.
+ */
+time_t
+notmuch_directory_get_mtime (notmuch_directory_t *directory);
+
+/**
+ * Get a notmuch_filenames_t iterator listing all the filenames of
+ * messages in the database within the given directory.
+ *
+ * The returned filenames will be the basename-entries only (not
+ * complete paths).
+ *
+ * Returns NULL if it triggers a Xapian exception
+ */
+notmuch_filenames_t *
+notmuch_directory_get_child_files (notmuch_directory_t *directory);
+
+/**
+ * Get a notmuch_filenames_t iterator listing all the filenames of
+ * sub-directories in the database within the given directory.
+ *
+ * The returned filenames will be the basename-entries only (not
+ * complete paths).
+ *
+ * Returns NULL if it triggers a Xapian exception
+ */
+notmuch_filenames_t *
+notmuch_directory_get_child_directories (notmuch_directory_t *directory);
+
+/**
+ * Delete directory document from the database, and destroy the
+ * notmuch_directory_t object. Assumes any child directories and files
+ * have been deleted by the caller.
+ *
+ * @since libnotmuch 4.3 (notmuch 0.21)
+ */
+notmuch_status_t
+notmuch_directory_delete (notmuch_directory_t *directory);
+
+/**
+ * Destroy a notmuch_directory_t object.
+ */
+void
+notmuch_directory_destroy (notmuch_directory_t *directory);
+
+/**
+ * Is the given 'filenames' iterator pointing at a valid filename.
+ *
+ * When this function returns TRUE, notmuch_filenames_get will return
+ * a valid string. Whereas when this function returns FALSE,
+ * notmuch_filenames_get will return NULL.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will always return FALSE.
+ */
+notmuch_bool_t
+notmuch_filenames_valid (notmuch_filenames_t *filenames);
+
+/**
+ * Get the current filename from 'filenames' as a string.
+ *
+ * Note: The returned string belongs to 'filenames' and has a lifetime
+ * identical to it (and the directory to which it ultimately belongs).
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will always return NULL.
+ */
+const char *
+notmuch_filenames_get (notmuch_filenames_t *filenames);
+
+/**
+ * Move the 'filenames' iterator to the next filename.
+ *
+ * If 'filenames' is already pointing at the last filename then the
+ * iterator will be moved to a point just beyond that last filename,
+ * (where notmuch_filenames_valid will return FALSE and
+ * notmuch_filenames_get will return NULL).
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will do nothing.
+ */
+void
+notmuch_filenames_move_to_next (notmuch_filenames_t *filenames);
+
+/**
+ * Destroy a notmuch_filenames_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_filenames_t object will be reclaimed when the
+ * containing directory object is destroyed.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will do nothing.
+ */
+void
+notmuch_filenames_destroy (notmuch_filenames_t *filenames);
+
+
+/**
+ * set config 'key' to 'value'
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ * @retval #NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *	read-only mode so message cannot be modified.
+ * @retval #NOTMUCH_STATUS_XAPIAN_EXCEPTION: an exception was thrown
+ *      accessing the database.
+ * @retval #NOTMUCH_STATUS_SUCCESS
+ */
+notmuch_status_t
+notmuch_database_set_config (notmuch_database_t *db, const char *key, const char *value);
+
+/**
+ * retrieve config item 'key', assign to  'value'
+ *
+ * keys which have not been previously set with n_d_set_config will
+ * return an empty string.
+ *
+ * return value is allocated by malloc and should be freed by the
+ * caller.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ *
+ */
+notmuch_status_t
+notmuch_database_get_config (notmuch_database_t *db, const char *key, char **value);
+
+/**
+ * Create an iterator for all config items with keys matching a given prefix
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_get_config_list (notmuch_database_t *db, const char *prefix,
+				  notmuch_config_list_t **out);
+
+/**
+ * Is 'config_list' iterator valid (i.e. _key, _value, _move_to_next can be called).
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_config_list_valid (notmuch_config_list_t *config_list);
+
+/**
+ * return key for current config pair
+ *
+ * return value is owned by the iterator, and will be destroyed by the
+ * next call to notmuch_config_list_key or notmuch_config_list_destroy.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_config_list_key (notmuch_config_list_t *config_list);
+
+/**
+ * return 'value' for current config pair
+ *
+ * return value is owned by the iterator, and will be destroyed by the
+ * next call to notmuch_config_list_value or notmuch config_list_destroy
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ * @retval NULL for errors
+ */
+const char *
+notmuch_config_list_value (notmuch_config_list_t *config_list);
+
+
+/**
+ * move 'config_list' iterator to the next pair
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_config_list_move_to_next (notmuch_config_list_t *config_list);
+
+/**
+ * free any resources held by 'config_list'
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_config_list_destroy (notmuch_config_list_t *config_list);
+
+/**
+ * Configuration keys known to libnotmuch
+ */
+typedef enum {
+    NOTMUCH_CONFIG_FIRST,
+    NOTMUCH_CONFIG_DATABASE_PATH = NOTMUCH_CONFIG_FIRST,
+    NOTMUCH_CONFIG_MAIL_ROOT,
+    NOTMUCH_CONFIG_HOOK_DIR,
+    NOTMUCH_CONFIG_BACKUP_DIR,
+    NOTMUCH_CONFIG_EXCLUDE_TAGS,
+    NOTMUCH_CONFIG_NEW_TAGS,
+    NOTMUCH_CONFIG_NEW_IGNORE,
+    NOTMUCH_CONFIG_SYNC_MAILDIR_FLAGS,
+    NOTMUCH_CONFIG_PRIMARY_EMAIL,
+    NOTMUCH_CONFIG_OTHER_EMAIL,
+    NOTMUCH_CONFIG_USER_NAME,
+    NOTMUCH_CONFIG_AUTOCOMMIT,
+    NOTMUCH_CONFIG_EXTRA_HEADERS,
+    NOTMUCH_CONFIG_INDEX_AS_TEXT,
+    NOTMUCH_CONFIG_LAST
+} notmuch_config_key_t;
+
+/**
+ * get a configuration value from an open database.
+ *
+ * This value reflects all configuration information given at the time
+ * the database was opened.
+ *
+ * @param[in] notmuch database
+ * @param[in] key configuration key
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval NULL if 'key' unknown or if no value is known for
+ *         'key'.  Otherwise returns a string owned by notmuch which should
+ *         not be modified nor freed by the caller.
+ */
+const char *
+notmuch_config_get (notmuch_database_t *notmuch, notmuch_config_key_t key);
+
+/**
+ * set a configuration value from in an open database.
+ *
+ * This value reflects all configuration information given at the time
+ * the database was opened.
+ *
+ * @param[in,out] notmuch database open read/write
+ * @param[in] key configuration key
+ * @param[in] val configuration value
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval returns any return value for notmuch_database_set_config.
+ */
+notmuch_status_t
+notmuch_config_set (notmuch_database_t *notmuch, notmuch_config_key_t key, const char *val);
+
+/**
+ * Returns an iterator for a ';'-delimited list of configuration values
+ *
+ * These values reflect all configuration information given at the
+ * time the database was opened.
+ *
+ * @param[in] notmuch database
+ * @param[in] key configuration key
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval NULL in case of error.
+ */
+notmuch_config_values_t *
+notmuch_config_get_values (notmuch_database_t *notmuch, notmuch_config_key_t key);
+
+/**
+ * Returns an iterator for a ';'-delimited list of configuration values
+ *
+ * These values reflect all configuration information given at the
+ * time the database was opened.
+ *
+ * @param[in] notmuch database
+ * @param[in] key configuration key
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval NULL in case of error.
+ */
+notmuch_config_values_t *
+notmuch_config_get_values_string (notmuch_database_t *notmuch, const char *key);
+
+/**
+ * Is the given 'config_values' iterator pointing at a valid element.
+ *
+ * @param[in] values iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval FALSE if passed a NULL pointer, or the iterator is exhausted.
+ *
+ */
+notmuch_bool_t
+notmuch_config_values_valid (notmuch_config_values_t *values);
+
+/**
+ * Get the current value from the 'values' iterator
+ *
+ * @param[in] values iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval a string with the same lifetime as the iterator
+ */
+const char *
+notmuch_config_values_get (notmuch_config_values_t *values);
+
+/**
+ * Move the 'values' iterator to the next element
+ *
+ * @param[in,out] values iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ */
+void
+notmuch_config_values_move_to_next (notmuch_config_values_t *values);
+
+
+/**
+ * reset the 'values' iterator to the first element
+ *
+ * @param[in,out] values iterator. A NULL value is ignored.
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ */
+void
+notmuch_config_values_start (notmuch_config_values_t *values);
+
+/**
+ * Destroy a config values iterator, along with any associated
+ * resources.
+ *
+ * @param[in,out] values iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ */
+void
+notmuch_config_values_destroy (notmuch_config_values_t *values);
+
+
+/**
+ * Returns an iterator for a (key, value) configuration pairs
+ *
+ * @param[in] notmuch database
+ * @param[in] prefix prefix for keys. Pass "" for all keys.
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval NULL in case of error.
+ */
+notmuch_config_pairs_t *
+notmuch_config_get_pairs (notmuch_database_t *notmuch,
+			  const char *prefix);
+
+/**
+ * Is the given 'config_pairs' iterator pointing at a valid element.
+ *
+ * @param[in] pairs iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval FALSE if passed a NULL pointer, or the iterator is exhausted.
+ *
+ */
+notmuch_bool_t
+notmuch_config_pairs_valid (notmuch_config_pairs_t *pairs);
+
+/**
+ * Move the 'config_pairs' iterator to the next element
+ *
+ * @param[in,out] pairs iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ */
+void
+notmuch_config_pairs_move_to_next (notmuch_config_pairs_t *pairs);
+
+/**
+ * Get the current key from the 'config_pairs' iterator
+ *
+ * @param[in] pairs iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval a string with the same lifetime as the iterator
+ */
+const char *
+notmuch_config_pairs_key (notmuch_config_pairs_t *pairs);
+
+/**
+ * Get the current value from the 'config_pairs' iterator
+ *
+ * @param[in] pairs iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval a string with the same lifetime as the iterator
+ */
+const char *
+notmuch_config_pairs_value (notmuch_config_pairs_t *pairs);
+
+/**
+ * Destroy a config_pairs iterator, along with any associated
+ * resources.
+ *
+ * @param[in,out] pairs iterator
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ */
+void
+notmuch_config_pairs_destroy (notmuch_config_pairs_t *pairs);
+
+/**
+ * get a configuration value from an open database as Boolean
+ *
+ * This value reflects all configuration information given at the time
+ * the database was opened.
+ *
+ * @param[in] notmuch database
+ * @param[in] key configuration key
+ * @param[out] val configuration value, converted to Boolean
+ *
+ * @since libnotmuch 5.4 (notmuch 0.32)
+ *
+ * @retval #NOTMUCH_STATUS_ILLEGAL_ARGUMENT if either key is unknown
+ * or the corresponding value does not convert to Boolean.
+ */
+notmuch_status_t
+notmuch_config_get_bool (notmuch_database_t *notmuch,
+			 notmuch_config_key_t key,
+			 notmuch_bool_t *val);
+
+/**
+ * return the path of the config file loaded, if any
+ *
+ * @retval NULL if no config file was loaded
+ */
+const char *
+notmuch_config_path (notmuch_database_t *notmuch);
+
+/**
+ * get the current default indexing options for a given database.
+ *
+ * This object will survive until the database itself is destroyed,
+ * but the caller may also release it earlier with
+ * notmuch_indexopts_destroy.
+ *
+ * This object represents a set of options on how a message can be
+ * added to the index.  At the moment it is a featureless stub.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ * @retval NULL in case of error
+ */
+notmuch_indexopts_t *
+notmuch_database_get_default_indexopts (notmuch_database_t *db);
+
+/**
+ * Stating a policy about how to decrypt messages.
+ *
+ * See index.decrypt in notmuch-config(1) for more details.
+ */
+typedef enum {
+    NOTMUCH_DECRYPT_FALSE,
+    NOTMUCH_DECRYPT_TRUE,
+    NOTMUCH_DECRYPT_AUTO,
+    NOTMUCH_DECRYPT_NOSTASH,
+} notmuch_decryption_policy_t;
+
+/**
+ * Specify whether to decrypt encrypted parts while indexing.
+ *
+ * Be aware that the index is likely sufficient to reconstruct the
+ * cleartext of the message itself, so please ensure that the notmuch
+ * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
+ * without considering the security of your index.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_indexopts_set_decrypt_policy (notmuch_indexopts_t *indexopts,
+				      notmuch_decryption_policy_t decrypt_policy);
+
+/**
+ * Return whether to decrypt encrypted parts while indexing.
+ * see notmuch_indexopts_set_decrypt_policy.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_decryption_policy_t
+notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts);
+
+/**
+ * Destroy a notmuch_indexopts_t object.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+void
+notmuch_indexopts_destroy (notmuch_indexopts_t *options);
+
+
+/**
+ * interrogate the library for compile time features
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_built_with (const char *name);
+/**@}*/
+
+#pragma GCC visibility pop
+
+NOTMUCH_END_DECLS
+
+#endif
diff --git a/lib/notmuch.sym b/lib/notmuch.sym
new file mode 100644
index 00000000..7d0c0af4
--- /dev/null
+++ b/lib/notmuch.sym
@@ -0,0 +1,7 @@
+{
+global:
+	_ZTI*;
+	_ZTS*;
+	notmuch_*;
+local: *;
+};
diff --git a/lib/open.cc b/lib/open.cc
new file mode 100644
index 00000000..005872dc
--- /dev/null
+++ b/lib/open.cc
@@ -0,0 +1,990 @@
+#include <unistd.h>
+#include <libgen.h>
+
+#include "database-private.h"
+#include "parse-time-vrp.h"
+#include "lastmod-fp.h"
+#include "path-util.h"
+
+#if HAVE_XAPIAN_DB_RETRY_LOCK
+#define DB_ACTION (Xapian::DB_CREATE_OR_OPEN | Xapian::DB_RETRY_LOCK)
+#else
+#define DB_ACTION Xapian::DB_CREATE_OR_OPEN
+#endif
+
+notmuch_status_t
+notmuch_database_open (const char *path,
+		       notmuch_database_mode_t mode,
+		       notmuch_database_t **database)
+{
+    char *status_string = NULL;
+    notmuch_status_t status;
+
+    status = notmuch_database_open_with_config (path, mode, "", NULL,
+						database, &status_string);
+    if (status_string) {
+	fputs (status_string, stderr);
+	free (status_string);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_open_verbose (const char *path,
+			       notmuch_database_mode_t mode,
+			       notmuch_database_t **database,
+			       char **status_string)
+{
+    return notmuch_database_open_with_config (path, mode, "", NULL,
+					      database, status_string);
+}
+
+static const char *
+_xdg_dir (void *ctx,
+	  const char *xdg_root_variable,
+	  const char *xdg_prefix,
+	  const char *profile_name)
+{
+    const char *xdg_root = getenv (xdg_root_variable);
+
+    if (! xdg_root) {
+	const char *home = getenv ("HOME");
+
+	if (! home) return NULL;
+
+	xdg_root = talloc_asprintf (ctx,
+				    "%s/%s",
+				    home,
+				    xdg_prefix);
+    }
+
+    if (! profile_name)
+	profile_name = getenv ("NOTMUCH_PROFILE");
+
+    if (! profile_name)
+	profile_name = "default";
+
+    return talloc_asprintf (ctx,
+			    "%s/notmuch/%s",
+			    xdg_root,
+			    profile_name);
+}
+
+static notmuch_status_t
+_choose_dir (notmuch_database_t *notmuch,
+	     const char *profile,
+	     notmuch_config_key_t key,
+	     const char *xdg_var,
+	     const char *xdg_subdir,
+	     const char *subdir,
+	     char **message = NULL)
+{
+    const char *parent;
+    const char *dir;
+    struct stat st;
+    int err;
+
+    dir = notmuch_config_get (notmuch, key);
+
+    if (dir)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    parent = _xdg_dir (notmuch, xdg_var, xdg_subdir, profile);
+    if (! parent)
+	return NOTMUCH_STATUS_PATH_ERROR;
+
+    dir = talloc_asprintf (notmuch, "%s/%s", parent, subdir);
+
+    err = stat (dir, &st);
+    if (err) {
+	if (errno == ENOENT) {
+	    char *notmuch_path = dirname (talloc_strdup (notmuch, notmuch->xapian_path));
+	    dir = talloc_asprintf (notmuch, "%s/%s", notmuch_path, subdir);
+	} else {
+	    IGNORE_RESULT (asprintf (message, "Error: Cannot stat %s: %s.\n",
+				     dir, strerror (errno)));
+	    return NOTMUCH_STATUS_FILE_ERROR;
+	}
+    }
+
+    _notmuch_config_cache (notmuch, key, dir);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_load_key_file (notmuch_database_t *notmuch,
+		const char *path,
+		const char *profile,
+		GKeyFile **key_file)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    if (path && EMPTY_STRING (path))
+	goto DONE;
+
+    if (! path)
+	path = getenv ("NOTMUCH_CONFIG");
+
+    if (path)
+	path = talloc_strdup (notmuch, path);
+    else {
+	const char *dir = _xdg_dir (notmuch, "XDG_CONFIG_HOME", ".config", profile);
+
+	if (dir) {
+	    path = talloc_asprintf (notmuch, "%s/config", dir);
+	    if (access (path, R_OK) != 0)
+		path = NULL;
+	}
+    }
+
+    if (! path) {
+	const char *home = getenv ("HOME");
+
+	path = talloc_asprintf (notmuch, "%s/.notmuch-config", home);
+
+	if (! profile)
+	    profile = getenv ("NOTMUCH_PROFILE");
+
+	if (profile)
+	    path = talloc_asprintf (notmuch, "%s.%s", path, profile);
+    }
+
+    *key_file = g_key_file_new ();
+    if (! g_key_file_load_from_file (*key_file, path, G_KEY_FILE_NONE, NULL)) {
+	status = NOTMUCH_STATUS_NO_CONFIG;
+    }
+
+  DONE:
+    if (path)
+	notmuch->config_path = path;
+
+    return status;
+}
+
+static notmuch_status_t
+_db_dir_exists (const char *database_path, char **message)
+{
+    struct stat st;
+    int err;
+
+    err = stat (database_path, &st);
+    if (err) {
+	IGNORE_RESULT (asprintf (message, "Error: Cannot open database at %s: %s.\n",
+				 database_path, strerror (errno)));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    if (! S_ISDIR (st.st_mode)) {
+	IGNORE_RESULT (asprintf (message, "Error: Cannot open database at %s: "
+				 "Not a directory.\n",
+				 database_path));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_choose_database_path (notmuch_database_t *notmuch,
+		       const char *profile,
+		       GKeyFile *key_file,
+		       const char **database_path,
+		       char **message)
+{
+    notmuch_status_t status;
+
+    if (! *database_path) {
+	*database_path = getenv ("NOTMUCH_DATABASE");
+    }
+
+    if (! *database_path && key_file) {
+	char *path = g_key_file_get_string (key_file, "database", "path", NULL);
+	if (path) {
+	    if (path[0] == '/')
+		*database_path = talloc_strdup (notmuch, path);
+	    else
+		*database_path = talloc_asprintf (notmuch, "%s/%s", getenv ("HOME"), path);
+	    g_free (path);
+	}
+    }
+    if (! *database_path) {
+	*database_path = _xdg_dir (notmuch, "XDG_DATA_HOME", ".local/share", profile);
+	status = _db_dir_exists (*database_path, message);
+	if (status) {
+	    *database_path = NULL;
+	} else {
+	    notmuch->params |= NOTMUCH_PARAM_SPLIT;
+	}
+    }
+
+    if (! *database_path) {
+	*database_path = getenv ("MAILDIR");
+    }
+
+    if (! *database_path) {
+	*database_path = talloc_asprintf (notmuch, "%s/mail", getenv ("HOME"));
+	status = _db_dir_exists (*database_path, message);
+	if (status) {
+	    *database_path = NULL;
+	}
+    }
+
+    if (*database_path == NULL) {
+	*message = strdup ("Error: could not locate database.\n");
+	return NOTMUCH_STATUS_NO_DATABASE;
+    }
+
+    if (*database_path[0] != '/') {
+	*message = strdup ("Error: Database path must be absolute.\n");
+	return NOTMUCH_STATUS_PATH_ERROR;
+    }
+
+    status = _db_dir_exists (*database_path, message);
+    if (status) {
+	IGNORE_RESULT (asprintf (message,
+				 "Error: database path '%s' does not exist or is not a directory.\n",
+				 *database_path));
+	return NOTMUCH_STATUS_NO_DATABASE;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_mkdir (const char *path, char **message)
+{
+    if (g_mkdir_with_parents (path, 0755)) {
+	IGNORE_RESULT (asprintf (message, "Error: Cannot create directory %s: %s.\n",
+				 path, strerror (errno)));
+	return NOTMUCH_STATUS_FILE_ERROR;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_create_database_path (notmuch_database_t *notmuch,
+		       const char *profile,
+		       GKeyFile *key_file,
+		       const char **database_path,
+		       char **message)
+{
+    notmuch_status_t status;
+
+    if (! *database_path) {
+	*database_path = getenv ("NOTMUCH_DATABASE");
+    }
+
+    if (! *database_path && key_file) {
+	char *path = g_key_file_get_string (key_file, "database", "path", NULL);
+	if (path) {
+	    if (path[0] == '/')
+		*database_path = talloc_strdup (notmuch, path);
+	    else
+		*database_path = talloc_asprintf (notmuch, "%s/%s", getenv ("HOME"), path);
+	    g_free (path);
+	}
+    }
+
+    if (! *database_path) {
+	*database_path = _xdg_dir (notmuch, "XDG_DATA_HOME", ".local/share", profile);
+	notmuch->params |= NOTMUCH_PARAM_SPLIT;
+    }
+
+    if (*database_path[0] != '/') {
+	*message = strdup ("Error: Database path must be absolute.\n");
+	return NOTMUCH_STATUS_PATH_ERROR;
+    }
+
+    if ((status = _mkdir (*database_path, message)))
+	return status;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_database_t *
+_alloc_notmuch (const char *database_path, const char *config_path, const char *profile)
+{
+    notmuch_database_t *notmuch;
+
+    notmuch = talloc_zero (NULL, notmuch_database_t);
+    if (! notmuch)
+	return NULL;
+
+    notmuch->exception_reported = false;
+    notmuch->status_string = NULL;
+    notmuch->writable_xapian_db = NULL;
+    notmuch->config_path = NULL;
+    notmuch->atomic_nesting = 0;
+    notmuch->transaction_count = 0;
+    notmuch->transaction_threshold = 0;
+    notmuch->view = 1;
+    notmuch->index_as_text = NULL;
+    notmuch->index_as_text_length = 0;
+
+    notmuch->params = NOTMUCH_PARAM_NONE;
+    if (database_path)
+	notmuch->params |= NOTMUCH_PARAM_DATABASE;
+    if (config_path)
+	notmuch->params |= NOTMUCH_PARAM_CONFIG;
+    if (profile)
+	notmuch->params |= NOTMUCH_PARAM_PROFILE;
+
+    return notmuch;
+}
+
+static notmuch_status_t
+_trial_open (const char *xapian_path, char **message_ptr)
+{
+    try {
+	Xapian::Database db (xapian_path);
+    } catch (const Xapian::DatabaseOpeningError &error) {
+	IGNORE_RESULT (asprintf (message_ptr,
+				 "Cannot open Xapian database at %s: %s\n",
+				 xapian_path,
+				 error.get_msg ().c_str ()));
+	return NOTMUCH_STATUS_PATH_ERROR;
+    } catch (const Xapian::Error &error) {
+	IGNORE_RESULT (asprintf (message_ptr,
+				 "A Xapian exception occurred opening database: %s\n",
+				 error.get_msg ().c_str ()));
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+_notmuch_choose_xapian_path (void *ctx, const char *database_path,
+			     const char **xapian_path, char **message_ptr)
+{
+    notmuch_status_t status;
+    const char *trial_path, *notmuch_path;
+
+    status = _db_dir_exists (database_path, message_ptr);
+    if (status)
+	goto DONE;
+
+    trial_path = talloc_asprintf (ctx, "%s/xapian", database_path);
+    status = _trial_open (trial_path, message_ptr);
+    if (status != NOTMUCH_STATUS_PATH_ERROR)
+	goto DONE;
+
+    if (*message_ptr)
+	free (*message_ptr);
+
+    notmuch_path = talloc_asprintf (ctx, "%s/.notmuch", database_path);
+    status = _db_dir_exists (notmuch_path, message_ptr);
+    if (status)
+	goto DONE;
+
+    trial_path = talloc_asprintf (ctx, "%s/xapian", notmuch_path);
+    status = _trial_open (trial_path, message_ptr);
+
+  DONE:
+    if (status == NOTMUCH_STATUS_SUCCESS)
+	*xapian_path = trial_path;
+    return status;
+}
+
+static void
+_set_database_path (notmuch_database_t *notmuch,
+		    const char *database_path)
+{
+    char *path = talloc_strdup (notmuch, database_path);
+
+    strip_trailing (path, '/');
+
+    _notmuch_config_cache (notmuch, NOTMUCH_CONFIG_DATABASE_PATH, path);
+}
+
+static void
+_load_database_state (notmuch_database_t *notmuch)
+{
+    std::string last_thread_id;
+    std::string last_mod;
+
+    notmuch->last_doc_id = notmuch->xapian_db->get_lastdocid ();
+    last_thread_id = notmuch->xapian_db->get_metadata ("last_thread_id");
+    if (last_thread_id.empty ()) {
+	notmuch->last_thread_id = 0;
+    } else {
+	const char *str;
+	char *end;
+
+	str = last_thread_id.c_str ();
+	notmuch->last_thread_id = strtoull (str, &end, 16);
+	if (*end != '\0')
+	    INTERNAL_ERROR ("Malformed database last_thread_id: %s", str);
+    }
+
+    /* Get current highest revision number. */
+    last_mod = notmuch->xapian_db->get_value_upper_bound (
+	NOTMUCH_VALUE_LAST_MOD);
+    if (last_mod.empty ())
+	notmuch->revision = 0;
+    else
+	notmuch->revision = Xapian::sortable_unserialise (last_mod);
+    notmuch->uuid = talloc_strdup (
+	notmuch, notmuch->xapian_db->get_uuid ().c_str ());
+}
+
+/* XXX This should really be done lazily, but the error reporting path in the indexing code
+ * would need to be redone to report any errors.
+ */
+notmuch_status_t
+_ensure_index_as_text (notmuch_database_t *notmuch, char **message)
+{
+    int nregex = 0;
+    regex_t *regexv = NULL;
+
+    if (notmuch->index_as_text)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    for (notmuch_config_values_t *list = notmuch_config_get_values (notmuch,
+								    NOTMUCH_CONFIG_INDEX_AS_TEXT);
+	 notmuch_config_values_valid (list);
+	 notmuch_config_values_move_to_next (list)) {
+	regex_t *new_regex;
+	int rerr;
+	const char *str = notmuch_config_values_get (list);
+	size_t len = strlen (str);
+
+	/* str must be non-empty, because n_c_get_values skips empty
+	 * strings */
+	assert (len > 0);
+
+	regexv = talloc_realloc (notmuch, regexv, regex_t, nregex + 1);
+	new_regex = &regexv[nregex];
+
+	rerr = regcomp (new_regex, str, REG_EXTENDED | REG_NOSUB);
+	if (rerr) {
+	    size_t error_size = regerror (rerr, new_regex, NULL, 0);
+	    char *error = (char *) talloc_size (str, error_size);
+
+	    regerror (rerr, new_regex, error, error_size);
+	    IGNORE_RESULT (asprintf (message, "Error in index.as_text: %s: %s\n", error, str));
+
+	    return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+	}
+	nregex++;
+    }
+
+    notmuch->index_as_text = regexv;
+    notmuch->index_as_text_length = nregex;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_finish_open (notmuch_database_t *notmuch,
+	      const char *profile,
+	      notmuch_database_mode_t mode,
+	      GKeyFile *key_file,
+	      char **message_ptr)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    char *incompat_features;
+    char *message = NULL;
+    const char *autocommit_str;
+    char *autocommit_end;
+    unsigned int version;
+    const char *database_path = notmuch_database_get_path (notmuch);
+
+    try {
+
+	if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
+	    notmuch->writable_xapian_db = new Xapian::WritableDatabase (notmuch->xapian_path,
+									DB_ACTION);
+	    notmuch->xapian_db = notmuch->writable_xapian_db;
+	} else {
+	    notmuch->xapian_db = new Xapian::Database (notmuch->xapian_path);
+	}
+
+	/* Check version.  As of database version 3, we represent
+	 * changes in terms of features, so assume a version bump
+	 * means a dramatically incompatible change. */
+	version = notmuch_database_get_version (notmuch);
+	if (version > NOTMUCH_DATABASE_VERSION) {
+	    IGNORE_RESULT (asprintf (&message,
+				     "Error: Notmuch database at %s\n"
+				     "       has a newer database format version (%u) than supported by this\n"
+				     "       version of notmuch (%u).\n",
+				     database_path, version, NOTMUCH_DATABASE_VERSION));
+	    status = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	/* Check features. */
+	incompat_features = NULL;
+	notmuch->features = _notmuch_database_parse_features (
+	    notmuch, notmuch->xapian_db->get_metadata ("features").c_str (),
+	    version, mode == NOTMUCH_DATABASE_MODE_READ_WRITE ? 'w' : 'r',
+	    &incompat_features);
+	if (incompat_features) {
+	    IGNORE_RESULT (asprintf (&message,
+				     "Error: Notmuch database at %s\n"
+				     "       requires features (%s)\n"
+				     "       not supported by this version of notmuch.\n",
+				     database_path, incompat_features));
+	    status = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	_load_database_state (notmuch);
+
+	notmuch->query_parser = new Xapian::QueryParser;
+	notmuch->term_gen = new Xapian::TermGenerator;
+	notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
+	notmuch->value_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
+	notmuch->date_range_processor = new ParseTimeRangeProcessor (NOTMUCH_VALUE_TIMESTAMP,
+								     "date:");
+	notmuch->last_mod_range_processor = new LastModRangeProcessor (notmuch, "lastmod:");
+	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
+	notmuch->query_parser->set_database (*notmuch->xapian_db);
+	notmuch->stemmer = new Xapian::Stem ("english");
+	notmuch->query_parser->set_stemmer (*notmuch->stemmer);
+	notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);
+	notmuch->query_parser->add_rangeprocessor (notmuch->value_range_processor);
+	notmuch->query_parser->add_rangeprocessor (notmuch->date_range_processor);
+	notmuch->query_parser->add_rangeprocessor (notmuch->last_mod_range_processor);
+
+	/* Configuration information is needed to set up query parser */
+	status = _notmuch_config_load_from_database (notmuch);
+	if (status)
+	    goto DONE;
+
+	if (key_file)
+	    status = _notmuch_config_load_from_file (notmuch, key_file, &message);
+	if (status)
+	    goto DONE;
+
+	status = _choose_dir (notmuch, profile,
+			      NOTMUCH_CONFIG_HOOK_DIR,
+			      "XDG_CONFIG_HOME",
+			      ".config",
+			      "hooks",
+			      &message);
+	if (status)
+	    goto DONE;
+
+	status = _choose_dir (notmuch, profile,
+			      NOTMUCH_CONFIG_BACKUP_DIR,
+			      "XDG_DATA_HOME",
+			      ".local/share",
+			      "backups",
+			      &message);
+	if (status)
+	    goto DONE;
+	status = _notmuch_config_load_defaults (notmuch);
+	if (status)
+	    goto DONE;
+
+	status = _ensure_index_as_text (notmuch, &message);
+	if (status)
+	    goto DONE;
+
+	autocommit_str = notmuch_config_get (notmuch, NOTMUCH_CONFIG_AUTOCOMMIT);
+	if (unlikely (! autocommit_str)) {
+	    INTERNAL_ERROR ("missing configuration for autocommit");
+	}
+	notmuch->transaction_threshold = strtoul (autocommit_str, &autocommit_end, 10);
+	if (*autocommit_end != '\0')
+	    INTERNAL_ERROR ("Malformed database database.autocommit value: %s", autocommit_str);
+
+	status = _notmuch_database_setup_standard_query_fields (notmuch);
+	if (status)
+	    goto DONE;
+
+	status = _notmuch_database_setup_user_query_fields (notmuch);
+	if (status)
+	    goto DONE;
+
+    } catch (const Xapian::Error &error) {
+	IGNORE_RESULT (asprintf (&message, "A Xapian exception occurred opening database: %s\n",
+				 error.get_msg ().c_str ()));
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+  DONE:
+    if (message_ptr)
+	*message_ptr = message;
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_open_with_config (const char *database_path,
+				   notmuch_database_mode_t mode,
+				   const char *config_path,
+				   const char *profile,
+				   notmuch_database_t **database,
+				   char **status_string)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    notmuch_database_t *notmuch = NULL;
+    char *message = NULL;
+    GKeyFile *key_file = NULL;
+
+    _notmuch_init ();
+
+    notmuch = _alloc_notmuch (database_path, config_path, profile);
+    if (! notmuch) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    status = _load_key_file (notmuch, config_path, profile, &key_file);
+    if (status) {
+	message = strdup ("Error: cannot load config file.\n");
+	goto DONE;
+    }
+
+    if ((status = _choose_database_path (notmuch, profile, key_file,
+					 &database_path,
+					 &message)))
+	goto DONE;
+
+    status = _db_dir_exists (database_path, &message);
+    if (status)
+	goto DONE;
+
+    _set_database_path (notmuch, database_path);
+
+    status = _notmuch_choose_xapian_path (notmuch, database_path,
+					  &notmuch->xapian_path, &message);
+    if (status)
+	goto DONE;
+
+    status = _finish_open (notmuch, profile, mode, key_file, &message);
+
+  DONE:
+    if (key_file)
+	g_key_file_free (key_file);
+
+    if (message) {
+	if (status_string)
+	    *status_string = message;
+	else
+	    free (message);
+    }
+
+    if (status && notmuch) {
+	notmuch_database_destroy (notmuch);
+	notmuch = NULL;
+    }
+
+    if (database)
+	*database = notmuch;
+
+    if (notmuch)
+	notmuch->open = true;
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_create (const char *path, notmuch_database_t **database)
+{
+    char *status_string = NULL;
+    notmuch_status_t status;
+
+    status = notmuch_database_create_verbose (path, database,
+					      &status_string);
+
+    if (status_string) {
+	fputs (status_string, stderr);
+	free (status_string);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_create_verbose (const char *path,
+				 notmuch_database_t **database,
+				 char **status_string)
+{
+    return notmuch_database_create_with_config (path, "", NULL, database, status_string);
+}
+
+notmuch_status_t
+notmuch_database_create_with_config (const char *database_path,
+				     const char *config_path,
+				     const char *profile,
+				     notmuch_database_t **database,
+				     char **status_string)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    notmuch_database_t *notmuch = NULL;
+    const char *notmuch_path = NULL;
+    char *message = NULL;
+    GKeyFile *key_file = NULL;
+
+    _notmuch_init ();
+
+    notmuch = _alloc_notmuch (database_path, config_path, profile);
+    if (! notmuch) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    status = _load_key_file (notmuch, config_path, profile, &key_file);
+    if (status) {
+	message = strdup ("Error: cannot load config file.\n");
+	goto DONE;
+    }
+
+    status = _choose_database_path (notmuch, profile, key_file,
+				    &database_path, &message);
+    switch (status) {
+    case NOTMUCH_STATUS_SUCCESS:
+	break;
+    case NOTMUCH_STATUS_NO_DATABASE:
+	if ((status = _create_database_path (notmuch, profile, key_file,
+					     &database_path, &message)))
+	    goto DONE;
+	break;
+    default:
+	goto DONE;
+    }
+
+    _set_database_path (notmuch, database_path);
+
+    if (key_file && ! (notmuch->params & NOTMUCH_PARAM_SPLIT)) {
+	char *mail_root = notmuch_canonicalize_file_name (
+	    g_key_file_get_string (key_file, "database", "mail_root", NULL));
+	char *db_path = notmuch_canonicalize_file_name (database_path);
+
+	if (mail_root && (0 != strcmp (mail_root, db_path)))
+	    notmuch->params |= NOTMUCH_PARAM_SPLIT;
+
+	free (mail_root);
+	free (db_path);
+    }
+
+    if (notmuch->params & NOTMUCH_PARAM_SPLIT) {
+	notmuch_path = database_path;
+    } else {
+	if (! (notmuch_path = talloc_asprintf (notmuch, "%s/%s", database_path, ".notmuch"))) {
+	    status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	    goto DONE;
+	}
+
+	status = _mkdir (notmuch_path, &message);
+	if (status)
+	    goto DONE;
+    }
+
+    if (! (notmuch->xapian_path = talloc_asprintf (notmuch, "%s/%s", notmuch_path, "xapian"))) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    status = _trial_open (notmuch->xapian_path, &message);
+    if (status == NOTMUCH_STATUS_SUCCESS) {
+	notmuch_database_destroy (notmuch);
+	notmuch = NULL;
+	status = NOTMUCH_STATUS_DATABASE_EXISTS;
+	goto DONE;
+    }
+
+    if (message)
+	free (message);
+
+    status = _finish_open (notmuch,
+			   profile,
+			   NOTMUCH_DATABASE_MODE_READ_WRITE,
+			   key_file,
+			   &message);
+    if (status)
+	goto DONE;
+
+    /* Upgrade doesn't add these feature to existing databases, but
+     * new databases have them. */
+    notmuch->features |= NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES;
+    notmuch->features |= NOTMUCH_FEATURE_INDEXED_MIMETYPES;
+    notmuch->features |= NOTMUCH_FEATURE_UNPREFIX_BODY_ONLY;
+
+    status = notmuch_database_upgrade (notmuch, NULL, NULL);
+    if (status) {
+	notmuch_database_close (notmuch);
+	notmuch = NULL;
+    }
+
+  DONE:
+    if (key_file)
+	g_key_file_free (key_file);
+
+    if (message) {
+	if (status_string)
+	    *status_string = message;
+	else
+	    free (message);
+    }
+    if (status && notmuch) {
+	notmuch_database_destroy (notmuch);
+	notmuch = NULL;
+    }
+
+    if (database)
+	*database = notmuch;
+
+    if (notmuch)
+	notmuch->open = true;
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_reopen (notmuch_database_t *notmuch,
+			 notmuch_database_mode_t new_mode)
+{
+    notmuch_database_mode_t cur_mode = _notmuch_database_mode (notmuch);
+
+    if (notmuch->xapian_db == NULL) {
+	_notmuch_database_log (notmuch, "Cannot reopen closed or nonexistent database\n");
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+    }
+
+    try {
+	if (cur_mode == new_mode &&
+	    new_mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
+	    notmuch->xapian_db->reopen ();
+	} else {
+	    notmuch->xapian_db->close ();
+
+	    delete notmuch->xapian_db;
+	    notmuch->xapian_db = NULL;
+	    /* no need to free the same object twice */
+	    notmuch->writable_xapian_db = NULL;
+
+	    if (new_mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
+		notmuch->writable_xapian_db = new Xapian::WritableDatabase (notmuch->xapian_path,
+									    DB_ACTION);
+		notmuch->xapian_db = notmuch->writable_xapian_db;
+	    } else {
+		notmuch->xapian_db = new Xapian::Database (notmuch->xapian_path,
+							   DB_ACTION);
+	    }
+	}
+
+	_load_database_state (notmuch);
+    } catch (const Xapian::Error &error) {
+	if (! notmuch->exception_reported) {
+	    _notmuch_database_log (notmuch, "Error: A Xapian exception reopening database: %s\n",
+				   error.get_msg ().c_str ());
+	    notmuch->exception_reported = true;
+	}
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    notmuch->view++;
+    notmuch->open = true;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_maybe_load_config_from_database (notmuch_database_t *notmuch,
+				  GKeyFile *key_file,
+				  const char *database_path,
+				  const char *profile)
+{
+    char *message; /* ignored */
+
+    if (_db_dir_exists (database_path, &message))
+	return NOTMUCH_STATUS_NO_DATABASE;
+
+    _set_database_path (notmuch, database_path);
+
+    if (_notmuch_choose_xapian_path (notmuch, database_path, &notmuch->xapian_path, &message))
+	return NOTMUCH_STATUS_NO_DATABASE;
+
+    (void) _finish_open (notmuch, profile, NOTMUCH_DATABASE_MODE_READ_ONLY, key_file, &message);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_load_config (const char *database_path,
+			      const char *config_path,
+			      const char *profile,
+			      notmuch_database_t **database,
+			      char **status_string)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS, warning = NOTMUCH_STATUS_SUCCESS;
+    notmuch_database_t *notmuch = NULL;
+    char *message = NULL;
+    GKeyFile *key_file = NULL;
+
+    _notmuch_init ();
+
+    notmuch = _alloc_notmuch (database_path, config_path, profile);
+    if (! notmuch) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    status = _load_key_file (notmuch, config_path, profile, &key_file);
+    switch (status) {
+    case NOTMUCH_STATUS_SUCCESS:
+	break;
+    case NOTMUCH_STATUS_NO_CONFIG:
+	warning = status;
+	break;
+    default:
+	message = strdup ("Error: cannot load config file.\n");
+	goto DONE;
+    }
+
+    status = _choose_database_path (notmuch, profile, key_file,
+				    &database_path, &message);
+    switch (status) {
+    case NOTMUCH_STATUS_NO_DATABASE:
+    case NOTMUCH_STATUS_SUCCESS:
+	if (! warning)
+	    warning = status;
+	break;
+    default:
+	goto DONE;
+    }
+
+
+    if (database_path) {
+	status = _maybe_load_config_from_database (notmuch, key_file, database_path, profile);
+	switch (status) {
+	case NOTMUCH_STATUS_NO_DATABASE:
+	case NOTMUCH_STATUS_SUCCESS:
+	    if (! warning)
+		warning = status;
+	    break;
+	default:
+	    goto DONE;
+	}
+    }
+
+    if (key_file) {
+	status = _notmuch_config_load_from_file (notmuch, key_file, &message);
+	if (status)
+	    goto DONE;
+    }
+    status = _notmuch_config_load_defaults (notmuch);
+    if (status)
+	goto DONE;
+
+  DONE:
+    if (status_string)
+	*status_string = message;
+
+    if (status &&
+	status != NOTMUCH_STATUS_NO_DATABASE
+	&& status != NOTMUCH_STATUS_NO_CONFIG) {
+	notmuch_database_destroy (notmuch);
+	notmuch = NULL;
+    }
+
+    if (database)
+	*database = notmuch;
+
+    if (status)
+	return status;
+    else
+	return warning;
+}
diff --git a/lib/parse-sexp.cc b/lib/parse-sexp.cc
new file mode 100644
index 00000000..9cadbc13
--- /dev/null
+++ b/lib/parse-sexp.cc
@@ -0,0 +1,731 @@
+#include "database-private.h"
+
+#if HAVE_SFSEXP
+#include "sexp.h"
+#include "unicode-util.h"
+
+/* _sexp is used for file scope symbols to avoid clashing with
+ * definitions from sexp.h */
+
+/* sexp_binding structs attach name to a sexp and a defining
+ * context. The latter allows lazy evaluation of parameters whose
+ * definition contains other parameters.  Lazy evaluation is needed
+ * because a primary goal of macros is to change the parent field for
+ * a sexp.
+ */
+
+typedef struct sexp_binding {
+    const char *name;
+    const sexp_t *sx;
+    const struct sexp_binding *context;
+    const struct sexp_binding *next;
+} _sexp_binding_t;
+
+typedef enum {
+    SEXP_FLAG_NONE	= 0,
+    SEXP_FLAG_FIELD	= 1 << 0,
+    SEXP_FLAG_BOOLEAN	= 1 << 1,
+    SEXP_FLAG_SINGLE	= 1 << 2,
+    SEXP_FLAG_WILDCARD	= 1 << 3,
+    SEXP_FLAG_REGEX	= 1 << 4,
+    SEXP_FLAG_DO_REGEX	= 1 << 5,
+    SEXP_FLAG_EXPAND	= 1 << 6,
+    SEXP_FLAG_DO_EXPAND = 1 << 7,
+    SEXP_FLAG_ORPHAN	= 1 << 8,
+    SEXP_FLAG_RANGE	= 1 << 9,
+    SEXP_FLAG_PATHNAME	= 1 << 10,
+} _sexp_flag_t;
+
+/*
+ * define bitwise operators to hide casts */
+
+inline _sexp_flag_t
+operator| (_sexp_flag_t a, _sexp_flag_t b)
+{
+    return static_cast<_sexp_flag_t>(
+	static_cast<unsigned>(a) | static_cast<unsigned>(b));
+}
+
+inline _sexp_flag_t
+operator& (_sexp_flag_t a, _sexp_flag_t b)
+{
+    return static_cast<_sexp_flag_t>(
+	static_cast<unsigned>(a) & static_cast<unsigned>(b));
+}
+
+typedef struct  {
+    const char *name;
+    Xapian::Query::op xapian_op;
+    Xapian::Query initial;
+    _sexp_flag_t flags;
+} _sexp_prefix_t;
+
+static _sexp_prefix_t prefixes[] =
+{
+    { "and",            Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_NONE },
+    { "attachment",     Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND },
+    { "body",           Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD },
+    { "date",           Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
+      SEXP_FLAG_RANGE },
+    { "from",           Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
+    { "folder",         Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND |
+      SEXP_FLAG_PATHNAME },
+    { "id",             Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX },
+    { "infix",          Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
+      SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN },
+    { "is",             Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
+    { "lastmod",           Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
+      SEXP_FLAG_RANGE },
+    { "matching",       Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_DO_EXPAND },
+    { "mid",            Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX },
+    { "mimetype",       Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND },
+    { "not",            Xapian::Query::OP_AND_NOT,      Xapian::Query::MatchAll,
+      SEXP_FLAG_NONE },
+    { "of",             Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_DO_EXPAND },
+    { "or",             Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
+      SEXP_FLAG_NONE },
+    { "path",           Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX |
+      SEXP_FLAG_PATHNAME },
+    { "property",       Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
+    { "query",          Xapian::Query::OP_INVALID,      Xapian::Query::MatchNothing,
+      SEXP_FLAG_SINGLE | SEXP_FLAG_ORPHAN },
+    { "regex",          Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
+      SEXP_FLAG_SINGLE | SEXP_FLAG_DO_REGEX },
+    { "rx",             Xapian::Query::OP_INVALID,      Xapian::Query::MatchAll,
+      SEXP_FLAG_SINGLE | SEXP_FLAG_DO_REGEX },
+    { "starts-with",    Xapian::Query::OP_WILDCARD,     Xapian::Query::MatchAll,
+      SEXP_FLAG_SINGLE },
+    { "subject",        Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
+    { "tag",            Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
+    { "thread",         Xapian::Query::OP_OR,           Xapian::Query::MatchNothing,
+      SEXP_FLAG_FIELD | SEXP_FLAG_BOOLEAN | SEXP_FLAG_WILDCARD | SEXP_FLAG_REGEX | SEXP_FLAG_EXPAND },
+    { "to",             Xapian::Query::OP_AND,          Xapian::Query::MatchAll,
+      SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD | SEXP_FLAG_EXPAND },
+    { }
+};
+
+static notmuch_status_t _sexp_to_xapian_query (notmuch_database_t *notmuch,
+					       const _sexp_prefix_t *parent,
+					       const _sexp_binding_t *env,
+					       const sexp_t *sx,
+					       Xapian::Query &output);
+
+static notmuch_status_t
+_sexp_combine_query (notmuch_database_t *notmuch,
+		     const _sexp_prefix_t *parent,
+		     const _sexp_binding_t *env,
+		     Xapian::Query::op operation,
+		     Xapian::Query left,
+		     const sexp_t *sx,
+		     Xapian::Query &output)
+{
+    Xapian::Query subquery;
+
+    notmuch_status_t status;
+
+    /* if we run out elements, return accumulator */
+
+    if (! sx) {
+	output = left;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    status = _sexp_to_xapian_query (notmuch, parent, env, sx, subquery);
+    if (status)
+	return status;
+
+    return _sexp_combine_query (notmuch,
+				parent,
+				env,
+				operation,
+				Xapian::Query (operation, left, subquery),
+				sx->next, output);
+}
+
+static notmuch_status_t
+_sexp_parse_phrase (std::string term_prefix, const char *phrase, Xapian::Query &output)
+{
+    Xapian::Utf8Iterator p (phrase);
+    Xapian::Utf8Iterator end;
+    std::vector<std::string> terms;
+
+    while (p != end) {
+	Xapian::Utf8Iterator start;
+	while (p != end && ! Xapian::Unicode::is_wordchar (*p))
+	    p++;
+
+	if (p == end)
+	    break;
+
+	start = p;
+
+	while (p != end && Xapian::Unicode::is_wordchar (*p))
+	    p++;
+
+	if (p != start) {
+	    std::string word (start, p);
+	    word = Xapian::Unicode::tolower (word);
+	    terms.push_back (term_prefix + word);
+	}
+    }
+    output = Xapian::Query (Xapian::Query::OP_PHRASE, terms.begin (), terms.end ());
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+resolve_binding (notmuch_database_t *notmuch, const _sexp_binding_t *env, const char *name,
+		 const _sexp_binding_t **out)
+{
+    for (; env; env = env->next) {
+	if (strcmp (name, env->name) == 0) {
+	    *out = env;
+	    return NOTMUCH_STATUS_SUCCESS;
+	}
+    }
+
+    _notmuch_database_log (notmuch, "undefined parameter '%s'\n", name);
+    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+}
+
+static notmuch_status_t
+_sexp_expand_term (notmuch_database_t *notmuch,
+		   const _sexp_prefix_t *prefix,
+		   const _sexp_binding_t *env,
+		   const sexp_t *sx,
+		   const char **out)
+{
+    notmuch_status_t status;
+
+    if (! out)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    while (sx->ty == SEXP_VALUE && sx->aty == SEXP_BASIC && sx->val[0] == ',') {
+	const char *name = sx->val + 1;
+	const _sexp_binding_t *binding;
+
+	status = resolve_binding (notmuch, env, name, &binding);
+	if (status)
+	    return status;
+
+	sx = binding->sx;
+	env = binding->context;
+    }
+
+    if (sx->ty != SEXP_VALUE) {
+	_notmuch_database_log (notmuch, "'%s' expects single atom as argument\n",
+			       prefix->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    *out = sx->val;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_sexp_parse_wildcard (notmuch_database_t *notmuch,
+		      const _sexp_prefix_t *parent,
+		      unused(const _sexp_binding_t *env),
+		      std::string match,
+		      Xapian::Query &output)
+{
+
+    std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : "";
+
+    if (parent && ! (parent->flags & SEXP_FLAG_WILDCARD)) {
+	_notmuch_database_log (notmuch, "'%s' does not support wildcard queries\n", parent->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    output = Xapian::Query (Xapian::Query::OP_WILDCARD,
+			    term_prefix + Xapian::Unicode::tolower (match));
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_sexp_parse_one_term (notmuch_database_t *notmuch, std::string term_prefix, const sexp_t *sx,
+		      Xapian::Query &output)
+{
+    Xapian::Stem stem = *(notmuch->stemmer);
+
+    if (sx->aty == SEXP_BASIC && unicode_word_utf8 (sx->val)) {
+	std::string term = Xapian::Unicode::tolower (sx->val);
+
+	output = Xapian::Query ("Z" + term_prefix + stem (term));
+	return NOTMUCH_STATUS_SUCCESS;
+    } else {
+	return _sexp_parse_phrase (term_prefix, sx->val, output);
+    }
+
+}
+
+notmuch_status_t
+_sexp_parse_regex (notmuch_database_t *notmuch,
+		   const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent,
+		   const _sexp_binding_t *env,
+		   const sexp_t *term, Xapian::Query &output)
+{
+    if (! parent) {
+	_notmuch_database_log (notmuch, "illegal '%s' outside field\n",
+			       prefix->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    if (! (parent->flags & SEXP_FLAG_REGEX)) {
+	_notmuch_database_log (notmuch, "'%s' not supported in field '%s'\n",
+			       prefix->name, parent->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    std::string msg; /* ignored */
+    const char *str;
+    notmuch_status_t status;
+
+    status = _sexp_expand_term (notmuch, prefix, env, term, &str);
+    if (status)
+	return status;
+
+    return _notmuch_regexp_to_query (notmuch, Xapian::BAD_VALUENO, parent->name,
+				     str, output, msg);
+}
+
+
+static notmuch_status_t
+_sexp_expand_query (notmuch_database_t *notmuch,
+		    const _sexp_prefix_t *prefix, const _sexp_prefix_t *parent,
+		    unused(const _sexp_binding_t *env), const sexp_t *sx, Xapian::Query &output)
+{
+    Xapian::Query subquery;
+    notmuch_status_t status;
+    std::string msg;
+
+    if (! (parent->flags & SEXP_FLAG_EXPAND)) {
+	_notmuch_database_log (notmuch, "'%s' unsupported inside '%s'\n", prefix->name, parent->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    status = _sexp_combine_query (notmuch, NULL, NULL, prefix->xapian_op, prefix->initial, sx,
+				  subquery);
+    if (status)
+	return status;
+
+    status = _notmuch_query_expand (notmuch, parent->name, subquery, output, msg);
+    if (status) {
+	_notmuch_database_log (notmuch, "error expanding query %s\n", msg.c_str ());
+    }
+    return status;
+}
+
+static notmuch_status_t
+_sexp_parse_infix (notmuch_database_t *notmuch, const sexp_t *sx, Xapian::Query &output)
+{
+    try {
+	output = notmuch->query_parser->parse_query (sx->val, NOTMUCH_QUERY_PARSER_FLAGS);
+    } catch (const Xapian::QueryParserError &error) {
+	_notmuch_database_log (notmuch, "Syntax error in infix query: %s\n", sx->val);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    } catch (const Xapian::Error &error) {
+	if (! notmuch->exception_reported) {
+	    _notmuch_database_log (notmuch,
+				   "A Xapian exception occurred parsing query: %s\n",
+				   error.get_msg ().c_str ());
+	    _notmuch_database_log_append (notmuch,
+					  "Query string was: %s\n",
+					  sx->val);
+	    notmuch->exception_reported = true;
+	    return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	}
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_sexp_parse_header (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		    const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
+{
+    _sexp_prefix_t user_prefix;
+
+    user_prefix.name = sx->list->val;
+    user_prefix.flags = SEXP_FLAG_FIELD | SEXP_FLAG_WILDCARD;
+
+    if (parent) {
+	_notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n",
+			       sx->list->val, parent->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    parent = &user_prefix;
+
+    return _sexp_combine_query (notmuch, parent, env, Xapian::Query::OP_AND, Xapian::Query::MatchAll,
+				sx->list->next, output);
+}
+
+static _sexp_binding_t *
+_sexp_bind (void *ctx, const _sexp_binding_t *env, const char *name, const sexp_t *sx, const
+	    _sexp_binding_t *context)
+{
+    _sexp_binding_t *binding = talloc (ctx, _sexp_binding_t);
+
+    binding->name = talloc_strdup (ctx, name);
+    binding->sx = sx;
+    binding->context = context;
+    binding->next = env;
+    return binding;
+}
+
+static notmuch_status_t
+maybe_apply_macro (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		   const _sexp_binding_t *env, const sexp_t *sx, const sexp_t *args,
+		   Xapian::Query &output)
+{
+    const sexp_t *params, *param, *arg, *body;
+    void *local = talloc_new (notmuch);
+    _sexp_binding_t *new_env = NULL;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    if (sx->list->ty != SEXP_VALUE || strcmp (sx->list->val, "macro") != 0) {
+	status = NOTMUCH_STATUS_IGNORED;
+	goto DONE;
+    }
+
+    params = sx->list->next;
+
+    if (! params || (params->ty != SEXP_LIST)) {
+	_notmuch_database_log (notmuch, "missing (possibly empty) list of arguments to macro\n");
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    body = params->next;
+
+    if (! body) {
+	_notmuch_database_log (notmuch, "missing body of macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    for (param = params->list, arg = args;
+	 param && arg;
+	 param = param->next, arg = arg->next) {
+	if (param->ty != SEXP_VALUE || param->aty != SEXP_BASIC) {
+	    _notmuch_database_log (notmuch, "macro parameters must be unquoted atoms\n");
+	    status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	    goto DONE;
+	}
+	new_env = _sexp_bind (local, new_env, param->val, arg, env);
+    }
+
+    if (param && ! arg) {
+	_notmuch_database_log (notmuch, "too few arguments to macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    if (! param && arg) {
+	_notmuch_database_log (notmuch, "too many arguments to macro\n");
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    status = _sexp_to_xapian_query (notmuch, parent, new_env, body, output);
+
+  DONE:
+    if (local)
+	talloc_free (local);
+
+    return status;
+}
+
+static notmuch_status_t
+maybe_saved_squery (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		    const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
+{
+    char *key;
+    char *expansion = NULL;
+    notmuch_status_t status;
+    sexp_t *saved_sexp;
+    void *local = talloc_new (notmuch);
+    char *buf;
+
+    key = talloc_asprintf (local, "squery.%s", sx->list->val);
+    if (! key) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    status = notmuch_database_get_config (notmuch, key, &expansion);
+    if (status)
+	goto DONE;
+    if (EMPTY_STRING (expansion)) {
+	status = NOTMUCH_STATUS_IGNORED;
+	goto DONE;
+    }
+
+    buf = talloc_strdup (local, expansion);
+    /* XXX TODO: free this memory */
+    saved_sexp = parse_sexp (buf, strlen (expansion));
+    if (! saved_sexp) {
+	_notmuch_database_log (notmuch, "invalid saved s-expression query: '%s'\n", expansion);
+	status = NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	goto DONE;
+    }
+
+    status = maybe_apply_macro (notmuch, parent, env, saved_sexp, sx->list->next, output);
+    if (status == NOTMUCH_STATUS_IGNORED)
+	status =  _sexp_to_xapian_query (notmuch, parent, env, saved_sexp, output);
+
+  DONE:
+    if (local)
+	talloc_free (local);
+
+    return status;
+}
+
+static notmuch_status_t
+_sexp_expand_param (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		    const _sexp_binding_t *env, const char *name,
+		    Xapian::Query &output)
+{
+    notmuch_status_t status;
+
+    const _sexp_binding_t *binding;
+
+    status = resolve_binding (notmuch, env, name, &binding);
+    if (status)
+	return status;
+
+    return _sexp_to_xapian_query (notmuch, parent, binding->context, binding->sx,
+				  output);
+}
+
+static notmuch_status_t
+_sexp_parse_range (notmuch_database_t *notmuch,  const _sexp_prefix_t *prefix,
+		   const sexp_t *sx, Xapian::Query &output)
+{
+    const char *from, *to;
+    std::string msg;
+
+    /* empty range matches everything */
+    if (! sx) {
+	output = Xapian::Query::MatchAll;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    if (sx->ty == SEXP_LIST) {
+	_notmuch_database_log (notmuch, "expected atom as first argument of '%s'\n", prefix->name);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    from = sx->val;
+    if (strcmp (from, "*") == 0)
+	from = "";
+
+    to = from;
+
+    if (sx->next) {
+	if (sx->next->ty == SEXP_LIST) {
+	    _notmuch_database_log (notmuch, "expected atom as second argument of '%s'\n",
+				   prefix->name);
+	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	}
+
+	if (sx->next->next) {
+	    _notmuch_database_log (notmuch, "'%s' expects maximum of two arguments\n", prefix->name);
+	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	}
+
+	to = sx->next->val;
+	if (strcmp (to, "*") == 0)
+	    to = "";
+    }
+
+    if (strcmp (prefix->name, "date") == 0) {
+	notmuch_status_t status;
+	status = _notmuch_date_strings_to_query (NOTMUCH_VALUE_TIMESTAMP, from, to, output, msg);
+	if (status) {
+	    if (! msg.empty ())
+		_notmuch_database_log (notmuch, "%s\n", msg.c_str ());
+	}
+	return status;
+    }
+
+    if (strcmp (prefix->name, "lastmod") == 0) {
+	notmuch_status_t status;
+	status = _notmuch_lastmod_strings_to_query (notmuch, from, to, output, msg);
+	if (status) {
+	    if (! msg.empty ())
+		_notmuch_database_log (notmuch, "%s\n", msg.c_str ());
+	}
+	return status;
+    }
+
+    _notmuch_database_log (notmuch, "unimplimented range prefix: '%s'\n", prefix->name);
+    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+}
+
+/* Here we expect the s-expression to be a proper list, with first
+ * element defining and operation, or as a special case the empty
+ * list */
+
+static notmuch_status_t
+_sexp_to_xapian_query (notmuch_database_t *notmuch, const _sexp_prefix_t *parent,
+		       const _sexp_binding_t *env, const sexp_t *sx, Xapian::Query &output)
+{
+    notmuch_status_t status;
+
+    if (sx->ty == SEXP_VALUE && sx->aty == SEXP_BASIC && sx->val[0] == ',') {
+	return _sexp_expand_param (notmuch, parent, env, sx->val + 1, output);
+    }
+
+    if (sx->ty == SEXP_VALUE) {
+	std::string term_prefix = parent ? _notmuch_database_prefix (notmuch, parent->name) : "";
+
+	if (sx->aty == SEXP_BASIC && strcmp (sx->val, "*") == 0) {
+	    return _sexp_parse_wildcard (notmuch, parent, env, "", output);
+	}
+
+	char *atom = sx->val;
+
+	if (parent && parent->flags & SEXP_FLAG_PATHNAME)
+	    strip_trailing (atom, '/');
+
+	if (parent && (parent->flags & SEXP_FLAG_BOOLEAN)) {
+	    output = Xapian::Query (term_prefix + atom);
+	    return NOTMUCH_STATUS_SUCCESS;
+	}
+
+	if (parent) {
+	    return _sexp_parse_one_term (notmuch, term_prefix, sx, output);
+	} else {
+	    Xapian::Query accumulator;
+	    for (_sexp_prefix_t *prefix = prefixes; prefix->name; prefix++) {
+		if (prefix->flags & SEXP_FLAG_FIELD) {
+		    Xapian::Query subquery;
+		    term_prefix = _notmuch_database_prefix (notmuch, prefix->name);
+		    status = _sexp_parse_one_term (notmuch, term_prefix, sx, subquery);
+		    if (status)
+			return status;
+		    accumulator = Xapian::Query (Xapian::Query::OP_OR, accumulator, subquery);
+		}
+	    }
+	    output = accumulator;
+	    return NOTMUCH_STATUS_SUCCESS;
+	}
+    }
+
+    /* Empty list */
+    if (! sx->list) {
+	output = Xapian::Query::MatchAll;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    if (sx->list->ty == SEXP_LIST) {
+	_notmuch_database_log (notmuch, "unexpected list in field/operation position\n",
+			       sx->list->val);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    status = maybe_saved_squery (notmuch, parent, env, sx, output);
+    if (status != NOTMUCH_STATUS_IGNORED)
+	return status;
+
+    /* Check for user defined field */
+    if (_notmuch_string_map_get (notmuch->user_prefix, sx->list->val)) {
+	return _sexp_parse_header (notmuch, parent, env, sx, output);
+    }
+
+    if (strcmp (sx->list->val, "macro") == 0) {
+	_notmuch_database_log (notmuch, "macro definition not permitted here\n");
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    for (_sexp_prefix_t *prefix = prefixes; prefix && prefix->name; prefix++) {
+	if (strcmp (prefix->name, sx->list->val) == 0) {
+	    if (prefix->flags & (SEXP_FLAG_FIELD | SEXP_FLAG_RANGE)) {
+		if (parent) {
+		    _notmuch_database_log (notmuch, "nested field: '%s' inside '%s'\n",
+					   prefix->name, parent->name);
+		    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+		}
+		parent = prefix;
+	    }
+
+	    if (parent && (prefix->flags & SEXP_FLAG_ORPHAN)) {
+		_notmuch_database_log (notmuch, "'%s' not supported inside '%s'\n",
+				       prefix->name, parent->name);
+		return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	    }
+
+	    if ((prefix->flags & SEXP_FLAG_SINGLE) &&
+		(! sx->list->next || sx->list->next->next || sx->list->next->ty != SEXP_VALUE)) {
+		_notmuch_database_log (notmuch, "'%s' expects single atom as argument\n",
+				       prefix->name);
+		return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	    }
+
+	    if (prefix->flags & SEXP_FLAG_RANGE)
+		return _sexp_parse_range (notmuch, prefix, sx->list->next, output);
+
+	    if (strcmp (prefix->name, "infix") == 0) {
+		return _sexp_parse_infix (notmuch, sx->list->next, output);
+	    }
+
+	    if (strcmp (prefix->name, "query") == 0) {
+		return _notmuch_query_name_to_query (notmuch, sx->list->next->val, output);
+	    }
+
+	    if (prefix->xapian_op == Xapian::Query::OP_WILDCARD) {
+		const char *str;
+		status = _sexp_expand_term (notmuch, prefix, env, sx->list->next, &str);
+		if (status)
+		    return status;
+
+		return _sexp_parse_wildcard (notmuch, parent, env, str, output);
+	    }
+
+	    if (prefix->flags & SEXP_FLAG_DO_REGEX) {
+		return _sexp_parse_regex (notmuch, prefix, parent, env, sx->list->next, output);
+	    }
+
+	    if (prefix->flags & SEXP_FLAG_DO_EXPAND) {
+		return _sexp_expand_query (notmuch, prefix, parent, env, sx->list->next, output);
+	    }
+
+	    return _sexp_combine_query (notmuch, parent, env, prefix->xapian_op, prefix->initial,
+					sx->list->next, output);
+	}
+    }
+
+    _notmuch_database_log (notmuch, "unknown prefix '%s'\n", sx->list->val);
+    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+}
+
+notmuch_status_t
+_notmuch_sexp_string_to_xapian_query (notmuch_database_t *notmuch, const char *querystr,
+				      Xapian::Query &output)
+{
+    const sexp_t *sx = NULL;
+    char *buf = talloc_strdup (notmuch, querystr);
+
+    sx = parse_sexp (buf, strlen (querystr));
+    if (! sx) {
+	_notmuch_database_log (notmuch, "invalid s-expression: '%s'\n", querystr);
+	return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+    }
+
+    return _sexp_to_xapian_query (notmuch, NULL, NULL, sx, output);
+}
+#endif
diff --git a/lib/parse-time-vrp.cc b/lib/parse-time-vrp.cc
new file mode 100644
index 00000000..6b07970b
--- /dev/null
+++ b/lib/parse-time-vrp.cc
@@ -0,0 +1,107 @@
+/* parse-time-vrp.cc - date range query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2012 Jani Nikula
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Jani Nikula <jani@nikula.org>
+ */
+
+#include "database-private.h"
+#include "parse-time-vrp.h"
+#include "parse-time-string.h"
+
+notmuch_status_t
+_notmuch_date_strings_to_query (Xapian::valueno slot,
+				const std::string &begin, const std::string &end,
+				Xapian::Query &output, std::string &msg)
+{
+    double from = DBL_MIN, to = DBL_MAX;
+    time_t parsed_time, now;
+    std::string str;
+
+    /* Use the same 'now' for begin and end. */
+    if (time (&now) == (time_t) -1) {
+	msg = "unable to get current time";
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+    }
+
+    if (! begin.empty ()) {
+	if (parse_time_string (begin.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_DOWN)) {
+	    msg = "Didn't understand date specification '" + begin + "'";
+	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	}
+
+	from = (double) parsed_time;
+    }
+
+    if (! end.empty ()) {
+	if (end == "!" && ! begin.empty ())
+	    str = begin;
+	else
+	    str = end;
+
+	if (parse_time_string (str.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_UP_INCLUSIVE)) {
+	    msg = "Didn't understand date specification '" + str + "'";
+	    return NOTMUCH_STATUS_BAD_QUERY_SYNTAX;
+	}
+	to = (double) parsed_time;
+    }
+
+    output = Xapian::Query (Xapian::Query::OP_VALUE_RANGE, slot,
+			    Xapian::sortable_serialise (from),
+			    Xapian::sortable_serialise (to));
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+Xapian::Query
+ParseTimeRangeProcessor::operator() (const std::string &begin, const std::string &end)
+{
+
+    Xapian::Query output;
+    std::string msg;
+
+    if (_notmuch_date_strings_to_query (slot, begin, end, output, msg))
+	throw Xapian::QueryParserError (msg);
+
+    return output;
+}
+
+/* XXX TODO: is throwing an exception the right thing to do here? */
+Xapian::Query
+DateFieldProcessor::operator() (const std::string & str)
+{
+    double from = DBL_MIN, to = DBL_MAX;
+    time_t parsed_time, now;
+
+    /* Use the same 'now' for begin and end. */
+    if (time (&now) == (time_t) -1)
+	throw Xapian::QueryParserError ("Unable to get current time");
+
+    if (parse_time_string (str.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_DOWN))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+    else
+	from = (double) parsed_time;
+
+    if (parse_time_string (str.c_str (), &parsed_time, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+    else
+	to = (double) parsed_time;
+
+    return Xapian::Query (Xapian::Query::OP_VALUE_RANGE, slot,
+			  Xapian::sortable_serialise (from),
+			  Xapian::sortable_serialise (to));
+}
diff --git a/lib/parse-time-vrp.h b/lib/parse-time-vrp.h
new file mode 100644
index 00000000..f495e716
--- /dev/null
+++ b/lib/parse-time-vrp.h
@@ -0,0 +1,46 @@
+/* parse-time-vrp.h - date range query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2012 Jani Nikula
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Jani Nikula <jani@nikula.org>
+ */
+
+#ifndef NOTMUCH_PARSE_TIME_VRP_H
+#define NOTMUCH_PARSE_TIME_VRP_H
+
+#include <xapian.h>
+
+/* see *ValueRangeProcessor in xapian-core/include/xapian/queryparser.h */
+class ParseTimeRangeProcessor : public Xapian::RangeProcessor {
+
+public:
+    ParseTimeRangeProcessor (Xapian::valueno slot_, const std::string prefix_)
+	:  Xapian::RangeProcessor(slot_, prefix_, 0) { }
+
+    Xapian::Query operator() (const std::string &begin, const std::string &end);
+};
+
+class DateFieldProcessor : public Xapian::FieldProcessor {
+private:
+    Xapian::valueno slot;
+public:
+    DateFieldProcessor(Xapian::valueno slot_) : slot(slot_) { };
+    Xapian::Query operator()(const std::string & str);
+};
+
+#endif /* NOTMUCH_PARSE_TIME_VRP_H */
diff --git a/lib/prefix.cc b/lib/prefix.cc
new file mode 100644
index 00000000..06e2333a
--- /dev/null
+++ b/lib/prefix.cc
@@ -0,0 +1,215 @@
+#include "database-private.h"
+#include "query-fp.h"
+#include "thread-fp.h"
+#include "regexp-fields.h"
+#include "parse-time-vrp.h"
+#include "sexp-fp.h"
+
+typedef struct {
+    const char *name;
+    const char *prefix;
+    notmuch_field_flag_t flags;
+} prefix_t;
+
+/* With these prefix values we follow the conventions published here:
+ *
+ * https://xapian.org/docs/omega/termprefixes.html
+ *
+ * as much as makes sense. Note that I took some liberty in matching
+ * the reserved prefix values to notmuch concepts, (for example, 'G'
+ * is documented as "newsGroup (or similar entity - e.g. a web forum
+ * name)", for which I think the thread is the closest analogue in
+ * notmuch. This in spite of the fact that we will eventually be
+ * storing mailing-list messages where 'G' for "mailing list name"
+ * might be even a closer analogue. I'm treating the single-character
+ * prefixes preferentially for core notmuch concepts (which will be
+ * nearly universal to all mail messages).
+ */
+
+static const
+prefix_t prefix_table[] = {
+    /* name			term prefix	flags */
+    { "type",                   "T",            NOTMUCH_FIELD_NO_FLAGS },
+    { "reference",              "XREFERENCE",   NOTMUCH_FIELD_NO_FLAGS },
+    { "replyto",                "XREPLYTO",     NOTMUCH_FIELD_NO_FLAGS },
+    { "directory",              "XDIRECTORY",   NOTMUCH_FIELD_NO_FLAGS },
+    { "file-direntry",          "XFDIRENTRY",   NOTMUCH_FIELD_NO_FLAGS },
+    { "directory-direntry",     "XDDIRENTRY",   NOTMUCH_FIELD_NO_FLAGS },
+    { "body",                   "",             NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROBABILISTIC },
+    { "thread",                 "G",            NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "tag",                    "K",            NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "is",                     "K",            NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "id",                     "Q",            NOTMUCH_FIELD_EXTERNAL },
+    { "mid",                    "Q",            NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "path",                   "P",            NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR | NOTMUCH_FIELD_STRIP_TRAILING_SLASH },
+    { "property",               "XPROPERTY",    NOTMUCH_FIELD_EXTERNAL },
+    /*
+     * Unconditionally add ':' to reduce potential ambiguity with
+     * overlapping prefixes and/or terms that start with capital
+     * letters. See Xapian document termprefixes.html for related
+     * discussion.
+     */
+    { "folder",                 "XFOLDER:",     NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR | NOTMUCH_FIELD_STRIP_TRAILING_SLASH },
+    { "date",                   NULL,           NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "query",                  NULL,           NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "sexp",                  NULL,            NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "from",                   "XFROM",        NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROBABILISTIC |
+      NOTMUCH_FIELD_PROCESSOR },
+    { "to",                     "XTO",          NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROBABILISTIC },
+    { "attachment",             "XATTACHMENT",  NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROBABILISTIC },
+    { "mimetype",               "XMIMETYPE",    NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROBABILISTIC },
+    { "subject",                "XSUBJECT",     NOTMUCH_FIELD_EXTERNAL |
+      NOTMUCH_FIELD_PROBABILISTIC |
+      NOTMUCH_FIELD_PROCESSOR },
+};
+
+static const char *
+_user_prefix (void *ctx, const char *name)
+{
+    return talloc_asprintf (ctx, "XU%s:", name);
+}
+
+const char *
+_find_prefix (const char *name)
+{
+    unsigned int i;
+
+    for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
+	if (strcmp (name, prefix_table[i].name) == 0)
+	    return prefix_table[i].prefix;
+    }
+
+    INTERNAL_ERROR ("No prefix exists for '%s'\n", name);
+
+    return "";
+}
+
+/* Like find prefix, but include the possibility of user defined
+ * prefixes specific to this database */
+
+const char *
+_notmuch_database_prefix (notmuch_database_t *notmuch, const char *name)
+{
+    unsigned int i;
+
+    /*XXX TODO: reduce code duplication */
+    for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
+	if (strcmp (name, prefix_table[i].name) == 0)
+	    return prefix_table[i].prefix;
+    }
+
+    if (notmuch->user_prefix)
+	return _notmuch_string_map_get (notmuch->user_prefix, name);
+
+    return NULL;
+}
+
+static void
+_setup_query_field_default (const prefix_t *prefix, notmuch_database_t *notmuch)
+{
+    if (prefix->prefix)
+	notmuch->query_parser->add_prefix ("", prefix->prefix);
+    if (prefix->flags & NOTMUCH_FIELD_PROBABILISTIC)
+	notmuch->query_parser->add_prefix (prefix->name, prefix->prefix);
+    else
+	notmuch->query_parser->add_boolean_prefix (prefix->name, prefix->prefix);
+}
+
+static void
+_setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
+{
+    if (prefix->flags & NOTMUCH_FIELD_PROCESSOR) {
+	Xapian::FieldProcessor *fp;
+
+	if (STRNCMP_LITERAL (prefix->name, "date") == 0)
+	    fp = (new DateFieldProcessor (NOTMUCH_VALUE_TIMESTAMP))->release ();
+	else if (STRNCMP_LITERAL (prefix->name, "query") == 0)
+	    fp = (new QueryFieldProcessor (*notmuch->query_parser, notmuch))->release ();
+	else if (STRNCMP_LITERAL (prefix->name, "thread") == 0)
+	    fp = (new ThreadFieldProcessor (*notmuch->query_parser, notmuch))->release ();
+	else if (STRNCMP_LITERAL (prefix->name, "sexp") == 0)
+	    fp = (new SexpFieldProcessor (notmuch))->release ();
+	else
+	    fp = (new RegexpFieldProcessor (prefix->name, prefix->flags,
+					    *notmuch->query_parser, notmuch))->release ();
+
+	/* we treat all field-processor fields as boolean in order to get the raw input */
+	if (prefix->prefix)
+	    notmuch->query_parser->add_prefix ("", prefix->prefix);
+	notmuch->query_parser->add_boolean_prefix (prefix->name, fp);
+    } else {
+	_setup_query_field_default (prefix, notmuch);
+    }
+}
+
+notmuch_status_t
+_notmuch_database_setup_standard_query_fields (notmuch_database_t *notmuch)
+{
+    for (unsigned int i = 0; i < ARRAY_SIZE (prefix_table); i++) {
+	const prefix_t *prefix = &prefix_table[i];
+	if (prefix->flags & NOTMUCH_FIELD_EXTERNAL) {
+	    _setup_query_field (prefix, notmuch);
+	}
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+_notmuch_database_setup_user_query_fields (notmuch_database_t *notmuch)
+{
+    notmuch_string_map_iterator_t *list;
+
+    notmuch->user_prefix = _notmuch_string_map_create (notmuch);
+    if (notmuch->user_prefix == NULL)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    notmuch->user_header = _notmuch_string_map_create (notmuch);
+    if (notmuch->user_header == NULL)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    list = _notmuch_string_map_iterator_create (notmuch->config, CONFIG_HEADER_PREFIX, FALSE);
+    if (! list)
+	INTERNAL_ERROR ("unable to read headers from configuration");
+
+    for (; _notmuch_string_map_iterator_valid (list);
+	 _notmuch_string_map_iterator_move_to_next (list)) {
+
+	prefix_t query_field;
+
+	const char *key = _notmuch_string_map_iterator_key (list)
+			  + sizeof (CONFIG_HEADER_PREFIX) - 1;
+
+	_notmuch_string_map_append (notmuch->user_prefix,
+				    key,
+				    _user_prefix (notmuch, key));
+
+	_notmuch_string_map_append (notmuch->user_header,
+				    key,
+				    _notmuch_string_map_iterator_value (list));
+
+	query_field.name = talloc_strdup (notmuch, key);
+	query_field.prefix = _user_prefix (notmuch, key);
+	query_field.flags = NOTMUCH_FIELD_PROBABILISTIC
+			    | NOTMUCH_FIELD_EXTERNAL;
+
+	_setup_query_field_default (&query_field, notmuch);
+    }
+
+    _notmuch_string_map_iterator_destroy (list);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/query-fp.cc b/lib/query-fp.cc
new file mode 100644
index 00000000..75b1d875
--- /dev/null
+++ b/lib/query-fp.cc
@@ -0,0 +1,56 @@
+/* query-fp.cc - "query:" field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "query-fp.h"
+#include <iostream>
+
+notmuch_status_t
+_notmuch_query_name_to_query (notmuch_database_t *notmuch, const std::string name,
+			      Xapian::Query &output)
+{
+    std::string key = "query." + name;
+    char *expansion;
+    notmuch_status_t status;
+
+    status = notmuch_database_get_config (notmuch, key.c_str (), &expansion);
+    if (status)
+	return status;
+
+    output = notmuch->query_parser->parse_query (expansion, NOTMUCH_QUERY_PARSER_FLAGS);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+Xapian::Query
+QueryFieldProcessor::operator() (const std::string & name)
+{
+    notmuch_status_t status;
+    Xapian::Query output;
+
+    status = _notmuch_query_name_to_query (notmuch, name, output);
+    if (status) {
+	throw Xapian::QueryParserError ("error looking up key" + name);
+    }
+
+    return output;
+
+}
diff --git a/lib/query-fp.h b/lib/query-fp.h
new file mode 100644
index 00000000..beaaf405
--- /dev/null
+++ b/lib/query-fp.h
@@ -0,0 +1,43 @@
+/* query-fp.h - query field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_QUERY_FP_H
+#define NOTMUCH_QUERY_FP_H
+
+#include <xapian.h>
+#include "notmuch.h"
+
+class QueryFieldProcessor : public Xapian::FieldProcessor {
+protected:
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+public:
+    QueryFieldProcessor (Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)
+	: parser (parser_), notmuch (notmuch_)
+    {
+    };
+
+    Xapian::Query operator() (const std::string & str);
+};
+
+#endif /* NOTMUCH_QUERY_FP_H */
diff --git a/lib/query.cc b/lib/query.cc
new file mode 100644
index 00000000..1c60c122
--- /dev/null
+++ b/lib/query.cc
@@ -0,0 +1,874 @@
+/* query.cc - Support for searching a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "xapian-extra.h"
+
+#include <glib.h> /* GHashTable, GPtrArray */
+
+struct _notmuch_query {
+    notmuch_database_t *notmuch;
+    const char *query_string;
+    notmuch_sort_t sort;
+    notmuch_string_list_t *exclude_terms;
+    notmuch_exclude_t omit_excluded;
+    bool parsed;
+    notmuch_query_syntax_t syntax;
+    Xapian::Query xapian_query;
+    std::set<std::string> terms;
+};
+
+typedef struct _notmuch_mset_messages {
+    notmuch_messages_t base;
+    notmuch_database_t *notmuch;
+    Xapian::MSetIterator iterator;
+    Xapian::MSetIterator iterator_end;
+} notmuch_mset_messages_t;
+
+struct _notmuch_doc_id_set {
+    unsigned char *bitmap;
+    unsigned int bound;
+};
+
+#define DOCIDSET_WORD(bit) ((bit) / CHAR_BIT)
+#define DOCIDSET_BIT(bit) ((bit) % CHAR_BIT)
+
+struct _notmuch_threads {
+    notmuch_query_t *query;
+
+    /* The ordered list of doc ids matched by the query. */
+    GArray *doc_ids;
+    /* Our iterator's current position in doc_ids. */
+    unsigned int doc_id_pos;
+    /* The set of matched docid's that have not been assigned to a
+     * thread. Initially, this contains every docid in doc_ids. */
+    notmuch_doc_id_set_t match_set;
+};
+
+/* We need this in the message functions so forward declare. */
+static bool
+_notmuch_doc_id_set_init (void *ctx,
+			  notmuch_doc_id_set_t *doc_ids,
+			  GArray *arr);
+
+static bool
+_debug_query (void)
+{
+    char *env = getenv ("NOTMUCH_DEBUG_QUERY");
+
+    return (env && strcmp (env, "") != 0);
+}
+
+/* Explicit destructor call for placement new */
+static int
+_notmuch_query_destructor (notmuch_query_t *query)
+{
+    query->xapian_query.~Query();
+    query->terms.~set<std::string>();
+    return 0;
+}
+
+static notmuch_query_t *
+_notmuch_query_constructor (notmuch_database_t *notmuch,
+			    const char *query_string)
+{
+    notmuch_query_t *query;
+
+    if (_debug_query ())
+	fprintf (stderr, "Query string is:\n%s\n", query_string);
+
+    query = talloc (notmuch, notmuch_query_t);
+    if (unlikely (query == NULL))
+	return NULL;
+
+    new (&query->xapian_query) Xapian::Query ();
+    new (&query->terms) std::set<std::string> ();
+    query->parsed = false;
+
+    talloc_set_destructor (query, _notmuch_query_destructor);
+
+    query->notmuch = notmuch;
+
+    if (query_string)
+	query->query_string = talloc_strdup (query, query_string);
+    else
+	query->query_string = NULL;
+
+    query->sort = NOTMUCH_SORT_NEWEST_FIRST;
+
+    query->exclude_terms = _notmuch_string_list_create (query);
+
+    query->omit_excluded = NOTMUCH_EXCLUDE_TRUE;
+
+    return query;
+}
+
+notmuch_query_t *
+notmuch_query_create (notmuch_database_t *notmuch,
+		      const char *query_string)
+{
+
+    notmuch_query_t *query;
+    notmuch_status_t status;
+
+    status = notmuch_query_create_with_syntax (notmuch, query_string,
+					       NOTMUCH_QUERY_SYNTAX_XAPIAN,
+					       &query);
+    if (status)
+	return NULL;
+
+    return query;
+}
+
+notmuch_status_t
+notmuch_query_create_with_syntax (notmuch_database_t *notmuch,
+				  const char *query_string,
+				  notmuch_query_syntax_t syntax,
+				  notmuch_query_t **output)
+{
+
+    notmuch_query_t *query;
+
+    if (! output)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    query = _notmuch_query_constructor (notmuch, query_string);
+    if (! query)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    if (syntax == NOTMUCH_QUERY_SYNTAX_SEXP && ! HAVE_SFSEXP) {
+	_notmuch_database_log (notmuch, "sexp query parser not available");
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+    }
+
+    query->syntax = syntax;
+
+    *output = query;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static void
+_notmuch_query_cache_terms (notmuch_query_t *query)
+{
+    /* Xapian doesn't support skip_to on terms from a query since
+     *  they are unordered, so cache a copy of all terms in
+     *  something searchable.
+     */
+
+    for (Xapian::TermIterator t = query->xapian_query.get_terms_begin ();
+	 t != query->xapian_query.get_terms_end (); ++t)
+	query->terms.insert (*t);
+}
+
+notmuch_status_t
+_notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch,
+				       std::string query_string,
+				       Xapian::Query &output,
+				       std::string &msg)
+{
+    try {
+	if (query_string == "" || query_string == "*") {
+	    output = xapian_query_match_all ();
+	} else {
+	    output =
+		notmuch->query_parser->
+		parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
+	}
+    } catch (const Xapian::Error &error) {
+	if (! notmuch->exception_reported) {
+	    _notmuch_database_log (notmuch,
+				   "A Xapian exception occurred parsing query: %s\n",
+				   error.get_msg ().c_str ());
+	    _notmuch_database_log_append (notmuch,
+					  "Query string was: %s\n",
+					  query_string.c_str ());
+	    notmuch->exception_reported = true;
+	}
+
+	msg = error.get_msg ();
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_notmuch_query_ensure_parsed_xapian (notmuch_query_t *query)
+{
+    notmuch_status_t status;
+    std::string msg; /* ignored */
+
+    status =  _notmuch_query_string_to_xapian_query (query->notmuch, query->query_string,
+						     query->xapian_query, msg);
+    if (status)
+	return status;
+
+    query->parsed = true;
+
+    _notmuch_query_cache_terms (query);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+#if HAVE_SFSEXP
+static notmuch_status_t
+_notmuch_query_ensure_parsed_sexpr (notmuch_query_t *query)
+{
+    notmuch_status_t status;
+
+    if (query->parsed)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    status = _notmuch_sexp_string_to_xapian_query (query->notmuch, query->query_string,
+						   query->xapian_query);
+    if (status)
+	return status;
+
+    _notmuch_query_cache_terms (query);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+#endif
+
+static notmuch_status_t
+_notmuch_query_ensure_parsed (notmuch_query_t *query)
+{
+    if (query->parsed)
+	return NOTMUCH_STATUS_SUCCESS;
+
+#if HAVE_SFSEXP
+    if (query->syntax == NOTMUCH_QUERY_SYNTAX_SEXP)
+	return _notmuch_query_ensure_parsed_sexpr (query);
+#endif
+
+    return _notmuch_query_ensure_parsed_xapian (query);
+}
+
+const char *
+notmuch_query_get_query_string (const notmuch_query_t *query)
+{
+    return query->query_string;
+}
+
+void
+notmuch_query_set_omit_excluded (notmuch_query_t *query,
+				 notmuch_exclude_t omit_excluded)
+{
+    query->omit_excluded = omit_excluded;
+}
+
+void
+notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort)
+{
+    query->sort = sort;
+}
+
+notmuch_sort_t
+notmuch_query_get_sort (const notmuch_query_t *query)
+{
+    return query->sort;
+}
+
+notmuch_status_t
+notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag)
+{
+    notmuch_status_t status;
+    char *term;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
+
+    term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
+    if (query->terms.count (term) != 0)
+	return NOTMUCH_STATUS_IGNORED;
+
+    _notmuch_string_list_append (query->exclude_terms, term);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* We end up having to call the destructors explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_messages_destructor (notmuch_mset_messages_t *messages)
+{
+    messages->iterator.~MSetIterator ();
+    messages->iterator_end.~MSetIterator ();
+
+    return 0;
+}
+
+/* Return a query that matches messages with the excluded tags
+ * registered with query. The caller of this function has to combine the returned
+ * query appropriately.*/
+static Xapian::Query
+_notmuch_exclude_tags (notmuch_query_t *query)
+{
+    Xapian::Query exclude_query = Xapian::Query::MatchNothing;
+
+    for (notmuch_string_node_t *term = query->exclude_terms->head; term;
+	 term = term->next) {
+	exclude_query = Xapian::Query (Xapian::Query::OP_OR,
+				       exclude_query, Xapian::Query (term->string));
+    }
+    return exclude_query;
+}
+
+
+notmuch_status_t
+notmuch_query_search_messages_st (notmuch_query_t *query,
+				  notmuch_messages_t **out)
+{
+    return notmuch_query_search_messages (query, out);
+}
+
+notmuch_status_t
+notmuch_query_search_messages (notmuch_query_t *query,
+			       notmuch_messages_t **out)
+{
+    return _notmuch_query_search_documents (query, "mail", out);
+}
+
+notmuch_status_t
+_notmuch_query_search_documents (notmuch_query_t *query,
+				 const char *type,
+				 notmuch_messages_t **out)
+{
+    notmuch_database_t *notmuch = query->notmuch;
+    notmuch_mset_messages_t *messages;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
+
+    messages = talloc (query, notmuch_mset_messages_t);
+    if (unlikely (messages == NULL))
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    try {
+
+	messages->base.is_of_list_type = false;
+	messages->base.iterator = NULL;
+	messages->notmuch = notmuch;
+	new (&messages->iterator) Xapian::MSetIterator ();
+	new (&messages->iterator_end) Xapian::MSetIterator ();
+
+	talloc_set_destructor (messages, _notmuch_messages_destructor);
+
+	Xapian::Enquire enquire (*notmuch->xapian_db);
+	Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
+						   _find_prefix ("type"),
+						   type));
+	Xapian::Query final_query, exclude_query;
+	Xapian::MSet mset;
+	Xapian::MSetIterator iterator;
+
+	final_query = Xapian::Query (Xapian::Query::OP_AND,
+				     mail_query, query->xapian_query);
+
+	messages->base.excluded_doc_ids = NULL;
+
+	if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) {
+	    exclude_query = _notmuch_exclude_tags (query);
+
+	    if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE ||
+		query->omit_excluded == NOTMUCH_EXCLUDE_ALL) {
+		final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
+					     final_query, exclude_query);
+	    } else { /* NOTMUCH_EXCLUDE_FLAG */
+		exclude_query = Xapian::Query (Xapian::Query::OP_AND,
+					       exclude_query, final_query);
+
+		enquire.set_weighting_scheme (Xapian::BoolWeight ());
+		enquire.set_query (exclude_query);
+
+		mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
+
+		GArray *excluded_doc_ids = g_array_new (false, false, sizeof (unsigned int));
+
+		for (iterator = mset.begin (); iterator != mset.end (); iterator++) {
+		    unsigned int doc_id = *iterator;
+		    g_array_append_val (excluded_doc_ids, doc_id);
+		}
+		messages->base.excluded_doc_ids = talloc (messages, _notmuch_doc_id_set);
+		_notmuch_doc_id_set_init (query, messages->base.excluded_doc_ids,
+					  excluded_doc_ids);
+		g_array_unref (excluded_doc_ids);
+	    }
+	}
+
+
+	enquire.set_weighting_scheme (Xapian::BoolWeight ());
+
+	switch (query->sort) {
+	case NOTMUCH_SORT_OLDEST_FIRST:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, false);
+	    break;
+	case NOTMUCH_SORT_NEWEST_FIRST:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, true);
+	    break;
+	case NOTMUCH_SORT_MESSAGE_ID:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, false);
+	    break;
+	case NOTMUCH_SORT_UNSORTED:
+	    break;
+	}
+
+	if (_debug_query ()) {
+	    fprintf (stderr, "Exclude query is:\n%s\n",
+		     exclude_query.get_description ().c_str ());
+	    fprintf (stderr, "Final query is:\n%s\n",
+		     final_query.get_description ().c_str ());
+	}
+
+	enquire.set_query (final_query);
+
+	mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
+
+	messages->iterator = mset.begin ();
+	messages->iterator_end = mset.end ();
+
+	*out = &messages->base;
+	return NOTMUCH_STATUS_SUCCESS;
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred performing query: %s\n",
+			       error.get_msg ().c_str ());
+	_notmuch_database_log_append (notmuch,
+				      "Query string was: %s\n",
+				      query->query_string);
+
+	notmuch->exception_reported = true;
+	talloc_free (messages);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+}
+
+bool
+_notmuch_mset_messages_valid (notmuch_messages_t *messages)
+{
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    return (mset_messages->iterator != mset_messages->iterator_end);
+}
+
+static Xapian::docid
+_notmuch_mset_messages_get_doc_id (notmuch_messages_t *messages)
+{
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    if (! _notmuch_mset_messages_valid (&mset_messages->base))
+	return 0;
+
+    return *mset_messages->iterator;
+}
+
+notmuch_message_t *
+_notmuch_mset_messages_get (notmuch_messages_t *messages)
+{
+    notmuch_message_t *message;
+    Xapian::docid doc_id;
+    notmuch_private_status_t status;
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    if (! _notmuch_mset_messages_valid (&mset_messages->base))
+	return NULL;
+
+    doc_id = *mset_messages->iterator;
+
+    message = _notmuch_message_create (mset_messages,
+				       mset_messages->notmuch, doc_id,
+				       &status);
+
+    if (message == NULL &&
+	status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	INTERNAL_ERROR ("a messages iterator contains a non-existent document ID.\n");
+    }
+
+    if (messages->excluded_doc_ids &&
+	_notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id))
+	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
+
+    return message;
+}
+
+void
+_notmuch_mset_messages_move_to_next (notmuch_messages_t *messages)
+{
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    mset_messages->iterator++;
+}
+
+static bool
+_notmuch_doc_id_set_init (void *ctx,
+			  notmuch_doc_id_set_t *doc_ids,
+			  GArray *arr)
+{
+    unsigned int max = 0;
+    unsigned char *bitmap;
+
+    for (unsigned int i = 0; i < arr->len; i++)
+	max = MAX (max, g_array_index (arr, unsigned int, i));
+    bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD (max) + 1);
+
+    if (bitmap == NULL)
+	return false;
+
+    doc_ids->bitmap = bitmap;
+    doc_ids->bound = max + 1;
+
+    for (unsigned int i = 0; i < arr->len; i++) {
+	unsigned int doc_id = g_array_index (arr, unsigned int, i);
+	bitmap[DOCIDSET_WORD (doc_id)] |= 1 << DOCIDSET_BIT (doc_id);
+    }
+
+    return true;
+}
+
+bool
+_notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
+			      unsigned int doc_id)
+{
+    if (doc_id >= doc_ids->bound)
+	return false;
+    return doc_ids->bitmap[DOCIDSET_WORD (doc_id)] & (1 << DOCIDSET_BIT (doc_id));
+}
+
+void
+_notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
+			    unsigned int doc_id)
+{
+    if (doc_id < doc_ids->bound)
+	doc_ids->bitmap[DOCIDSET_WORD (doc_id)] &= ~(1 << DOCIDSET_BIT (doc_id));
+}
+
+/* Glib objects force use to use a talloc destructor as well, (but not
+ * nearly as ugly as the for messages due to C++ objects). At
+ * this point, I'd really like to have some talloc-friendly
+ * equivalents for the few pieces of glib that I'm using. */
+static int
+_notmuch_threads_destructor (notmuch_threads_t *threads)
+{
+    if (threads->doc_ids)
+	g_array_unref (threads->doc_ids);
+
+    return 0;
+}
+
+notmuch_status_t
+notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out)
+{
+    return notmuch_query_search_threads (query, out);
+}
+
+notmuch_status_t
+notmuch_query_search_threads (notmuch_query_t *query,
+			      notmuch_threads_t **out)
+{
+    notmuch_threads_t *threads;
+    notmuch_messages_t *messages;
+    notmuch_status_t status;
+
+    threads = talloc (query, notmuch_threads_t);
+    if (threads == NULL)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    threads->doc_ids = NULL;
+    talloc_set_destructor (threads, _notmuch_threads_destructor);
+
+    threads->query = query;
+
+    status = notmuch_query_search_messages (query, &messages);
+    if (status) {
+	talloc_free (threads);
+	return status;
+    }
+
+    threads->doc_ids = g_array_new (false, false, sizeof (unsigned int));
+    while (notmuch_messages_valid (messages)) {
+	unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages);
+	g_array_append_val (threads->doc_ids, doc_id);
+	notmuch_messages_move_to_next (messages);
+    }
+    threads->doc_id_pos = 0;
+
+    talloc_free (messages);
+
+    if (! _notmuch_doc_id_set_init (threads, &threads->match_set,
+				    threads->doc_ids)) {
+	talloc_free (threads);
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    }
+
+    *out = threads;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+void
+notmuch_query_destroy (notmuch_query_t *query)
+{
+    talloc_free (query);
+}
+
+notmuch_bool_t
+notmuch_threads_valid (notmuch_threads_t *threads)
+{
+    unsigned int doc_id;
+
+    if (! threads)
+	return false;
+
+    while (threads->doc_id_pos < threads->doc_ids->len) {
+	doc_id = g_array_index (threads->doc_ids, unsigned int,
+				threads->doc_id_pos);
+	if (_notmuch_doc_id_set_contains (&threads->match_set, doc_id))
+	    break;
+
+	threads->doc_id_pos++;
+    }
+
+    return threads->doc_id_pos < threads->doc_ids->len;
+}
+
+notmuch_thread_t *
+notmuch_threads_get (notmuch_threads_t *threads)
+{
+    unsigned int doc_id;
+
+    if (! notmuch_threads_valid (threads))
+	return NULL;
+
+    doc_id = g_array_index (threads->doc_ids, unsigned int,
+			    threads->doc_id_pos);
+    return _notmuch_thread_create (threads->query,
+				   threads->query->notmuch,
+				   doc_id,
+				   &threads->match_set,
+				   threads->query->exclude_terms,
+				   threads->query->omit_excluded,
+				   threads->query->sort);
+}
+
+void
+notmuch_threads_move_to_next (notmuch_threads_t *threads)
+{
+    threads->doc_id_pos++;
+}
+
+void
+notmuch_threads_destroy (notmuch_threads_t *threads)
+{
+    talloc_free (threads);
+}
+
+notmuch_status_t
+notmuch_query_count_messages_st (notmuch_query_t *query, unsigned *count_out)
+{
+    return notmuch_query_count_messages (query, count_out);
+}
+
+notmuch_status_t
+notmuch_query_count_messages (notmuch_query_t *query, unsigned *count_out)
+{
+    return _notmuch_query_count_documents (query, "mail", count_out);
+}
+
+notmuch_status_t
+_notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsigned *count_out)
+{
+    notmuch_database_t *notmuch = query->notmuch;
+    Xapian::doccount count = 0;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
+
+    try {
+	Xapian::Enquire enquire (*notmuch->xapian_db);
+	Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
+						   _find_prefix ("type"),
+						   type));
+	Xapian::Query final_query, exclude_query;
+	Xapian::MSet mset;
+
+	final_query = Xapian::Query (Xapian::Query::OP_AND,
+				     mail_query, query->xapian_query);
+
+	exclude_query = _notmuch_exclude_tags (query);
+
+	final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
+				     final_query, exclude_query);
+
+	enquire.set_weighting_scheme (Xapian::BoolWeight ());
+	enquire.set_docid_order (Xapian::Enquire::ASCENDING);
+
+	if (_debug_query ()) {
+	    fprintf (stderr, "Exclude query is:\n%s\n",
+		     exclude_query.get_description ().c_str ());
+	    fprintf (stderr, "Final query is:\n%s\n",
+		     final_query.get_description ().c_str ());
+	}
+
+	enquire.set_query (final_query);
+
+	/*
+	 * Set the checkatleast parameter to the number of documents
+	 * in the database to make get_matches_estimated() exact.
+	 * Set the max parameter to 1 to avoid fetching documents we will discard.
+	 */
+	mset = enquire.get_mset (0, 1,
+				 notmuch->xapian_db->get_doccount ());
+
+	count = mset.get_matches_estimated ();
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred performing query: %s\n",
+			       error.get_msg ().c_str ());
+	_notmuch_database_log_append (notmuch,
+				      "Query string was: %s\n",
+				      query->query_string);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    *count_out = count;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
+{
+    return notmuch_query_count_threads (query, count);
+}
+
+notmuch_status_t
+notmuch_query_count_threads (notmuch_query_t *query, unsigned *count)
+{
+    notmuch_messages_t *messages;
+    GHashTable *hash;
+    notmuch_sort_t sort;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    sort = query->sort;
+    query->sort = NOTMUCH_SORT_UNSORTED;
+    ret = notmuch_query_search_messages (query, &messages);
+    if (ret)
+	return ret;
+    query->sort = sort;
+    if (messages == NULL)
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+
+    hash = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
+    if (hash == NULL) {
+	talloc_free (messages);
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    }
+
+    while (notmuch_messages_valid (messages)) {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	const char *thread_id = notmuch_message_get_thread_id (message);
+	char *thread_id_copy = talloc_strdup (messages, thread_id);
+	if (unlikely (thread_id_copy == NULL)) {
+	    notmuch_message_destroy (message);
+	    ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	    goto DONE;
+	}
+	g_hash_table_insert (hash, thread_id_copy, NULL);
+	notmuch_message_destroy (message);
+	notmuch_messages_move_to_next (messages);
+    }
+
+    *count = g_hash_table_size (hash);
+
+  DONE:
+    g_hash_table_unref (hash);
+    talloc_free (messages);
+
+    return ret;
+}
+
+notmuch_database_t *
+notmuch_query_get_database (const notmuch_query_t *query)
+{
+    return query->notmuch;
+}
+
+notmuch_status_t
+_notmuch_query_expand (notmuch_database_t *notmuch, const char *field, Xapian::Query subquery,
+		       Xapian::Query &output, std::string &msg)
+{
+    std::set<std::string> terms;
+    const std::string term_prefix =  _find_prefix (field);
+
+    if (_debug_query ()) {
+	fprintf (stderr, "Expanding subquery:\n%s\n",
+		 subquery.get_description ().c_str ());
+    }
+
+    try {
+	Xapian::Enquire enquire (*notmuch->xapian_db);
+	Xapian::MSet mset;
+
+	enquire.set_weighting_scheme (Xapian::BoolWeight ());
+	enquire.set_query (subquery);
+
+	mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
+
+	for (Xapian::MSetIterator iterator = mset.begin (); iterator != mset.end (); iterator++) {
+	    Xapian::docid doc_id = *iterator;
+	    Xapian::Document doc = notmuch->xapian_db->get_document (doc_id);
+	    Xapian::TermIterator i = doc.termlist_begin ();
+
+	    for (i.skip_to (term_prefix);
+		 i != doc.termlist_end () && ((*i).rfind (term_prefix, 0) == 0); i++) {
+		terms.insert (*i);
+	    }
+	}
+	output = Xapian::Query (Xapian::Query::OP_OR, terms.begin (), terms.end ());
+	if (_debug_query ()) {
+	    fprintf (stderr, "Expanded query:\n%s\n",
+		     subquery.get_description ().c_str ());
+	}
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred expanding query: %s\n",
+			       error.get_msg ().c_str ());
+	msg = error.get_msg ();
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc
new file mode 100644
index 00000000..3a775261
--- /dev/null
+++ b/lib/regexp-fields.cc
@@ -0,0 +1,252 @@
+/* regexp-fields.cc - field processor glue for regex supporting fields
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2015 Austin Clements
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Austin Clements <aclements@csail.mit.edu>
+ *                David Bremner <david@tethera.net>
+ */
+
+#include "regexp-fields.h"
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "xapian-extra.h"
+
+notmuch_status_t
+compile_regex (regex_t &regexp, const char *str, std::string &msg)
+{
+    int err = regcomp (&regexp, str, REG_EXTENDED | REG_NOSUB);
+
+    if (err != 0) {
+	size_t len = regerror (err, &regexp, NULL, 0);
+	char *buffer = new char[len];
+	msg = "Regexp error: ";
+	(void) regerror (err, &regexp, buffer, len);
+	msg.append (buffer, len);
+	delete[] buffer;
+
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+RegexpPostingSource::RegexpPostingSource (Xapian::valueno slot, const std::string &regexp)
+    : slot_ (slot)
+{
+    std::string msg;
+    notmuch_status_t status = compile_regex (regexp_, regexp.c_str (), msg);
+
+    if (status)
+	throw Xapian::QueryParserError (msg);
+}
+
+RegexpPostingSource::~RegexpPostingSource ()
+{
+    regfree (&regexp_);
+}
+
+void
+RegexpPostingSource::init (const Xapian::Database &db)
+{
+    db_ = db;
+    it_ = db_.valuestream_begin (slot_);
+    end_ = db.valuestream_end (slot_);
+    started_ = false;
+}
+
+Xapian::doccount
+RegexpPostingSource::get_termfreq_min () const
+{
+    return 0;
+}
+
+Xapian::doccount
+RegexpPostingSource::get_termfreq_est () const
+{
+    return get_termfreq_max () / 2;
+}
+
+Xapian::doccount
+RegexpPostingSource::get_termfreq_max () const
+{
+    return db_.get_value_freq (slot_);
+}
+
+Xapian::docid
+RegexpPostingSource::get_docid () const
+{
+    return it_.get_docid ();
+}
+
+bool
+RegexpPostingSource::at_end () const
+{
+    return it_ == end_;
+}
+
+void
+RegexpPostingSource::next (unused (double min_wt))
+{
+    if (started_ && ! at_end ())
+	++it_;
+    started_ = true;
+
+    for (; ! at_end (); ++it_) {
+	std::string value = *it_;
+	if (regexec (&regexp_, value.c_str (), 0, NULL, 0) == 0)
+	    break;
+    }
+}
+
+void
+RegexpPostingSource::skip_to (Xapian::docid did, unused (double min_wt))
+{
+    started_ = true;
+    it_.skip_to (did);
+    for (; ! at_end (); ++it_) {
+	std::string value = *it_;
+	if (regexec (&regexp_, value.c_str (), 0, NULL, 0) == 0)
+	    break;
+    }
+}
+
+bool
+RegexpPostingSource::check (Xapian::docid did, unused (double min_wt))
+{
+    started_ = true;
+    if (! it_.check (did) || at_end ())
+	return false;
+    return (regexec (&regexp_, (*it_).c_str (), 0, NULL, 0) == 0);
+}
+
+static inline Xapian::valueno
+_find_slot (std::string prefix)
+{
+    if (prefix == "from")
+	return NOTMUCH_VALUE_FROM;
+    else if (prefix == "subject")
+	return NOTMUCH_VALUE_SUBJECT;
+    else if (prefix == "mid")
+	return NOTMUCH_VALUE_MESSAGE_ID;
+    else
+	return Xapian::BAD_VALUENO;
+}
+
+RegexpFieldProcessor::RegexpFieldProcessor (std::string field_,
+					    notmuch_field_flag_t options_,
+					    Xapian::QueryParser &parser_,
+					    notmuch_database_t *notmuch_)
+    : slot (_find_slot (field_)),
+    field (field_),
+    term_prefix (_find_prefix (field_.c_str ())),
+    options (options_),
+    parser (parser_),
+    notmuch (notmuch_)
+{
+};
+
+notmuch_status_t
+_notmuch_regexp_to_query (notmuch_database_t *notmuch, Xapian::valueno slot, std::string field,
+			  std::string regexp_str,
+			  Xapian::Query &output, std::string &msg)
+{
+    regex_t regexp;
+    notmuch_status_t status;
+
+    status = compile_regex (regexp, regexp_str.c_str (), msg);
+    if (status) {
+	_notmuch_database_log_append (notmuch, "error compiling regex %s", msg.c_str ());
+	return status;
+    }
+
+    if (slot == Xapian::BAD_VALUENO)
+	slot = _find_slot (field);
+
+    if (slot == Xapian::BAD_VALUENO) {
+	std::string term_prefix = _find_prefix (field.c_str ());
+	std::vector<std::string> terms;
+
+	for (Xapian::TermIterator it = notmuch->xapian_db->allterms_begin (term_prefix);
+	     it != notmuch->xapian_db->allterms_end (); ++it) {
+	    if (regexec (&regexp, (*it).c_str () + term_prefix.size (),
+			 0, NULL, 0) == 0)
+		terms.push_back (*it);
+	}
+	output = Xapian::Query (Xapian::Query::OP_OR, terms.begin (), terms.end ());
+    } else {
+	RegexpPostingSource *postings = new RegexpPostingSource (slot, regexp_str);
+	output = Xapian::Query (postings->release ());
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+Xapian::Query
+RegexpFieldProcessor::operator() (const std::string & str)
+{
+    if (str.empty ()) {
+	if (options & NOTMUCH_FIELD_PROBABILISTIC) {
+	    return Xapian::Query (Xapian::Query::OP_AND_NOT,
+				  xapian_query_match_all (),
+				  Xapian::Query (Xapian::Query::OP_WILDCARD, term_prefix));
+	} else {
+	    return Xapian::Query (term_prefix);
+	}
+    }
+
+    if (str.at (0) == '/') {
+	if (str.length () > 1 && str.at (str.size () - 1) == '/') {
+	    Xapian::Query query;
+	    std::string regexp_str = str.substr (1, str.size () - 2);
+	    std::string msg;
+	    notmuch_status_t status;
+
+	    status = _notmuch_regexp_to_query (notmuch, slot, field, regexp_str, query, msg);
+	    if (status)
+		throw Xapian::QueryParserError (msg);
+	    return query;
+	} else {
+	    throw Xapian::QueryParserError ("unmatched regex delimiter in '" + str + "'");
+	}
+    } else {
+	if (options & NOTMUCH_FIELD_PROBABILISTIC) {
+	    /* TODO replace this with a nicer API level triggering of
+	     * phrase parsing, when possible */
+	    std::string query_str;
+
+	    if ((str.at (0) != '(' || *str.rbegin () != ')') &&
+		(*str.rbegin () != '*' || str.find (' ') != std::string::npos))
+		query_str = '"' + str + '"';
+	    else
+		query_str = str;
+
+	    return parser.parse_query (query_str, NOTMUCH_QUERY_PARSER_FLAGS, term_prefix);
+	} else {
+	    /* Boolean prefix */
+	    std::string query_str;
+	    std::string term;
+
+	    if (str.length () > 1 && str.at (str.size () - 1) == '/')
+		query_str = str.substr (0, str.size () - 1);
+	    else
+		query_str = str;
+
+	    term = term_prefix + query_str;
+	    return Xapian::Query (term);
+	}
+    }
+}
diff --git a/lib/regexp-fields.h b/lib/regexp-fields.h
new file mode 100644
index 00000000..9c871de7
--- /dev/null
+++ b/lib/regexp-fields.h
@@ -0,0 +1,89 @@
+/* regex-fields.h - xapian glue for semi-bruteforce regexp search
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2015 Austin Clements
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Austin Clements <aclements@csail.mit.edu>
+ *                David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_REGEXP_FIELDS_H
+#define NOTMUCH_REGEXP_FIELDS_H
+
+#include <sys/types.h>
+#include <regex.h>
+#include "database-private.h"
+#include "notmuch-private.h"
+
+notmuch_status_t
+_notmuch_regex_to_query (notmuch_database_t *notmuch, Xapian::valueno slot, std::string field,
+			 std::string regexp_str,
+			 Xapian::Query &output, std::string &msg);
+
+/* A posting source that returns documents where a value matches a
+ * regexp.
+ */
+class RegexpPostingSource : public Xapian::PostingSource
+{
+protected:
+    const Xapian::valueno slot_;
+    regex_t regexp_;
+    Xapian::Database db_;
+    bool started_;
+    Xapian::ValueIterator it_, end_;
+
+/* No copying */
+    RegexpPostingSource (const RegexpPostingSource &);
+    RegexpPostingSource &operator= (const RegexpPostingSource &);
+
+public:
+    RegexpPostingSource (Xapian::valueno slot, const std::string &regexp);
+    ~RegexpPostingSource ();
+    void init (const Xapian::Database &db);
+    Xapian::doccount get_termfreq_min () const;
+    Xapian::doccount get_termfreq_est () const;
+    Xapian::doccount get_termfreq_max () const;
+    Xapian::docid get_docid () const;
+    bool at_end () const;
+    void next (unused (double min_wt));
+    void skip_to (Xapian::docid did, unused (double min_wt));
+    bool check (Xapian::docid did, unused (double min_wt));
+};
+
+
+class RegexpFieldProcessor : public Xapian::FieldProcessor {
+protected:
+    Xapian::valueno slot;
+    std::string field;
+    std::string term_prefix;
+    notmuch_field_flag_t options;
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+public:
+    RegexpFieldProcessor (std::string prefix, notmuch_field_flag_t options,
+			  Xapian::QueryParser &parser_, notmuch_database_t *notmuch_);
+
+    ~RegexpFieldProcessor ()
+    {
+    };
+
+    Xapian::Query operator() (const std::string & str);
+};
+
+#endif /* NOTMUCH_REGEXP_FIELDS_H */
diff --git a/lib/sexp-fp.cc b/lib/sexp-fp.cc
new file mode 100644
index 00000000..1fdf5225
--- /dev/null
+++ b/lib/sexp-fp.cc
@@ -0,0 +1,44 @@
+/* sexp-fp.cc - "sexp:" field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2022 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "sexp-fp.h"
+#include <iostream>
+
+Xapian::Query
+SexpFieldProcessor::operator() (const std::string & query_string)
+{
+    notmuch_status_t status;
+    Xapian::Query output;
+
+#if HAVE_SFSEXP
+    status = _notmuch_sexp_string_to_xapian_query (notmuch, query_string.c_str (), output);
+    if (status) {
+	throw Xapian::QueryParserError ("error parsing " + query_string);
+    }
+#else
+    throw Xapian::QueryParserError ("sexp query parser not available");
+#endif
+
+    return output;
+
+}
diff --git a/lib/sexp-fp.h b/lib/sexp-fp.h
new file mode 100644
index 00000000..341dfa7e
--- /dev/null
+++ b/lib/sexp-fp.h
@@ -0,0 +1,41 @@
+/* sexp-fp.h - sexp field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2022 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_SEXP_FP_H
+#define NOTMUCH_SEXP_FP_H
+
+#include <xapian.h>
+#include "notmuch.h"
+
+class SexpFieldProcessor : public Xapian::FieldProcessor {
+protected:
+    notmuch_database_t *notmuch;
+
+public:
+    SexpFieldProcessor (notmuch_database_t *notmuch_) : notmuch (notmuch_)
+    {
+    };
+
+    Xapian::Query operator() (const std::string & query_string);
+};
+
+#endif /* NOTMUCH_SEXP_FP_H */
diff --git a/lib/sha1.c b/lib/sha1.c
new file mode 100644
index 00000000..d1a76ee6
--- /dev/null
+++ b/lib/sha1.c
@@ -0,0 +1,94 @@
+/* sha1.c - Interfaces to SHA-1 hash for the notmuch mail system
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <glib.h>
+
+/* Create a hexadecimal string version of the SHA-1 digest of 'str'
+ * (including its null terminating character).
+ *
+ * This function returns a newly allocated string which the caller
+ * should free() when finished.
+ */
+char *
+_notmuch_sha1_of_string (const char *str)
+{
+    GChecksum *sha1;
+    char *digest;
+
+    sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+    g_checksum_update (sha1, (const guchar *) str, strlen (str) + 1);
+    digest = xstrdup (g_checksum_get_string (sha1));
+    g_checksum_free (sha1);
+
+    return digest;
+}
+
+/* Create a hexadecimal string version of the SHA-1 digest of the
+ * contents of the named file.
+ *
+ * This function returns a newly allocated string which the caller
+ * should free() when finished.
+ *
+ * If any error occurs while reading the file, (permission denied,
+ * file not found, etc.), this function returns NULL.
+ */
+char *
+_notmuch_sha1_of_file (const char *filename)
+{
+    FILE *file;
+
+#define BLOCK_SIZE 4096
+    unsigned char block[BLOCK_SIZE];
+    size_t bytes_read;
+    GChecksum *sha1;
+    char *digest = NULL;
+
+    file = fopen (filename, "r");
+    if (file == NULL)
+	return NULL;
+
+    sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+    if (sha1 == NULL)
+	goto DONE;
+
+    while (1) {
+	bytes_read = fread (block, 1, 4096, file);
+	if (bytes_read == 0) {
+	    if (feof (file))
+		break;
+	    else if (ferror (file))
+		goto DONE;
+	} else {
+	    g_checksum_update (sha1, block, bytes_read);
+	}
+    }
+
+    digest = xstrdup (g_checksum_get_string (sha1));
+
+  DONE:
+    if (sha1)
+	g_checksum_free (sha1);
+    if (file)
+	fclose (file);
+
+    return digest;
+}
diff --git a/lib/string-list.c b/lib/string-list.c
new file mode 100644
index 00000000..f3dac675
--- /dev/null
+++ b/lib/string-list.c
@@ -0,0 +1,101 @@
+/* strings.c - Iterator for a list of strings
+ *
+ * Copyright © 2010 Intel Corporation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ *         Austin Clements <aclements@csail.mit.edu>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_list_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+notmuch_string_list_t *
+_notmuch_string_list_create (const void *ctx)
+{
+    notmuch_string_list_t *list;
+
+    list = talloc (ctx, notmuch_string_list_t);
+    if (unlikely (list == NULL))
+	return NULL;
+
+    list->length = 0;
+    list->head = NULL;
+    list->tail = &list->head;
+
+    return list;
+}
+
+int
+_notmuch_string_list_length (notmuch_string_list_t *list)
+{
+    return list->length;
+}
+
+void
+_notmuch_string_list_append (notmuch_string_list_t *list,
+			     const char *string)
+{
+    /* Create and initialize new node. */
+    notmuch_string_node_t *node = talloc (list, notmuch_string_node_t);
+
+    node->string = talloc_strdup (node, string);
+    node->next = NULL;
+
+    /* Append the node to the list. */
+    *(list->tail) = node;
+    list->tail = &node->next;
+    list->length++;
+}
+
+static int
+cmpnode (const void *pa, const void *pb)
+{
+    notmuch_string_node_t *a = *(notmuch_string_node_t *const *) pa;
+    notmuch_string_node_t *b = *(notmuch_string_node_t *const *) pb;
+
+    return strcmp (a->string, b->string);
+}
+
+void
+_notmuch_string_list_sort (notmuch_string_list_t *list)
+{
+    notmuch_string_node_t **nodes, *node;
+    int i;
+
+    if (list->length == 0)
+	return;
+
+    nodes = talloc_array (list, notmuch_string_node_t *, list->length);
+    if (unlikely (nodes == NULL))
+	INTERNAL_ERROR ("Could not allocate memory for list sort");
+
+    for (i = 0, node = list->head; node; i++, node = node->next)
+	nodes[i] = node;
+
+    qsort (nodes, list->length, sizeof (*nodes), cmpnode);
+
+    for (i = 0; i < list->length - 1; ++i)
+	nodes[i]->next = nodes[i + 1];
+    nodes[i]->next = NULL;
+    list->head = nodes[0];
+    list->tail = &nodes[i]->next;
+
+    talloc_free (nodes);
+}
diff --git a/lib/string-map.c b/lib/string-map.c
new file mode 100644
index 00000000..99bc2ea2
--- /dev/null
+++ b/lib/string-map.c
@@ -0,0 +1,250 @@
+/* string-map.c - associative arrays of strings
+ *
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_map_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+
+typedef struct _notmuch_string_pair_t {
+    char *key;
+    char *value;
+} notmuch_string_pair_t;
+
+struct _notmuch_string_map {
+    bool sorted;
+    size_t length;
+    notmuch_string_pair_t *pairs;
+};
+
+struct _notmuch_string_map_iterator {
+    notmuch_string_pair_t *current;
+    bool exact;
+    const char *key;
+};
+
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx)
+{
+    notmuch_string_map_t *map;
+
+    map = talloc (ctx, notmuch_string_map_t);
+    if (unlikely (map == NULL))
+	return NULL;
+
+    map->length = 0;
+    map->pairs = NULL;
+    map->sorted = true;
+
+    return map;
+}
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value)
+{
+
+    map->length++;
+    map->sorted = false;
+
+    if (map->pairs)
+	map->pairs = talloc_realloc (map, map->pairs, notmuch_string_pair_t, map->length + 1);
+    else
+	map->pairs = talloc_array (map, notmuch_string_pair_t, map->length + 1);
+
+    map->pairs[map->length - 1].key = talloc_strdup (map, key);
+    map->pairs[map->length - 1].value = talloc_strdup (map, value);
+
+    /* Add sentinel */
+    map->pairs[map->length].key = NULL;
+    map->pairs[map->length].value = NULL;
+
+}
+
+static int
+cmppair (const void *pa, const void *pb)
+{
+    int cmp = 0;
+    notmuch_string_pair_t *a = (notmuch_string_pair_t *) pa;
+    notmuch_string_pair_t *b = (notmuch_string_pair_t *) pb;
+
+    cmp = strcmp (a->key, b->key);
+    if (cmp == 0)
+	cmp = strcmp (a->value, b->value);
+    return cmp;
+}
+
+static void
+_notmuch_string_map_sort (notmuch_string_map_t *map)
+{
+    if (map->length == 0)
+	return;
+
+    if (map->sorted)
+	return;
+
+    qsort (map->pairs, map->length, sizeof (notmuch_string_pair_t), cmppair);
+
+    map->sorted = true;
+}
+
+static int
+string_cmp (const char *a, const char *b, bool exact)
+{
+    if (exact)
+	return (strcmp (a, b));
+    else
+	return (strncmp (a, b, strlen (a)));
+}
+
+static notmuch_string_pair_t *
+bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, bool exact)
+{
+    size_t first = 0;
+    size_t last = len - 1;
+    size_t mid;
+
+    if (len <= 0)
+	return NULL;
+
+    while (last > first) {
+	mid = (first + last) / 2;
+	int sign = string_cmp (key, array[mid].key, exact);
+
+	if (sign <= 0)
+	    last = mid;
+	else
+	    first = mid + 1;
+    }
+
+
+    if (string_cmp (key, array[first].key, exact) == 0)
+	return array + first;
+    else
+	return NULL;
+
+}
+
+void
+_notmuch_string_map_set (notmuch_string_map_t *map,
+			 const char *key,
+			 const char *val)
+{
+    notmuch_string_pair_t *pair;
+
+    /* this means that calling string_map_set invalidates iterators */
+    _notmuch_string_map_sort (map);
+    pair = bsearch_first (map->pairs, map->length, key, true);
+    if (! pair)
+	_notmuch_string_map_append (map, key, val);
+    else {
+	talloc_free (pair->value);
+	pair->value = talloc_strdup (map->pairs, val);
+    }
+}
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
+{
+    notmuch_string_pair_t *pair;
+
+    /* this means that calling append invalidates iterators */
+    _notmuch_string_map_sort (map);
+
+    pair = bsearch_first (map->pairs, map->length, key, true);
+    if (! pair)
+	return NULL;
+
+    return pair->value;
+}
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     bool exact)
+{
+    notmuch_string_map_iterator_t *iter;
+
+    _notmuch_string_map_sort (map);
+
+    iter = talloc (map, notmuch_string_map_iterator_t);
+    if (unlikely (iter == NULL))
+	return NULL;
+
+    if (unlikely (talloc_reference (iter, map) == NULL))
+	return NULL;
+
+    iter->key = talloc_strdup (iter, key);
+    iter->exact = exact;
+    iter->current = bsearch_first (map->pairs, map->length, key, exact);
+    return iter;
+}
+
+bool
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iterator)
+{
+    if (iterator->current == NULL)
+	return false;
+
+    /* sentinel */
+    if (iterator->current->key == NULL)
+	return false;
+
+    return (0 == string_cmp (iterator->key, iterator->current->key, iterator->exact));
+
+}
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iterator)
+{
+
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return;
+
+    (iterator->current)++;
+}
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->key;
+}
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->value;
+}
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator)
+{
+    talloc_free (iterator);
+}
diff --git a/lib/tags.c b/lib/tags.c
new file mode 100644
index 00000000..ec5366ff
--- /dev/null
+++ b/lib/tags.c
@@ -0,0 +1,76 @@
+/* tags.c - Iterator for tags returned from message or thread
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+struct _notmuch_tags {
+    notmuch_string_node_t *iterator;
+};
+
+/* Create a new notmuch_tags_t object, with 'ctx' as its talloc owner.
+ * The returned iterator will talloc_steal the 'list', since the list
+ * is almost always transient.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+notmuch_tags_t *
+_notmuch_tags_create (const void *ctx, notmuch_string_list_t *list)
+{
+    notmuch_tags_t *tags;
+
+    tags = talloc (ctx, notmuch_tags_t);
+    if (unlikely (tags == NULL))
+	return NULL;
+
+    tags->iterator = list->head;
+    (void) talloc_steal (tags, list);
+
+    return tags;
+}
+
+notmuch_bool_t
+notmuch_tags_valid (notmuch_tags_t *tags)
+{
+    return tags && (tags->iterator != NULL);
+}
+
+const char *
+notmuch_tags_get (notmuch_tags_t *tags)
+{
+    if (tags->iterator == NULL)
+	return NULL;
+
+    return (char *) tags->iterator->string;
+}
+
+void
+notmuch_tags_move_to_next (notmuch_tags_t *tags)
+{
+    if (tags->iterator == NULL)
+	return;
+
+    tags->iterator = tags->iterator->next;
+}
+
+void
+notmuch_tags_destroy (notmuch_tags_t *tags)
+{
+    talloc_free (tags);
+}
diff --git a/lib/thread-fp.cc b/lib/thread-fp.cc
new file mode 100644
index 00000000..3aa9c423
--- /dev/null
+++ b/lib/thread-fp.cc
@@ -0,0 +1,58 @@
+/* thread-fp.cc - "thread:" field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2018 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "thread-fp.h"
+#include <iostream>
+
+Xapian::Query
+ThreadFieldProcessor::operator() (const std::string & str)
+{
+    notmuch_status_t status;
+    const char *thread_prefix = _find_prefix ("thread");
+
+    if (str.at (0) == '{') {
+	if (str.size () <= 1 || str.at (str.size () - 1) != '}') {
+	    throw Xapian::QueryParserError ("missing } in '" + str + "'");
+	} else {
+	    Xapian::Query subquery;
+	    Xapian::Query query;
+	    std::string msg;
+	    std::string subquery_str = str.substr (1, str.size () - 2);
+
+	    status = _notmuch_query_string_to_xapian_query (notmuch, subquery_str, subquery, msg);
+	    if (status)
+		throw Xapian::QueryParserError (msg);
+
+	    status = _notmuch_query_expand (notmuch, "thread", subquery, query, msg);
+	    if (status)
+		throw Xapian::QueryParserError (msg);
+
+	    return query;
+	}
+    } else {
+	/* literal thread id */
+	std::string term = thread_prefix + str;
+	return Xapian::Query (term);
+    }
+
+}
diff --git a/lib/thread-fp.h b/lib/thread-fp.h
new file mode 100644
index 00000000..00bf1aa2
--- /dev/null
+++ b/lib/thread-fp.h
@@ -0,0 +1,43 @@
+/* thread-fp.h - thread field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2018 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_THREAD_FP_H
+#define NOTMUCH_THREAD_FP_H
+
+#include <xapian.h>
+#include "notmuch.h"
+
+class ThreadFieldProcessor : public Xapian::FieldProcessor {
+protected:
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+public:
+    ThreadFieldProcessor (Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)
+	: parser (parser_), notmuch (notmuch_)
+    {
+    };
+
+    Xapian::Query operator() (const std::string & str);
+};
+
+#endif /* NOTMUCH_THREAD_FP_H */
diff --git a/lib/thread.cc b/lib/thread.cc
new file mode 100644
index 00000000..60e9a666
--- /dev/null
+++ b/lib/thread.cc
@@ -0,0 +1,742 @@
+/* thread.cc - Results of thread-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <gmime/gmime.h>
+#include <glib.h>                                       /* GHashTable */
+
+#ifdef DEBUG_THREADING
+#define THREAD_DEBUG(format, ...) fprintf (stderr, "DT: " format " (%s).\n", ##__VA_ARGS__, \
+					   __location__)
+#else
+#define THREAD_DEBUG(format, ...) do {} while (0)       /* ignored */
+#endif
+
+struct _notmuch_thread {
+    notmuch_database_t *notmuch;
+    char *thread_id;
+    char *subject;
+    GHashTable *authors_hash;
+    GPtrArray *authors_array;
+    GHashTable *matched_authors_hash;
+    GPtrArray *matched_authors_array;
+    char *authors;
+    GHashTable *tags;
+
+    /* All messages, oldest first. */
+    notmuch_message_list_t *message_list;
+    /* Top-level messages, oldest first. */
+    notmuch_message_list_t *toplevel_list;
+
+    GHashTable *message_hash;
+    int total_messages;
+    int total_files;
+    int matched_messages;
+    time_t oldest;
+    time_t newest;
+};
+
+static int
+_notmuch_thread_destructor (notmuch_thread_t *thread)
+{
+    g_hash_table_unref (thread->authors_hash);
+    g_hash_table_unref (thread->matched_authors_hash);
+    g_hash_table_unref (thread->tags);
+    g_hash_table_unref (thread->message_hash);
+
+    if (thread->authors_array) {
+	g_ptr_array_free (thread->authors_array, true);
+	thread->authors_array = NULL;
+    }
+
+    if (thread->matched_authors_array) {
+	g_ptr_array_free (thread->matched_authors_array, true);
+	thread->matched_authors_array = NULL;
+    }
+
+    return 0;
+}
+
+/* Add each author of the thread to the thread's authors_hash and to
+ * the thread's authors_array. */
+static void
+_thread_add_author (notmuch_thread_t *thread,
+		    const char *author)
+{
+    char *author_copy;
+
+    if (author == NULL)
+	return;
+
+    if (g_hash_table_lookup_extended (thread->authors_hash,
+				      author, NULL, NULL))
+	return;
+
+    author_copy = talloc_strdup (thread, author);
+
+    g_hash_table_insert (thread->authors_hash, author_copy, NULL);
+
+    g_ptr_array_add (thread->authors_array, author_copy);
+}
+
+/* Add each matched author of the thread to the thread's
+ * matched_authors_hash and to the thread's matched_authors_array. */
+static void
+_thread_add_matched_author (notmuch_thread_t *thread,
+			    const char *author)
+{
+    char *author_copy;
+
+    if (author == NULL)
+	return;
+
+    if (g_hash_table_lookup_extended (thread->matched_authors_hash,
+				      author, NULL, NULL))
+	return;
+
+    author_copy = talloc_strdup (thread, author);
+
+    g_hash_table_insert (thread->matched_authors_hash, author_copy, NULL);
+
+    g_ptr_array_add (thread->matched_authors_array, author_copy);
+}
+
+/* Construct an authors string from matched_authors_array and
+ * authors_array. The string contains matched authors first, then
+ * non-matched authors (with the two groups separated by '|'). Within
+ * each group, authors are listed in date order. */
+static void
+_resolve_thread_authors_string (notmuch_thread_t *thread)
+{
+    unsigned int i;
+    char *author;
+    int first_non_matched_author = 1;
+
+    /* First, list all matched authors in date order. */
+    for (i = 0; i < thread->matched_authors_array->len; i++) {
+	author = (char *) g_ptr_array_index (thread->matched_authors_array, i);
+	if (thread->authors)
+	    thread->authors = talloc_asprintf (thread, "%s, %s",
+					       thread->authors,
+					       author);
+	else
+	    thread->authors = author;
+    }
+
+    /* Next, append any non-matched authors that haven't already appeared. */
+    for (i = 0; i < thread->authors_array->len; i++) {
+	author = (char *) g_ptr_array_index (thread->authors_array, i);
+	if (g_hash_table_lookup_extended (thread->matched_authors_hash,
+					  author, NULL, NULL))
+	    continue;
+	if (first_non_matched_author) {
+	    thread->authors = talloc_asprintf (thread, "%s| %s",
+					       thread->authors,
+					       author);
+	} else {
+	    thread->authors = talloc_asprintf (thread, "%s, %s",
+					       thread->authors,
+					       author);
+	}
+
+	first_non_matched_author = 0;
+    }
+
+    g_ptr_array_free (thread->authors_array, true);
+    thread->authors_array = NULL;
+    g_ptr_array_free (thread->matched_authors_array, true);
+    thread->matched_authors_array = NULL;
+
+    if (! thread->authors)
+	thread->authors = talloc_strdup (thread, "");
+}
+
+/* clean up the ugly "Lastname, Firstname" format that some mail systems
+ * (most notably, Exchange) are creating to be "Firstname Lastname"
+ * To make sure that we don't change other potential situations where a
+ * comma is in the name, we check that we match one of these patterns
+ * "Last, First" <first.last@company.com>
+ * "Last, First MI" <first.mi.last@company.com>
+ */
+static char *
+_thread_cleanup_author (notmuch_thread_t *thread,
+			const char *author, const char *from)
+{
+    char *clean_author, *test_author;
+    const char *comma;
+    char *blank;
+    int fname, lname;
+
+    if (author == NULL)
+	return NULL;
+    clean_author = talloc_strdup (thread, author);
+    if (clean_author == NULL)
+	return NULL;
+    /* check if there's a comma in the name and that there's a
+     * component of the name behind it (so the name doesn't end with
+     * the comma - in which case the string that strchr finds is just
+     * one character long ",\0").
+     * Otherwise just return the copy of the original author name that
+     * we just made*/
+    comma = strchr (author, ',');
+    if (comma && strlen (comma) > 1) {
+	/* let's assemble what we think is the correct name */
+	lname = comma - author;
+
+	/* Skip all the spaces after the comma */
+	fname = strlen (author) - lname - 1;
+	comma += 1;
+	while (*comma == ' ') {
+	    fname -= 1;
+	    comma += 1;
+	}
+	strncpy (clean_author, comma, fname);
+
+	*(clean_author + fname) = ' ';
+	strncpy (clean_author + fname + 1, author, lname);
+	*(clean_author + fname + 1 + lname) = '\0';
+	/* make a temporary copy and see if it matches the email */
+	test_author = talloc_strdup (thread, clean_author);
+
+	blank = strchr (test_author, ' ');
+	while (blank != NULL) {
+	    *blank = '.';
+	    blank = strchr (test_author, ' ');
+	}
+	if (strcasestr (from, test_author) == NULL)
+	    /* we didn't identify this as part of the email address
+	     * so let's punt and return the original author */
+	    strcpy (clean_author, author);
+    }
+    return clean_author;
+}
+
+/* Add 'message' as a message that belongs to 'thread'.
+ *
+ * The 'thread' will talloc_steal the 'message' and hold onto a
+ * reference to it.
+ */
+static void
+_thread_add_message (notmuch_thread_t *thread,
+		     notmuch_message_t *message,
+		     notmuch_string_list_t *exclude_terms,
+		     notmuch_exclude_t omit_exclude)
+{
+    notmuch_tags_t *tags;
+    const char *tag;
+    InternetAddressList *list = NULL;
+    InternetAddress *address;
+    const char *from, *author;
+    char *clean_author;
+    bool message_excluded = false;
+
+    if (omit_exclude != NOTMUCH_EXCLUDE_FALSE) {
+	for (tags = notmuch_message_get_tags (message);
+	     notmuch_tags_valid (tags);
+	     notmuch_tags_move_to_next (tags)) {
+	    tag = notmuch_tags_get (tags);
+	    /* Is message excluded? */
+	    for (notmuch_string_node_t *term = exclude_terms->head;
+		 term != NULL;
+		 term = term->next) {
+		/* Check for an empty string, and then ignore initial 'K'. */
+		if (*(term->string) && strcmp (tag, (term->string + 1)) == 0) {
+		    message_excluded = true;
+		    break;
+		}
+	    }
+	}
+    }
+
+    if (message_excluded && omit_exclude == NOTMUCH_EXCLUDE_ALL)
+	return;
+
+    _notmuch_message_list_add_message (thread->message_list,
+				       talloc_steal (thread, message));
+    thread->total_messages++;
+    thread->total_files += notmuch_message_count_files (message);
+
+    g_hash_table_insert (thread->message_hash,
+			 xstrdup (notmuch_message_get_message_id (message)),
+			 message);
+
+    from = notmuch_message_get_header (message, "from");
+    if (from)
+	list = internet_address_list_parse (NULL, from);
+
+    if (list) {
+	address = internet_address_list_get_address (list, 0);
+	if (address) {
+	    author = internet_address_get_name (address);
+	    /* We treat quoted empty names as if they were empty. */
+	    if (author == NULL || author[0] == '\0') {
+		InternetAddressMailbox *mailbox;
+		mailbox = INTERNET_ADDRESS_MAILBOX (address);
+		author = internet_address_mailbox_get_addr (mailbox);
+	    }
+	    clean_author = _thread_cleanup_author (thread, author, from);
+	    _thread_add_author (thread, clean_author);
+	    _notmuch_message_set_author (message, clean_author);
+	}
+	g_object_unref (G_OBJECT (list));
+    }
+
+    if (! thread->subject) {
+	const char *subject;
+	subject = notmuch_message_get_header (message, "subject");
+	thread->subject = talloc_strdup (thread, subject ? subject : "");
+    }
+
+    for (tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags)) {
+	tag = notmuch_tags_get (tags);
+	g_hash_table_insert (thread->tags, xstrdup (tag), NULL);
+    }
+
+    /* Mark excluded messages. */
+    if (message_excluded)
+	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
+}
+
+static void
+_thread_set_subject_from_message (notmuch_thread_t *thread,
+				  notmuch_message_t *message)
+{
+    const char *subject;
+    const char *cleaned_subject;
+
+    subject = notmuch_message_get_header (message, "subject");
+    if (! subject)
+	return;
+
+    if ((strncasecmp (subject, "Re: ", 4) == 0) ||
+	(strncasecmp (subject, "Aw: ", 4) == 0) ||
+	(strncasecmp (subject, "Vs: ", 4) == 0) ||
+	(strncasecmp (subject, "Sv: ", 4) == 0)) {
+
+	cleaned_subject = talloc_strndup (thread,
+					  subject + 4,
+					  strlen (subject) - 4);
+    } else {
+	cleaned_subject = talloc_strdup (thread, subject);
+    }
+
+    if (! EMPTY_STRING (cleaned_subject)) {
+	if (thread->subject)
+	    talloc_free (thread->subject);
+
+	thread->subject = talloc_strdup (thread, cleaned_subject);
+    }
+}
+
+/* Add a message to this thread which is known to match the original
+ * search specification. The 'sort' parameter controls whether the
+ * oldest or newest matching subject is applied to the thread as a
+ * whole. Returns 0 on success.
+ */
+static int
+_thread_add_matched_message (notmuch_thread_t *thread,
+			     notmuch_message_t *message,
+			     notmuch_sort_t sort)
+{
+    time_t date;
+    notmuch_message_t *hashed_message;
+    notmuch_bool_t is_set;
+
+    date = notmuch_message_get_date (message);
+
+    if (date < thread->oldest || ! thread->matched_messages) {
+	thread->oldest = date;
+	if (sort == NOTMUCH_SORT_OLDEST_FIRST)
+	    _thread_set_subject_from_message (thread, message);
+    }
+
+    if (date > thread->newest || ! thread->matched_messages) {
+	thread->newest = date;
+	const char *cur_subject = notmuch_thread_get_subject (thread);
+	if (sort != NOTMUCH_SORT_OLDEST_FIRST || EMPTY_STRING (cur_subject))
+	    _thread_set_subject_from_message (thread, message);
+    }
+
+    if (notmuch_message_get_flag_st (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, &is_set))
+	return -1;
+    if (! is_set)
+	thread->matched_messages++;
+
+    if (g_hash_table_lookup_extended (thread->message_hash,
+				      notmuch_message_get_message_id (message), NULL,
+				      (void **) &hashed_message)) {
+	notmuch_message_set_flag (hashed_message,
+				  NOTMUCH_MESSAGE_FLAG_MATCH, 1);
+    }
+
+    _thread_add_matched_author (thread, _notmuch_message_get_author (hashed_message));
+    return 0;
+}
+
+static bool
+_parent_via_in_reply_to (notmuch_thread_t *thread, notmuch_message_t *message)
+{
+    notmuch_message_t *parent;
+    const char *in_reply_to;
+
+    in_reply_to = _notmuch_message_get_in_reply_to (message);
+    THREAD_DEBUG ("checking message = %s in_reply_to=%s",
+		  notmuch_message_get_message_id (message), in_reply_to);
+
+    if (in_reply_to && (! EMPTY_STRING (in_reply_to)) &&
+	g_hash_table_lookup_extended (thread->message_hash,
+				      in_reply_to, NULL,
+				      (void **) &parent)) {
+	_notmuch_message_add_reply (parent, message);
+	return true;
+    } else {
+	return false;
+    }
+}
+
+static void
+_parent_or_toplevel (notmuch_thread_t *thread, notmuch_message_t *message)
+{
+    size_t max_depth = 0;
+    notmuch_message_t *new_parent;
+    notmuch_message_t *parent = NULL;
+    const notmuch_string_list_t *references =
+	_notmuch_message_get_references (message);
+
+    THREAD_DEBUG ("trying to reparent via references: %s",
+		  notmuch_message_get_message_id (message));
+
+    for (notmuch_string_node_t *ref_node = references->head;
+	 ref_node; ref_node = ref_node->next) {
+	THREAD_DEBUG ("checking reference=%s", ref_node->string);
+	if ((g_hash_table_lookup_extended (thread->message_hash,
+					   ref_node->string, NULL,
+					   (void **) &new_parent))) {
+	    size_t new_depth = _notmuch_message_get_thread_depth (new_parent);
+	    THREAD_DEBUG ("got depth %lu", new_depth);
+	    if (new_depth > max_depth || ! parent) {
+		THREAD_DEBUG ("adding at depth %lu parent=%s", new_depth, ref_node->string);
+		max_depth = new_depth;
+		parent = new_parent;
+	    }
+	}
+    }
+    if (parent) {
+	THREAD_DEBUG ("adding reply %s to parent=%s",
+		      notmuch_message_get_message_id (message),
+		      notmuch_message_get_message_id (parent));
+	_notmuch_message_add_reply (parent, message);
+    } else {
+	THREAD_DEBUG ("adding as toplevel %s",
+		      notmuch_message_get_message_id (message));
+	_notmuch_message_list_add_message (thread->toplevel_list, message);
+    }
+}
+
+static void
+_resolve_thread_relationships (notmuch_thread_t *thread)
+{
+    notmuch_message_node_t *node, *first_node;
+    notmuch_message_t *message;
+    void *local;
+    notmuch_message_list_t *maybe_toplevel_list;
+
+    first_node = thread->message_list->head;
+    if (! first_node)
+	return;
+
+    local = talloc_new (thread);
+    maybe_toplevel_list = _notmuch_message_list_create (local);
+
+    for (node = first_node->next; node; node = node->next) {
+	message = node->message;
+	if (! _parent_via_in_reply_to (thread, message))
+	    _notmuch_message_list_add_message (maybe_toplevel_list, message);
+    }
+
+    /*
+     * if we reach the end of the list without finding a top-level
+     * message, that means the thread is a cycle (or set of cycles)
+     * and any message can be considered top-level.  Choose the oldest
+     * message, which happens to be first in our list.
+     */
+    if (first_node) {
+	message = first_node->message;
+	THREAD_DEBUG ("checking first message  %s",
+		      notmuch_message_get_message_id (message));
+
+	if (_notmuch_message_list_empty (maybe_toplevel_list) ||
+	    ! _parent_via_in_reply_to (thread, message)) {
+
+	    THREAD_DEBUG ("adding first message as toplevel = %s",
+			  notmuch_message_get_message_id (message));
+	    _notmuch_message_list_add_message (maybe_toplevel_list, message);
+	}
+    }
+
+    for (notmuch_messages_t *messages = _notmuch_messages_create (maybe_toplevel_list);
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	_notmuch_message_label_depths (message, 0);
+    }
+
+    for (notmuch_messages_t *roots = _notmuch_messages_create (maybe_toplevel_list);
+	 notmuch_messages_valid (roots);
+	 notmuch_messages_move_to_next (roots)) {
+	notmuch_message_t *message = notmuch_messages_get (roots);
+	if (_notmuch_messages_has_next (roots) || ! _notmuch_message_list_empty (
+		thread->toplevel_list))
+	    _parent_or_toplevel (thread, message);
+	else
+	    _notmuch_message_list_add_message (thread->toplevel_list, message);
+    }
+
+    /* XXX this could be made conditional on messages being inserted
+     * (out of order) in later passes
+     */
+    thread->toplevel_list = _notmuch_message_sort_subtrees (thread, thread->toplevel_list);
+
+    talloc_free (local);
+}
+
+/* Create a new notmuch_thread_t object by finding the thread
+ * containing the message with the given doc ID, treating any messages
+ * contained in match_set as "matched".  Remove all messages in the
+ * thread from match_set.
+ *
+ * Creating the thread will perform a database search to get all
+ * messages belonging to the thread and will get the first subject
+ * line, the total count of messages, and all authors in the thread.
+ * Each message in the thread is checked against match_set to allow
+ * for a separate count of matched messages, and to allow a viewer to
+ * display these messages differently.
+ *
+ * Here, 'ctx' is talloc context for the resulting thread object.
+ *
+ * This function returns NULL in the case of any error.
+ */
+notmuch_thread_t *
+_notmuch_thread_create (void *ctx,
+			notmuch_database_t *notmuch,
+			unsigned int seed_doc_id,
+			notmuch_doc_id_set_t *match_set,
+			notmuch_string_list_t *exclude_terms,
+			notmuch_exclude_t omit_excluded,
+			notmuch_sort_t sort)
+{
+    void *local = talloc_new (ctx);
+    notmuch_thread_t *thread = NULL;
+    notmuch_message_t *seed_message;
+    const char *thread_id;
+    char *thread_id_query_string;
+    notmuch_query_t *thread_id_query;
+
+    notmuch_messages_t *messages;
+    notmuch_message_t *message;
+    notmuch_status_t status;
+
+    seed_message = _notmuch_message_create (local, notmuch, seed_doc_id, NULL);
+    if (! seed_message)
+	INTERNAL_ERROR ("Thread seed message %u does not exist", seed_doc_id);
+
+    thread_id = notmuch_message_get_thread_id (seed_message);
+    thread_id_query_string = talloc_asprintf (local, "thread:%s", thread_id);
+    if (unlikely (thread_id_query_string == NULL))
+	goto DONE;
+
+    thread_id_query = talloc_steal (
+	local, notmuch_query_create (notmuch, thread_id_query_string));
+    if (unlikely (thread_id_query == NULL))
+	goto DONE;
+
+    thread = talloc (local, notmuch_thread_t);
+    if (unlikely (thread == NULL))
+	goto DONE;
+
+    talloc_set_destructor (thread, _notmuch_thread_destructor);
+
+    thread->notmuch = notmuch;
+    thread->thread_id = talloc_strdup (thread, thread_id);
+    thread->subject = NULL;
+    thread->authors_hash = g_hash_table_new_full (g_str_hash, g_str_equal,
+						  NULL, NULL);
+    thread->authors_array = g_ptr_array_new ();
+    thread->matched_authors_hash = g_hash_table_new_full (g_str_hash,
+							  g_str_equal,
+							  NULL, NULL);
+    thread->matched_authors_array = g_ptr_array_new ();
+    thread->authors = NULL;
+    thread->tags = g_hash_table_new_full (g_str_hash, g_str_equal,
+					  free, NULL);
+
+    thread->message_list = _notmuch_message_list_create (thread);
+    thread->toplevel_list = _notmuch_message_list_create (thread);
+    if (unlikely (thread->message_list == NULL ||
+		  thread->toplevel_list == NULL)) {
+	thread = NULL;
+	goto DONE;
+    }
+
+    thread->message_hash = g_hash_table_new_full (g_str_hash, g_str_equal,
+						  free, NULL);
+
+    thread->total_messages = 0;
+    thread->total_files = 0;
+    thread->matched_messages = 0;
+    thread->oldest = 0;
+    thread->newest = 0;
+
+    /* We use oldest-first order unconditionally here to obtain the
+     * proper author ordering for the thread. The 'sort' parameter
+     * passed to this function is used only to indicate whether the
+     * oldest or newest subject is desired. */
+    notmuch_query_set_sort (thread_id_query, NOTMUCH_SORT_OLDEST_FIRST);
+
+    status = notmuch_query_search_messages (thread_id_query, &messages);
+    if (status)
+	goto DONE;
+
+    for (;
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	unsigned int doc_id;
+
+	message = notmuch_messages_get (messages);
+	doc_id = _notmuch_message_get_doc_id (message);
+	if (doc_id == seed_doc_id)
+	    message = seed_message;
+
+	_thread_add_message (thread, message, exclude_terms, omit_excluded);
+
+	if ( _notmuch_doc_id_set_contains (match_set, doc_id)) {
+	    _notmuch_doc_id_set_remove (match_set, doc_id);
+	    if (_thread_add_matched_message (thread, message, sort)) {
+		thread = NULL;
+		goto DONE;
+	    }
+	}
+
+	_notmuch_message_close (message);
+    }
+
+    _resolve_thread_authors_string (thread);
+
+    _resolve_thread_relationships (thread);
+
+    /* Commit to returning thread. */
+    (void) talloc_steal (ctx, thread);
+
+  DONE:
+    talloc_free (local);
+    return thread;
+}
+
+notmuch_messages_t *
+notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread)
+{
+    return _notmuch_messages_create (thread->toplevel_list);
+}
+
+notmuch_messages_t *
+notmuch_thread_get_messages (notmuch_thread_t *thread)
+{
+    return _notmuch_messages_create (thread->message_list);
+}
+
+const char *
+notmuch_thread_get_thread_id (notmuch_thread_t *thread)
+{
+    return thread->thread_id;
+}
+
+int
+notmuch_thread_get_total_messages (notmuch_thread_t *thread)
+{
+    return thread->total_messages;
+}
+
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread)
+{
+    return thread->total_files;
+}
+
+int
+notmuch_thread_get_matched_messages (notmuch_thread_t *thread)
+{
+    return thread->matched_messages;
+}
+
+const char *
+notmuch_thread_get_authors (notmuch_thread_t *thread)
+{
+    return thread->authors;
+}
+
+const char *
+notmuch_thread_get_subject (notmuch_thread_t *thread)
+{
+    return thread->subject;
+}
+
+time_t
+notmuch_thread_get_oldest_date (notmuch_thread_t *thread)
+{
+    return thread->oldest;
+}
+
+time_t
+notmuch_thread_get_newest_date (notmuch_thread_t *thread)
+{
+    return thread->newest;
+}
+
+notmuch_tags_t *
+notmuch_thread_get_tags (notmuch_thread_t *thread)
+{
+    notmuch_string_list_t *tags;
+    GList *keys, *l;
+
+    tags = _notmuch_string_list_create (thread);
+    if (unlikely (tags == NULL))
+	return NULL;
+
+    keys = g_hash_table_get_keys (thread->tags);
+
+    for (l = keys; l; l = l->next)
+	_notmuch_string_list_append (tags, (char *) l->data);
+
+    g_list_free (keys);
+
+    _notmuch_string_list_sort (tags);
+
+    return _notmuch_tags_create (thread, tags);
+}
+
+void
+notmuch_thread_destroy (notmuch_thread_t *thread)
+{
+    talloc_free (thread);
+}