Import notmuch_0.28.2.orig.tar.gz

[dgit import orig notmuch_0.28.2.orig.tar.gz]

34 files changed, 13080 insertions, 0 deletions

diff --git a/lib/Makefile b/lib/Makefile
new file mode 100644
index 00000000..de492a7c
--- /dev/null
+++ b/lib/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C .. all
+
+.DEFAULT:
+	$(MAKE) -C .. $@
diff --git a/lib/Makefile.local b/lib/Makefile.local
new file mode 100644
index 00000000..5dc057c0
--- /dev/null
+++ b/lib/Makefile.local
@@ -0,0 +1,92 @@
+# -*- makefile -*-
+
+dir := lib
+
+# The (often-reused) $dir works fine within targets/prerequisites,
+# but cannot be used reliably within commands, so copy its value to a
+# variable that is not reused.
+lib := $(dir)
+
+ifeq ($(PLATFORM),MACOSX)
+LIBRARY_SUFFIX = dylib
+# On OS X, library version numbers go before suffix.
+LINKER_NAME = libnotmuch.$(LIBRARY_SUFFIX)
+SONAME = libnotmuch.$(LIBNOTMUCH_VERSION_MAJOR).$(LIBRARY_SUFFIX)
+LIBNAME = libnotmuch.$(LIBNOTMUCH_VERSION_MAJOR).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE).$(LIBRARY_SUFFIX)
+LIBRARY_LINK_FLAG = -dynamiclib -install_name $(libdir)/$(SONAME) -compatibility_version $(LIBNOTMUCH_VERSION_MAJOR).$(LIBNOTMUCH_VERSION_MINOR) -current_version $(LIBNOTMUCH_VERSION_MAJOR).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE)
+else
+LIBRARY_SUFFIX = so
+LINKER_NAME = libnotmuch.$(LIBRARY_SUFFIX)
+SONAME = $(LINKER_NAME).$(LIBNOTMUCH_VERSION_MAJOR)
+LIBNAME = $(SONAME).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE)
+LIBRARY_LINK_FLAG = -shared -Wl,--version-script=$(srcdir)/$(lib)/notmuch.sym,-soname=$(SONAME) $(NO_UNDEFINED_LDFLAGS)
+ifeq ($(PLATFORM),OPENBSD)
+LIBRARY_LINK_FLAG += -lc
+endif
+ifeq ($(LIBDIR_IN_LDCONFIG),1)
+ifeq ($(DESTDIR),)
+LIBRARY_INSTALL_POST_COMMAND=ldconfig
+endif
+endif
+endif
+
+extra_cflags += -I$(srcdir)/$(dir) -fPIC -fvisibility=hidden
+extra_cxxflags += -fvisibility-inlines-hidden
+
+libnotmuch_c_srcs =		\
+	$(notmuch_compat_srcs)	\
+	$(dir)/filenames.c	\
+	$(dir)/string-list.c	\
+	$(dir)/message-file.c	\
+	$(dir)/message-id.c	\
+	$(dir)/messages.c	\
+	$(dir)/sha1.c		\
+	$(dir)/built-with.c	\
+	$(dir)/string-map.c	\
+	$(dir)/indexopts.c	\
+	$(dir)/tags.c
+
+libnotmuch_cxx_srcs =		\
+	$(dir)/database.cc	\
+	$(dir)/parse-time-vrp.cc	\
+	$(dir)/directory.cc	\
+	$(dir)/index.cc		\
+	$(dir)/message.cc	\
+	$(dir)/add-message.cc	\
+	$(dir)/message-property.cc \
+	$(dir)/query.cc		\
+	$(dir)/query-fp.cc      \
+	$(dir)/config.cc	\
+	$(dir)/regexp-fields.cc	\
+	$(dir)/thread.cc \
+	$(dir)/thread-fp.cc
+
+libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o)
+
+$(dir)/libnotmuch.a: $(libnotmuch_modules)
+	$(call quiet,AR) rcs $@ $^
+
+$(dir)/$(LIBNAME): $(libnotmuch_modules) util/libnotmuch_util.a parse-time-string/libparse-time-string.a
+	$(call quiet,CXX $(CXXFLAGS)) $(libnotmuch_modules) $(FINAL_LIBNOTMUCH_LDFLAGS) $(LIBRARY_LINK_FLAG) -o $@ util/libnotmuch_util.a parse-time-string/libparse-time-string.a
+
+$(dir)/$(SONAME): $(dir)/$(LIBNAME)
+	ln -sf $(LIBNAME) $@
+
+$(dir)/$(LINKER_NAME): $(dir)/$(SONAME)
+	ln -sf $(LIBNAME) $@
+
+install: install-$(dir)
+
+install-$(dir): $(dir)/$(LIBNAME)
+	mkdir -p "$(DESTDIR)$(libdir)/"
+	install -m0644 "$(lib)/$(LIBNAME)" "$(DESTDIR)$(libdir)/"
+	ln -sf $(LIBNAME) "$(DESTDIR)$(libdir)/$(SONAME)"
+	ln -sf $(LIBNAME) "$(DESTDIR)$(libdir)/$(LINKER_NAME)"
+	mkdir -p "$(DESTDIR)$(includedir)"
+	install -m0644 "$(srcdir)/$(lib)/notmuch.h" "$(DESTDIR)$(includedir)/"
+	$(LIBRARY_INSTALL_POST_COMMAND)
+
+SRCS  := $(SRCS) $(libnotmuch_c_srcs) $(libnotmuch_cxx_srcs)
+CLEAN += $(libnotmuch_modules) $(dir)/$(SONAME) $(dir)/$(LINKER_NAME)
+CLEAN += $(dir)/$(LIBNAME) $(dir)/libnotmuch.a
+CLEAN += $(dir)/notmuch.h.gch
diff --git a/lib/add-message.cc b/lib/add-message.cc
new file mode 100644
index 00000000..da37032c
--- /dev/null
+++ b/lib/add-message.cc
@@ -0,0 +1,608 @@
+#include "database-private.h"
+
+/* Parse a References header value, putting a (talloc'ed under 'ctx')
+ * copy of each referenced message-id into 'hash'.
+ *
+ * We explicitly avoid including any reference identical to
+ * 'message_id' in the result (to avoid mass confusion when a single
+ * message references itself cyclically---and yes, mail messages are
+ * not infrequent in the wild that do this---don't ask me why).
+ *
+ * Return the last reference parsed, if it is not equal to message_id.
+ */
+static char *
+parse_references (void *ctx,
+		  const char *message_id,
+		  GHashTable *hash,
+		  const char *refs)
+{
+    char *ref, *last_ref = NULL;
+
+    if (refs == NULL || *refs == '\0')
+	return NULL;
+
+    while (*refs) {
+	ref = _notmuch_message_id_parse (ctx, refs, &refs);
+
+	if (ref && strcmp (ref, message_id)) {
+	    g_hash_table_add (hash, ref);
+	    last_ref = ref;
+	}
+    }
+
+    /* The return value of this function is used to add a parent
+     * reference to the database.  We should avoid making a message
+     * its own parent, thus the above check.
+     */
+    return talloc_strdup(ctx, last_ref);
+}
+
+static const char *
+_notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
+{
+    /* 16 bytes (+ terminator) for hexadecimal representation of
+     * a 64-bit integer. */
+    static char thread_id[17];
+    Xapian::WritableDatabase *db;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    notmuch->last_thread_id++;
+
+    sprintf (thread_id, "%016" PRIx64, notmuch->last_thread_id);
+
+    db->set_metadata ("last_thread_id", thread_id);
+
+    return thread_id;
+}
+
+static char *
+_get_metadata_thread_id_key (void *ctx, const char *message_id)
+{
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (ctx, message_id);
+
+    return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
+			    message_id);
+}
+
+
+static notmuch_status_t
+_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
+				      void *ctx,
+				      const char *message_id,
+				      const char **thread_id_ret);
+
+
+/* Find the thread ID to which the message with 'message_id' belongs.
+ *
+ * Note: 'thread_id_ret' must not be NULL!
+ * On success '*thread_id_ret' is set to a newly talloced string belonging to
+ * 'ctx'.
+ *
+ * Note: If there is no message in the database with the given
+ * 'message_id' then a new thread_id will be allocated for this
+ * message ID and stored in the database metadata so that the
+ * thread ID can be looked up if the message is added to the database
+ * later.
+ */
+static notmuch_status_t
+_resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
+				  void *ctx,
+				  const char *message_id,
+				  const char **thread_id_ret)
+{
+    notmuch_private_status_t status;
+    notmuch_message_t *message;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
+	return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
+						     thread_id_ret);
+
+    /* Look for this message (regular or ghost) */
+    message = _notmuch_message_create_for_message_id (
+	notmuch, message_id, &status);
+    if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+	/* Message exists */
+	*thread_id_ret = talloc_steal (
+	    ctx, notmuch_message_get_thread_id (message));
+    } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	/* Message did not exist.  Give it a fresh thread ID and
+	 * populate this message as a ghost message. */
+	*thread_id_ret = talloc_strdup (
+	    ctx, _notmuch_database_generate_thread_id (notmuch));
+	if (! *thread_id_ret) {
+	    status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	} else {
+	    status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
+	    if (status == 0)
+		/* Commit the new ghost message */
+		_notmuch_message_sync (message);
+	}
+    } else {
+	/* Create failed. Fall through. */
+    }
+
+    notmuch_message_destroy (message);
+
+    return COERCE_STATUS (status, "Error creating ghost message");
+}
+
+/* Pre-ghost messages _resolve_message_id_to_thread_id */
+static notmuch_status_t
+_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
+				      void *ctx,
+				      const char *message_id,
+				      const char **thread_id_ret)
+{
+    notmuch_status_t status;
+    notmuch_message_t *message;
+    std::string thread_id_string;
+    char *metadata_key;
+    Xapian::WritableDatabase *db;
+
+    status = notmuch_database_find_message (notmuch, message_id, &message);
+
+    if (status)
+	return status;
+
+    if (message) {
+	*thread_id_ret = talloc_steal (ctx,
+				       notmuch_message_get_thread_id (message));
+
+	notmuch_message_destroy (message);
+
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    /* Message has not been seen yet.
+     *
+     * We may have seen a reference to it already, in which case, we
+     * can return the thread ID stored in the metadata. Otherwise, we
+     * generate a new thread ID and store it there.
+     */
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
+    thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
+
+    if (thread_id_string.empty()) {
+	*thread_id_ret = talloc_strdup (ctx,
+					_notmuch_database_generate_thread_id (notmuch));
+	db->set_metadata (metadata_key, *thread_id_ret);
+    } else {
+	*thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str());
+    }
+
+    talloc_free (metadata_key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_merge_threads (notmuch_database_t *notmuch,
+		const char *winner_thread_id,
+		const char *loser_thread_id)
+{
+    Xapian::PostingIterator loser, loser_end;
+    notmuch_message_t *message = NULL;
+    notmuch_private_status_t private_status;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    _notmuch_database_find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);
+
+    for ( ; loser != loser_end; loser++) {
+	message = _notmuch_message_create (notmuch, notmuch,
+					   *loser, &private_status);
+	if (message == NULL) {
+	    ret = COERCE_STATUS (private_status,
+				 "Cannot find document for doc_id from query");
+	    goto DONE;
+	}
+
+	_notmuch_message_remove_term (message, "thread", loser_thread_id);
+	_notmuch_message_add_term (message, "thread", winner_thread_id);
+	_notmuch_message_sync (message);
+
+	notmuch_message_destroy (message);
+	message = NULL;
+    }
+
+  DONE:
+    if (message)
+	notmuch_message_destroy (message);
+
+    return ret;
+}
+
+static void
+_my_talloc_free_for_g_hash (void *ptr)
+{
+    talloc_free (ptr);
+}
+
+notmuch_status_t
+_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
+					   notmuch_message_t *message,
+					   notmuch_message_file_t *message_file,
+					   const char **thread_id)
+{
+    GHashTable *parents = NULL;
+    const char *refs, *in_reply_to, *in_reply_to_message_id, *strict_message_id = NULL;
+    const char *last_ref_message_id, *this_message_id;
+    GList *l, *keys = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    parents = g_hash_table_new_full (g_str_hash, g_str_equal,
+				     _my_talloc_free_for_g_hash, NULL);
+    this_message_id = notmuch_message_get_message_id (message);
+
+    refs = _notmuch_message_file_get_header (message_file, "references");
+    last_ref_message_id = parse_references (message,
+					    this_message_id,
+					    parents, refs);
+
+    in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
+    if (in_reply_to)
+	strict_message_id = _notmuch_message_id_parse_strict (message,
+							      in_reply_to);
+
+    in_reply_to_message_id = parse_references (message,
+					       this_message_id,
+					       parents, in_reply_to);
+
+    /* For the parent of this message, use
+     * 1) the In-Reply-To header, if it looks sane, otherwise
+     * 2) the last message ID of the References header, if available.
+     * 3) Otherwise, fall back to the first message ID in
+     * the In-Reply-To header.
+     */
+
+    if (strict_message_id) {
+	_notmuch_message_add_term (message, "replyto", strict_message_id);
+    } else if (last_ref_message_id) {
+	_notmuch_message_add_term (message, "replyto",
+				   last_ref_message_id);
+    } else if (in_reply_to_message_id) {
+	_notmuch_message_add_term (message, "replyto",
+			     in_reply_to_message_id);
+    }
+
+    keys = g_hash_table_get_keys (parents);
+    for (l = keys; l; l = l->next) {
+	char *parent_message_id;
+	const char *parent_thread_id = NULL;
+
+	parent_message_id = (char *) l->data;
+
+	_notmuch_message_add_term (message, "reference",
+				   parent_message_id);
+
+	ret = _resolve_message_id_to_thread_id (notmuch,
+						message,
+						parent_message_id,
+						&parent_thread_id);
+	if (ret)
+	    goto DONE;
+
+	if (*thread_id == NULL) {
+	    *thread_id = talloc_strdup (message, parent_thread_id);
+	    _notmuch_message_add_term (message, "thread", *thread_id);
+	} else if (strcmp (*thread_id, parent_thread_id)) {
+	    ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
+	    if (ret)
+		goto DONE;
+	}
+    }
+
+  DONE:
+    if (keys)
+	g_list_free (keys);
+    if (parents)
+	g_hash_table_unref (parents);
+
+    return ret;
+}
+
+static notmuch_status_t
+_notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
+					    notmuch_message_t *message,
+					    const char **thread_id)
+{
+    const char *message_id = notmuch_message_get_message_id (message);
+    Xapian::PostingIterator child, children_end;
+    notmuch_message_t *child_message = NULL;
+    const char *child_thread_id;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_private_status_t private_status;
+
+    _notmuch_database_find_doc_ids (notmuch, "reference", message_id, &child, &children_end);
+
+    for ( ; child != children_end; child++) {
+
+	child_message = _notmuch_message_create (message, notmuch,
+						 *child, &private_status);
+	if (child_message == NULL) {
+	    ret = COERCE_STATUS (private_status,
+				 "Cannot find document for doc_id from query");
+	    goto DONE;
+	}
+
+	child_thread_id = notmuch_message_get_thread_id (child_message);
+	if (*thread_id == NULL) {
+	    *thread_id = talloc_strdup (message, child_thread_id);
+	    _notmuch_message_add_term (message, "thread", *thread_id);
+	} else if (strcmp (*thread_id, child_thread_id)) {
+	    _notmuch_message_remove_term (child_message, "reference",
+					  message_id);
+	    _notmuch_message_sync (child_message);
+	    ret = _merge_threads (notmuch, *thread_id, child_thread_id);
+	    if (ret)
+		goto DONE;
+	}
+
+	notmuch_message_destroy (child_message);
+	child_message = NULL;
+    }
+
+  DONE:
+    if (child_message)
+	notmuch_message_destroy (child_message);
+
+    return ret;
+}
+
+/* Fetch and clear the stored thread_id for message, or NULL if none. */
+static char *
+_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
+			     notmuch_message_t *message)
+{
+    const char *message_id;
+    std::string stored_id;
+    char *metadata_key;
+
+    message_id = notmuch_message_get_message_id (message);
+    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
+
+    /* Check if we have already seen related messages to this one.
+     * If we have then use the thread_id that we stored at that time.
+     */
+    stored_id = notmuch->xapian_db->get_metadata (metadata_key);
+    if (stored_id.empty ()) {
+	return NULL;
+    } else {
+	Xapian::WritableDatabase *db;
+
+	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+	/* Clear the metadata for this message ID. We don't need it
+	 * anymore. */
+	db->set_metadata (metadata_key, "");
+
+	return talloc_strdup (ctx, stored_id.c_str ());
+    }
+}
+
+/* Given a blank or ghost 'message' and its corresponding
+ * 'message_file' link it to existing threads in the database.
+ *
+ * First, if is_ghost, this retrieves the thread ID already stored in
+ * the message (which will be the case if a message was previously
+ * added that referenced this one).  If the message is blank
+ * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
+ * later in this function).  If the database does not support ghost
+ * messages, this checks for a thread ID stored in database metadata
+ * for this message ID.
+ *
+ * Second, we look at 'message_file' and its link-relevant headers
+ * (References and In-Reply-To) for message IDs.
+ *
+ * Finally, we look in the database for existing message that
+ * reference 'message'.
+ *
+ * In all cases, we assign to the current message the first thread ID
+ * found. We will also merge any existing, distinct threads where this
+ * message belongs to both, (which is not uncommon when messages are
+ * processed out of order).
+ *
+ * Finally, if no thread ID has been found through referenced messages, we
+ * call _notmuch_message_generate_thread_id to generate a new thread
+ * ID. This should only happen for new, top-level messages, (no
+ * References or In-Reply-To header in this message, and no previously
+ * added message refers to this message).
+ */
+static notmuch_status_t
+_notmuch_database_link_message (notmuch_database_t *notmuch,
+				notmuch_message_t *message,
+				notmuch_message_file_t *message_file,
+				bool is_ghost)
+{
+    void *local = talloc_new (NULL);
+    notmuch_status_t status;
+    const char *thread_id = NULL;
+
+    /* Check if the message already had a thread ID */
+    if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
+	if (is_ghost)
+	    thread_id = notmuch_message_get_thread_id (message);
+    } else {
+	thread_id = _consume_metadata_thread_id (local, notmuch, message);
+	if (thread_id)
+	    _notmuch_message_add_term (message, "thread", thread_id);
+    }
+
+    status = _notmuch_database_link_message_to_parents (notmuch, message,
+							message_file,
+							&thread_id);
+    if (status)
+	goto DONE;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
+	/* In general, it shouldn't be necessary to link children,
+	 * since the earlier indexing of those children will have
+	 * stored a thread ID for the missing parent.  However, prior
+	 * to ghost messages, these stored thread IDs were NOT
+	 * rewritten during thread merging (and there was no
+	 * performant way to do so), so if indexed children were
+	 * pulled into a different thread ID by a merge, it was
+	 * necessary to pull them *back* into the stored thread ID of
+	 * the parent.  With ghost messages, we just rewrite the
+	 * stored thread IDs during merging, so this workaround isn't
+	 * necessary. */
+	status = _notmuch_database_link_message_to_children (notmuch, message,
+							     &thread_id);
+	if (status)
+	    goto DONE;
+    }
+
+    /* If not part of any existing thread, generate a new thread ID. */
+    if (thread_id == NULL) {
+	thread_id = _notmuch_database_generate_thread_id (notmuch);
+
+	_notmuch_message_add_term (message, "thread", thread_id);
+    }
+
+ DONE:
+    talloc_free (local);
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_index_file (notmuch_database_t *notmuch,
+			     const char *filename,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_t **message_ret)
+{
+    notmuch_message_file_t *message_file;
+    notmuch_message_t *message = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
+    notmuch_private_status_t private_status;
+    bool is_ghost = false, is_new = false;
+    notmuch_indexopts_t *def_indexopts = NULL;
+
+    const char *date;
+    const char *from, *to, *subject;
+    char *message_id = NULL;
+
+    if (message_ret)
+	*message_ret = NULL;
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    message_file = _notmuch_message_file_open (notmuch, filename);
+    if (message_file == NULL)
+	return NOTMUCH_STATUS_FILE_ERROR;
+
+    /* Adding a message may change many documents.  Do this all
+     * atomically. */
+    ret = notmuch_database_begin_atomic (notmuch);
+    if (ret)
+	goto DONE;
+
+    ret = _notmuch_message_file_get_headers (message_file,
+					     &from, &subject, &to, &date,
+					     &message_id);
+    if (ret)
+	goto DONE;
+
+    try {
+	/* Now that we have a message ID, we get a message object,
+	 * (which may or may not reference an existing document in the
+	 * database). */
+
+	message = _notmuch_message_create_for_message_id (notmuch,
+							  message_id,
+							  &private_status);
+
+	talloc_free (message_id);
+
+	/* We cannot call notmuch_message_get_flag for a new message */
+	switch (private_status) {
+	case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+	    is_ghost = false;
+	    is_new = true;
+	    break;
+	case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+	    is_ghost = notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_GHOST);
+	    is_new = false;
+	    break;
+	default:
+	    ret = COERCE_STATUS (private_status,
+				 "Unexpected status value from _notmuch_message_create_for_message_id");
+	    goto DONE;
+	}
+
+	_notmuch_message_add_filename (message, filename);
+
+	if (is_new || is_ghost) {
+	    _notmuch_message_add_term (message, "type", "mail");
+	    if (is_ghost)
+		/* Convert ghost message to a regular message */
+		_notmuch_message_remove_term (message, "type", "ghost");
+	}
+
+	ret = _notmuch_database_link_message (notmuch, message,
+						  message_file, is_ghost);
+	if (ret)
+	    goto DONE;
+
+	if (is_new || is_ghost)
+	    _notmuch_message_set_header_values (message, date, from, subject);
+
+	if (!indexopts) {
+	    def_indexopts = notmuch_database_get_default_indexopts (notmuch);
+	    indexopts = def_indexopts;
+	}
+
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
+	if (ret)
+	    goto DONE;
+
+	if (! is_new && !is_ghost)
+	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	_notmuch_message_sync (message);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	goto DONE;
+    }
+
+  DONE:
+    if (def_indexopts)
+	notmuch_indexopts_destroy (def_indexopts);
+
+    if (message) {
+	if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
+	    *message_ret = message;
+	else
+	    notmuch_message_destroy (message);
+    }
+
+    if (message_file)
+	_notmuch_message_file_close (message_file);
+
+    ret2 = notmuch_database_end_atomic (notmuch);
+    if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
+	ret2 != NOTMUCH_STATUS_SUCCESS)
+	ret = ret2;
+
+    return ret;
+}
+
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *notmuch,
+			      const char *filename,
+			      notmuch_message_t **message_ret)
+{
+    return notmuch_database_index_file (notmuch, filename,
+					NULL,
+					message_ret);
+
+}
diff --git a/lib/built-with.c b/lib/built-with.c
new file mode 100644
index 00000000..9cffd9f9
--- /dev/null
+++ b/lib/built-with.c
@@ -0,0 +1,38 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+
+notmuch_bool_t
+notmuch_built_with (const char *name)
+{
+    if (STRNCMP_LITERAL (name, "compact") == 0) {
+	return HAVE_XAPIAN_COMPACT;
+    } else if (STRNCMP_LITERAL (name, "field_processor") == 0) {
+	return HAVE_XAPIAN_FIELD_PROCESSOR;
+    } else if (STRNCMP_LITERAL (name, "retry_lock") == 0) {
+	return HAVE_XAPIAN_DB_RETRY_LOCK;
+    } else if (STRNCMP_LITERAL (name, "session_key") == 0) {
+	return HAVE_GMIME_SESSION_KEYS;
+    } else {
+	return false;
+    }
+}
diff --git a/lib/config.cc b/lib/config.cc
new file mode 100644
index 00000000..da71c16e
--- /dev/null
+++ b/lib/config.cc
@@ -0,0 +1,193 @@
+/* config.cc - API for database metadata
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+#include "database-private.h"
+
+static const std::string CONFIG_PREFIX = "C";
+
+struct _notmuch_config_list {
+    notmuch_database_t *notmuch;
+    Xapian::TermIterator iterator;
+    char *current_key;
+    char *current_val;
+};
+
+static int
+_notmuch_config_list_destroy (notmuch_config_list_t *list)
+{
+    /* invoke destructor w/o deallocating memory */
+    list->iterator.~TermIterator();
+    return 0;
+}
+
+notmuch_status_t
+notmuch_database_set_config (notmuch_database_t *notmuch,
+			     const char *key,
+			     const char *value)
+{
+    notmuch_status_t status;
+    Xapian::WritableDatabase *db;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    try {
+	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+	db->set_metadata (CONFIG_PREFIX + key, value);
+    } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	notmuch->exception_reported = true;
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred setting metadata: %s\n",
+			       error.get_msg().c_str());
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_metadata_value (notmuch_database_t *notmuch,
+		 const char *key,
+		 std::string &value)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    try {
+	value = notmuch->xapian_db->get_metadata (CONFIG_PREFIX + key);
+    } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	notmuch->exception_reported = true;
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred getting metadata: %s\n",
+			       error.get_msg().c_str());
+    }
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_get_config (notmuch_database_t *notmuch,
+			     const char *key,
+			     char **value)
+{
+    std::string strval;
+    notmuch_status_t status;
+
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    status = _metadata_value (notmuch, key, strval);
+    if (status)
+	return status;
+
+    *value = strdup (strval.c_str ());
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_get_config_list (notmuch_database_t *notmuch,
+				  const char *prefix,
+				  notmuch_config_list_t **out)
+{
+    notmuch_config_list_t *list = NULL;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    list = talloc (notmuch, notmuch_config_list_t);
+    if (! list) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    talloc_set_destructor (list, _notmuch_config_list_destroy);
+    list->notmuch = notmuch;
+    list->current_key = NULL;
+    list->current_val = NULL;
+
+    try {
+
+	new(&(list->iterator)) Xapian::TermIterator (notmuch->xapian_db->metadata_keys_begin
+						     (CONFIG_PREFIX + (prefix ? prefix : "")));
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred getting metadata iterator: %s.\n",
+			       error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    *out = list;
+
+  DONE:
+    if (status && list)
+	talloc_free (list);
+
+    return status;
+}
+
+notmuch_bool_t
+notmuch_config_list_valid (notmuch_config_list_t *metadata)
+{
+    if (metadata->iterator == metadata->notmuch->xapian_db->metadata_keys_end ())
+	return false;
+
+    return true;
+}
+
+const char *
+notmuch_config_list_key (notmuch_config_list_t *list)
+{
+    if (list->current_key)
+	talloc_free (list->current_key);
+
+    list->current_key = talloc_strdup (list, (*list->iterator).c_str () + CONFIG_PREFIX.length ());
+
+    return list->current_key;
+}
+
+const char *
+notmuch_config_list_value (notmuch_config_list_t *list)
+{
+    std::string strval;
+    notmuch_status_t status;
+    const char *key = notmuch_config_list_key (list);
+
+    /* TODO: better error reporting?? */
+    status = _metadata_value (list->notmuch, key, strval);
+    if (status)
+	return NULL;
+
+    if (list->current_val)
+	talloc_free (list->current_val);
+
+    list->current_val = talloc_strdup (list, strval.c_str ());
+    return list->current_val;
+}
+
+void
+notmuch_config_list_move_to_next (notmuch_config_list_t *list)
+{
+    list->iterator++;
+}
+
+void
+notmuch_config_list_destroy (notmuch_config_list_t *list)
+{
+    talloc_free (list);
+}
diff --git a/lib/database-private.h b/lib/database-private.h
new file mode 100644
index 00000000..a499b259
--- /dev/null
+++ b/lib/database-private.h
@@ -0,0 +1,255 @@
+/* database-private.h - For peeking into the internals of notmuch_database_t
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_DATABASE_PRIVATE_H
+#define NOTMUCH_DATABASE_PRIVATE_H
+
+/* According to WG14/N1124, a C++ implementation won't provide us a
+ * macro like PRIx64 (which gives a printf format string for
+ * formatting a uint64_t as hexadecimal) unless we define
+ * __STDC_FORMAT_MACROS before including inttypes.h. That's annoying,
+ * but there it is.
+ */
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+
+#include "notmuch-private.h"
+
+#ifdef SILENCE_XAPIAN_DEPRECATION_WARNINGS
+#define XAPIAN_DEPRECATED(D) D
+#endif
+
+#include <xapian.h>
+
+/* Bit masks for _notmuch_database::features.  Features are named,
+ * independent aspects of the database schema.
+ *
+ * A database stores the set of features that it "uses" (implicitly
+ * before database version 3 and explicitly as of version 3).
+ *
+ * A given library version will "recognize" a particular set of
+ * features; if a database uses a feature that the library does not
+ * recognize, the library will refuse to open it.  It is assumed the
+ * set of recognized features grows monotonically over time.  A
+ * library version will "implement" some subset of the recognized
+ * features: some operations may require that the database use (or not
+ * use) some feature, while other operations may support both
+ * databases that use and that don't use some feature.
+ *
+ * On disk, the database stores string names for these features (see
+ * the feature_names array).  These enum bit values are never
+ * persisted to disk and may change freely.
+ */
+enum _notmuch_features {
+    /* If set, file names are stored in "file-direntry" terms.  If
+     * unset, file names are stored in document data.
+     *
+     * Introduced: version 1. */
+    NOTMUCH_FEATURE_FILE_TERMS = 1 << 0,
+
+    /* If set, directory timestamps are stored in documents with
+     * XDIRECTORY terms and relative paths.  If unset, directory
+     * timestamps are stored in documents with XTIMESTAMP terms and
+     * absolute paths.
+     *
+     * Introduced: version 1. */
+    NOTMUCH_FEATURE_DIRECTORY_DOCS = 1 << 1,
+
+    /* If set, the from, subject, and message-id headers are stored in
+     * message document values.  If unset, message documents *may*
+     * have these values, but if the value is empty, it must be
+     * retrieved from the message file.
+     *
+     * Introduced: optional in version 1, required as of version 3.
+     */
+    NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES = 1 << 2,
+
+    /* If set, folder terms are boolean and path terms exist.  If
+     * unset, folder terms are probabilistic and stemmed and path
+     * terms do not exist.
+     *
+     * Introduced: version 2. */
+    NOTMUCH_FEATURE_BOOL_FOLDER = 1 << 3,
+
+    /* If set, missing messages are stored in ghost mail documents.
+     * If unset, thread IDs of ghost messages are stored as database
+     * metadata instead of in ghost documents.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_GHOSTS = 1 << 4,
+
+
+    /* If set, then the database was created after the introduction of
+     * indexed mime types. If unset, then the database may contain a
+     * mixture of messages with indexed and non-indexed mime types.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_INDEXED_MIMETYPES = 1 << 5,
+
+    /* If set, messages store the revision number of the last
+     * modification in NOTMUCH_VALUE_LAST_MOD.
+     *
+     * Introduced: version 3. */
+    NOTMUCH_FEATURE_LAST_MOD = 1 << 6,
+};
+
+/* In C++, a named enum is its own type, so define bitwise operators
+ * on _notmuch_features. */
+inline _notmuch_features
+operator|(_notmuch_features a, _notmuch_features b)
+{
+    return static_cast<_notmuch_features>(
+	static_cast<unsigned>(a) | static_cast<unsigned>(b));
+}
+
+inline _notmuch_features
+operator&(_notmuch_features a, _notmuch_features b)
+{
+    return static_cast<_notmuch_features>(
+	static_cast<unsigned>(a) & static_cast<unsigned>(b));
+}
+
+inline _notmuch_features
+operator~(_notmuch_features a)
+{
+    return static_cast<_notmuch_features>(~static_cast<unsigned>(a));
+}
+
+inline _notmuch_features&
+operator|=(_notmuch_features &a, _notmuch_features b)
+{
+    a = a | b;
+    return a;
+}
+
+inline _notmuch_features&
+operator&=(_notmuch_features &a, _notmuch_features b)
+{
+    a = a & b;
+    return a;
+}
+
+/*
+ * Configuration options for xapian database fields */
+typedef enum notmuch_field_flags {
+    NOTMUCH_FIELD_NO_FLAGS = 0,
+    NOTMUCH_FIELD_EXTERNAL = 1 << 0,
+    NOTMUCH_FIELD_PROBABILISTIC = 1 << 1,
+    NOTMUCH_FIELD_PROCESSOR = 1 << 2,
+} notmuch_field_flag_t;
+
+/*
+ * define bitwise operators to hide casts */
+inline notmuch_field_flag_t
+operator|(notmuch_field_flag_t a, notmuch_field_flag_t b)
+{
+    return static_cast<notmuch_field_flag_t>(
+	static_cast<unsigned>(a) | static_cast<unsigned>(b));
+}
+
+inline notmuch_field_flag_t
+operator&(notmuch_field_flag_t a, notmuch_field_flag_t b)
+{
+    return static_cast<notmuch_field_flag_t>(
+	static_cast<unsigned>(a) & static_cast<unsigned>(b));
+}
+
+#define NOTMUCH_QUERY_PARSER_FLAGS (Xapian::QueryParser::FLAG_BOOLEAN | \
+				    Xapian::QueryParser::FLAG_PHRASE | \
+				    Xapian::QueryParser::FLAG_LOVEHATE | \
+				    Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE | \
+				    Xapian::QueryParser::FLAG_WILDCARD | \
+				    Xapian::QueryParser::FLAG_PURE_NOT)
+
+struct _notmuch_database {
+    bool exception_reported;
+
+    char *path;
+
+    notmuch_database_mode_t mode;
+    int atomic_nesting;
+    /* true if changes have been made in this atomic section */
+    bool atomic_dirty;
+    Xapian::Database *xapian_db;
+
+    /* Bit mask of features used by this database.  This is a
+     * bitwise-OR of NOTMUCH_FEATURE_* values (above). */
+    enum _notmuch_features features;
+
+    unsigned int last_doc_id;
+    uint64_t last_thread_id;
+
+    /* error reporting; this value persists only until the
+     * next library call. May be NULL */
+    char *status_string;
+
+    /* Highest committed revision number.  Modifications are recorded
+     * under a higher revision number, which can be generated with
+     * notmuch_database_new_revision. */
+    unsigned long revision;
+    const char *uuid;
+
+    /* Keep track of the number of times the database has been re-opened
+     * (or other global invalidations of notmuch's caching)
+     */
+    unsigned long view;
+    Xapian::QueryParser *query_parser;
+    Xapian::TermGenerator *term_gen;
+    Xapian::ValueRangeProcessor *value_range_processor;
+    Xapian::ValueRangeProcessor *date_range_processor;
+    Xapian::ValueRangeProcessor *last_mod_range_processor;
+};
+
+/* Prior to database version 3, features were implied by the database
+ * version number, so hard-code them for earlier versions. */
+#define NOTMUCH_FEATURES_V0 ((enum _notmuch_features)0)
+#define NOTMUCH_FEATURES_V1 (NOTMUCH_FEATURES_V0 | NOTMUCH_FEATURE_FILE_TERMS | \
+			     NOTMUCH_FEATURE_DIRECTORY_DOCS)
+#define NOTMUCH_FEATURES_V2 (NOTMUCH_FEATURES_V1 | NOTMUCH_FEATURE_BOOL_FOLDER)
+
+/* Current database features.  If any of these are missing from a
+ * database, request an upgrade.
+ * NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES and
+ * NOTMUCH_FEATURE_INDEXED_MIMETYPES are not included because upgrade
+ * doesn't currently introduce the features (though brand new databases
+ * will have it). */
+#define NOTMUCH_FEATURES_CURRENT \
+    (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_DIRECTORY_DOCS | \
+     NOTMUCH_FEATURE_BOOL_FOLDER | NOTMUCH_FEATURE_GHOSTS | \
+     NOTMUCH_FEATURE_LAST_MOD)
+
+/* Return the list of terms from the given iterator matching a prefix.
+ * The prefix will be stripped from the strings in the returned list.
+ * The list will be allocated using ctx as the talloc context.
+ *
+ * The function returns NULL on failure.
+ */
+notmuch_string_list_t *
+_notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
+					 Xapian::TermIterator &end,
+					 const char *prefix);
+
+void
+_notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
+				const char *prefix_name,
+				const char *value,
+				Xapian::PostingIterator *begin,
+				Xapian::PostingIterator *end);
+#endif
diff --git a/lib/database.cc b/lib/database.cc
new file mode 100644
index 00000000..9cf8062c
--- /dev/null
+++ b/lib/database.cc
@@ -0,0 +1,2043 @@
+/* database.cc - The database interfaces of the notmuch mail library
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "database-private.h"
+#include "parse-time-vrp.h"
+#include "query-fp.h"
+#include "thread-fp.h"
+#include "regexp-fields.h"
+#include "string-util.h"
+
+#include <iostream>
+
+#include <sys/time.h>
+#include <sys/stat.h>
+#include <signal.h>
+#include <ftw.h>
+
+#include <glib.h> /* g_free, GPtrArray, GHashTable */
+#include <glib-object.h> /* g_type_init */
+
+#include <gmime/gmime.h> /* g_mime_init */
+
+using namespace std;
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
+typedef struct {
+    const char *name;
+    const char *prefix;
+    notmuch_field_flag_t flags;
+} prefix_t;
+
+#define NOTMUCH_DATABASE_VERSION 3
+
+#define STRINGIFY(s) _SUB_STRINGIFY(s)
+#define _SUB_STRINGIFY(s) #s
+
+#if HAVE_XAPIAN_DB_RETRY_LOCK
+#define DB_ACTION (Xapian::DB_CREATE_OR_OPEN | Xapian::DB_RETRY_LOCK)
+#else
+#define DB_ACTION Xapian::DB_CREATE_OR_OPEN
+#endif
+
+/* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
+ *
+ * We currently have three different types of documents (mail, ghost,
+ * and directory) and also some metadata.
+ *
+ * Mail document
+ * -------------
+ * A mail document is associated with a particular email message. It
+ * is stored in one or more files on disk (though only one has its
+ * content indexed) and is uniquely identified  by its "id" field
+ * (which is generally the message ID). It is indexed with the
+ * following prefixed terms which the database uses to construct
+ * threads, etc.:
+ *
+ *    Single terms of given prefix:
+ *
+ *	type:	mail
+ *
+ *	id:	Unique ID of mail. This is from the Message-ID header
+ *		if present and not too long (see NOTMUCH_MESSAGE_ID_MAX).
+ *		If it's present and too long, then we use
+ *		"notmuch-sha1-<sha1_sum_of_message_id>".
+ *              If this header is not present, we use
+ *		"notmuch-sha1-<sha1_sum_of_entire_file>".
+ *
+ *	thread:	The ID of the thread to which the mail belongs
+ *
+ *	replyto: The ID from the In-Reply-To header of the mail (if any).
+ *
+ *    Multiple terms of given prefix:
+ *
+ *	reference: All message IDs from In-Reply-To and References
+ *		   headers in the message.
+ *
+ *	tag:	   Any tags associated with this message by the user.
+ *
+ *	file-direntry:  A colon-separated pair of values
+ *		        (INTEGER:STRING), where INTEGER is the
+ *		        document ID of a directory document, and
+ *		        STRING is the name of a file within that
+ *		        directory for this mail message.
+ *
+ *      property:       Has a property with key=value
+ *                 FIXME: if no = is present, should match on any value
+ *
+ *    A mail document also has four values:
+ *
+ *	TIMESTAMP:	The time_t value corresponding to the message's
+ *			Date header.
+ *
+ *	MESSAGE_ID:	The unique ID of the mail mess (see "id" above)
+ *
+ *	FROM:		The value of the "From" header
+ *
+ *	SUBJECT:	The value of the "Subject" header
+ *
+ *	LAST_MOD:	The revision number as of the last tag or
+ *			filename change.
+ *
+ * In addition, terms from the content of the message are added with
+ * "from", "to", "attachment", and "subject" prefixes for use by the
+ * user in searching. Similarly, terms from the path of the mail
+ * message are added with "folder" and "path" prefixes. But the
+ * database doesn't really care itself about any of these.
+ *
+ * The data portion of a mail document is empty.
+ *
+ * Ghost mail document [if NOTMUCH_FEATURE_GHOSTS]
+ * -----------------------------------------------
+ * A ghost mail document is like a mail document, but where we don't
+ * have the message content.  These are used to track thread reference
+ * information for messages we haven't received.
+ *
+ * A ghost mail document has type: ghost; id and thread fields that
+ * are identical to the mail document fields; and a MESSAGE_ID value.
+ *
+ * Directory document
+ * ------------------
+ * A directory document is used by a client of the notmuch library to
+ * maintain data necessary to allow for efficient polling of mail
+ * directories.
+ *
+ * All directory documents contain one term:
+ *
+ *	directory:	The directory path (relative to the database path)
+ *			Or the SHA1 sum of the directory path (if the
+ *			path itself is too long to fit in a Xapian
+ *			term).
+ *
+ * And all directory documents for directories other than top-level
+ * directories also contain the following term:
+ *
+ *	directory-direntry: A colon-separated pair of values
+ *		            (INTEGER:STRING), where INTEGER is the
+ *		            document ID of the parent directory
+ *		            document, and STRING is the name of this
+ *		            directory within that parent.
+ *
+ * All directory documents have a single value:
+ *
+ *	TIMESTAMP:	The mtime of the directory (at last scan)
+ *
+ * The data portion of a directory document contains the path of the
+ * directory (relative to the database path).
+ *
+ * Database metadata
+ * -----------------
+ * Xapian allows us to store arbitrary name-value pairs as
+ * "metadata". We currently use the following metadata names with the
+ * given meanings:
+ *
+ *	version		The database schema version, (which is distinct
+ *			from both the notmuch package version (see
+ *			notmuch --version) and the libnotmuch library
+ *			version. The version is stored as an base-10
+ *			ASCII integer. The initial database version
+ *			was 1, (though a schema existed before that
+ *			were no "version" database value existed at
+ *			all). Successive versions are allocated as
+ *			changes are made to the database (such as by
+ *			indexing new fields).
+ *
+ *	features	The set of features supported by this
+ *			database. This consists of a set of
+ *			'\n'-separated lines, where each is a feature
+ *			name, a '\t', and compatibility flags.  If the
+ *			compatibility flags contain 'w', then the
+ *			opener must support this feature to safely
+ *			write this database.  If the compatibility
+ *			flags contain 'r', then the opener must
+ *			support this feature to read this database.
+ *			Introduced in database version 3.
+ *
+ *	last_thread_id	The last thread ID generated. This is stored
+ *			as a 16-byte hexadecimal ASCII representation
+ *			of a 64-bit unsigned integer. The first ID
+ *			generated is 1 and the value will be
+ *			incremented for each thread ID.
+ *
+ *	C*		metadata keys starting with C indicate
+ *			configuration data. It can be managed with the
+ *			n_database_*config* API.  There is a convention
+ *			of hierarchical keys separated by '.' (e.g.
+ *			query.notmuch stores the value for the named
+ *			query 'notmuch'), but it is not enforced by the
+ *			API.
+ *
+ * Obsolete metadata
+ * -----------------
+ *
+ * If ! NOTMUCH_FEATURE_GHOSTS, there are no ghost mail documents.
+ * Instead, the database has the following additional database
+ * metadata:
+ *
+ *	thread_id_*	A pre-allocated thread ID for a particular
+ *			message. This is actually an arbitrarily large
+ *			family of metadata name. Any particular name is
+ *			formed by concatenating "thread_id_" with a message
+ *			ID (or the SHA1 sum of a message ID if it is very
+ *			long---see description of 'id' in the mail
+ *			document). The value stored is a thread ID.
+ *
+ *			These thread ID metadata values are stored
+ *			whenever a message references a parent message
+ *			that does not yet exist in the database. A
+ *			thread ID will be allocated and stored, and if
+ *			the message is later added, the stored thread
+ *			ID will be used (and the metadata value will
+ *			be cleared).
+ *
+ *			Even before a message is added, it's
+ *			pre-allocated thread ID is useful so that all
+ *			descendant messages that reference this common
+ *			parent can be recognized as belonging to the
+ *			same thread.
+ */
+
+/* With these prefix values we follow the conventions published here:
+ *
+ * https://xapian.org/docs/omega/termprefixes.html
+ *
+ * as much as makes sense. Note that I took some liberty in matching
+ * the reserved prefix values to notmuch concepts, (for example, 'G'
+ * is documented as "newsGroup (or similar entity - e.g. a web forum
+ * name)", for which I think the thread is the closest analogue in
+ * notmuch. This in spite of the fact that we will eventually be
+ * storing mailing-list messages where 'G' for "mailing list name"
+ * might be even a closer analogue. I'm treating the single-character
+ * prefixes preferentially for core notmuch concepts (which will be
+ * nearly universal to all mail messages).
+ */
+
+static const
+prefix_t prefix_table[] = {
+    /* name			term prefix	flags */
+    { "type",			"T",		NOTMUCH_FIELD_NO_FLAGS },
+    { "reference",		"XREFERENCE",	NOTMUCH_FIELD_NO_FLAGS },
+    { "replyto",		"XREPLYTO",	NOTMUCH_FIELD_NO_FLAGS },
+    { "directory",		"XDIRECTORY",	NOTMUCH_FIELD_NO_FLAGS },
+    { "file-direntry",		"XFDIRENTRY",	NOTMUCH_FIELD_NO_FLAGS },
+    { "directory-direntry",	"XDDIRENTRY",	NOTMUCH_FIELD_NO_FLAGS },
+    { "thread",			"G",		NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROCESSOR },
+    { "tag",			"K",		NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROCESSOR },
+    { "is",			"K",		NOTMUCH_FIELD_EXTERNAL |
+					        NOTMUCH_FIELD_PROCESSOR },
+    { "id",			"Q",		NOTMUCH_FIELD_EXTERNAL },
+    { "mid",			"Q",		NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROCESSOR },
+    { "path",			"P",		NOTMUCH_FIELD_EXTERNAL|
+						NOTMUCH_FIELD_PROCESSOR },
+    { "property",		"XPROPERTY",	NOTMUCH_FIELD_EXTERNAL },
+    /*
+     * Unconditionally add ':' to reduce potential ambiguity with
+     * overlapping prefixes and/or terms that start with capital
+     * letters. See Xapian document termprefixes.html for related
+     * discussion.
+     */
+    { "folder",			"XFOLDER:",	NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROCESSOR },
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+    { "date",			NULL,		NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROCESSOR },
+    { "query",			NULL,		NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROCESSOR },
+#endif
+    { "from",			"XFROM",	NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROBABILISTIC |
+						NOTMUCH_FIELD_PROCESSOR },
+    { "to",			"XTO",		NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROBABILISTIC },
+    { "attachment",		"XATTACHMENT",	NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROBABILISTIC },
+    { "mimetype",		"XMIMETYPE",	NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROBABILISTIC },
+    { "subject",		"XSUBJECT",	NOTMUCH_FIELD_EXTERNAL |
+						NOTMUCH_FIELD_PROBABILISTIC |
+						NOTMUCH_FIELD_PROCESSOR},
+};
+
+static void
+_setup_query_field_default (const prefix_t *prefix, notmuch_database_t *notmuch)
+{
+    if (prefix->flags & NOTMUCH_FIELD_PROBABILISTIC)
+	notmuch->query_parser->add_prefix (prefix->name, prefix->prefix);
+    else
+	notmuch->query_parser->add_boolean_prefix (prefix->name, prefix->prefix);
+}
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+static void
+_setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
+{
+    if (prefix->flags & NOTMUCH_FIELD_PROCESSOR) {
+	Xapian::FieldProcessor *fp;
+
+	if (STRNCMP_LITERAL (prefix->name, "date") == 0)
+	    fp = (new DateFieldProcessor())->release ();
+	else if (STRNCMP_LITERAL(prefix->name, "query") == 0)
+	    fp = (new QueryFieldProcessor (*notmuch->query_parser, notmuch))->release ();
+	else if (STRNCMP_LITERAL(prefix->name, "thread") == 0)
+	    fp = (new ThreadFieldProcessor (*notmuch->query_parser, notmuch))->release ();
+	else
+	    fp = (new RegexpFieldProcessor (prefix->name, prefix->flags,
+					    *notmuch->query_parser, notmuch))->release ();
+
+	/* we treat all field-processor fields as boolean in order to get the raw input */
+	notmuch->query_parser->add_boolean_prefix (prefix->name, fp);
+    } else {
+	_setup_query_field_default (prefix, notmuch);
+    }
+}
+#else
+static inline void
+_setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
+{
+    _setup_query_field_default (prefix, notmuch);
+}
+#endif
+
+const char *
+_find_prefix (const char *name)
+{
+    unsigned int i;
+
+    for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
+	if (strcmp (name, prefix_table[i].name) == 0)
+	    return prefix_table[i].prefix;
+    }
+
+    INTERNAL_ERROR ("No prefix exists for '%s'\n", name);
+
+    return "";
+}
+
+static const struct {
+    /* NOTMUCH_FEATURE_* value. */
+    _notmuch_features value;
+    /* Feature name as it appears in the database.  This name should
+     * be appropriate for displaying to the user if an older version
+     * of notmuch doesn't support this feature. */
+    const char *name;
+    /* Compatibility flags when this feature is declared. */
+    const char *flags;
+} feature_names[] = {
+    { NOTMUCH_FEATURE_FILE_TERMS,
+      "multiple paths per message", "rw" },
+    { NOTMUCH_FEATURE_DIRECTORY_DOCS,
+      "relative directory paths", "rw" },
+    /* Header values are not required for reading a database because a
+     * reader can just refer to the message file. */
+    { NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES,
+      "from/subject/message-ID in database", "w" },
+    { NOTMUCH_FEATURE_BOOL_FOLDER,
+      "exact folder:/path: search", "rw" },
+    { NOTMUCH_FEATURE_GHOSTS,
+      "mail documents for missing messages", "w"},
+    /* Knowledge of the index mime-types are not required for reading
+     * a database because a reader will just be unable to query
+     * them. */
+    { NOTMUCH_FEATURE_INDEXED_MIMETYPES,
+      "indexed MIME types", "w"},
+    { NOTMUCH_FEATURE_LAST_MOD,
+      "modification tracking", "w"},
+};
+
+const char *
+notmuch_status_to_string (notmuch_status_t status)
+{
+    switch (status) {
+    case NOTMUCH_STATUS_SUCCESS:
+	return "No error occurred";
+    case NOTMUCH_STATUS_OUT_OF_MEMORY:
+	return "Out of memory";
+    case NOTMUCH_STATUS_READ_ONLY_DATABASE:
+	return "Attempt to write to a read-only database";
+    case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
+	return "A Xapian exception occurred";
+    case NOTMUCH_STATUS_FILE_ERROR:
+	return "Something went wrong trying to read or write a file";
+    case NOTMUCH_STATUS_FILE_NOT_EMAIL:
+	return "File is not an email";
+    case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
+	return "Message ID is identical to a message in database";
+    case NOTMUCH_STATUS_NULL_POINTER:
+	return "Erroneous NULL pointer";
+    case NOTMUCH_STATUS_TAG_TOO_LONG:
+	return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
+    case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
+	return "Unbalanced number of calls to notmuch_message_freeze/thaw";
+    case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
+	return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic";
+    case NOTMUCH_STATUS_UNSUPPORTED_OPERATION:
+	return "Unsupported operation";
+    case NOTMUCH_STATUS_UPGRADE_REQUIRED:
+	return "Operation requires a database upgrade";
+    case NOTMUCH_STATUS_PATH_ERROR:
+	return "Path supplied is illegal for this function";
+    case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
+	return "Crypto protocol missing, malformed, or unintelligible";
+    case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
+	return "Crypto engine initialization failure";
+    case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
+	return "Unknown crypto protocol";
+    default:
+    case NOTMUCH_STATUS_LAST_STATUS:
+	return "Unknown error status value";
+    }
+}
+
+void
+_notmuch_database_log (notmuch_database_t *notmuch,
+		      const char *format,
+		      ...)
+{
+    va_list va_args;
+
+    va_start (va_args, format);
+
+    if (notmuch->status_string)
+	talloc_free (notmuch->status_string);
+
+    notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
+    va_end (va_args);
+}
+
+void
+_notmuch_database_log_append (notmuch_database_t *notmuch,
+		      const char *format,
+		      ...)
+{
+    va_list va_args;
+
+    va_start (va_args, format);
+
+    if (notmuch->status_string)
+	notmuch->status_string = talloc_vasprintf_append (notmuch->status_string, format, va_args);
+    else
+	notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
+
+    va_end (va_args);
+}
+
+static void
+find_doc_ids_for_term (notmuch_database_t *notmuch,
+		       const char *term,
+		       Xapian::PostingIterator *begin,
+		       Xapian::PostingIterator *end)
+{
+    *begin = notmuch->xapian_db->postlist_begin (term);
+
+    *end = notmuch->xapian_db->postlist_end (term);
+}
+
+void
+_notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
+				const char *prefix_name,
+				const char *value,
+				Xapian::PostingIterator *begin,
+				Xapian::PostingIterator *end)
+{
+    char *term;
+
+    term = talloc_asprintf (notmuch, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    find_doc_ids_for_term (notmuch, term, begin, end);
+
+    talloc_free (term);
+}
+
+notmuch_private_status_t
+_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
+				      const char *prefix_name,
+				      const char *value,
+				      unsigned int *doc_id)
+{
+    Xapian::PostingIterator i, end;
+
+    _notmuch_database_find_doc_ids (notmuch, prefix_name, value, &i, &end);
+
+    if (i == end) {
+	*doc_id = 0;
+	return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+    }
+
+    *doc_id = *i;
+
+#if DEBUG_DATABASE_SANITY
+    i++;
+
+    if (i != end)
+	INTERNAL_ERROR ("Term %s:%s is not unique as expected.\n",
+			prefix_name, value);
+#endif
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+static Xapian::Document
+find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
+{
+    return notmuch->xapian_db->get_document (doc_id);
+}
+
+/* Generate a compressed version of 'message_id' of the form:
+ *
+ *	notmuch-sha1-<sha1_sum_of_message_id>
+ */
+char *
+_notmuch_message_id_compressed (void *ctx, const char *message_id)
+{
+    char *sha1, *compressed;
+
+    sha1 = _notmuch_sha1_of_string (message_id);
+
+    compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1);
+    free (sha1);
+
+    return compressed;
+}
+
+notmuch_status_t
+notmuch_database_find_message (notmuch_database_t *notmuch,
+			       const char *message_id,
+			       notmuch_message_t **message_ret)
+{
+    notmuch_private_status_t status;
+    unsigned int doc_id;
+
+    if (message_ret == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (notmuch, message_id);
+
+    try {
+	status = _notmuch_database_find_unique_doc_id (notmuch, "id",
+						       message_id, &doc_id);
+
+	if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
+	    *message_ret = NULL;
+	else {
+	    *message_ret = _notmuch_message_create (notmuch, notmuch, doc_id,
+						    NULL);
+	    if (*message_ret == NULL)
+		return NOTMUCH_STATUS_OUT_OF_MEMORY;
+	}
+
+	return NOTMUCH_STATUS_SUCCESS;
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred finding message: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	*message_ret = NULL;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+}
+
+notmuch_status_t
+notmuch_database_create (const char *path, notmuch_database_t **database)
+{
+    char *status_string = NULL;
+    notmuch_status_t status;
+
+    status = notmuch_database_create_verbose (path, database,
+					      &status_string);
+
+    if (status_string) {
+	fputs (status_string, stderr);
+	free (status_string);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_create_verbose (const char *path,
+				 notmuch_database_t **database,
+				 char **status_string)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    notmuch_database_t *notmuch = NULL;
+    char *notmuch_path = NULL;
+    char *message = NULL;
+    struct stat st;
+    int err;
+
+    if (path == NULL) {
+	message = strdup ("Error: Cannot create a database for a NULL path.\n");
+	status = NOTMUCH_STATUS_NULL_POINTER;
+	goto DONE;
+    }
+
+    if (path[0] != '/') {
+	message = strdup ("Error: Database path must be absolute.\n");
+	status = NOTMUCH_STATUS_PATH_ERROR;
+	goto DONE;
+    }
+
+    err = stat (path, &st);
+    if (err) {
+	IGNORE_RESULT (asprintf (&message, "Error: Cannot create database at %s: %s.\n",
+				path, strerror (errno)));
+	status = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (! S_ISDIR (st.st_mode)) {
+	IGNORE_RESULT (asprintf (&message, "Error: Cannot create database at %s: "
+				 "Not a directory.\n",
+				 path));
+	status = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    notmuch_path = talloc_asprintf (NULL, "%s/%s", path, ".notmuch");
+
+    err = mkdir (notmuch_path, 0755);
+
+    if (err) {
+	IGNORE_RESULT (asprintf (&message, "Error: Cannot create directory %s: %s.\n",
+				 notmuch_path, strerror (errno)));
+	status = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    status = notmuch_database_open_verbose (path,
+					    NOTMUCH_DATABASE_MODE_READ_WRITE,
+					    &notmuch, &message);
+    if (status)
+	goto DONE;
+
+    /* Upgrade doesn't add these feature to existing databases, but
+     * new databases have them. */
+    notmuch->features |= NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES;
+    notmuch->features |= NOTMUCH_FEATURE_INDEXED_MIMETYPES;
+
+    status = notmuch_database_upgrade (notmuch, NULL, NULL);
+    if (status) {
+	notmuch_database_close(notmuch);
+	notmuch = NULL;
+    }
+
+  DONE:
+    if (notmuch_path)
+	talloc_free (notmuch_path);
+
+    if (message) {
+	if (status_string)
+	    *status_string = message;
+	else
+	    free (message);
+    }
+    if (database)
+	*database = notmuch;
+    else
+	talloc_free (notmuch);
+    return status;
+}
+
+notmuch_status_t
+_notmuch_database_ensure_writable (notmuch_database_t *notmuch)
+{
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
+	_notmuch_database_log (notmuch, "Cannot write to a read-only database.\n");
+	return NOTMUCH_STATUS_READ_ONLY_DATABASE;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Allocate a revision number for the next change. */
+unsigned long
+_notmuch_database_new_revision (notmuch_database_t *notmuch)
+{
+    unsigned long new_revision = notmuch->revision + 1;
+
+    /* If we're in an atomic section, hold off on updating the
+     * committed revision number until we commit the atomic section.
+     */
+    if (notmuch->atomic_nesting)
+	notmuch->atomic_dirty = true;
+    else
+	notmuch->revision = new_revision;
+
+    return new_revision;
+}
+
+/* Parse a database features string from the given database version.
+ * Returns the feature bit set.
+ *
+ * For version < 3, this ignores the features string and returns a
+ * hard-coded set of features.
+ *
+ * If there are unrecognized features that are required to open the
+ * database in mode (which should be 'r' or 'w'), return a
+ * comma-separated list of unrecognized but required features in
+ * *incompat_out suitable for presenting to the user.  *incompat_out
+ * will be allocated from ctx.
+ */
+static _notmuch_features
+_parse_features (const void *ctx, const char *features, unsigned int version,
+		 char mode, char **incompat_out)
+{
+    _notmuch_features res = static_cast<_notmuch_features>(0);
+    unsigned int namelen, i;
+    size_t llen = 0;
+    const char *flags;
+
+    /* Prior to database version 3, features were implied by the
+     * version number. */
+    if (version == 0)
+	return NOTMUCH_FEATURES_V0;
+    else if (version == 1)
+	return NOTMUCH_FEATURES_V1;
+    else if (version == 2)
+	return NOTMUCH_FEATURES_V2;
+
+    /* Parse the features string */
+    while ((features = strtok_len_c (features + llen, "\n", &llen)) != NULL) {
+	flags = strchr (features, '\t');
+	if (! flags || flags > features + llen)
+	    continue;
+	namelen = flags - features;
+
+	for (i = 0; i < ARRAY_SIZE (feature_names); ++i) {
+	    if (strlen (feature_names[i].name) == namelen &&
+		strncmp (feature_names[i].name, features, namelen) == 0) {
+		res |= feature_names[i].value;
+		break;
+	    }
+	}
+
+	if (i == ARRAY_SIZE (feature_names) && incompat_out) {
+	    /* Unrecognized feature */
+	    const char *have = strchr (flags, mode);
+	    if (have && have < features + llen) {
+		/* This feature is required to access this database in
+		 * 'mode', but we don't understand it. */
+		if (! *incompat_out)
+		    *incompat_out = talloc_strdup (ctx, "");
+		*incompat_out = talloc_asprintf_append_buffer (
+		    *incompat_out, "%s%.*s", **incompat_out ? ", " : "",
+		    namelen, features);
+	    }
+	}
+    }
+
+    return res;
+}
+
+static char *
+_print_features (const void *ctx, unsigned int features)
+{
+    unsigned int i;
+    char *res = talloc_strdup (ctx, "");
+
+    for (i = 0; i < ARRAY_SIZE (feature_names); ++i)
+	if (features & feature_names[i].value)
+	    res = talloc_asprintf_append_buffer (
+		res, "%s\t%s\n", feature_names[i].name, feature_names[i].flags);
+
+    return res;
+}
+
+notmuch_status_t
+notmuch_database_open (const char *path,
+		       notmuch_database_mode_t mode,
+		       notmuch_database_t **database)
+{
+    char *status_string = NULL;
+    notmuch_status_t status;
+
+    status = notmuch_database_open_verbose (path, mode, database,
+					   &status_string);
+
+    if (status_string) {
+	fputs (status_string, stderr);
+	free (status_string);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_open_verbose (const char *path,
+			       notmuch_database_mode_t mode,
+			       notmuch_database_t **database,
+			       char **status_string)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    void *local = talloc_new (NULL);
+    notmuch_database_t *notmuch = NULL;
+    char *notmuch_path, *xapian_path, *incompat_features;
+    char *message = NULL;
+    struct stat st;
+    int err;
+    unsigned int i, version;
+    static int initialized = 0;
+
+    if (path == NULL) {
+	message = strdup ("Error: Cannot open a database for a NULL path.\n");
+	status = NOTMUCH_STATUS_NULL_POINTER;
+	goto DONE;
+    }
+
+    if (path[0] != '/') {
+	message = strdup ("Error: Database path must be absolute.\n");
+	status = NOTMUCH_STATUS_PATH_ERROR;
+	goto DONE;
+    }
+
+    if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
+	message = strdup ("Out of memory\n");
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    err = stat (notmuch_path, &st);
+    if (err) {
+	IGNORE_RESULT (asprintf (&message, "Error opening database at %s: %s\n",
+				 notmuch_path, strerror (errno)));
+	status = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) {
+	message = strdup ("Out of memory\n");
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    /* Initialize the GLib type system and threads */
+#if !GLIB_CHECK_VERSION(2, 35, 1)
+    g_type_init ();
+#endif
+
+    /* Initialize gmime */
+    if (! initialized) {
+	g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS);
+	initialized = 1;
+    }
+
+    notmuch = talloc_zero (NULL, notmuch_database_t);
+    notmuch->exception_reported = false;
+    notmuch->status_string = NULL;
+    notmuch->path = talloc_strdup (notmuch, path);
+
+    strip_trailing(notmuch->path, '/');
+
+    notmuch->mode = mode;
+    notmuch->atomic_nesting = 0;
+    notmuch->view = 1;
+    try {
+	string last_thread_id;
+	string last_mod;
+
+	if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
+	    notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
+							       DB_ACTION);
+	} else {
+	    notmuch->xapian_db = new Xapian::Database (xapian_path);
+	}
+
+	/* Check version.  As of database version 3, we represent
+	 * changes in terms of features, so assume a version bump
+	 * means a dramatically incompatible change. */
+	version = notmuch_database_get_version (notmuch);
+	if (version > NOTMUCH_DATABASE_VERSION) {
+	    IGNORE_RESULT (asprintf (&message,
+		      "Error: Notmuch database at %s\n"
+		      "       has a newer database format version (%u) than supported by this\n"
+		      "       version of notmuch (%u).\n",
+				     notmuch_path, version, NOTMUCH_DATABASE_VERSION));
+	    notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY;
+	    notmuch_database_destroy (notmuch);
+	    notmuch = NULL;
+	    status = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	/* Check features. */
+	incompat_features = NULL;
+	notmuch->features = _parse_features (
+	    local, notmuch->xapian_db->get_metadata ("features").c_str (),
+	    version, mode == NOTMUCH_DATABASE_MODE_READ_WRITE ? 'w' : 'r',
+	    &incompat_features);
+	if (incompat_features) {
+	    IGNORE_RESULT (asprintf (&message,
+		"Error: Notmuch database at %s\n"
+		"       requires features (%s)\n"
+		"       not supported by this version of notmuch.\n",
+				     notmuch_path, incompat_features));
+	    notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY;
+	    notmuch_database_destroy (notmuch);
+	    notmuch = NULL;
+	    status = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	notmuch->last_doc_id = notmuch->xapian_db->get_lastdocid ();
+	last_thread_id = notmuch->xapian_db->get_metadata ("last_thread_id");
+	if (last_thread_id.empty ()) {
+	    notmuch->last_thread_id = 0;
+	} else {
+	    const char *str;
+	    char *end;
+
+	    str = last_thread_id.c_str ();
+	    notmuch->last_thread_id = strtoull (str, &end, 16);
+	    if (*end != '\0')
+		INTERNAL_ERROR ("Malformed database last_thread_id: %s", str);
+	}
+
+	/* Get current highest revision number. */
+	last_mod = notmuch->xapian_db->get_value_upper_bound (
+	    NOTMUCH_VALUE_LAST_MOD);
+	if (last_mod.empty ())
+	    notmuch->revision = 0;
+	else
+	    notmuch->revision = Xapian::sortable_unserialise (last_mod);
+	notmuch->uuid = talloc_strdup (
+	    notmuch, notmuch->xapian_db->get_uuid ().c_str ());
+
+	notmuch->query_parser = new Xapian::QueryParser;
+	notmuch->term_gen = new Xapian::TermGenerator;
+	notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
+	notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
+	notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
+	notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
+
+	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
+	notmuch->query_parser->set_database (*notmuch->xapian_db);
+	notmuch->query_parser->set_stemmer (Xapian::Stem ("english"));
+	notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);
+	notmuch->query_parser->add_valuerangeprocessor (notmuch->value_range_processor);
+	notmuch->query_parser->add_valuerangeprocessor (notmuch->date_range_processor);
+	notmuch->query_parser->add_valuerangeprocessor (notmuch->last_mod_range_processor);
+
+	for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
+	    const prefix_t *prefix = &prefix_table[i];
+	    if (prefix->flags & NOTMUCH_FIELD_EXTERNAL) {
+		_setup_query_field (prefix, notmuch);
+	    }
+	}
+    } catch (const Xapian::Error &error) {
+	IGNORE_RESULT (asprintf (&message, "A Xapian exception occurred opening database: %s\n",
+				 error.get_msg().c_str()));
+	notmuch_database_destroy (notmuch);
+	notmuch = NULL;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    talloc_free (local);
+
+    if (message) {
+	if (status_string)
+	    *status_string = message;
+	else
+	    free (message);
+    }
+
+    if (database)
+	*database = notmuch;
+    else
+	talloc_free (notmuch);
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_close (notmuch_database_t *notmuch)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    /* Many Xapian objects (and thus notmuch objects) hold references to
+     * the database, so merely deleting the database may not suffice to
+     * close it.  Thus, we explicitly close it here. */
+    if (notmuch->xapian_db != NULL) {
+	try {
+	    /* If there's an outstanding transaction, it's unclear if
+	     * closing the Xapian database commits everything up to
+	     * that transaction, or may discard committed (but
+	     * unflushed) transactions.  To be certain, explicitly
+	     * cancel any outstanding transaction before closing. */
+	    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE &&
+		notmuch->atomic_nesting)
+		(static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))
+		    ->cancel_transaction ();
+
+	    /* Close the database.  This implicitly flushes
+	     * outstanding changes. */
+	    notmuch->xapian_db->close();
+	} catch (const Xapian::Error &error) {
+	    status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	    if (! notmuch->exception_reported) {
+		_notmuch_database_log (notmuch, "Error: A Xapian exception occurred closing database: %s\n",
+			 error.get_msg().c_str());
+	    }
+	}
+    }
+
+    delete notmuch->term_gen;
+    notmuch->term_gen = NULL;
+    delete notmuch->query_parser;
+    notmuch->query_parser = NULL;
+    delete notmuch->xapian_db;
+    notmuch->xapian_db = NULL;
+    delete notmuch->value_range_processor;
+    notmuch->value_range_processor = NULL;
+    delete notmuch->date_range_processor;
+    notmuch->date_range_processor = NULL;
+    delete notmuch->last_mod_range_processor;
+    notmuch->last_mod_range_processor = NULL;
+
+    return status;
+}
+
+notmuch_status_t
+_notmuch_database_reopen (notmuch_database_t *notmuch)
+{
+    if (notmuch->mode != NOTMUCH_DATABASE_MODE_READ_ONLY)
+	return NOTMUCH_STATUS_UNSUPPORTED_OPERATION;
+
+    try {
+	notmuch->xapian_db->reopen ();
+    } catch (const Xapian::Error &error) {
+	if (! notmuch->exception_reported) {
+	    _notmuch_database_log (notmuch, "Error: A Xapian exception reopening database: %s\n",
+				   error.get_msg ().c_str ());
+	    notmuch->exception_reported = true;
+	}
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    notmuch->view++;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static int
+unlink_cb (const char *path,
+	   unused (const struct stat *sb),
+	   unused (int type),
+	   unused (struct FTW *ftw))
+{
+    return remove (path);
+}
+
+static int
+rmtree (const char *path)
+{
+    return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
+}
+
+class NotmuchCompactor : public Xapian::Compactor
+{
+    notmuch_compact_status_cb_t status_cb;
+    void *status_closure;
+
+public:
+    NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) :
+	status_cb (cb), status_closure (closure) { }
+
+    virtual void
+    set_status (const std::string &table, const std::string &status)
+    {
+	char *msg;
+
+	if (status_cb == NULL)
+	    return;
+
+	if (status.length () == 0)
+	    msg = talloc_asprintf (NULL, "compacting table %s", table.c_str());
+	else
+	    msg = talloc_asprintf (NULL, "     %s", status.c_str());
+
+	if (msg == NULL) {
+	    return;
+	}
+
+	status_cb (msg, status_closure);
+	talloc_free (msg);
+    }
+};
+
+/* Compacts the given database, optionally saving the original database
+ * in backup_path. Additionally, a callback function can be provided to
+ * give the user feedback on the progress of the (likely long-lived)
+ * compaction process.
+ *
+ * The backup path must point to a directory on the same volume as the
+ * original database. Passing a NULL backup_path will result in the
+ * uncompacted database being deleted after compaction has finished.
+ * Note that the database write lock will be held during the
+ * compaction process to protect data integrity.
+ */
+notmuch_status_t
+notmuch_database_compact (const char *path,
+			  const char *backup_path,
+			  notmuch_compact_status_cb_t status_cb,
+			  void *closure)
+{
+    void *local;
+    char *notmuch_path, *xapian_path, *compact_xapian_path;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_database_t *notmuch = NULL;
+    struct stat statbuf;
+    bool keep_backup;
+    char *message = NULL;
+
+    local = talloc_new (NULL);
+    if (! local)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    ret = notmuch_database_open_verbose (path,
+					 NOTMUCH_DATABASE_MODE_READ_WRITE,
+					 &notmuch,
+					 &message);
+    if (ret) {
+	if (status_cb) status_cb (message, closure);
+	goto DONE;
+    }
+
+    if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
+	ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) {
+	ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    if (! (compact_xapian_path = talloc_asprintf (local, "%s.compact", xapian_path))) {
+	ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    if (backup_path == NULL) {
+	if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) {
+	    ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	    goto DONE;
+	}
+	keep_backup = false;
+    }
+    else {
+	keep_backup = true;
+    }
+
+    if (stat (backup_path, &statbuf) != -1) {
+	_notmuch_database_log (notmuch, "Path already exists: %s\n", backup_path);
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+    if (errno != ENOENT) {
+	_notmuch_database_log (notmuch, "Unknown error while stat()ing path: %s\n",
+		 strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    /* Unconditionally attempt to remove old work-in-progress database (if
+     * any). This is "protected" by database lock. If this fails due to write
+     * errors (etc), the following code will fail and provide error message.
+     */
+    (void) rmtree (compact_xapian_path);
+
+    try {
+	NotmuchCompactor compactor (status_cb, closure);
+
+	compactor.set_renumber (false);
+	compactor.add_source (xapian_path);
+	compactor.set_destdir (compact_xapian_path);
+	compactor.compact ();
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "Error while compacting: %s\n", error.get_msg().c_str());
+	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	goto DONE;
+    }
+
+    if (rename (xapian_path, backup_path)) {
+	_notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
+		 xapian_path, backup_path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (rename (compact_xapian_path, xapian_path)) {
+	_notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
+		 compact_xapian_path, xapian_path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+
+    if (! keep_backup) {
+	if (rmtree (backup_path)) {
+	    _notmuch_database_log (notmuch, "Error removing old database %s: %s\n",
+		     backup_path, strerror (errno));
+	    ret = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+    }
+
+  DONE:
+    if (notmuch) {
+	notmuch_status_t ret2;
+
+	const char *str = notmuch_database_status_string (notmuch);
+	if (status_cb && str)
+	    status_cb (str, closure);
+
+	ret2 = notmuch_database_destroy (notmuch);
+
+	/* don't clobber previous error status */
+	if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS)
+	    ret = ret2;
+    }
+
+    talloc_free (local);
+
+    return ret;
+}
+
+notmuch_status_t
+notmuch_database_destroy (notmuch_database_t *notmuch)
+{
+    notmuch_status_t status;
+
+    status = notmuch_database_close (notmuch);
+    talloc_free (notmuch);
+
+    return status;
+}
+
+const char *
+notmuch_database_get_path (notmuch_database_t *notmuch)
+{
+    return notmuch->path;
+}
+
+unsigned int
+notmuch_database_get_version (notmuch_database_t *notmuch)
+{
+    unsigned int version;
+    string version_string;
+    const char *str;
+    char *end;
+
+    version_string = notmuch->xapian_db->get_metadata ("version");
+    if (version_string.empty ())
+	return 0;
+
+    str = version_string.c_str ();
+    if (str == NULL || *str == '\0')
+	return 0;
+
+    version = strtoul (str, &end, 10);
+    if (*end != '\0')
+	INTERNAL_ERROR ("Malformed database version: %s", str);
+
+    return version;
+}
+
+notmuch_bool_t
+notmuch_database_needs_upgrade (notmuch_database_t *notmuch)
+{
+    return notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE &&
+	((NOTMUCH_FEATURES_CURRENT & ~notmuch->features) ||
+	 (notmuch_database_get_version (notmuch) < NOTMUCH_DATABASE_VERSION));
+}
+
+static volatile sig_atomic_t do_progress_notify = 0;
+
+static void
+handle_sigalrm (unused (int signal))
+{
+    do_progress_notify = 1;
+}
+
+/* Upgrade the current database.
+ *
+ * After opening a database in read-write mode, the client should
+ * check if an upgrade is needed (notmuch_database_needs_upgrade) and
+ * if so, upgrade with this function before making any modifications.
+ *
+ * The optional progress_notify callback can be used by the caller to
+ * provide progress indication to the user. If non-NULL it will be
+ * called periodically with 'count' as the number of messages upgraded
+ * so far and 'total' the overall number of messages that will be
+ * converted.
+ */
+notmuch_status_t
+notmuch_database_upgrade (notmuch_database_t *notmuch,
+			  void (*progress_notify) (void *closure,
+						   double progress),
+			  void *closure)
+{
+    void *local = talloc_new (NULL);
+    Xapian::TermIterator t, t_end;
+    Xapian::WritableDatabase *db;
+    struct sigaction action;
+    struct itimerval timerval;
+    bool timer_is_active = false;
+    enum _notmuch_features target_features, new_features;
+    notmuch_status_t status;
+    notmuch_private_status_t private_status;
+    notmuch_query_t *query = NULL;
+    unsigned int count = 0, total = 0;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    target_features = notmuch->features | NOTMUCH_FEATURES_CURRENT;
+    new_features = NOTMUCH_FEATURES_CURRENT & ~notmuch->features;
+
+    if (! notmuch_database_needs_upgrade (notmuch))
+	return NOTMUCH_STATUS_SUCCESS;
+
+    if (progress_notify) {
+	/* Set up our handler for SIGALRM */
+	memset (&action, 0, sizeof (struct sigaction));
+	action.sa_handler = handle_sigalrm;
+	sigemptyset (&action.sa_mask);
+	action.sa_flags = SA_RESTART;
+	sigaction (SIGALRM, &action, NULL);
+
+	/* Then start a timer to send SIGALRM once per second. */
+	timerval.it_interval.tv_sec = 1;
+	timerval.it_interval.tv_usec = 0;
+	timerval.it_value.tv_sec = 1;
+	timerval.it_value.tv_usec = 0;
+	setitimer (ITIMER_REAL, &timerval, NULL);
+
+	timer_is_active = true;
+    }
+
+    /* Figure out how much total work we need to do. */
+    if (new_features &
+	(NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
+	 NOTMUCH_FEATURE_LAST_MOD)) {
+	query = notmuch_query_create (notmuch, "");
+	unsigned msg_count;
+
+	status = notmuch_query_count_messages (query, &msg_count);
+	if (status)
+	    goto DONE;
+
+	total += msg_count;
+	notmuch_query_destroy (query);
+	query = NULL;
+    }
+    if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
+	t_end = db->allterms_end ("XTIMESTAMP");
+	for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++)
+	    ++total;
+    }
+    if (new_features & NOTMUCH_FEATURE_GHOSTS) {
+	/* The ghost message upgrade converts all thread_id_*
+	 * metadata values into ghost message documents. */
+	t_end = db->metadata_keys_end ("thread_id_");
+	for (t = db->metadata_keys_begin ("thread_id_"); t != t_end; ++t)
+	    ++total;
+    }
+
+    /* Perform the upgrade in a transaction. */
+    db->begin_transaction (true);
+
+    /* Set the target features so we write out changes in the desired
+     * format. */
+    notmuch->features = target_features;
+
+    /* Perform per-message upgrades. */
+    if (new_features &
+	(NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
+	 NOTMUCH_FEATURE_LAST_MOD)) {
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+	char *filename;
+
+	query = notmuch_query_create (notmuch, "");
+
+	status = notmuch_query_search_messages (query, &messages);
+	if (status)
+	    goto DONE;
+	for (;
+	     notmuch_messages_valid (messages);
+	     notmuch_messages_move_to_next (messages))
+	{
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    /* Before version 1, each message document had its
+	     * filename in the data field. Copy that into the new
+	     * format by calling notmuch_message_add_filename.
+	     */
+	    if (new_features & NOTMUCH_FEATURE_FILE_TERMS) {
+		filename = _notmuch_message_talloc_copy_data (message);
+		if (filename && *filename != '\0') {
+		    _notmuch_message_add_filename (message, filename);
+		    _notmuch_message_clear_data (message);
+		}
+		talloc_free (filename);
+	    }
+
+	    /* Prior to version 2, the "folder:" prefix was
+	     * probabilistic and stemmed. Change it to the current
+	     * boolean prefix. Add "path:" prefixes while at it.
+	     */
+	    if (new_features & NOTMUCH_FEATURE_BOOL_FOLDER)
+		_notmuch_message_upgrade_folder (message);
+
+	    /* Prior to NOTMUCH_FEATURE_LAST_MOD, messages did not
+	     * track modification revisions.  Give all messages the
+	     * next available revision; since we just started tracking
+	     * revisions for this database, that will be 1.
+	     */
+	    if (new_features & NOTMUCH_FEATURE_LAST_MOD)
+		_notmuch_message_upgrade_last_mod (message);
+
+	    _notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+	query = NULL;
+    }
+
+    /* Perform per-directory upgrades. */
+
+    /* Before version 1 we stored directory timestamps in
+     * XTIMESTAMP documents instead of the current XDIRECTORY
+     * documents. So copy those as well. */
+    if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
+	t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
+
+	for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
+	     t != t_end;
+	     t++)
+	{
+	    Xapian::PostingIterator p, p_end;
+	    std::string term = *t;
+
+	    p_end = notmuch->xapian_db->postlist_end (term);
+
+	    for (p = notmuch->xapian_db->postlist_begin (term);
+		 p != p_end;
+		 p++)
+	    {
+		Xapian::Document document;
+		time_t mtime;
+		notmuch_directory_t *directory;
+
+		if (do_progress_notify) {
+		    progress_notify (closure, (double) count / total);
+		    do_progress_notify = 0;
+		}
+
+		document = find_document_for_doc_id (notmuch, *p);
+		mtime = Xapian::sortable_unserialise (
+		    document.get_value (NOTMUCH_VALUE_TIMESTAMP));
+
+		directory = _notmuch_directory_create (notmuch, term.c_str() + 10,
+						       NOTMUCH_FIND_CREATE, &status);
+		notmuch_directory_set_mtime (directory, mtime);
+		notmuch_directory_destroy (directory);
+
+		db->delete_document (*p);
+	    }
+
+	    ++count;
+	}
+    }
+
+    /* Perform metadata upgrades. */
+
+    /* Prior to NOTMUCH_FEATURE_GHOSTS, thread IDs for missing
+     * messages were stored as database metadata. Change these to
+     * ghost messages.
+     */
+    if (new_features & NOTMUCH_FEATURE_GHOSTS) {
+	notmuch_message_t *message;
+	std::string message_id, thread_id;
+
+	t_end = db->metadata_keys_end (NOTMUCH_METADATA_THREAD_ID_PREFIX);
+	for (t = db->metadata_keys_begin (NOTMUCH_METADATA_THREAD_ID_PREFIX);
+	     t != t_end; ++t) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message_id = (*t).substr (
+		strlen (NOTMUCH_METADATA_THREAD_ID_PREFIX));
+	    thread_id = db->get_metadata (*t);
+
+	    /* Create ghost message */
+	    message = _notmuch_message_create_for_message_id (
+		notmuch, message_id.c_str (), &private_status);
+	    if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+		/* Document already exists; ignore the stored thread ID */
+	    } else if (private_status ==
+		       NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+		private_status = _notmuch_message_initialize_ghost (
+		    message, thread_id.c_str ());
+		if (! private_status)
+		    _notmuch_message_sync (message);
+	    }
+
+	    if (private_status) {
+		_notmuch_database_log (notmuch,
+			 "Upgrade failed while creating ghost messages.\n");
+		status = COERCE_STATUS (private_status, "Unexpected status from _notmuch_message_initialize_ghost");
+		goto DONE;
+	    }
+
+	    /* Clear saved metadata thread ID */
+	    db->set_metadata (*t, "");
+
+	    ++count;
+	}
+    }
+
+    status = NOTMUCH_STATUS_SUCCESS;
+    db->set_metadata ("features", _print_features (local, notmuch->features));
+    db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
+
+ DONE:
+    if (status == NOTMUCH_STATUS_SUCCESS)
+	db->commit_transaction ();
+    else
+	db->cancel_transaction ();
+
+    if (timer_is_active) {
+	/* Now stop the timer. */
+	timerval.it_interval.tv_sec = 0;
+	timerval.it_interval.tv_usec = 0;
+	timerval.it_value.tv_sec = 0;
+	timerval.it_value.tv_usec = 0;
+	setitimer (ITIMER_REAL, &timerval, NULL);
+
+	/* And disable the signal handler. */
+	action.sa_handler = SIG_IGN;
+	sigaction (SIGALRM, &action, NULL);
+    }
+
+    if (query)
+	notmuch_query_destroy (query);
+
+    talloc_free (local);
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_begin_atomic (notmuch_database_t *notmuch)
+{
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY ||
+	notmuch->atomic_nesting > 0)
+	goto DONE;
+
+    if (notmuch_database_needs_upgrade (notmuch))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    try {
+	(static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))->begin_transaction (false);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred beginning transaction: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+DONE:
+    notmuch->atomic_nesting++;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_end_atomic (notmuch_database_t *notmuch)
+{
+    Xapian::WritableDatabase *db;
+
+    if (notmuch->atomic_nesting == 0)
+	return NOTMUCH_STATUS_UNBALANCED_ATOMIC;
+
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY ||
+	notmuch->atomic_nesting > 1)
+	goto DONE;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+    try {
+	db->commit_transaction ();
+
+	/* This is a hack for testing.  Xapian never flushes on a
+	 * non-flushed commit, even if the flush threshold is 1.
+	 * However, we rely on flushing to test atomicity. */
+	const char *thresh = getenv ("XAPIAN_FLUSH_THRESHOLD");
+	if (thresh && atoi (thresh) == 1)
+	    db->commit ();
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred committing transaction: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    if (notmuch->atomic_dirty) {
+	++notmuch->revision;
+	notmuch->atomic_dirty = false;
+    }
+
+DONE:
+    notmuch->atomic_nesting--;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+unsigned long
+notmuch_database_get_revision (notmuch_database_t *notmuch,
+				const char **uuid)
+{
+    if (uuid)
+	*uuid = notmuch->uuid;
+    return notmuch->revision;
+}
+
+/* We allow the user to use arbitrarily long paths for directories. But
+ * we have a term-length limit. So if we exceed that, we'll use the
+ * SHA-1 of the path for the database term.
+ *
+ * Note: This function may return the original value of 'path'. If it
+ * does not, then the caller is responsible to free() the returned
+ * value.
+ */
+const char *
+_notmuch_database_get_directory_db_path (const char *path)
+{
+    int term_len = strlen (_find_prefix ("directory")) + strlen (path);
+
+    if (term_len > NOTMUCH_TERM_MAX)
+	return _notmuch_sha1_of_string (path);
+    else
+	return path;
+}
+
+/* Given a path, split it into two parts: the directory part is all
+ * components except for the last, and the basename is that last
+ * component. Getting the return-value for either part is optional
+ * (the caller can pass NULL).
+ *
+ * The original 'path' can represent either a regular file or a
+ * directory---the splitting will be carried out in the same way in
+ * either case. Trailing slashes on 'path' will be ignored, and any
+ * cases of multiple '/' characters appearing in series will be
+ * treated as a single '/'.
+ *
+ * Allocation (if any) will have 'ctx' as the talloc owner. But
+ * pointers will be returned within the original path string whenever
+ * possible.
+ *
+ * Note: If 'path' is non-empty and contains no non-trailing slash,
+ * (that is, consists of a filename with no parent directory), then
+ * the directory returned will be an empty string. However, if 'path'
+ * is an empty string, then both directory and basename will be
+ * returned as NULL.
+ */
+notmuch_status_t
+_notmuch_database_split_path (void *ctx,
+			      const char *path,
+			      const char **directory,
+			      const char **basename)
+{
+    const char *slash;
+
+    if (path == NULL || *path == '\0') {
+	if (directory)
+	    *directory = NULL;
+	if (basename)
+	    *basename = NULL;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    /* Find the last slash (not counting a trailing slash), if any. */
+
+    slash = path + strlen (path) - 1;
+
+    /* First, skip trailing slashes. */
+    while (slash != path && *slash == '/')
+	--slash;
+
+    /* Then, find a slash. */
+    while (slash != path && *slash != '/') {
+	if (basename)
+	    *basename = slash;
+
+	--slash;
+    }
+
+    /* Finally, skip multiple slashes. */
+    while (slash != path && *(slash - 1) == '/')
+	--slash;
+
+    if (slash == path) {
+	if (directory)
+	    *directory = talloc_strdup (ctx, "");
+	if (basename)
+	    *basename = path;
+    } else {
+	if (directory)
+	    *directory = talloc_strndup (ctx, path, slash - path);
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Find the document ID of the specified directory.
+ *
+ * If (flags & NOTMUCH_FIND_CREATE), a new directory document will be
+ * created if one does not exist for 'path'.  Otherwise, if the
+ * directory document does not exist, this sets *directory_id to
+ * ((unsigned int)-1) and returns NOTMUCH_STATUS_SUCCESS.
+ */
+notmuch_status_t
+_notmuch_database_find_directory_id (notmuch_database_t *notmuch,
+				     const char *path,
+				     notmuch_find_flags_t flags,
+				     unsigned int *directory_id)
+{
+    notmuch_directory_t *directory;
+    notmuch_status_t status;
+
+    if (path == NULL) {
+	*directory_id = 0;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    directory = _notmuch_directory_create (notmuch, path, flags, &status);
+    if (status || !directory) {
+	*directory_id = -1;
+	return status;
+    }
+
+    *directory_id = _notmuch_directory_get_document_id (directory);
+
+    notmuch_directory_destroy (directory);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+const char *
+_notmuch_database_get_directory_path (void *ctx,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id)
+{
+    Xapian::Document document;
+
+    document = find_document_for_doc_id (notmuch, doc_id);
+
+    return talloc_strdup (ctx, document.get_data ().c_str ());
+}
+
+/* Given a legal 'filename' for the database, (either relative to
+ * database path or absolute with initial components identical to
+ * database path), return a new string (with 'ctx' as the talloc
+ * owner) suitable for use as a direntry term value.
+ *
+ * If (flags & NOTMUCH_FIND_CREATE), the necessary directory documents
+ * will be created in the database as needed.  Otherwise, if the
+ * necessary directory documents do not exist, this sets
+ * *direntry to NULL and returns NOTMUCH_STATUS_SUCCESS.
+ */
+notmuch_status_t
+_notmuch_database_filename_to_direntry (void *ctx,
+					notmuch_database_t *notmuch,
+					const char *filename,
+					notmuch_find_flags_t flags,
+					char **direntry)
+{
+    const char *relative, *directory, *basename;
+    Xapian::docid directory_id;
+    notmuch_status_t status;
+
+    relative = _notmuch_database_relative_path (notmuch, filename);
+
+    status = _notmuch_database_split_path (ctx, relative,
+					   &directory, &basename);
+    if (status)
+	return status;
+
+    status = _notmuch_database_find_directory_id (notmuch, directory, flags,
+						  &directory_id);
+    if (status || directory_id == (unsigned int)-1) {
+	*direntry = NULL;
+	return status;
+    }
+
+    *direntry = talloc_asprintf (ctx, "%u:%s", directory_id, basename);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Given a legal 'path' for the database, return the relative path.
+ *
+ * The return value will be a pointer to the original path contents,
+ * and will be either the original string (if 'path' was relative) or
+ * a portion of the string (if path was absolute and begins with the
+ * database path).
+ */
+const char *
+_notmuch_database_relative_path (notmuch_database_t *notmuch,
+				 const char *path)
+{
+    const char *db_path, *relative;
+    unsigned int db_path_len;
+
+    db_path = notmuch_database_get_path (notmuch);
+    db_path_len = strlen (db_path);
+
+    relative = path;
+
+    if (*relative == '/') {
+	while (*relative == '/' && *(relative+1) == '/')
+	    relative++;
+
+	if (strncmp (relative, db_path, db_path_len) == 0)
+	{
+	    relative += db_path_len;
+	    while (*relative == '/')
+		relative++;
+	}
+    }
+
+    return relative;
+}
+
+notmuch_status_t
+notmuch_database_get_directory (notmuch_database_t *notmuch,
+				const char *path,
+				notmuch_directory_t **directory)
+{
+    notmuch_status_t status;
+
+    if (directory == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    *directory = NULL;
+
+    try {
+	*directory = _notmuch_directory_create (notmuch, path,
+						NOTMUCH_FIND_LOOKUP, &status);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred getting directory: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return status;
+}
+
+/* Allocate a document ID that satisfies the following criteria:
+ *
+ * 1. The ID does not exist for any document in the Xapian database
+ *
+ * 2. The ID was not previously returned from this function
+ *
+ * 3. The ID is the smallest integer satisfying (1) and (2)
+ *
+ * This function will trigger an internal error if these constraints
+ * cannot all be satisfied, (that is, the pool of available document
+ * IDs has been exhausted).
+ */
+unsigned int
+_notmuch_database_generate_doc_id (notmuch_database_t *notmuch)
+{
+    assert (notmuch->last_doc_id >= notmuch->xapian_db->get_lastdocid ());
+
+    notmuch->last_doc_id++;
+
+    if (notmuch->last_doc_id == 0)
+	INTERNAL_ERROR ("Xapian document IDs are exhausted.\n");
+
+    return notmuch->last_doc_id;
+}
+
+notmuch_status_t
+notmuch_database_remove_message (notmuch_database_t *notmuch,
+				 const char *filename)
+{
+    notmuch_status_t status;
+    notmuch_message_t *message;
+
+    status = notmuch_database_find_message_by_filename (notmuch, filename,
+							&message);
+
+    if (status == NOTMUCH_STATUS_SUCCESS && message) {
+	    status = _notmuch_message_remove_filename (message, filename);
+	    if (status == NOTMUCH_STATUS_SUCCESS)
+		_notmuch_message_delete (message);
+	    else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
+		_notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
+					   const char *filename,
+					   notmuch_message_t **message_ret)
+{
+    void *local;
+    const char *prefix = _find_prefix ("file-direntry");
+    char *direntry, *term;
+    Xapian::PostingIterator i, end;
+    notmuch_status_t status;
+
+    if (message_ret == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_FILE_TERMS))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    /* return NULL on any failure */
+    *message_ret = NULL;
+
+    local = talloc_new (notmuch);
+
+    try {
+	status = _notmuch_database_filename_to_direntry (
+	    local, notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
+	if (status || !direntry)
+	    goto DONE;
+
+	term = talloc_asprintf (local, "%s%s", prefix, direntry);
+
+	find_doc_ids_for_term (notmuch, term, &i, &end);
+
+	if (i != end) {
+	    notmuch_private_status_t private_status;
+
+	    *message_ret = _notmuch_message_create (notmuch, notmuch, *i,
+						    &private_status);
+	    if (*message_ret == NULL)
+		status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	}
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred finding message by filename: %s\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    talloc_free (local);
+
+    if (status && *message_ret) {
+	notmuch_message_destroy (*message_ret);
+	*message_ret = NULL;
+    }
+    return status;
+}
+
+notmuch_string_list_t *
+_notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
+					 Xapian::TermIterator &end,
+					 const char *prefix)
+{
+    int prefix_len = strlen (prefix);
+    notmuch_string_list_t *list;
+
+    list = _notmuch_string_list_create (ctx);
+    if (unlikely (list == NULL))
+	return NULL;
+
+    for (i.skip_to (prefix); i != end; i++) {
+	/* Terminate loop at first term without desired prefix. */
+	if (strncmp ((*i).c_str (), prefix, prefix_len))
+	    break;
+
+	_notmuch_string_list_append (list, (*i).c_str () + prefix_len);
+    }
+
+    return list;
+}
+
+notmuch_tags_t *
+notmuch_database_get_all_tags (notmuch_database_t *db)
+{
+    Xapian::TermIterator i, end;
+    notmuch_string_list_t *tags;
+
+    try {
+	i = db->xapian_db->allterms_begin();
+	end = db->xapian_db->allterms_end();
+	tags = _notmuch_database_get_terms_with_prefix (db, i, end,
+							_find_prefix ("tag"));
+	_notmuch_string_list_sort (tags);
+	return _notmuch_tags_create (db, tags);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (db, "A Xapian exception occurred getting tags: %s.\n",
+		 error.get_msg().c_str());
+	db->exception_reported = true;
+	return NULL;
+    }
+}
+
+const char *
+notmuch_database_status_string (const notmuch_database_t *notmuch)
+{
+    return notmuch->status_string;
+}
diff --git a/lib/directory.cc b/lib/directory.cc
new file mode 100644
index 00000000..4fcb0177
--- /dev/null
+++ b/lib/directory.cc
@@ -0,0 +1,316 @@
+/* directory.cc - Results of directory-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+/* Create an iterator to iterate over the basenames of files (or
+ * directories) that all share a common parent directory.
+ */
+static notmuch_filenames_t *
+_create_filenames_for_terms_with_prefix (void *ctx,
+					 notmuch_database_t *notmuch,
+					 const char *prefix)
+{
+    notmuch_string_list_t *filename_list;
+    Xapian::TermIterator i, end;
+
+    i = notmuch->xapian_db->allterms_begin();
+    end = notmuch->xapian_db->allterms_end();
+    filename_list = _notmuch_database_get_terms_with_prefix (ctx, i, end,
+							     prefix);
+    if (unlikely (filename_list == NULL))
+	return NULL;
+
+    return _notmuch_filenames_create (ctx, filename_list);
+}
+
+struct _notmuch_directory {
+    notmuch_database_t *notmuch;
+    Xapian::docid document_id;
+    Xapian::Document doc;
+    time_t mtime;
+};
+
+/* We end up having to call the destructor explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_directory_destructor (notmuch_directory_t *directory)
+{
+    directory->doc.~Document ();
+
+    return 0;
+}
+
+static notmuch_private_status_t
+find_directory_document (notmuch_database_t *notmuch,
+			 const char *db_path,
+			 Xapian::Document *document)
+{
+    notmuch_private_status_t status;
+    Xapian::docid doc_id;
+
+    status = _notmuch_database_find_unique_doc_id (notmuch, "directory",
+						   db_path, &doc_id);
+    if (status) {
+	*document = Xapian::Document ();
+	return status;
+    }
+
+    *document = notmuch->xapian_db->get_document (doc_id);
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Find or create a directory document.
+ *
+ * 'path' should be a path relative to the path of 'database', or else
+ * should be an absolute path with initial components that match the
+ * path of 'database'.
+ *
+ * If (flags & NOTMUCH_FIND_CREATE), then the directory document will
+ * be created if it does not exist.  Otherwise, if the directory
+ * document does not exist, *status_ret is set to
+ * NOTMUCH_STATUS_SUCCESS and this returns NULL.
+ */
+notmuch_directory_t *
+_notmuch_directory_create (notmuch_database_t *notmuch,
+			   const char *path,
+			   notmuch_find_flags_t flags,
+			   notmuch_status_t *status_ret)
+{
+    Xapian::WritableDatabase *db;
+    notmuch_directory_t *directory;
+    notmuch_private_status_t private_status;
+    const char *db_path;
+    bool create = (flags & NOTMUCH_FIND_CREATE);
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_DIRECTORY_DOCS)) {
+	*status_ret = NOTMUCH_STATUS_UPGRADE_REQUIRED;
+	return NULL;
+    }
+
+    *status_ret = NOTMUCH_STATUS_SUCCESS;
+
+    path = _notmuch_database_relative_path (notmuch, path);
+
+    if (create && notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	INTERNAL_ERROR ("Failure to ensure database is writable");
+
+    directory = talloc (notmuch, notmuch_directory_t);
+    if (unlikely (directory == NULL)) {
+	*status_ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    directory->notmuch = notmuch;
+
+    /* "placement new"---not actually allocating memory */
+    new (&directory->doc) Xapian::Document;
+
+    talloc_set_destructor (directory, _notmuch_directory_destructor);
+
+    db_path = _notmuch_database_get_directory_db_path (path);
+
+    try {
+	Xapian::TermIterator i, end;
+
+	private_status = find_directory_document (notmuch, db_path,
+						  &directory->doc);
+	directory->document_id = directory->doc.get_docid ();
+
+	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	    if (!create) {
+		notmuch_directory_destroy (directory);
+		directory = NULL;
+		*status_ret = NOTMUCH_STATUS_SUCCESS;
+		goto DONE;
+	    }
+
+	    void *local = talloc_new (directory);
+	    const char *parent, *basename;
+	    Xapian::docid parent_id;
+	    char *term = talloc_asprintf (local, "%s%s",
+					  _find_prefix ("directory"), db_path);
+	    directory->doc.add_term (term, 0);
+
+	    directory->doc.set_data (path);
+
+	    _notmuch_database_split_path (local, path, &parent, &basename);
+
+	    *status_ret = _notmuch_database_find_directory_id (
+		notmuch, parent, NOTMUCH_FIND_CREATE, &parent_id);
+	    if (*status_ret) {
+		notmuch_directory_destroy (directory);
+		directory = NULL;
+		goto DONE;
+	    }
+
+	    if (basename) {
+		term = talloc_asprintf (local, "%s%u:%s",
+					_find_prefix ("directory-direntry"),
+					parent_id, basename);
+		directory->doc.add_term (term, 0);
+	    }
+
+	    directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+				      Xapian::sortable_serialise (0));
+
+	    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+	    directory->document_id = _notmuch_database_generate_doc_id (notmuch);
+	    db->replace_document (directory->document_id, directory->doc);
+	    talloc_free (local);
+	}
+
+	directory->mtime = Xapian::sortable_unserialise (
+	    directory->doc.get_value (NOTMUCH_VALUE_TIMESTAMP));
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+		 "A Xapian exception occurred creating a directory: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	notmuch_directory_destroy (directory);
+	directory = NULL;
+	*status_ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+  DONE:
+    if (db_path != path)
+	free ((char *) db_path);
+
+    return directory;
+}
+
+unsigned int
+_notmuch_directory_get_document_id (notmuch_directory_t *directory)
+{
+    return directory->document_id;
+}
+
+notmuch_status_t
+notmuch_directory_set_mtime (notmuch_directory_t *directory,
+			     time_t mtime)
+{
+    notmuch_database_t *notmuch = directory->notmuch;
+    Xapian::WritableDatabase *db;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    try {
+	directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+				   Xapian::sortable_serialise (mtime));
+
+	db->replace_document (directory->document_id, directory->doc);
+
+	directory->mtime = mtime;
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+		 "A Xapian exception occurred setting directory mtime: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+time_t
+notmuch_directory_get_mtime (notmuch_directory_t *directory)
+{
+    return directory->mtime;
+}
+
+notmuch_filenames_t *
+notmuch_directory_get_child_files (notmuch_directory_t *directory)
+{
+    char *term;
+    notmuch_filenames_t *child_files;
+
+    term = talloc_asprintf (directory, "%s%u:",
+			    _find_prefix ("file-direntry"),
+			    directory->document_id);
+
+    child_files = _create_filenames_for_terms_with_prefix (directory,
+							   directory->notmuch,
+							   term);
+
+    talloc_free (term);
+
+    return child_files;
+}
+
+notmuch_filenames_t *
+notmuch_directory_get_child_directories (notmuch_directory_t *directory)
+{
+    char *term;
+    notmuch_filenames_t *child_directories;
+
+    term = talloc_asprintf (directory, "%s%u:",
+			    _find_prefix ("directory-direntry"),
+			    directory->document_id);
+
+    child_directories = _create_filenames_for_terms_with_prefix (directory,
+						 directory->notmuch, term);
+
+    talloc_free (term);
+
+    return child_directories;
+}
+
+notmuch_status_t
+notmuch_directory_delete (notmuch_directory_t *directory)
+{
+    notmuch_status_t status;
+    Xapian::WritableDatabase *db;
+
+    status = _notmuch_database_ensure_writable (directory->notmuch);
+    if (status)
+	return status;
+
+    try {
+	db = static_cast <Xapian::WritableDatabase *> (directory->notmuch->xapian_db);
+	db->delete_document (directory->document_id);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (directory->notmuch,
+			       "A Xapian exception occurred deleting directory entry: %s.\n",
+			       error.get_msg().c_str());
+	directory->notmuch->exception_reported = true;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    notmuch_directory_destroy (directory);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+void
+notmuch_directory_destroy (notmuch_directory_t *directory)
+{
+    talloc_free (directory);
+}
diff --git a/lib/filenames.c b/lib/filenames.c
new file mode 100644
index 00000000..37d631d6
--- /dev/null
+++ b/lib/filenames.c
@@ -0,0 +1,76 @@
+/* filenames.c - Iterator for a list of filenames
+ *
+ * Copyright © 2010 Intel Corporation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+struct _notmuch_filenames {
+    notmuch_string_node_t *iterator;
+};
+
+/* The notmuch_filenames_t iterates over a notmuch_string_list_t of
+ * file names */
+notmuch_filenames_t *
+_notmuch_filenames_create (const void *ctx,
+			   notmuch_string_list_t *list)
+{
+    notmuch_filenames_t *filenames;
+
+    filenames = talloc (ctx, notmuch_filenames_t);
+    if (unlikely (filenames == NULL))
+	return NULL;
+
+    filenames->iterator = list->head;
+    (void) talloc_reference (filenames, list);
+
+    return filenames;
+}
+
+notmuch_bool_t
+notmuch_filenames_valid (notmuch_filenames_t *filenames)
+{
+    if (filenames == NULL)
+	return false;
+
+    return (filenames->iterator != NULL);
+}
+
+const char *
+notmuch_filenames_get (notmuch_filenames_t *filenames)
+{
+    if ((filenames == NULL) || (filenames->iterator == NULL))
+	return NULL;
+
+    return filenames->iterator->string;
+}
+
+void
+notmuch_filenames_move_to_next (notmuch_filenames_t *filenames)
+{
+    if ((filenames == NULL) || (filenames->iterator == NULL))
+	return;
+
+    filenames->iterator = filenames->iterator->next;
+}
+
+void
+notmuch_filenames_destroy (notmuch_filenames_t *filenames)
+{
+    talloc_free (filenames);
+}
diff --git a/lib/index.cc b/lib/index.cc
new file mode 100644
index 00000000..3f694387
--- /dev/null
+++ b/lib/index.cc
@@ -0,0 +1,630 @@
+/*
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <gmime/gmime.h>
+#include <gmime/gmime-filter.h>
+
+#include <xapian.h>
+
+
+typedef struct {
+    int state;
+    int a;
+    int b;
+    int next_if_match;
+    int next_if_not_match;
+} scanner_state_t;
+
+/* Simple, linear state-transition diagram for the uuencode filter.
+ *
+ * If the character being processed is within the range of [a, b]
+ * for the current state then we transition next_if_match
+ * state. If not, we transition to the next_if_not_match state.
+ *
+ * The final two states are special in that they are the states in
+ * which we discard data. */
+static const int first_uuencode_skipping_state = 11;
+static const scanner_state_t uuencode_states[] = {
+    {0,  'b',  'b',  1,  0},
+    {1,  'e',  'e',  2,  0},
+    {2,  'g',  'g',  3,  0},
+    {3,  'i',  'i',  4,  0},
+    {4,  'n',  'n',  5,  0},
+    {5,  ' ',  ' ',  6,  0},
+    {6,  '0',  '7',  7,  0},
+    {7,  '0',  '7',  8,  0},
+    {8,  '0',  '7',  9,  0},
+    {9,  ' ',  ' ',  10, 0},
+    {10, '\n', '\n', 11, 10},
+    {11, 'M',  'M',  12, 0},
+    {12, ' ',  '`',  12, 11}
+};
+
+/* The following table is intended to implement this DFA (in 'dot'
+   format). Note that 2 and 3 are "hidden" states used to step through
+   the possible out edges of state 1.
+
+digraph html_filter {
+       0 -> 1  [label="<"];
+       0 -> 0;
+       1 -> 4 [label="'"];
+       1 -> 5 [label="\""];
+       1 -> 0 [label=">"];
+       1 -> 1;
+       4 -> 1 [label="'"];
+       4 -> 4;
+       5 -> 1 [label="\""];
+       5 -> 5;
+}
+*/
+static const int first_html_skipping_state = 1;
+static const scanner_state_t html_states[] = {
+    {0,  '<',  '<',  1,  0},
+    {1,  '\'', '\'', 4,  2},  /* scanning for quote or > */
+    {1,  '"',  '"',  5,  3},
+    {1,  '>',  '>',  0,  1},
+    {4,  '\'', '\'', 1,  4},  /* inside single quotes */
+    {5,  '"', '"',   1,  5},  /* inside double quotes */
+};
+
+/* Oh, how I wish that gobject didn't require so much noisy boilerplate!
+ * (Though I have at least eliminated some of the stock set...) */
+typedef struct _NotmuchFilterDiscardNonTerm NotmuchFilterDiscardNonTerm;
+typedef struct _NotmuchFilterDiscardNonTermClass NotmuchFilterDiscardNonTermClass;
+
+/**
+ * NotmuchFilterDiscardNonTerm:
+ *
+ * @parent_object: parent #GMimeFilter
+ * @encode: encoding vs decoding
+ * @state: State of the parser
+ *
+ * A filter to discard uuencoded portions of an email.
+ *
+ * A uuencoded portion is identified as beginning with a line
+ * matching:
+ *
+ *	begin [0-7][0-7][0-7] .*
+ *
+ * After that detection, and beginning with the following line,
+ * characters will be discarded as long as the first character of each
+ * line begins with M and subsequent characters on the line are within
+ * the range of ASCII characters from ' ' to '`'.
+ *
+ * This is not a perfect UUencode filter. It's possible to have a
+ * message that will legitimately match that pattern, (so that some
+ * legitimate content is discarded). And for most UUencoded files, the
+ * final line of encoded data (the line not starting with M) will be
+ * indexed.
+ **/
+struct _NotmuchFilterDiscardNonTerm {
+    GMimeFilter parent_object;
+    GMimeContentType *content_type;
+    int state;
+    int first_skipping_state;
+    const scanner_state_t *states;
+};
+
+struct _NotmuchFilterDiscardNonTermClass {
+    GMimeFilterClass parent_class;
+};
+
+static GMimeFilter *notmuch_filter_discard_non_term_new (GMimeContentType *content);
+
+static void notmuch_filter_discard_non_term_finalize (GObject *object);
+
+static GMimeFilter *filter_copy (GMimeFilter *filter);
+static void filter_filter (GMimeFilter *filter, char *in, size_t len, size_t prespace,
+			   char **out, size_t *outlen, size_t *outprespace);
+static void filter_complete (GMimeFilter *filter, char *in, size_t len, size_t prespace,
+			     char **out, size_t *outlen, size_t *outprespace);
+static void filter_reset (GMimeFilter *filter);
+
+
+static GMimeFilterClass *parent_class = NULL;
+
+static void
+notmuch_filter_discard_non_term_class_init (NotmuchFilterDiscardNonTermClass *klass)
+{
+    GObjectClass *object_class = G_OBJECT_CLASS (klass);
+    GMimeFilterClass *filter_class = GMIME_FILTER_CLASS (klass);
+
+    parent_class = (GMimeFilterClass *) g_type_class_ref (GMIME_TYPE_FILTER);
+
+    object_class->finalize = notmuch_filter_discard_non_term_finalize;
+
+    filter_class->copy = filter_copy;
+    filter_class->filter = filter_filter;
+    filter_class->complete = filter_complete;
+    filter_class->reset = filter_reset;
+}
+
+static void
+notmuch_filter_discard_non_term_finalize (GObject *object)
+{
+    G_OBJECT_CLASS (parent_class)->finalize (object);
+}
+
+static GMimeFilter *
+filter_copy (GMimeFilter *gmime_filter)
+{
+    NotmuchFilterDiscardNonTerm *filter = (NotmuchFilterDiscardNonTerm *) gmime_filter;
+    return notmuch_filter_discard_non_term_new (filter->content_type);
+}
+
+static void
+filter_filter (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t prespace,
+	       char **outbuf, size_t *outlen, size_t *outprespace)
+{
+    NotmuchFilterDiscardNonTerm *filter = (NotmuchFilterDiscardNonTerm *) gmime_filter;
+    const scanner_state_t *states = filter->states;
+    const char *inptr = inbuf;
+    const char *inend = inbuf + inlen;
+    char *outptr;
+
+    (void) prespace;
+
+    int next;
+
+    g_mime_filter_set_size (gmime_filter, inlen, false);
+    outptr = gmime_filter->outbuf;
+
+    next = filter->state;
+    while (inptr < inend) {
+	 /* Each state is defined by a contiguous set of rows of the
+	 * state table marked by a common value for '.state'. The
+	 * state numbers must be equal to the index of the first row
+	 * in a given state; thus the loop condition here looks for a
+	 * jump to a first row of a state, which is a real transition
+	 * in the underlying DFA.
+	 */
+	do {
+	    if (*inptr >= states[next].a && *inptr <= states[next].b)  {
+		next = states[next].next_if_match;
+	    } else  {
+		next = states[next].next_if_not_match;
+	    }
+
+	} while (next != states[next].state);
+
+	if (filter->state < filter->first_skipping_state)
+	    *outptr++ = *inptr;
+
+	filter->state = next;
+	inptr++;
+    }
+
+    *outlen = outptr - gmime_filter->outbuf;
+    *outprespace = gmime_filter->outpre;
+    *outbuf = gmime_filter->outbuf;
+}
+
+static void
+filter_complete (GMimeFilter *filter, char *inbuf, size_t inlen, size_t prespace,
+		 char **outbuf, size_t *outlen, size_t *outprespace)
+{
+    if (inbuf && inlen)
+	filter_filter (filter, inbuf, inlen, prespace, outbuf, outlen, outprespace);
+}
+
+static void
+filter_reset (GMimeFilter *gmime_filter)
+{
+    NotmuchFilterDiscardNonTerm *filter = (NotmuchFilterDiscardNonTerm *) gmime_filter;
+
+    filter->state = 0;
+}
+
+/**
+ * notmuch_filter_discard_non_term_new:
+ *
+ * Returns: a new #NotmuchFilterDiscardNonTerm filter.
+ **/
+static GMimeFilter *
+notmuch_filter_discard_non_term_new (GMimeContentType *content_type)
+{
+    static GType type = 0;
+    NotmuchFilterDiscardNonTerm *filter;
+
+    if (!type) {
+	static const GTypeInfo info = {
+	    sizeof (NotmuchFilterDiscardNonTermClass),
+	    NULL, /* base_class_init */
+	    NULL, /* base_class_finalize */
+	    (GClassInitFunc) notmuch_filter_discard_non_term_class_init,
+	    NULL, /* class_finalize */
+	    NULL, /* class_data */
+	    sizeof (NotmuchFilterDiscardNonTerm),
+	    0,    /* n_preallocs */
+	    NULL, /* instance_init */
+	    NULL  /* value_table */
+	};
+
+	type = g_type_register_static (GMIME_TYPE_FILTER, "NotmuchFilterDiscardNonTerm", &info, (GTypeFlags) 0);
+    }
+
+    filter = (NotmuchFilterDiscardNonTerm *) g_object_new (type, NULL);
+    filter->content_type = content_type;
+    filter->state = 0;
+    if (g_mime_content_type_is_type (content_type, "text", "html")) {
+      filter->states = html_states;
+      filter->first_skipping_state = first_html_skipping_state;
+    } else {
+      filter->states = uuencode_states;
+      filter->first_skipping_state = first_uuencode_skipping_state;
+    }
+
+    return (GMimeFilter *) filter;
+}
+
+/* We're finally down to a single (NAME + address) email "mailbox". */
+static void
+_index_address_mailbox (notmuch_message_t *message,
+			const char *prefix_name,
+			InternetAddress *address)
+{
+    InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
+    const char *name, *addr, *combined;
+    void *local = talloc_new (message);
+
+    name = internet_address_get_name (address);
+    addr = internet_address_mailbox_get_addr (mailbox);
+
+    /* Combine the name and address and index them as a phrase. */
+    if (name && addr)
+	combined = talloc_asprintf (local, "%s %s", name, addr);
+    else if (name)
+	combined = name;
+    else
+	combined = addr;
+
+    if (combined)
+	_notmuch_message_gen_terms (message, prefix_name, combined);
+
+    talloc_free (local);
+}
+
+static void
+_index_address_list (notmuch_message_t *message,
+		     const char *prefix_name,
+		     InternetAddressList *addresses);
+
+/* The outer loop over the InternetAddressList wasn't quite enough.
+ * There can actually be a tree here where a single member of the list
+ * is a "group" containing another list. Recurse please.
+ */
+static void
+_index_address_group (notmuch_message_t *message,
+		      const char *prefix_name,
+		      InternetAddress *address)
+{
+    InternetAddressGroup *group;
+    InternetAddressList *list;
+
+    group = INTERNET_ADDRESS_GROUP (address);
+    list = internet_address_group_get_members (group);
+
+    if (! list)
+	return;
+
+    _index_address_list (message, prefix_name, list);
+}
+
+static void
+_index_address_list (notmuch_message_t *message,
+		     const char *prefix_name,
+		     InternetAddressList *addresses)
+{
+    int i;
+    InternetAddress *address;
+
+    if (addresses == NULL)
+	return;
+
+    for (i = 0; i < internet_address_list_length (addresses); i++) {
+	address = internet_address_list_get_address (addresses, i);
+	if (INTERNET_ADDRESS_IS_MAILBOX (address)) {
+	    _index_address_mailbox (message, prefix_name, address);
+	} else if (INTERNET_ADDRESS_IS_GROUP (address)) {
+	    _index_address_group (message, prefix_name, address);
+	} else {
+	    INTERNAL_ERROR ("GMime InternetAddress is neither a mailbox nor a group.\n");
+	}
+    }
+}
+
+static void
+_index_content_type (notmuch_message_t *message, GMimeObject *part)
+{
+    GMimeContentType *content_type = g_mime_object_get_content_type (part);
+    if (content_type) {
+	char *mime_string = g_mime_content_type_to_string (content_type);
+	if (mime_string) {
+	    _notmuch_message_gen_terms (message, "mimetype", mime_string);
+	    g_free (mime_string);
+	}
+    }
+}
+
+static void
+_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts,
+			    GMimeContentType *content_type,
+			    GMimeMultipartEncrypted *part);
+
+/* Callback to generate terms for each mime part of a message. */
+static void
+_index_mime_part (notmuch_message_t *message,
+		  notmuch_indexopts_t *indexopts,
+		  GMimeObject *part)
+{
+    GMimeStream *stream, *filter;
+    GMimeFilter *discard_non_term_filter;
+    GMimeDataWrapper *wrapper;
+    GByteArray *byte_array;
+    GMimeContentDisposition *disposition;
+    GMimeContentType *content_type;
+    char *body;
+    const char *charset;
+
+    if (! part) {
+	_notmuch_database_log (notmuch_message_get_database (message),
+			      "Warning: Not indexing empty mime part.\n");
+	return;
+    }
+
+    _index_content_type (message, part);
+    content_type = g_mime_object_get_content_type (part);
+
+    if (GMIME_IS_MULTIPART (part)) {
+	GMimeMultipart *multipart = GMIME_MULTIPART (part);
+	int i;
+
+	if (GMIME_IS_MULTIPART_SIGNED (multipart))
+	  _notmuch_message_add_term (message, "tag", "signed");
+
+	if (GMIME_IS_MULTIPART_ENCRYPTED (multipart))
+	  _notmuch_message_add_term (message, "tag", "encrypted");
+
+	for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
+	    if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
+		/* Don't index the signature, but index its content type. */
+		if (i == GMIME_MULTIPART_SIGNED_SIGNATURE) {
+		    _index_content_type (message,
+					 g_mime_multipart_get_part (multipart, i));
+		    continue;
+		} else if (i != GMIME_MULTIPART_SIGNED_CONTENT) {
+		    _notmuch_database_log (notmuch_message_get_database (message),
+					   "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+		}
+	    }
+	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
+		_index_content_type (message,
+				     g_mime_multipart_get_part (multipart, i));
+		if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) {
+		    _index_encrypted_mime_part(message, indexopts,
+					       content_type,
+					       GMIME_MULTIPART_ENCRYPTED (part));
+		} else {
+		    if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) {
+			_notmuch_database_log (notmuch_message_get_database (message),
+					       "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		    }
+		}
+		continue;
+	    }
+	    _index_mime_part (message, indexopts,
+			      g_mime_multipart_get_part (multipart, i));
+	}
+	return;
+    }
+
+    if (GMIME_IS_MESSAGE_PART (part)) {
+	GMimeMessage *mime_message;
+
+	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
+
+	_index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
+
+	return;
+    }
+
+    if (! (GMIME_IS_PART (part))) {
+	_notmuch_database_log (notmuch_message_get_database (message),
+			      "Warning: Not indexing unknown mime part: %s.\n",
+			      g_type_name (G_OBJECT_TYPE (part)));
+	return;
+    }
+
+    disposition = g_mime_object_get_content_disposition (part);
+    if (disposition &&
+	strcasecmp (g_mime_content_disposition_get_disposition (disposition),
+		    GMIME_DISPOSITION_ATTACHMENT) == 0)
+    {
+	const char *filename = g_mime_part_get_filename (GMIME_PART (part));
+
+	_notmuch_message_add_term (message, "tag", "attachment");
+	_notmuch_message_gen_terms (message, "attachment", filename);
+
+	/* XXX: Would be nice to call out to something here to parse
+	 * the attachment into text and then index that. */
+	return;
+    }
+
+    byte_array = g_byte_array_new ();
+
+    stream = g_mime_stream_mem_new_with_byte_array (byte_array);
+    g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), false);
+
+    filter = g_mime_stream_filter_new (stream);
+
+    discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
+
+    g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
+			      discard_non_term_filter);
+
+    charset = g_mime_object_get_content_type_parameter (part, "charset");
+    if (charset) {
+	GMimeFilter *charset_filter;
+	charset_filter = g_mime_filter_charset_new (charset, "UTF-8");
+	/* This result can be NULL for things like "unknown-8bit".
+	 * Don't set a NULL filter as that makes GMime print
+	 * annoying assertion-failure messages on stderr. */
+	if (charset_filter) {
+	    g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
+				      charset_filter);
+	    g_object_unref (charset_filter);
+	}
+    }
+
+    wrapper = g_mime_part_get_content_object (GMIME_PART (part));
+    if (wrapper)
+	g_mime_data_wrapper_write_to_stream (wrapper, filter);
+
+    g_object_unref (stream);
+    g_object_unref (filter);
+    g_object_unref (discard_non_term_filter);
+
+    g_byte_array_append (byte_array, (guint8 *) "\0", 1);
+    body = (char *) g_byte_array_free (byte_array, false);
+
+    if (body) {
+	_notmuch_message_gen_terms (message, NULL, body);
+
+	free (body);
+    }
+}
+
+/* descend (if desired) into the cleartext part of an encrypted MIME
+ * part while indexing. */
+static void
+_index_encrypted_mime_part (notmuch_message_t *message,
+			    notmuch_indexopts_t *indexopts,
+			    g_mime_3_unused(GMimeContentType *content_type),
+			    GMimeMultipartEncrypted *encrypted_data)
+{
+    notmuch_status_t status;
+    GError *err = NULL;
+    notmuch_database_t * notmuch = NULL;
+    GMimeObject *clear = NULL;
+
+    if (!indexopts || (notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_FALSE))
+	return;
+
+    notmuch = notmuch_message_get_database (message);
+
+    GMimeCryptoContext* crypto_ctx = NULL;
+#if (GMIME_MAJOR_VERSION < 3)
+    {
+	const char *protocol = NULL;
+	protocol = g_mime_content_type_get_parameter (content_type, "protocol");
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto),
+							 protocol, &crypto_ctx);
+	if (status) {
+	    _notmuch_database_log (notmuch, "Warning: setup failed for decrypting "
+				   "during indexing. (%d)\n", status);
+	    status = notmuch_message_add_property (message, "index.decryption", "failure");
+	    if (status)
+		_notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					      "property (%d)\n", status);
+	    return;
+	}
+    }
+#endif
+    bool attempted = false;
+    GMimeDecryptResult *decrypt_result = NULL;
+    bool get_sk = (HAVE_GMIME_SESSION_KEYS && notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_TRUE);
+    clear = _notmuch_crypto_decrypt (&attempted, notmuch_indexopts_get_decrypt_policy (indexopts),
+				     message, crypto_ctx, encrypted_data, get_sk ? &decrypt_result : NULL, &err);
+    if (!attempted)
+	return;
+    if (err || !clear) {
+	if (decrypt_result)
+	    g_object_unref (decrypt_result);
+	if (err) {
+	    _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n",
+				   err->domain, err->code, err->message);
+	    g_error_free(err);
+	} else {
+	    _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (unknown error)\n");
+	}
+	/* Indicate that we failed to decrypt during indexing */
+	status = notmuch_message_add_property (message, "index.decryption", "failure");
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					  "property (%d)\n", status);
+	return;
+    }
+    if (decrypt_result) {
+#if HAVE_GMIME_SESSION_KEYS
+	if (get_sk) {
+	    status = notmuch_message_add_property (message, "session-key",
+						   g_mime_decrypt_result_get_session_key (decrypt_result));
+	    if (status)
+		_notmuch_database_log (notmuch, "failed to add session-key "
+				       "property (%d)\n", status);
+	}
+#endif
+	g_object_unref (decrypt_result);
+    }
+    _index_mime_part (message, indexopts, clear);
+    g_object_unref (clear);
+
+    status = notmuch_message_add_property (message, "index.decryption", "success");
+    if (status)
+	_notmuch_database_log (notmuch, "failed to add index.decryption "
+			       "property (%d)\n", status);
+
+}
+
+notmuch_status_t
+_notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_file_t *message_file)
+{
+    GMimeMessage *mime_message;
+    InternetAddressList *addresses;
+    const char *subject;
+    notmuch_status_t status;
+
+    status = _notmuch_message_file_get_mime_message (message_file,
+						     &mime_message);
+    if (status)
+	return status;
+
+    addresses = g_mime_message_get_from (mime_message);
+    if (addresses) {
+	_index_address_list (message, "from", addresses);
+	g_mime_2_6_unref (addresses);
+    }
+
+    addresses = g_mime_message_get_all_recipients (mime_message);
+    if (addresses) {
+	_index_address_list (message, "to", addresses);
+	g_object_unref (addresses);
+    }
+
+    subject = g_mime_message_get_subject (mime_message);
+    _notmuch_message_gen_terms (message, "subject", subject);
+
+    _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
diff --git a/lib/indexopts.c b/lib/indexopts.c
new file mode 100644
index 00000000..b78a57b6
--- /dev/null
+++ b/lib/indexopts.c
@@ -0,0 +1,75 @@
+/* indexopts.c - options for indexing messages (currently a stub)
+ *
+ * Copyright © 2017 Daniel Kahn Gillmor
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
+ */
+
+#include "notmuch-private.h"
+
+notmuch_indexopts_t *
+notmuch_database_get_default_indexopts (notmuch_database_t *db)
+{
+    notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t);
+    if (!ret)
+	return ret;
+    ret->crypto.decrypt = NOTMUCH_DECRYPT_AUTO;
+
+    char * decrypt_policy;
+    notmuch_status_t err = notmuch_database_get_config (db, "index.decrypt", &decrypt_policy);
+    if (err)
+	return ret;
+
+    if (decrypt_policy) {
+	if ((!(strcasecmp(decrypt_policy, "true"))) ||
+	    (!(strcasecmp(decrypt_policy, "yes"))) ||
+	    (!(strcasecmp(decrypt_policy, "1"))))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_TRUE);
+	else if ((!(strcasecmp(decrypt_policy, "false"))) ||
+		 (!(strcasecmp(decrypt_policy, "no"))) ||
+		 (!(strcasecmp(decrypt_policy, "0"))))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_FALSE);
+	else if (!strcasecmp(decrypt_policy, "nostash"))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_NOSTASH);
+    }
+
+    free (decrypt_policy);
+    return ret;
+}
+
+notmuch_status_t
+notmuch_indexopts_set_decrypt_policy (notmuch_indexopts_t *indexopts,
+				      notmuch_decryption_policy_t decrypt_policy)
+{
+    if (!indexopts)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    indexopts->crypto.decrypt = decrypt_policy;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_decryption_policy_t
+notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts)
+{
+    if (!indexopts)
+	return false;
+    return indexopts->crypto.decrypt;
+}
+
+void
+notmuch_indexopts_destroy (notmuch_indexopts_t *indexopts)
+{
+    talloc_free (indexopts);
+}
diff --git a/lib/message-file.c b/lib/message-file.c
new file mode 100644
index 00000000..8f0dbbda
--- /dev/null
+++ b/lib/message-file.c
@@ -0,0 +1,433 @@
+/* message.c - Utility functions for parsing an email message for notmuch.
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include <stdarg.h>
+
+#include "notmuch-private.h"
+
+#include <gmime/gmime.h>
+
+#include <glib.h> /* GHashTable */
+
+struct _notmuch_message_file {
+    /* File object */
+    FILE *file;
+    char *filename;
+
+    /* Cache for decoded headers */
+    GHashTable *headers;
+
+    GMimeMessage *message;
+};
+
+static int
+_notmuch_message_file_destructor (notmuch_message_file_t *message)
+{
+    if (message->headers)
+	g_hash_table_destroy (message->headers);
+
+    if (message->message)
+	g_object_unref (message->message);
+
+    if (message->file)
+	fclose (message->file);
+
+    return 0;
+}
+
+/* Create a new notmuch_message_file_t for 'filename' with 'ctx' as
+ * the talloc owner. */
+notmuch_message_file_t *
+_notmuch_message_file_open_ctx (notmuch_database_t *notmuch,
+				void *ctx, const char *filename)
+{
+    notmuch_message_file_t *message;
+
+    message = talloc_zero (ctx, notmuch_message_file_t);
+    if (unlikely (message == NULL))
+	return NULL;
+
+    /* Only needed for error messages during parsing. */
+    message->filename = talloc_strdup (message, filename);
+    if (message->filename == NULL)
+	goto FAIL;
+
+    talloc_set_destructor (message, _notmuch_message_file_destructor);
+
+    message->file = fopen (filename, "r");
+    if (message->file == NULL)
+	goto FAIL;
+
+    return message;
+
+  FAIL:
+    _notmuch_database_log (notmuch, "Error opening %s: %s\n",
+			  filename, strerror (errno));
+    _notmuch_message_file_close (message);
+
+    return NULL;
+}
+
+notmuch_message_file_t *
+_notmuch_message_file_open (notmuch_database_t *notmuch,
+			    const char *filename)
+{
+    return _notmuch_message_file_open_ctx (notmuch, NULL, filename);
+}
+
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message_file)
+{
+    return message_file->filename;
+}
+
+void
+_notmuch_message_file_close (notmuch_message_file_t *message)
+{
+    talloc_free (message);
+}
+
+static bool
+_is_mbox (FILE *file)
+{
+    char from_buf[5];
+    bool ret = false;
+
+    /* Is this mbox? */
+    if (fread (from_buf, sizeof (from_buf), 1, file) == 1 &&
+	strncmp (from_buf, "From ", 5) == 0)
+	ret = true;
+
+    rewind (file);
+
+    return ret;
+}
+
+notmuch_status_t
+_notmuch_message_file_parse (notmuch_message_file_t *message)
+{
+    GMimeStream *stream;
+    GMimeParser *parser;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+    static int initialized = 0;
+    bool is_mbox;
+
+    if (message->message)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    is_mbox = _is_mbox (message->file);
+
+    if (! initialized) {
+	g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS);
+	initialized = 1;
+    }
+
+    message->headers = g_hash_table_new_full (strcase_hash, strcase_equal,
+					      free, g_free);
+    if (! message->headers)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    stream = g_mime_stream_file_new (message->file);
+
+    /* We'll own and fclose the FILE* ourselves. */
+    g_mime_stream_file_set_owner (GMIME_STREAM_FILE (stream), false);
+
+    parser = g_mime_parser_new_with_stream (stream);
+    g_mime_parser_set_scan_from (parser, is_mbox);
+
+    message->message = g_mime_parser_construct_message (parser);
+    if (! message->message) {
+	status = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	goto DONE;
+    }
+
+    if (is_mbox && ! g_mime_parser_eos (parser)) {
+	/*
+	 * This is a multi-message mbox. (For historical reasons, we
+	 * do support single-message mboxes.)
+	 */
+	status = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+    }
+
+  DONE:
+    g_object_unref (stream);
+    g_object_unref (parser);
+
+    if (status) {
+	g_hash_table_destroy (message->headers);
+	message->headers = NULL;
+
+	if (message->message) {
+	    g_object_unref (message->message);
+	    message->message = NULL;
+	}
+
+	rewind (message->file);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+_notmuch_message_file_get_mime_message (notmuch_message_file_t *message,
+					GMimeMessage **mime_message)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_message_file_parse (message);
+    if (status)
+	return status;
+
+    *mime_message = message->message;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/*
+ * Get all instances of a header decoded and concatenated.
+ *
+ * The result must be freed using g_free().
+ *
+ * Return NULL on errors, empty string for non-existing headers.
+ */
+
+static char *
+_extend_header (char *combined, const char *value) {
+    char *decoded;
+
+    decoded = g_mime_utils_header_decode_text (value);
+    if (! decoded) {
+	if (combined) {
+	    g_free (combined);
+	    combined = NULL;
+	}
+	goto DONE;
+    }
+
+    if (combined) {
+	char *tmp = g_strdup_printf ("%s %s", combined, decoded);
+	g_free (decoded);
+	g_free (combined);
+	if (! tmp) {
+	    combined = NULL;
+	    goto DONE;
+	}
+
+	combined = tmp;
+    } else {
+	combined = decoded;
+    }
+ DONE:
+    return combined;
+}
+
+#if (GMIME_MAJOR_VERSION < 3)
+static char *
+_notmuch_message_file_get_combined_header (notmuch_message_file_t *message,
+					   const char *header)
+{
+    GMimeHeaderList *headers;
+    GMimeHeaderIter *iter;
+    char *combined = NULL;
+
+    headers = g_mime_object_get_header_list (GMIME_OBJECT (message->message));
+    if (! headers)
+	return NULL;
+
+    iter = g_mime_header_iter_new ();
+    if (! iter)
+	return NULL;
+
+    if (! g_mime_header_list_get_iter (headers, iter))
+	goto DONE;
+
+    do {
+	const char *value;
+	if (strcasecmp (g_mime_header_iter_get_name (iter), header) != 0)
+	    continue;
+
+	/* Note that GMime retains ownership of value... */
+	value = g_mime_header_iter_get_value (iter);
+
+	combined = _extend_header (combined, value);
+    } while (g_mime_header_iter_next (iter));
+
+    /* Return empty string for non-existing headers. */
+    if (! combined)
+	combined = g_strdup ("");
+
+  DONE:
+    g_mime_header_iter_free (iter);
+
+    return combined;
+}
+#else
+static char *
+_notmuch_message_file_get_combined_header (notmuch_message_file_t *message,
+					   const char *header)
+{
+    char *combined = NULL;
+    GMimeHeaderList *headers;
+
+    headers = g_mime_object_get_header_list (GMIME_OBJECT (message->message));
+    if (! headers)
+	return NULL;
+
+
+    for (int i=0; i < g_mime_header_list_get_count (headers); i++) {
+	const char *value;
+	GMimeHeader *g_header = g_mime_header_list_get_header_at (headers, i);
+
+	if (strcasecmp (g_mime_header_get_name (g_header), header) != 0)
+	    continue;
+
+	/* GMime retains ownership of value, we hope */
+	value = g_mime_header_get_value (g_header);
+
+	combined = _extend_header (combined, value);
+    }
+
+    /* Return empty string for non-existing headers. */
+    if (! combined)
+	combined = g_strdup ("");
+
+    return combined;
+}
+#endif
+
+const char *
+_notmuch_message_file_get_header (notmuch_message_file_t *message,
+				 const char *header)
+{
+    const char *value;
+    char *decoded;
+
+    if (_notmuch_message_file_parse (message))
+	return NULL;
+
+    /* If we have a cached decoded value, use it. */
+    value = g_hash_table_lookup (message->headers, header);
+    if (value)
+	return value;
+
+    if (strcasecmp (header, "received") == 0) {
+	/*
+	 * The Received: header is special. We concatenate all
+	 * instances of the header as we use this when analyzing the
+	 * path the mail has taken from sender to recipient.
+	 */
+	decoded = _notmuch_message_file_get_combined_header (message, header);
+    } else {
+	value = g_mime_object_get_header (GMIME_OBJECT (message->message),
+					  header);
+	if (value)
+	    decoded = g_mime_utils_header_decode_text (value);
+	else
+	    decoded = g_strdup ("");
+    }
+
+    if (! decoded)
+	return NULL;
+
+    /* Cache the decoded value. We also own the strings. */
+    g_hash_table_insert (message->headers, xstrdup (header), decoded);
+
+    return decoded;
+}
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+				   const char **from_out,
+				   const char **subject_out,
+				   const char **to_out,
+				   const char **date_out,
+				   char **message_id_out)
+{
+    notmuch_status_t ret;
+    const char *header;
+    const char *from, *to, *subject, *date;
+    char *message_id = NULL;
+
+    /* Parse message up front to get better error status. */
+    ret = _notmuch_message_file_parse (message_file);
+    if (ret)
+	goto DONE;
+
+    /* Before we do any real work, (especially before doing a
+     * potential SHA-1 computation on the entire file's contents),
+     * let's make sure that what we're looking at looks like an
+     * actual email message.
+     */
+    from = _notmuch_message_file_get_header (message_file, "from");
+    subject = _notmuch_message_file_get_header (message_file, "subject");
+    to = _notmuch_message_file_get_header (message_file, "to");
+    date = _notmuch_message_file_get_header (message_file, "date");
+
+    if ((from == NULL || *from == '\0') &&
+	(subject == NULL || *subject == '\0') &&
+	(to == NULL || *to == '\0')) {
+	ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	goto DONE;
+    }
+
+    /* Now that we're sure it's mail, the first order of business
+     * is to find a message ID (or else create one ourselves).
+     */
+    header = _notmuch_message_file_get_header (message_file, "message-id");
+    if (header && *header != '\0') {
+	message_id = _notmuch_message_id_parse (message_file, header, NULL);
+
+	/* So the header value isn't RFC-compliant, but it's
+	 * better than no message-id at all.
+	 */
+	if (message_id == NULL)
+	    message_id = talloc_strdup (message_file, header);
+    }
+
+    if (message_id == NULL ) {
+	/* No message-id at all, let's generate one by taking a
+	 * hash over the file's contents.
+	 */
+	char *sha1 = _notmuch_sha1_of_file (_notmuch_message_file_get_filename (message_file));
+
+	/* If that failed too, something is really wrong. Give up. */
+	if (sha1 == NULL) {
+	    ret = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
+	free (sha1);
+    }
+ DONE:
+    if (ret == NOTMUCH_STATUS_SUCCESS) {
+	if (from_out)
+	    *from_out = from;
+	if (subject_out)
+	    *subject_out = subject;
+	if (to_out)
+	    *to_out = to;
+	if (date_out)
+	    *date_out = date;
+	if (message_id_out)
+	    *message_id_out = message_id;
+    }
+    return ret;
+}
diff --git a/lib/message-id.c b/lib/message-id.c
new file mode 100644
index 00000000..e71ce9f4
--- /dev/null
+++ b/lib/message-id.c
@@ -0,0 +1,126 @@
+#include "notmuch-private.h"
+#include "string-util.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+    const char *s;
+
+    s = *str;
+    while (*s && (isspace (*s) || *s == '(')) {
+	while (*s && isspace (*s))
+	    s++;
+	if (*s == '(') {
+	    int nesting = 1;
+	    s++;
+	    while (*s && nesting) {
+		if (*s == '(') {
+		    nesting++;
+		} else if (*s == ')') {
+		    nesting--;
+		} else if (*s == '\\') {
+		    if (*(s+1))
+			s++;
+		}
+		s++;
+	    }
+	}
+    }
+
+    *str = s;
+}
+
+char *
+_notmuch_message_id_parse (void *ctx, const char *message_id, const char **next)
+{
+    const char *s, *end;
+    char *result;
+
+    if (message_id == NULL || *message_id == '\0')
+	return NULL;
+
+    s = message_id;
+
+    skip_space_and_comments (&s);
+
+    /* Skip any unstructured text as well. */
+    while (*s && *s != '<')
+	s++;
+
+    if (*s == '<') {
+	s++;
+    } else {
+	if (next)
+	    *next = s;
+	return NULL;
+    }
+
+    skip_space_and_comments (&s);
+
+    end = s;
+    while (*end && *end != '>')
+	end++;
+    if (next) {
+	if (*end)
+	    *next = end + 1;
+	else
+	    *next = end;
+    }
+
+    if (end > s && *end == '>')
+	end--;
+    if (end <= s)
+	return NULL;
+
+    result = talloc_strndup (ctx, s, end - s + 1);
+
+    /* Finally, collapse any whitespace that is within the message-id
+     * itself. */
+    {
+	char *r;
+	int len;
+
+	for (r = result, len = strlen (r); *r; r++, len--)
+	    if (*r == ' ' || *r == '\t')
+		memmove (r, r+1, len);
+    }
+
+    return result;
+}
+
+char *
+_notmuch_message_id_parse_strict (void *ctx, const char *message_id)
+{
+    const char *s, *end;
+
+    if (message_id == NULL || *message_id == '\0')
+	return NULL;
+
+    s = skip_space (message_id);
+    if (*s == '<')
+	s++;
+    else
+	return NULL;
+
+    for (end = s; *end && *end != '>'; end++)
+	if (isspace (*end))
+	    return NULL;
+
+    if (*end != '>')
+	return NULL;
+    else {
+	const char *last = skip_space (end + 1);
+	if (*last != '\0')
+	    return NULL;
+    }
+
+    return talloc_strndup (ctx, s, end - s);
+}
diff --git a/lib/message-private.h b/lib/message-private.h
new file mode 100644
index 00000000..73b080e4
--- /dev/null
+++ b/lib/message-private.h
@@ -0,0 +1,16 @@
+#ifndef MESSAGE_PRIVATE_H
+#define MESSAGE_PRIVATE_H
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message);
+
+bool
+_notmuch_message_frozen (notmuch_message_t *message);
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix);
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message,  const char *prefix_name);
+
+#endif
diff --git a/lib/message-property.cc b/lib/message-property.cc
new file mode 100644
index 00000000..710ba046
--- /dev/null
+++ b/lib/message-property.cc
@@ -0,0 +1,185 @@
+/* message-property.cc - Properties are like tags, but (key,value) pairs.
+ * keys are allowed to repeat.
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value)
+{
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    *value = _notmuch_string_map_get (_notmuch_message_property_map (message), key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_count_properties (notmuch_message_t *message, const char *key, unsigned int *count)
+{
+    if (! count || ! key || ! message)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    notmuch_string_map_t *map;
+    map = _notmuch_message_property_map (message);
+    if (! map)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    notmuch_string_map_iterator_t *matcher = _notmuch_string_map_iterator_create (map, key, true);
+    if (! matcher)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    *count = 0;
+    while (_notmuch_string_map_iterator_valid (matcher)) {
+	(*count)++;
+	_notmuch_string_map_iterator_move_to_next (matcher);
+    }
+
+    _notmuch_string_map_iterator_destroy (matcher);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_notmuch_message_modify_property (notmuch_message_t *message, const char *key, const char *value,
+				  bool delete_it)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+    char *term = NULL;
+
+    status = _notmuch_database_ensure_writable (notmuch_message_get_database (message));
+    if (status)
+	return status;
+
+    if (key == NULL || value == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (strchr (key, '='))
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+
+    term = talloc_asprintf (message, "%s=%s", key, value);
+
+    if (delete_it)
+	private_status = _notmuch_message_remove_term (message, "property", term);
+    else
+	private_status = _notmuch_message_add_term (message, "property", term);
+
+    if (private_status)
+	return COERCE_STATUS (private_status,
+			      "Unhandled error modifying message property");
+    if (! _notmuch_message_frozen (message))
+	_notmuch_message_sync (message);
+
+    if (term)
+	talloc_free (term);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, true);
+}
+
+static
+notmuch_status_t
+_notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key, bool prefix)
+{
+    notmuch_status_t status;
+    const char * term_prefix;
+
+    status = _notmuch_database_ensure_writable (notmuch_message_get_database (message));
+    if (status)
+	return status;
+
+    _notmuch_message_invalidate_metadata (message, "property");
+    if (key)
+	term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key,
+				       prefix ? "" : "=");
+    else
+	term_prefix = _find_prefix ("property");
+
+    /* XXX better error reporting ? */
+    _notmuch_message_remove_terms (message, term_prefix);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    return _notmuch_message_remove_all_properties (message, key, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix)
+{
+    return _notmuch_message_remove_all_properties (message, prefix, true);
+}
+
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
+{
+    notmuch_string_map_t *map;
+    map = _notmuch_message_property_map (message);
+    return _notmuch_string_map_iterator_create (map, key, exact);
+}
+
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_valid (properties);
+}
+
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_move_to_next (properties);
+}
+
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_key (properties);
+}
+
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_value (properties);
+}
+
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties)
+{
+    _notmuch_string_map_iterator_destroy (properties);
+}
diff --git a/lib/message.cc b/lib/message.cc
new file mode 100644
index 00000000..6f2f6345
--- /dev/null
+++ b/lib/message.cc
@@ -0,0 +1,2185 @@
+/* message.cc - Results of message-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+#include <stdint.h>
+
+#include <gmime/gmime.h>
+
+struct _notmuch_message {
+    notmuch_database_t *notmuch;
+    Xapian::docid doc_id;
+    int frozen;
+    char *message_id;
+    char *thread_id;
+    size_t thread_depth;
+    char *in_reply_to;
+    notmuch_string_list_t *tag_list;
+    notmuch_string_list_t *filename_term_list;
+    notmuch_string_list_t *filename_list;
+    char *maildir_flags;
+    char *author;
+    notmuch_message_file_t *message_file;
+    notmuch_string_list_t *property_term_list;
+    notmuch_string_map_t *property_map;
+    notmuch_string_list_t *reference_list;
+    notmuch_message_list_t *replies;
+    unsigned long flags;
+    /* For flags that are initialized on-demand, lazy_flags indicates
+     * if each flag has been initialized. */
+    unsigned long lazy_flags;
+
+    /* Message document modified since last sync */
+    bool modified;
+
+    /* last view of database the struct is synced with */
+    unsigned long last_view;
+
+    Xapian::Document doc;
+    Xapian::termcount termpos;
+};
+
+#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
+
+struct maildir_flag_tag {
+    char flag;
+    const char *tag;
+    bool inverse;
+};
+
+/* ASCII ordered table of Maildir flags and associated tags */
+static struct maildir_flag_tag flag2tag[] = {
+    { 'D', "draft",   false},
+    { 'F', "flagged", false},
+    { 'P', "passed",  false},
+    { 'R', "replied", false},
+    { 'S', "unread",  true }
+};
+
+/* We end up having to call the destructor explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_message_destructor (notmuch_message_t *message)
+{
+    message->doc.~Document ();
+
+    return 0;
+}
+
+static notmuch_message_t *
+_notmuch_message_create_for_document (const void *talloc_owner,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id,
+				      Xapian::Document doc,
+				      notmuch_private_status_t *status)
+{
+    notmuch_message_t *message;
+
+    if (status)
+	*status = NOTMUCH_PRIVATE_STATUS_SUCCESS;
+
+    message = talloc (talloc_owner, notmuch_message_t);
+    if (unlikely (message == NULL)) {
+	if (status)
+	    *status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    message->notmuch = notmuch;
+    message->doc_id = doc_id;
+
+    message->frozen = 0;
+    message->flags = 0;
+    message->lazy_flags = 0;
+
+    /* the message is initially not synchronized with Xapian */
+    message->last_view = 0;
+
+    /* Calculated after the thread structure is computed */
+    message->thread_depth = 0;
+
+    /* Each of these will be lazily created as needed. */
+    message->message_id = NULL;
+    message->thread_id = NULL;
+    message->in_reply_to = NULL;
+    message->tag_list = NULL;
+    message->filename_term_list = NULL;
+    message->filename_list = NULL;
+    message->maildir_flags = NULL;
+    message->message_file = NULL;
+    message->author = NULL;
+    message->property_term_list = NULL;
+    message->property_map = NULL;
+    message->reference_list = NULL;
+
+    message->replies = _notmuch_message_list_create (message);
+    if (unlikely (message->replies == NULL)) {
+	if (status)
+	    *status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    /* This is C++'s creepy "placement new", which is really just an
+     * ugly way to call a constructor for a pre-allocated object. So
+     * it's really not an error to not be checking for OUT_OF_MEMORY
+     * here, since this "new" isn't actually allocating memory. This
+     * is language-design comedy of the wrong kind. */
+
+    new (&message->doc) Xapian::Document;
+
+    talloc_set_destructor (message, _notmuch_message_destructor);
+
+    message->doc = doc;
+    message->termpos = 0;
+
+    return message;
+}
+
+/* Create a new notmuch_message_t object for an existing document in
+ * the database.
+ *
+ * Here, 'talloc owner' is an optional talloc context to which the new
+ * message will belong. This allows for the caller to not bother
+ * calling notmuch_message_destroy on the message, and know that all
+ * memory will be reclaimed when 'talloc_owner' is freed. The caller
+ * still can call notmuch_message_destroy when finished with the
+ * message if desired.
+ *
+ * The 'talloc_owner' argument can also be NULL, in which case the
+ * caller *is* responsible for calling notmuch_message_destroy.
+ *
+ * If no document exists in the database with document ID of 'doc_id'
+ * then this function returns NULL and optionally sets *status to
+ * NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND.
+ *
+ * This function can also fail to due lack of available memory,
+ * returning NULL and optionally setting *status to
+ * NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY.
+ *
+ * The caller can pass NULL for status if uninterested in
+ * distinguishing these two cases.
+ */
+notmuch_message_t *
+_notmuch_message_create (const void *talloc_owner,
+			 notmuch_database_t *notmuch,
+			 unsigned int doc_id,
+			 notmuch_private_status_t *status)
+{
+    Xapian::Document doc;
+
+    try {
+	doc = notmuch->xapian_db->get_document (doc_id);
+    } catch (const Xapian::DocNotFoundError &error) {
+	if (status)
+	    *status = NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+	return NULL;
+    }
+
+    return _notmuch_message_create_for_document (talloc_owner, notmuch,
+						 doc_id, doc, status);
+}
+
+/* Create a new notmuch_message_t object for a specific message ID,
+ * (which may or may not already exist in the database).
+ *
+ * The 'notmuch' database will be the talloc owner of the returned
+ * message.
+ *
+ * This function returns a valid notmuch_message_t whether or not
+ * there is already a document in the database with the given message
+ * ID. These two cases can be distinguished by the value of *status:
+ *
+ *
+ *   NOTMUCH_PRIVATE_STATUS_SUCCESS:
+ *
+ *     There is already a document with message ID 'message_id' in the
+ *     database. The returned message can be used to query/modify the
+ *     document. The message may be a ghost message.
+ *
+ *   NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+ *
+ *     No document with 'message_id' exists in the database. The
+ *     returned message contains a newly created document (not yet
+ *     added to the database) and a document ID that is known not to
+ *     exist in the database.  This message is "blank"; that is, it
+ *     contains only a message ID and no other metadata. The caller
+ *     can modify the message, and a call to _notmuch_message_sync
+ *     will add the document to the database.
+ *
+ * If an error occurs, this function will return NULL and *status
+ * will be set as appropriate. (The status pointer argument must
+ * not be NULL.)
+ */
+notmuch_message_t *
+_notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
+					const char *message_id,
+					notmuch_private_status_t *status_ret)
+{
+    notmuch_message_t *message;
+    Xapian::Document doc;
+    unsigned int doc_id;
+    char *term;
+
+    *status_ret = (notmuch_private_status_t) notmuch_database_find_message (notmuch,
+									    message_id,
+									    &message);
+    if (message)
+	return talloc_steal (notmuch, message);
+    else if (*status_ret)
+	return NULL;
+
+    /* If the message ID is too long, substitute its sha1 instead. */
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (message, message_id);
+
+    term = talloc_asprintf (NULL, "%s%s",
+			    _find_prefix ("id"), message_id);
+    if (term == NULL) {
+	*status_ret = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	return NULL;
+    }
+
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	INTERNAL_ERROR ("Failure to ensure database is writable.");
+
+    try {
+	doc.add_term (term, 0);
+	talloc_free (term);
+
+	doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
+
+	doc_id = _notmuch_database_generate_doc_id (notmuch);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred creating message: %s\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	*status_ret = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+	return NULL;
+    }
+
+    message = _notmuch_message_create_for_document (notmuch, notmuch,
+						    doc_id, doc, status_ret);
+
+    /* We want to inform the caller that we had to create a new
+     * document. */
+    if (*status_ret == NOTMUCH_PRIVATE_STATUS_SUCCESS)
+	*status_ret = NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+
+    return message;
+}
+
+static char *
+_notmuch_message_get_term (notmuch_message_t *message,
+			   Xapian::TermIterator &i, Xapian::TermIterator &end,
+			   const char *prefix)
+{
+    int prefix_len = strlen (prefix);
+    char *value;
+
+    i.skip_to (prefix);
+
+    if (i == end)
+	return NULL;
+
+    const std::string &term = *i;
+    if (strncmp (term.c_str(), prefix, prefix_len))
+	return NULL;
+
+    value = talloc_strdup (message, term.c_str() + prefix_len);
+
+#if DEBUG_DATABASE_SANITY
+    i++;
+
+    if (i != end && strncmp ((*i).c_str (), prefix, prefix_len) == 0) {
+	INTERNAL_ERROR ("Mail (doc_id: %d) has duplicate %s terms: %s and %s\n",
+			message->doc_id, prefix, value,
+			(*i).c_str () + prefix_len);
+    }
+#endif
+
+    return value;
+}
+
+/*
+ * For special applications where we only want the thread id, reading
+ * in all metadata is a heavy I/O penalty.
+ */
+const char *
+_notmuch_message_get_thread_id_only (notmuch_message_t *message)
+{
+
+    Xapian::TermIterator i = message->doc.termlist_begin ();
+    Xapian::TermIterator end = message->doc.termlist_end ();
+
+    message->thread_id = _notmuch_message_get_term (message, i, end,
+						    _find_prefix ("thread"));
+    return message->thread_id;
+}
+
+
+static void
+_notmuch_message_ensure_metadata (notmuch_message_t *message, void *field)
+{
+    Xapian::TermIterator i, end;
+
+    if (field && (message->last_view >= message->notmuch->view))
+	return;
+
+    const char *thread_prefix = _find_prefix ("thread"),
+	*tag_prefix = _find_prefix ("tag"),
+	*id_prefix = _find_prefix ("id"),
+	*type_prefix = _find_prefix ("type"),
+	*filename_prefix = _find_prefix ("file-direntry"),
+	*property_prefix = _find_prefix ("property"),
+	*reference_prefix = _find_prefix ("reference"),
+	*replyto_prefix = _find_prefix ("replyto");
+
+    /* We do this all in a single pass because Xapian decompresses the
+     * term list every time you iterate over it.  Thus, while this is
+     * slightly more costly than looking up individual fields if only
+     * one field of the message object is actually used, it's a huge
+     * win as more fields are used. */
+    for (int count=0; count < 3; count++) {
+	try {
+	    i = message->doc.termlist_begin ();
+	    end = message->doc.termlist_end ();
+
+	    /* Get thread */
+	    if (!message->thread_id)
+		message->thread_id =
+		    _notmuch_message_get_term (message, i, end, thread_prefix);
+
+	    /* Get tags */
+	    assert (strcmp (thread_prefix, tag_prefix) < 0);
+	    if (!message->tag_list) {
+		message->tag_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     tag_prefix);
+		_notmuch_string_list_sort (message->tag_list);
+	    }
+
+	    /* Get id */
+	    assert (strcmp (tag_prefix, id_prefix) < 0);
+	    if (!message->message_id)
+		message->message_id =
+		    _notmuch_message_get_term (message, i, end, id_prefix);
+
+	    /* Get document type */
+	    assert (strcmp (id_prefix, type_prefix) < 0);
+	    if (! NOTMUCH_TEST_BIT (message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST)) {
+		i.skip_to (type_prefix);
+		/* "T" is the prefix "type" fields.  See
+		 * BOOLEAN_PREFIX_INTERNAL. */
+		if (*i == "Tmail")
+		    NOTMUCH_CLEAR_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+		else if (*i == "Tghost")
+		    NOTMUCH_SET_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+		else
+		    INTERNAL_ERROR ("Message without type term");
+		NOTMUCH_SET_BIT (&message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+	    }
+
+	    /* Get filename list.  Here we get only the terms.  We lazily
+	     * expand them to full file names when needed in
+	     * _notmuch_message_ensure_filename_list. */
+	    assert (strcmp (type_prefix, filename_prefix) < 0);
+	    if (!message->filename_term_list && !message->filename_list)
+		message->filename_term_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     filename_prefix);
+
+
+	    /* Get property terms. Mimic the setup with filenames above */
+	    assert (strcmp (filename_prefix, property_prefix) < 0);
+	    if (!message->property_map && !message->property_term_list)
+		message->property_term_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							 property_prefix);
+
+	    /* get references */
+	    assert (strcmp (property_prefix, reference_prefix) < 0);
+	    if (!message->reference_list) {
+		message->reference_list =
+		    _notmuch_database_get_terms_with_prefix (message, i, end,
+							     reference_prefix);
+	    }
+
+	    /* Get reply to */
+	    assert (strcmp (property_prefix, replyto_prefix) < 0);
+	    if (!message->in_reply_to)
+		message->in_reply_to =
+		    _notmuch_message_get_term (message, i, end, replyto_prefix);
+
+
+	    /* It's perfectly valid for a message to have no In-Reply-To
+	     * header. For these cases, we return an empty string. */
+	    if (!message->in_reply_to)
+		message->in_reply_to = talloc_strdup (message, "");
+
+	    /* all the way without an exception */
+	    break;
+	} catch (const Xapian::DatabaseModifiedError &error) {
+	    notmuch_status_t status = _notmuch_database_reopen (message->notmuch);
+	    if (status != NOTMUCH_STATUS_SUCCESS)
+		INTERNAL_ERROR ("unhandled error from notmuch_database_reopen: %s\n",
+				notmuch_status_to_string (status));
+	} catch (const Xapian::Error &error) {
+	    INTERNAL_ERROR ("A Xapian exception occurred fetching message metadata: %s\n",
+			    error.get_msg().c_str());
+	}
+    }
+    message->last_view = message->notmuch->view;
+}
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message,
+				      const char *prefix_name)
+{
+    if (strcmp ("thread", prefix_name) == 0) {
+	talloc_free (message->thread_id);
+	message->thread_id = NULL;
+    }
+
+    if (strcmp ("tag", prefix_name) == 0) {
+	talloc_unlink (message, message->tag_list);
+	message->tag_list = NULL;
+    }
+
+    if (strcmp ("type", prefix_name) == 0) {
+	NOTMUCH_CLEAR_BIT (&message->flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+	NOTMUCH_CLEAR_BIT (&message->lazy_flags, NOTMUCH_MESSAGE_FLAG_GHOST);
+    }
+
+    if (strcmp ("file-direntry", prefix_name) == 0) {
+	talloc_free (message->filename_term_list);
+	talloc_free (message->filename_list);
+	message->filename_term_list = message->filename_list = NULL;
+    }
+
+    if (strcmp ("property", prefix_name) == 0) {
+
+	if (message->property_term_list)
+	    talloc_free (message->property_term_list);
+	message->property_term_list = NULL;
+
+	if (message->property_map)
+	    talloc_unlink (message, message->property_map);
+
+	message->property_map = NULL;
+    }
+
+    if (strcmp ("replyto", prefix_name) == 0) {
+	talloc_free (message->in_reply_to);
+	message->in_reply_to = NULL;
+    }
+}
+
+unsigned int
+_notmuch_message_get_doc_id (notmuch_message_t *message)
+{
+    return message->doc_id;
+}
+
+const char *
+notmuch_message_get_message_id (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_metadata (message, message->message_id);
+    if (!message->message_id)
+	INTERNAL_ERROR ("Message with document ID of %u has no message ID.\n",
+			message->doc_id);
+    return message->message_id;
+}
+
+static void
+_notmuch_message_ensure_message_file (notmuch_message_t *message)
+{
+    const char *filename;
+
+    if (message->message_file)
+	return;
+
+    filename = notmuch_message_get_filename (message);
+    if (unlikely (filename == NULL))
+	return;
+
+    message->message_file = _notmuch_message_file_open_ctx (
+	notmuch_message_get_database (message), message, filename);
+}
+
+const char *
+notmuch_message_get_header (notmuch_message_t *message, const char *header)
+{
+    Xapian::valueno slot = Xapian::BAD_VALUENO;
+
+    /* Fetch header from the appropriate xapian value field if
+     * available */
+    if (strcasecmp (header, "from") == 0)
+	slot = NOTMUCH_VALUE_FROM;
+    else if (strcasecmp (header, "subject") == 0)
+	slot = NOTMUCH_VALUE_SUBJECT;
+    else if (strcasecmp (header, "message-id") == 0)
+	slot = NOTMUCH_VALUE_MESSAGE_ID;
+
+    if (slot != Xapian::BAD_VALUENO) {
+	try {
+	    std::string value = message->doc.get_value (slot);
+
+	    /* If we have NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES, then
+	     * empty values indicate empty headers.  If we don't, then
+	     * it could just mean we didn't record the header. */
+	    if ((message->notmuch->features &
+		 NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES) ||
+		! value.empty())
+		return talloc_strdup (message, value.c_str ());
+
+	} catch (Xapian::Error &error) {
+	    _notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred when reading header: %s\n",
+		     error.get_msg().c_str());
+	    message->notmuch->exception_reported = true;
+	    return NULL;
+	}
+    }
+
+    /* Otherwise fall back to parsing the file */
+    _notmuch_message_ensure_message_file (message);
+    if (message->message_file == NULL)
+	return NULL;
+
+    return _notmuch_message_file_get_header (message->message_file, header);
+}
+
+/* Return the message ID from the In-Reply-To header of 'message'.
+ *
+ * Returns an empty string ("") if 'message' has no In-Reply-To
+ * header.
+ *
+ * Returns NULL if any error occurs.
+ */
+const char *
+_notmuch_message_get_in_reply_to (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_metadata (message, message->in_reply_to);
+    return message->in_reply_to;
+}
+
+const char *
+notmuch_message_get_thread_id (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_metadata (message, message->thread_id);
+    if (!message->thread_id)
+	INTERNAL_ERROR ("Message with document ID of %u has no thread ID.\n",
+			message->doc_id);
+    return message->thread_id;
+}
+
+void
+_notmuch_message_add_reply (notmuch_message_t *message,
+			    notmuch_message_t *reply)
+{
+    _notmuch_message_list_add_message (message->replies, reply);
+}
+
+size_t
+_notmuch_message_get_thread_depth (notmuch_message_t *message) {
+    return message->thread_depth;
+}
+
+void
+_notmuch_message_label_depths (notmuch_message_t *message,
+			       size_t depth)
+{
+    message->thread_depth = depth;
+
+    for (notmuch_messages_t *messages = _notmuch_messages_create (message->replies);
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	notmuch_message_t *child = notmuch_messages_get (messages);
+	_notmuch_message_label_depths (child, depth+1);
+    }
+}
+
+const notmuch_string_list_t *
+_notmuch_message_get_references (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_metadata (message, message->reference_list);
+    return message->reference_list;
+}
+
+static int
+_cmpmsg (const void *pa, const void *pb)
+{
+    notmuch_message_t **a = (notmuch_message_t **) pa;
+    notmuch_message_t **b = (notmuch_message_t **) pb;
+    time_t time_a = notmuch_message_get_date (*a);
+    time_t time_b = notmuch_message_get_date (*b);
+
+    if (time_a == time_b)
+	return 0;
+    else if (time_a < time_b)
+	return -1;
+    else
+	return 1;
+}
+
+notmuch_message_list_t *
+_notmuch_message_sort_subtrees (void *ctx, notmuch_message_list_t *list)
+{
+
+    size_t count = 0;
+    size_t capacity = 16;
+
+    if (! list)
+	return list;
+
+    void *local = talloc_new (NULL);
+    notmuch_message_list_t *new_list = _notmuch_message_list_create (ctx);
+    notmuch_message_t **message_array = talloc_zero_array (local, notmuch_message_t *, capacity);
+
+    for (notmuch_messages_t *messages = _notmuch_messages_create (list);
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	notmuch_message_t *root = notmuch_messages_get (messages);
+	if (count >= capacity) {
+	    capacity *= 2;
+	    message_array = talloc_realloc (local, message_array, notmuch_message_t *, capacity);
+	}
+	message_array[count++] = root;
+	root->replies = _notmuch_message_sort_subtrees (root, root->replies);
+    }
+
+    qsort (message_array, count, sizeof (notmuch_message_t *), _cmpmsg);
+    for (size_t i = 0; i < count; i++) {
+	_notmuch_message_list_add_message (new_list, message_array[i]);
+    }
+
+    talloc_free (local);
+    talloc_free (list);
+    return new_list;
+}
+
+notmuch_messages_t *
+notmuch_message_get_replies (notmuch_message_t *message)
+{
+    return _notmuch_messages_create (message->replies);
+}
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
+{
+    Xapian::TermIterator i;
+    size_t prefix_len = 0;
+
+    prefix_len = strlen (prefix);
+
+    while (1) {
+	i = message->doc.termlist_begin ();
+	i.skip_to (prefix);
+
+	/* Terminate loop when no terms remain with desired prefix. */
+	if (i == message->doc.termlist_end () ||
+	    strncmp ((*i).c_str (), prefix, prefix_len))
+	    break;
+
+	try {
+	    message->doc.remove_term ((*i));
+	    message->modified = true;
+	} catch (const Xapian::InvalidArgumentError) {
+	    /* Ignore failure to remove non-existent term. */
+	}
+    }
+}
+
+
+/* Remove all terms generated by indexing, i.e. not tags or
+ * properties, along with any automatic tags*/
+notmuch_private_status_t
+_notmuch_message_remove_indexed_terms (notmuch_message_t *message)
+{
+    Xapian::TermIterator i;
+
+    const std::string
+	id_prefix = _find_prefix ("id"),
+	property_prefix = _find_prefix ("property"),
+	tag_prefix = _find_prefix ("tag"),
+	type_prefix = _find_prefix ("type");
+
+    for (i = message->doc.termlist_begin ();
+	 i != message->doc.termlist_end (); i++) {
+
+	const std::string term = *i;
+
+	if (term.compare (0, type_prefix.size (), type_prefix) == 0)
+	    continue;
+
+	if (term.compare (0, id_prefix.size (), id_prefix) == 0)
+	    continue;
+
+	if (term.compare (0, property_prefix.size (), property_prefix) == 0)
+	    continue;
+
+	if (term.compare (0, tag_prefix.size (), tag_prefix) == 0 &&
+	    term.compare (1, strlen("encrypted"), "encrypted") != 0 &&
+	    term.compare (1, strlen("signed"), "signed") != 0 &&
+	    term.compare (1, strlen("attachment"), "attachment") != 0)
+	    continue;
+
+	try {
+	    message->doc.remove_term ((*i));
+	    message->modified = true;
+	} catch (const Xapian::InvalidArgumentError) {
+	    /* Ignore failure to remove non-existent term. */
+	} catch (const Xapian::Error &error) {
+	    notmuch_database_t *notmuch = message->notmuch;
+
+	    if (!notmuch->exception_reported) {
+		_notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred creating message: %s\n",
+				      error.get_msg().c_str());
+		notmuch->exception_reported = true;
+	    }
+	    return NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+	}
+    }
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Return true if p points at "new" or "cur". */
+static bool is_maildir (const char *p)
+{
+    return strcmp (p, "cur") == 0 || strcmp (p, "new") == 0;
+}
+
+/* Add "folder:" term for directory. */
+static notmuch_status_t
+_notmuch_message_add_folder_terms (notmuch_message_t *message,
+				   const char *directory)
+{
+    char *folder, *last;
+
+    folder = talloc_strdup (NULL, directory);
+    if (! folder)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    /*
+     * If the message file is in a leaf directory named "new" or
+     * "cur", presume maildir and index the parent directory. Thus a
+     * "folder:" prefix search matches messages in the specified
+     * maildir folder, i.e. in the specified directory and its "new"
+     * and "cur" subdirectories.
+     *
+     * Note that this means the "folder:" prefix can't be used for
+     * distinguishing between message files in "new" or "cur". The
+     * "path:" prefix needs to be used for that.
+     *
+     * Note the deliberate difference to _filename_is_in_maildir(). We
+     * don't want to index different things depending on the existence
+     * or non-existence of all maildir sibling directories "new",
+     * "cur", and "tmp". Doing so would be surprising, and difficult
+     * for the user to fix in case all subdirectories were not in
+     * place during indexing.
+     */
+    last = strrchr (folder, '/');
+    if (last) {
+	if (is_maildir (last + 1))
+	    *last = '\0';
+    } else if (is_maildir (folder)) {
+	*folder = '\0';
+    }
+
+    _notmuch_message_add_term (message, "folder", folder);
+
+    talloc_free (folder);
+
+    message->modified = true;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+#define RECURSIVE_SUFFIX "/**"
+
+/* Add "path:" terms for directory. */
+static notmuch_status_t
+_notmuch_message_add_path_terms (notmuch_message_t *message,
+				 const char *directory)
+{
+    /* Add exact "path:" term. */
+    _notmuch_message_add_term (message, "path", directory);
+
+    if (strlen (directory)) {
+	char *path, *p;
+
+	path = talloc_asprintf (NULL, "%s%s", directory, RECURSIVE_SUFFIX);
+	if (! path)
+	    return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+	/* Add recursive "path:" terms for directory and all parents. */
+	for (p = path + strlen (path) - 1; p > path; p--) {
+	    if (*p == '/') {
+		strcpy (p, RECURSIVE_SUFFIX);
+		_notmuch_message_add_term (message, "path", path);
+	    }
+	}
+
+	talloc_free (path);
+    }
+
+    /* Recursive all-matching path:** for consistency. */
+    _notmuch_message_add_term (message, "path", "**");
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Add directory based terms for all filenames of the message. */
+static notmuch_status_t
+_notmuch_message_add_directory_terms (void *ctx, notmuch_message_t *message)
+{
+    const char *direntry_prefix = _find_prefix ("file-direntry");
+    int direntry_prefix_len = strlen (direntry_prefix);
+    Xapian::TermIterator i = message->doc.termlist_begin ();
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    for (i.skip_to (direntry_prefix); i != message->doc.termlist_end (); i++) {
+	unsigned int directory_id;
+	const char *direntry, *directory;
+	char *colon;
+	const std::string &term = *i;
+
+	/* Terminate loop at first term without desired prefix. */
+	if (strncmp (term.c_str (), direntry_prefix, direntry_prefix_len))
+	    break;
+
+	/* Indicate that there are filenames remaining. */
+	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	direntry = term.c_str ();
+	direntry += direntry_prefix_len;
+
+	directory_id = strtol (direntry, &colon, 10);
+
+	if (colon == NULL || *colon != ':')
+	    INTERNAL_ERROR ("malformed direntry");
+
+	directory = _notmuch_database_get_directory_path (ctx,
+							  message->notmuch,
+							  directory_id);
+
+	_notmuch_message_add_folder_terms (message, directory);
+	_notmuch_message_add_path_terms (message, directory);
+    }
+
+    return status;
+}
+
+/* Add an additional 'filename' for 'message'.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync. */
+notmuch_status_t
+_notmuch_message_add_filename (notmuch_message_t *message,
+			       const char *filename)
+{
+    const char *relative, *directory;
+    notmuch_status_t status;
+    void *local = talloc_new (message);
+    char *direntry;
+
+    if (filename == NULL)
+	INTERNAL_ERROR ("Message filename cannot be NULL.");
+
+    if (! (message->notmuch->features & NOTMUCH_FEATURE_FILE_TERMS) ||
+	! (message->notmuch->features & NOTMUCH_FEATURE_BOOL_FOLDER))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    relative = _notmuch_database_relative_path (message->notmuch, filename);
+
+    status = _notmuch_database_split_path (local, relative, &directory, NULL);
+    if (status)
+	return status;
+
+    status = _notmuch_database_filename_to_direntry (
+	local, message->notmuch, filename, NOTMUCH_FIND_CREATE, &direntry);
+    if (status)
+	return status;
+
+    /* New file-direntry allows navigating to this message with
+     * notmuch_directory_get_child_files() . */
+    _notmuch_message_add_term (message, "file-direntry", direntry);
+
+    _notmuch_message_add_folder_terms (message, directory);
+    _notmuch_message_add_path_terms (message, directory);
+
+    talloc_free (local);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Remove a particular 'filename' from 'message'.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync.
+ *
+ * If this message still has other filenames, returns
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID.
+ *
+ * Note: This function does not remove a document from the database,
+ * even if the specified filename is the only filename for this
+ * message. For that functionality, see
+ * notmuch_database_remove_message. */
+notmuch_status_t
+_notmuch_message_remove_filename (notmuch_message_t *message,
+				  const char *filename)
+{
+    void *local = talloc_new (message);
+    char *direntry;
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    if (! (message->notmuch->features & NOTMUCH_FEATURE_FILE_TERMS) ||
+	! (message->notmuch->features & NOTMUCH_FEATURE_BOOL_FOLDER))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+
+    status = _notmuch_database_filename_to_direntry (
+	local, message->notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
+    if (status || !direntry)
+	return status;
+
+    /* Unlink this file from its parent directory. */
+    private_status = _notmuch_message_remove_term (message,
+						   "file-direntry", direntry);
+    status = COERCE_STATUS (private_status,
+			    "Unexpected error from _notmuch_message_remove_term");
+    if (status)
+	return status;
+
+    /* Re-synchronize "folder:" and "path:" terms for this message. */
+
+    /* Remove all "folder:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("folder"));
+
+    /* Remove all "path:" terms. */
+    _notmuch_message_remove_terms (message, _find_prefix ("path"));
+
+    /* Add back terms for all remaining filenames of the message. */
+    status = _notmuch_message_add_directory_terms (local, message);
+
+    talloc_free (local);
+
+    return status;
+}
+
+/* Upgrade the "folder:" prefix from V1 to V2. */
+#define FOLDER_PREFIX_V1       "XFOLDER"
+#define ZFOLDER_PREFIX_V1      "Z" FOLDER_PREFIX_V1
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message)
+{
+    /* Remove all old "folder:" terms. */
+    _notmuch_message_remove_terms (message, FOLDER_PREFIX_V1);
+
+    /* Remove all old "folder:" stemmed terms. */
+    _notmuch_message_remove_terms (message, ZFOLDER_PREFIX_V1);
+
+    /* Add new boolean "folder:" and "path:" terms. */
+    _notmuch_message_add_directory_terms (message, message);
+}
+
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message)
+{
+    return talloc_strdup (message, message->doc.get_data ().c_str ());
+}
+
+void
+_notmuch_message_clear_data (notmuch_message_t *message)
+{
+    message->doc.set_data ("");
+    message->modified = true;
+}
+
+static void
+_notmuch_message_ensure_filename_list (notmuch_message_t *message)
+{
+    notmuch_string_node_t *node;
+
+    if (message->filename_list)
+	return;
+
+    _notmuch_message_ensure_metadata (message, message->filename_term_list);
+
+    message->filename_list = _notmuch_string_list_create (message);
+    node = message->filename_term_list->head;
+
+    if (!node) {
+	/* A message document created by an old version of notmuch
+	 * (prior to rename support) will have the filename in the
+	 * data of the document rather than as a file-direntry term.
+	 *
+	 * It would be nice to do the upgrade of the document directly
+	 * here, but the database is likely open in read-only mode. */
+
+	std::string datastr = message->doc.get_data ();
+	const char *data = datastr.c_str ();
+
+	if (data == NULL)
+	    INTERNAL_ERROR ("message with no filename");
+
+	_notmuch_string_list_append (message->filename_list, data);
+
+	return;
+    }
+
+    for (; node; node = node->next) {
+	void *local = talloc_new (message);
+	const char *db_path, *directory, *basename, *filename;
+	char *colon, *direntry = NULL;
+	unsigned int directory_id;
+
+	direntry = node->string;
+
+	directory_id = strtol (direntry, &colon, 10);
+
+	if (colon == NULL || *colon != ':')
+	    INTERNAL_ERROR ("malformed direntry");
+
+	basename = colon + 1;
+
+	*colon = '\0';
+
+	db_path = notmuch_database_get_path (message->notmuch);
+
+	directory = _notmuch_database_get_directory_path (local,
+							  message->notmuch,
+							  directory_id);
+
+	if (strlen (directory))
+	    filename = talloc_asprintf (message, "%s/%s/%s",
+					db_path, directory, basename);
+	else
+	    filename = talloc_asprintf (message, "%s/%s",
+					db_path, basename);
+
+	_notmuch_string_list_append (message->filename_list, filename);
+
+	talloc_free (local);
+    }
+
+    talloc_free (message->filename_term_list);
+    message->filename_term_list = NULL;
+}
+
+const char *
+notmuch_message_get_filename (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_filename_list (message);
+
+    if (message->filename_list == NULL)
+	return NULL;
+
+    if (message->filename_list->head == NULL ||
+	message->filename_list->head->string == NULL)
+    {
+	INTERNAL_ERROR ("message with no filename");
+    }
+
+    return message->filename_list->head->string;
+}
+
+notmuch_filenames_t *
+notmuch_message_get_filenames (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_filename_list (message);
+
+    return _notmuch_filenames_create (message, message->filename_list);
+}
+
+int
+notmuch_message_count_files (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_filename_list (message);
+
+    return _notmuch_string_list_length (message->filename_list);
+}
+
+notmuch_bool_t
+notmuch_message_get_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag)
+{
+    if (flag == NOTMUCH_MESSAGE_FLAG_GHOST &&
+	! NOTMUCH_TEST_BIT (message->lazy_flags, flag))
+	_notmuch_message_ensure_metadata (message, NULL);
+
+    return NOTMUCH_TEST_BIT (message->flags, flag);
+}
+
+void
+notmuch_message_set_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag, notmuch_bool_t enable)
+{
+    if (enable)
+	NOTMUCH_SET_BIT (&message->flags, flag);
+    else
+	NOTMUCH_CLEAR_BIT (&message->flags, flag);
+    NOTMUCH_SET_BIT (&message->lazy_flags, flag);
+}
+
+time_t
+notmuch_message_get_date (notmuch_message_t *message)
+{
+    std::string value;
+
+    try {
+	value = message->doc.get_value (NOTMUCH_VALUE_TIMESTAMP);
+    } catch (Xapian::Error &error) {
+	_notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred when reading date: %s\n",
+		 error.get_msg().c_str());
+	message->notmuch->exception_reported = true;
+	return 0;
+    }
+
+    if (value.empty ())
+	/* sortable_unserialise is undefined on empty string */
+	return 0;
+    return Xapian::sortable_unserialise (value);
+}
+
+notmuch_tags_t *
+notmuch_message_get_tags (notmuch_message_t *message)
+{
+    notmuch_tags_t *tags;
+
+    _notmuch_message_ensure_metadata (message, message->tag_list);
+
+    tags = _notmuch_tags_create (message, message->tag_list);
+    /* _notmuch_tags_create steals the reference to the tag_list, but
+     * in this case it's still used by the message, so we add an
+     * *additional* talloc reference to the list.  As a result, it's
+     * possible to modify the message tags (which talloc_unlink's the
+     * current list from the message) while still iterating because
+     * the iterator will keep the current list alive. */
+    if (!talloc_reference (message, message->tag_list))
+	return NULL;
+
+    return tags;
+}
+
+const char *
+_notmuch_message_get_author (notmuch_message_t *message)
+{
+    return message->author;
+}
+
+void
+_notmuch_message_set_author (notmuch_message_t *message,
+			    const char *author)
+{
+    if (message->author)
+	talloc_free(message->author);
+    message->author = talloc_strdup(message, author);
+    return;
+}
+
+void
+_notmuch_message_set_header_values (notmuch_message_t *message,
+				    const char *date,
+				    const char *from,
+				    const char *subject)
+{
+    time_t time_value;
+
+    /* GMime really doesn't want to see a NULL date, so protect its
+     * sensibilities. */
+    if (date == NULL || *date == '\0') {
+	time_value = 0;
+    } else {
+	time_value = g_mime_utils_header_decode_date_unix (date);
+	/*
+	 * Workaround for https://bugzilla.gnome.org/show_bug.cgi?id=779923
+	 */
+	if (time_value < 0)
+	    time_value = 0;
+    }
+
+    message->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+			    Xapian::sortable_serialise (time_value));
+    message->doc.add_value (NOTMUCH_VALUE_FROM, from);
+    message->doc.add_value (NOTMUCH_VALUE_SUBJECT, subject);
+    message->modified = true;
+}
+
+/* Upgrade a message to support NOTMUCH_FEATURE_LAST_MOD.  The caller
+ * must call _notmuch_message_sync. */
+void
+_notmuch_message_upgrade_last_mod (notmuch_message_t *message)
+{
+    /* _notmuch_message_sync will update the last modification
+     * revision; we just have to ask it to. */
+    message->modified = true;
+}
+
+/* Synchronize changes made to message->doc out into the database. */
+void
+_notmuch_message_sync (notmuch_message_t *message)
+{
+    Xapian::WritableDatabase *db;
+
+    if (message->notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	return;
+
+    if (! message->modified)
+	return;
+
+    /* Update the last modification of this message. */
+    if (message->notmuch->features & NOTMUCH_FEATURE_LAST_MOD)
+	/* sortable_serialise gives a reasonably compact encoding,
+	 * which directly translates to reduced IO when scanning the
+	 * value stream.  Since it's built for doubles, we only get 53
+	 * effective bits, but that's still enough for the database to
+	 * last a few centuries at 1 million revisions per second. */
+	message->doc.add_value (NOTMUCH_VALUE_LAST_MOD,
+				Xapian::sortable_serialise (
+				    _notmuch_database_new_revision (
+					message->notmuch)));
+
+    db = static_cast <Xapian::WritableDatabase *> (message->notmuch->xapian_db);
+    db->replace_document (message->doc_id, message->doc);
+    message->modified = false;
+}
+
+/* Delete a message document from the database, leaving a ghost
+ * message in its place */
+notmuch_status_t
+_notmuch_message_delete (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+    Xapian::WritableDatabase *db;
+    const char *mid, *tid, *query_string;
+    notmuch_message_t *ghost;
+    notmuch_private_status_t private_status;
+    notmuch_database_t *notmuch;
+    notmuch_query_t *query;
+    unsigned int count = 0;
+    bool is_ghost;
+
+    mid = notmuch_message_get_message_id (message);
+    tid = notmuch_message_get_thread_id (message);
+    notmuch = message->notmuch;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+    db->delete_document (message->doc_id);
+
+    /* if this was a ghost to begin with, we are done */
+    private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost);
+    if (private_status)
+	return COERCE_STATUS (private_status,
+			      "Error trying to determine whether message was a ghost");
+    if (is_ghost)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    query_string = talloc_asprintf (message, "thread:%s", tid);
+    query = notmuch_query_create (notmuch, query_string);
+    if (query == NULL)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    status = notmuch_query_count_messages (query, &count);
+    if (status) {
+	notmuch_query_destroy (query);
+	return status;
+    }
+
+    if (count > 0) {
+	/* reintroduce a ghost in its place because there are still
+	 * other active messages in this thread: */
+	ghost = _notmuch_message_create_for_message_id (notmuch, mid, &private_status);
+	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	    private_status = _notmuch_message_initialize_ghost (ghost, tid);
+	    if (! private_status)
+		_notmuch_message_sync (ghost);
+	} else if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+	    /* this is deeply weird, and we should not have gotten
+	       into this state.  is there a better error message to
+	       return here? */
+	    status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+	}
+
+	notmuch_message_destroy (ghost);
+	status = COERCE_STATUS (private_status, "Error converting to ghost message");
+    } else {
+	/* the thread is empty; drop all ghost messages from it */
+	notmuch_messages_t *messages;
+	status = _notmuch_query_search_documents (query,
+						  "ghost",
+						  &messages);
+	if (status == NOTMUCH_STATUS_SUCCESS) {
+	    notmuch_status_t last_error = NOTMUCH_STATUS_SUCCESS;
+	    while (notmuch_messages_valid (messages)) {
+		message = notmuch_messages_get (messages);
+		status = _notmuch_message_delete (message);
+		if (status) /* we'll report the last failure we see;
+			     * if there is more than one failure, we
+			     * forget about previous ones */
+		    last_error = status;
+		notmuch_message_destroy (message);
+		notmuch_messages_move_to_next (messages);
+	    }
+	    status = last_error;
+	}
+    }
+    notmuch_query_destroy (query);
+    return status;
+}
+
+/* Transform a blank message into a ghost message.  The caller must
+ * _notmuch_message_sync the message. */
+notmuch_private_status_t
+_notmuch_message_initialize_ghost (notmuch_message_t *message,
+				   const char *thread_id)
+{
+    notmuch_private_status_t status;
+
+    status = _notmuch_message_add_term (message, "type", "ghost");
+    if (status)
+	return status;
+    status = _notmuch_message_add_term (message, "thread", thread_id);
+    if (status)
+	return status;
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Ensure that 'message' is not holding any file object open. Future
+ * calls to various functions will still automatically open the
+ * message file as needed.
+ */
+void
+_notmuch_message_close (notmuch_message_t *message)
+{
+    if (message->message_file) {
+	_notmuch_message_file_close (message->message_file);
+	message->message_file = NULL;
+    }
+}
+
+/* Add a name:value term to 'message', (the actual term will be
+ * encoded by prefixing the value with a short prefix). See
+ * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term
+ * names to prefix values.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync. */
+notmuch_private_status_t
+_notmuch_message_add_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value)
+{
+
+    char *term;
+
+    if (value == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term = talloc_asprintf (message, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    if (strlen (term) > NOTMUCH_TERM_MAX)
+	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
+
+    message->doc.add_term (term, 0);
+    message->modified = true;
+
+    talloc_free (term);
+
+    _notmuch_message_invalidate_metadata (message, prefix_name);
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Parse 'text' and add a term to 'message' for each parsed word. Each
+ * term will be added both prefixed (if prefix_name is not NULL) and
+ * also non-prefixed). */
+notmuch_private_status_t
+_notmuch_message_gen_terms (notmuch_message_t *message,
+			    const char *prefix_name,
+			    const char *text)
+{
+    Xapian::TermGenerator *term_gen = message->notmuch->term_gen;
+
+    if (text == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term_gen->set_document (message->doc);
+
+    if (prefix_name) {
+	const char *prefix = _find_prefix (prefix_name);
+
+	term_gen->set_termpos (message->termpos);
+	term_gen->index_text (text, 1, prefix);
+	/* Create a gap between this an the next terms so they don't
+	 * appear to be a phrase. */
+	message->termpos = term_gen->get_termpos () + 100;
+
+	_notmuch_message_invalidate_metadata (message, prefix_name);
+    }
+
+    term_gen->set_termpos (message->termpos);
+    term_gen->index_text (text);
+    /* Create a term gap, as above. */
+    message->termpos = term_gen->get_termpos () + 100;
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+/* Remove a name:value term from 'message', (the actual term will be
+ * encoded by prefixing the value with a short prefix). See
+ * NORMAL_PREFIX and BOOLEAN_PREFIX arrays for the mapping of term
+ * names to prefix values.
+ *
+ * This change will not be reflected in the database until the next
+ * call to _notmuch_message_sync. */
+notmuch_private_status_t
+_notmuch_message_remove_term (notmuch_message_t *message,
+			      const char *prefix_name,
+			      const char *value)
+{
+    char *term;
+
+    if (value == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term = talloc_asprintf (message, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    if (strlen (term) > NOTMUCH_TERM_MAX)
+	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
+
+    try {
+	message->doc.remove_term (term);
+	message->modified = true;
+    } catch (const Xapian::InvalidArgumentError) {
+	/* We'll let the philosophers try to wrestle with the
+	 * question of whether failing to remove that which was not
+	 * there in the first place is failure. For us, we'll silently
+	 * consider it all good. */
+    }
+
+    talloc_free (term);
+
+    _notmuch_message_invalidate_metadata (message, prefix_name);
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+notmuch_private_status_t
+_notmuch_message_has_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value,
+			   bool *result)
+{
+    char *term;
+    bool out = false;
+    notmuch_private_status_t status = NOTMUCH_PRIVATE_STATUS_SUCCESS;
+
+    if (value == NULL)
+	return NOTMUCH_PRIVATE_STATUS_NULL_POINTER;
+
+    term = talloc_asprintf (message, "%s%s",
+			    _find_prefix (prefix_name), value);
+
+    if (strlen (term) > NOTMUCH_TERM_MAX)
+	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
+
+    try {
+	/* Look for the exact term */
+	Xapian::TermIterator i = message->doc.termlist_begin ();
+	i.skip_to (term);
+	if (i != message->doc.termlist_end () &&
+	    !strcmp ((*i).c_str (), term))
+	    out = true;
+    } catch (Xapian::Error &error) {
+	status = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+    }
+    talloc_free (term);
+
+    *result = out;
+    return status;
+}
+
+notmuch_status_t
+notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    if (tag == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (strlen (tag) > NOTMUCH_TAG_MAX)
+	return NOTMUCH_STATUS_TAG_TOO_LONG;
+
+    private_status = _notmuch_message_add_term (message, "tag", tag);
+    if (private_status) {
+	INTERNAL_ERROR ("_notmuch_message_add_term return unexpected value: %d\n",
+			private_status);
+    }
+
+    if (! message->frozen)
+	_notmuch_message_sync (message);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    if (tag == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (strlen (tag) > NOTMUCH_TAG_MAX)
+	return NOTMUCH_STATUS_TAG_TOO_LONG;
+
+    private_status = _notmuch_message_remove_term (message, "tag", tag);
+    if (private_status) {
+	INTERNAL_ERROR ("_notmuch_message_remove_term return unexpected value: %d\n",
+			private_status);
+    }
+
+    if (! message->frozen)
+	_notmuch_message_sync (message);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Is the given filename within a maildir directory?
+ *
+ * Specifically, is the final directory component of 'filename' either
+ * "cur" or "new". If so, return a pointer to that final directory
+ * component within 'filename'. If not, return NULL.
+ *
+ * A non-NULL return value is guaranteed to be a valid string pointer
+ * pointing to the characters "new/" or "cur/", (but not
+ * NUL-terminated).
+ */
+static const char *
+_filename_is_in_maildir (const char *filename)
+{
+    const char *slash, *dir = NULL;
+
+    /* Find the last '/' separating directory from filename. */
+    slash = strrchr (filename, '/');
+    if (slash == NULL)
+	return NULL;
+
+    /* Jump back 4 characters to where the previous '/' will be if the
+     * directory is named "cur" or "new". */
+    if (slash - filename < 4)
+	return NULL;
+
+    slash -= 4;
+
+    if (*slash != '/')
+	return NULL;
+
+    dir = slash + 1;
+
+    if (STRNCMP_LITERAL (dir, "cur/") == 0 ||
+	STRNCMP_LITERAL (dir, "new/") == 0)
+    {
+	return dir;
+    }
+
+    return NULL;
+}
+
+static void
+_ensure_maildir_flags (notmuch_message_t *message, bool force)
+{
+    const char *flags;
+    notmuch_filenames_t *filenames;
+    const char *filename, *dir;
+    char *combined_flags = talloc_strdup (message, "");
+    int seen_maildir_info = 0;
+
+    if (message->maildir_flags) {
+	if (force) {
+	    talloc_free (message->maildir_flags);
+	    message->maildir_flags = NULL;
+	}
+    }
+
+    for (filenames = notmuch_message_get_filenames (message);
+	 notmuch_filenames_valid (filenames);
+	 notmuch_filenames_move_to_next (filenames))
+    {
+	filename = notmuch_filenames_get (filenames);
+	dir = _filename_is_in_maildir (filename);
+
+	if (! dir)
+	    continue;
+
+	flags = strstr (filename, ":2,");
+	if (flags) {
+	    seen_maildir_info = 1;
+	    flags += 3;
+	    combined_flags = talloc_strdup_append (combined_flags, flags);
+	} else if (STRNCMP_LITERAL (dir, "new/") == 0) {
+	    /* Messages are delivered to new/ with no "info" part, but
+	     * they effectively have default maildir flags.  According
+	     * to the spec, we should ignore the info part for
+	     * messages in new/, but some MUAs (mutt) can set maildir
+	     * flags on messages in new/, so we're liberal in what we
+	     * accept. */
+	    seen_maildir_info = 1;
+	}
+    }
+    if (seen_maildir_info)
+	message->maildir_flags = combined_flags;
+}
+
+notmuch_bool_t
+notmuch_message_has_maildir_flag (notmuch_message_t *message, char flag)
+{
+    _ensure_maildir_flags (message, false);
+    return message->maildir_flags && (strchr (message->maildir_flags, flag) != NULL);
+}
+
+notmuch_status_t
+notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+    unsigned i;
+
+    _ensure_maildir_flags (message, true);
+    /* If none of the filenames have any maildir info field (not even
+     * an empty info with no flags set) then there's no information to
+     * go on, so do nothing. */
+    if (! message->maildir_flags)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    status = notmuch_message_freeze (message);
+    if (status)
+	return status;
+
+    for (i = 0; i < ARRAY_SIZE(flag2tag); i++) {
+	if ((strchr (message->maildir_flags, flag2tag[i].flag) != NULL)
+	    ^
+	    flag2tag[i].inverse)
+	{
+	    status = notmuch_message_add_tag (message, flag2tag[i].tag);
+	} else {
+	    status = notmuch_message_remove_tag (message, flag2tag[i].tag);
+	}
+	if (status)
+	    return status;
+    }
+    status = notmuch_message_thaw (message);
+
+    return status;
+}
+
+/* From the set of tags on 'message' and the flag2tag table, compute a
+ * set of maildir-flag actions to be taken, (flags that should be
+ * either set or cleared).
+ *
+ * The result is returned as two talloced strings: to_set, and to_clear
+ */
+static void
+_get_maildir_flag_actions (notmuch_message_t *message,
+			   char **to_set_ret,
+			   char **to_clear_ret)
+{
+    char *to_set, *to_clear;
+    notmuch_tags_t *tags;
+    const char *tag;
+    unsigned i;
+
+    to_set = talloc_strdup (message, "");
+    to_clear = talloc_strdup (message, "");
+
+    /* First, find flags for all set tags. */
+    for (tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags))
+    {
+	tag = notmuch_tags_get (tags);
+
+	for (i = 0; i < ARRAY_SIZE (flag2tag); i++) {
+	    if (strcmp (tag, flag2tag[i].tag) == 0) {
+		if (flag2tag[i].inverse)
+		    to_clear = talloc_asprintf_append (to_clear,
+						       "%c",
+						       flag2tag[i].flag);
+		else
+		    to_set = talloc_asprintf_append (to_set,
+						     "%c",
+						     flag2tag[i].flag);
+	    }
+	}
+    }
+
+    /* Then, find the flags for all tags not present. */
+    for (i = 0; i < ARRAY_SIZE (flag2tag); i++) {
+	if (flag2tag[i].inverse) {
+	    if (strchr (to_clear, flag2tag[i].flag) == NULL)
+		to_set = talloc_asprintf_append (to_set, "%c", flag2tag[i].flag);
+	} else {
+	    if (strchr (to_set, flag2tag[i].flag) == NULL)
+		to_clear = talloc_asprintf_append (to_clear, "%c", flag2tag[i].flag);
+	}
+    }
+
+    *to_set_ret = to_set;
+    *to_clear_ret = to_clear;
+}
+
+/* Given 'filename' and a set of maildir flags to set and to clear,
+ * compute the new maildir filename.
+ *
+ * If the existing filename is in the directory "new", the new
+ * filename will be in the directory "cur", except for the case when
+ * no flags are changed and the existing filename does not contain
+ * maildir info (starting with ",2:").
+ *
+ * After a sequence of ":2," in the filename, any subsequent
+ * single-character flags will be added or removed according to the
+ * characters in flags_to_set and flags_to_clear. Any existing flags
+ * not mentioned in either string will remain. The final list of flags
+ * will be in ASCII order.
+ *
+ * If the original flags seem invalid, (repeated characters or
+ * non-ASCII ordering of flags), this function will return NULL
+ * (meaning that renaming would not be safe and should not occur).
+ */
+static char*
+_new_maildir_filename (void *ctx,
+		       const char *filename,
+		       const char *flags_to_set,
+		       const char *flags_to_clear)
+{
+    const char *info, *flags;
+    unsigned int flag, last_flag;
+    char *filename_new, *dir;
+    char flag_map[128];
+    int flags_in_map = 0;
+    bool flags_changed = false;
+    unsigned int i;
+    char *s;
+
+    memset (flag_map, 0, sizeof (flag_map));
+
+    info = strstr (filename, ":2,");
+
+    if (info == NULL) {
+	info = filename + strlen(filename);
+    } else {
+	/* Loop through existing flags in filename. */
+	for (flags = info + 3, last_flag = 0;
+	     *flags;
+	     last_flag = flag, flags++)
+	{
+	    flag = *flags;
+
+	    /* Original flags not in ASCII order. Abort. */
+	    if (flag < last_flag)
+		return NULL;
+
+	    /* Non-ASCII flag. Abort. */
+	    if (flag > sizeof(flag_map) - 1)
+		return NULL;
+
+	    /* Repeated flag value. Abort. */
+	    if (flag_map[flag])
+		return NULL;
+
+	    flag_map[flag] = 1;
+	    flags_in_map++;
+	}
+    }
+
+    /* Then set and clear our flags from tags. */
+    for (flags = flags_to_set; *flags; flags++) {
+	flag = *flags;
+	if (flag_map[flag] == 0) {
+	    flag_map[flag] = 1;
+	    flags_in_map++;
+	    flags_changed = true;
+	}
+    }
+
+    for (flags = flags_to_clear; *flags; flags++) {
+	flag = *flags;
+	if (flag_map[flag]) {
+	    flag_map[flag] = 0;
+	    flags_in_map--;
+	    flags_changed = true;
+	}
+    }
+
+    /* Messages in new/ without maildir info can be kept in new/ if no
+     * flags have changed. */
+    dir = (char *) _filename_is_in_maildir (filename);
+    if (dir && STRNCMP_LITERAL (dir, "new/") == 0 && !*info && !flags_changed)
+	return talloc_strdup (ctx, filename);
+
+    filename_new = (char *) talloc_size (ctx,
+					 info - filename +
+					 strlen (":2,") + flags_in_map + 1);
+    if (unlikely (filename_new == NULL))
+	return NULL;
+
+    strncpy (filename_new, filename, info - filename);
+    filename_new[info - filename] = '\0';
+
+    strcat (filename_new, ":2,");
+
+    s = filename_new + strlen (filename_new);
+    for (i = 0; i < sizeof (flag_map); i++)
+    {
+	if (flag_map[i]) {
+	    *s = i;
+	    s++;
+	}
+    }
+    *s = '\0';
+
+    /* If message is in new/ move it under cur/. */
+    dir = (char *) _filename_is_in_maildir (filename_new);
+    if (dir && STRNCMP_LITERAL (dir, "new/") == 0)
+	memcpy (dir, "cur/", 4);
+
+    return filename_new;
+}
+
+notmuch_status_t
+notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
+{
+    notmuch_filenames_t *filenames;
+    const char *filename;
+    char *filename_new;
+    char *to_set, *to_clear;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    _get_maildir_flag_actions (message, &to_set, &to_clear);
+
+    for (filenames = notmuch_message_get_filenames (message);
+	 notmuch_filenames_valid (filenames);
+	 notmuch_filenames_move_to_next (filenames))
+    {
+	filename = notmuch_filenames_get (filenames);
+
+	if (! _filename_is_in_maildir (filename))
+	    continue;
+
+	filename_new = _new_maildir_filename (message, filename,
+					      to_set, to_clear);
+	if (filename_new == NULL)
+	    continue;
+
+	if (strcmp (filename, filename_new)) {
+	    int err;
+	    notmuch_status_t new_status;
+
+	    err = rename (filename, filename_new);
+	    if (err)
+		continue;
+
+	    new_status = _notmuch_message_remove_filename (message,
+							   filename);
+	    /* Hold on to only the first error. */
+	    if (! status && new_status
+		&& new_status != NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) {
+		status = new_status;
+		continue;
+	    }
+
+	    new_status = _notmuch_message_add_filename (message,
+							filename_new);
+	    /* Hold on to only the first error. */
+	    if (! status && new_status) {
+		status = new_status;
+		continue;
+	    }
+
+	    _notmuch_message_sync (message);
+	}
+
+	talloc_free (filename_new);
+    }
+
+    talloc_free (to_set);
+    talloc_free (to_clear);
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_message_remove_all_tags (notmuch_message_t *message)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+    notmuch_tags_t *tags;
+    const char *tag;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    for (tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags))
+    {
+	tag = notmuch_tags_get (tags);
+
+	private_status = _notmuch_message_remove_term (message, "tag", tag);
+	if (private_status) {
+	    INTERNAL_ERROR ("_notmuch_message_remove_term return unexpected value: %d\n",
+			    private_status);
+	}
+    }
+
+    if (! message->frozen)
+	_notmuch_message_sync (message);
+
+    talloc_free (tags);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_freeze (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    message->frozen++;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_thaw (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    if (message->frozen > 0) {
+	message->frozen--;
+	if (message->frozen == 0)
+	    _notmuch_message_sync (message);
+	return NOTMUCH_STATUS_SUCCESS;
+    } else {
+	return NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW;
+    }
+}
+
+void
+notmuch_message_destroy (notmuch_message_t *message)
+{
+    talloc_free (message);
+}
+
+notmuch_database_t *
+notmuch_message_get_database (const notmuch_message_t *message)
+{
+    return message->notmuch;
+}
+
+static void
+_notmuch_message_ensure_property_map (notmuch_message_t *message)
+{
+    notmuch_string_node_t *node;
+
+    if (message->property_map)
+	return;
+
+    _notmuch_message_ensure_metadata (message, message->property_term_list);
+
+    message->property_map = _notmuch_string_map_create (message);
+
+    for (node = message->property_term_list->head; node; node = node->next) {
+	const char *key;
+	char *value;
+
+	value = strchr(node->string, '=');
+	if (!value)
+	    INTERNAL_ERROR ("malformed property term");
+
+	*value = '\0';
+	value++;
+	key = node->string;
+
+	_notmuch_string_map_append (message->property_map, key, value);
+
+    }
+
+    talloc_free (message->property_term_list);
+    message->property_term_list = NULL;
+}
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_property_map (message);
+
+    return message->property_map;
+}
+
+bool
+_notmuch_message_frozen (notmuch_message_t *message)
+{
+    return message->frozen;
+}
+
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+			 notmuch_indexopts_t *indexopts)
+{
+    notmuch_database_t *notmuch = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_private_status_t private_status;
+    notmuch_filenames_t *orig_filenames = NULL;
+    const char *orig_thread_id = NULL;
+    notmuch_message_file_t *message_file = NULL;
+
+    int found = 0;
+
+    if (message == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    /* Save in case we need to delete message */
+    orig_thread_id = notmuch_message_get_thread_id (message);
+    if (!orig_thread_id) {
+	/* XXX TODO: make up new error return? */
+	INTERNAL_ERROR ("message without thread-id");
+    }
+
+    /* strdup it because the metadata may be invalidated */
+    orig_thread_id = talloc_strdup (message, orig_thread_id);
+
+    notmuch = notmuch_message_get_database (message);
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    orig_filenames = notmuch_message_get_filenames (message);
+
+    private_status = _notmuch_message_remove_indexed_terms (message);
+    if (private_status) {
+	ret = COERCE_STATUS(private_status, "error removing terms");
+	goto DONE;
+    }
+
+    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
+    if (ret)
+	goto DONE; /* XXX TODO: distinguish from other error returns above? */
+    if (indexopts && notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_FALSE) {
+	ret = notmuch_message_remove_all_properties (message, "session-key");
+	if (ret)
+	    goto DONE;
+    }
+
+    /* re-add the filenames with the associated indexopts */
+    for (; notmuch_filenames_valid (orig_filenames);
+	 notmuch_filenames_move_to_next (orig_filenames)) {
+
+	const char *date;
+	const char *from, *to, *subject;
+	char *message_id = NULL;
+	const char *thread_id = NULL;
+
+	const char *filename = notmuch_filenames_get (orig_filenames);
+
+	message_file = _notmuch_message_file_open (notmuch, filename);
+	if (message_file == NULL)
+	    continue;
+
+	ret = _notmuch_message_file_get_headers (message_file,
+						 &from, &subject, &to, &date,
+						 &message_id);
+	if (ret)
+	    goto DONE;
+
+	/* XXX TODO: deal with changing message id? */
+
+	_notmuch_message_add_filename (message, filename);
+
+	ret = _notmuch_database_link_message_to_parents (notmuch, message,
+							 message_file,
+							 &thread_id);
+	if (ret)
+	    goto DONE;
+
+	if (thread_id == NULL)
+	    thread_id = orig_thread_id;
+
+	_notmuch_message_add_term (message, "thread", thread_id);
+	/* Take header values only from first filename */
+	if (found == 0)
+	    _notmuch_message_set_header_values (message, date, from, subject);
+
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
+
+	if (ret == NOTMUCH_STATUS_FILE_ERROR)
+	    continue;
+	if (ret)
+	    goto DONE;
+
+	found++;
+	_notmuch_message_file_close (message_file);
+	message_file = NULL;
+    }
+    if (found == 0) {
+	/* put back thread id to help cleanup */
+	_notmuch_message_add_term (message, "thread", orig_thread_id);
+	ret = _notmuch_message_delete (message);
+    } else {
+	_notmuch_message_sync (message);
+    }
+
+ DONE:
+    if (message_file)
+	_notmuch_message_file_close (message_file);
+
+    /* XXX TODO destroy orig_filenames? */
+    return ret;
+}
diff --git a/lib/messages.c b/lib/messages.c
new file mode 100644
index 00000000..04fa19f8
--- /dev/null
+++ b/lib/messages.c
@@ -0,0 +1,194 @@
+/* messages.c - Iterator for a set of messages
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <glib.h>
+
+/* Create a new notmuch_message_list_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+notmuch_message_list_t *
+_notmuch_message_list_create (const void *ctx)
+{
+    notmuch_message_list_t *list;
+
+    list = talloc (ctx, notmuch_message_list_t);
+    if (unlikely (list == NULL))
+	return NULL;
+
+    list->head = NULL;
+    list->tail = &list->head;
+
+    return list;
+}
+
+/* Append 'message' to the end of 'list'. */
+void
+_notmuch_message_list_add_message (notmuch_message_list_t *list,
+				   notmuch_message_t *message)
+{
+    notmuch_message_node_t *node = talloc (list, notmuch_message_node_t);
+
+    node->message = message;
+    node->next = NULL;
+
+    *(list->tail) = node;
+    list->tail = &node->next;
+}
+
+bool
+_notmuch_message_list_empty (notmuch_message_list_t *list)
+{
+    if (list == NULL)
+	return TRUE;
+
+    return (list->head == NULL);
+}
+
+notmuch_messages_t *
+_notmuch_messages_create (notmuch_message_list_t *list)
+{
+    notmuch_messages_t *messages;
+
+    if (list->head == NULL)
+	return NULL;
+
+    messages = talloc (list, notmuch_messages_t);
+    if (unlikely (messages == NULL))
+	return NULL;
+
+    messages->is_of_list_type = true;
+    messages->iterator = list->head;
+
+    return messages;
+}
+
+/* We're using the "is_of_type_list" to conditionally defer to the
+ * notmuch_mset_messages_t implementation of notmuch_messages_t in
+ * query.cc. It's ugly that that's over in query.cc, and it's ugly
+ * that we're not using a union here. Both of those uglies are due to
+ * C++:
+ *
+ *	1. I didn't want to force a C++ header file onto
+ *	   notmuch-private.h and suddenly subject all our code to a
+ *	   C++ compiler and its rules.
+ *
+ *	2. C++ won't allow me to put C++ objects, (with non-trivial
+ *	   constructors) into a union anyway. Even though I'd
+ *	   carefully control object construction with placement new
+ *	   anyway. *sigh*
+ */
+notmuch_bool_t
+notmuch_messages_valid (notmuch_messages_t *messages)
+{
+    if (messages == NULL)
+	return false;
+
+    if (! messages->is_of_list_type)
+	return _notmuch_mset_messages_valid (messages);
+
+    return (messages->iterator != NULL);
+}
+
+bool
+_notmuch_messages_has_next (notmuch_messages_t *messages)
+{
+    if (! notmuch_messages_valid (messages))
+	return false;
+
+    if (! messages->is_of_list_type)
+	INTERNAL_ERROR("_notmuch_messages_has_next not implimented for msets");
+
+    return (messages->iterator->next != NULL);
+}
+
+notmuch_message_t *
+notmuch_messages_get (notmuch_messages_t *messages)
+{
+    if (! messages->is_of_list_type)
+	return _notmuch_mset_messages_get (messages);
+
+    if (messages->iterator == NULL)
+	return NULL;
+
+    return messages->iterator->message;
+}
+
+void
+notmuch_messages_move_to_next (notmuch_messages_t *messages)
+{
+    if (! messages->is_of_list_type) {
+	_notmuch_mset_messages_move_to_next (messages);
+	return;
+    }
+
+    if (messages->iterator == NULL)
+	return;
+
+    messages->iterator = messages->iterator->next;
+}
+
+void
+notmuch_messages_destroy (notmuch_messages_t *messages)
+{
+    talloc_free (messages);
+}
+
+
+notmuch_tags_t *
+notmuch_messages_collect_tags (notmuch_messages_t *messages)
+{
+    notmuch_string_list_t *tags;
+    notmuch_tags_t *msg_tags;
+    notmuch_message_t *msg;
+    GHashTable *htable;
+    GList *keys, *l;
+    const char *tag;
+
+    tags = _notmuch_string_list_create (messages);
+    if (tags == NULL) return NULL;
+
+    htable = g_hash_table_new_full (g_str_hash, g_str_equal, free, NULL);
+
+    while ((msg = notmuch_messages_get (messages))) {
+	msg_tags = notmuch_message_get_tags (msg);
+	while ((tag = notmuch_tags_get (msg_tags))) {
+	    g_hash_table_insert (htable, xstrdup (tag), NULL);
+	    notmuch_tags_move_to_next (msg_tags);
+	}
+	notmuch_tags_destroy (msg_tags);
+	notmuch_message_destroy (msg);
+	notmuch_messages_move_to_next (messages);
+    }
+
+    keys = g_hash_table_get_keys (htable);
+    for (l = keys; l; l = l->next) {
+	_notmuch_string_list_append (tags, (char *)l->data);
+    }
+
+    g_list_free (keys);
+    g_hash_table_destroy (htable);
+
+    _notmuch_string_list_sort (tags);
+    return _notmuch_tags_create (messages, tags);
+}
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
new file mode 100644
index 00000000..df32d39c
--- /dev/null
+++ b/lib/notmuch-private.h
@@ -0,0 +1,699 @@
+/* notmuch-private.h - Internal interfaces for notmuch.
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#ifndef NOTMUCH_PRIVATE_H
+#define NOTMUCH_PRIVATE_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE /* For getline and asprintf */
+#endif
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "compat.h"
+
+#include "notmuch.h"
+
+NOTMUCH_BEGIN_DECLS
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include <talloc.h>
+
+#include "gmime-extra.h"
+
+#include "xutil.h"
+#include "error_util.h"
+#include "string-util.h"
+#include "crypto.h"
+
+#ifdef DEBUG
+# define DEBUG_DATABASE_SANITY 1
+# define DEBUG_THREADING 1
+# define DEBUG_QUERY 1
+#endif
+
+#define COMPILE_TIME_ASSERT(pred) ((void)sizeof(char[1 - 2*!(pred)]))
+
+#define STRNCMP_LITERAL(var, literal) \
+    strncmp ((var), (literal), sizeof (literal) - 1)
+
+/* Robust bit test/set/reset macros */
+#define _NOTMUCH_VALID_BIT(bit) \
+    ((bit) >= 0 && ((unsigned long) bit) < CHAR_BIT * sizeof (unsigned long long))
+#define NOTMUCH_TEST_BIT(val, bit) \
+    (_NOTMUCH_VALID_BIT(bit) ? !!((val) & (1ull << (bit))) : 0)
+#define NOTMUCH_SET_BIT(valp, bit) \
+    (_NOTMUCH_VALID_BIT(bit) ? (*(valp) |= (1ull << (bit))) : *(valp))
+#define NOTMUCH_CLEAR_BIT(valp,  bit) \
+    (_NOTMUCH_VALID_BIT(bit) ? (*(valp) &= ~(1ull << (bit))) : *(valp))
+
+#define unused(x) x __attribute__ ((unused))
+
+/* Thanks to Andrew Tridgell's (SAMBA's) talloc for this definition of
+ * unlikely. The talloc source code comes to us via the GNU LGPL v. 3.
+ */
+/* these macros gain us a few percent of speed on gcc */
+#if (__GNUC__ >= 3)
+/* the strange !! is to ensure that __builtin_expect() takes either 0 or 1
+   as its first argument */
+#ifndef likely
+#define likely(x)   __builtin_expect(!!(x), 1)
+#endif
+#ifndef unlikely
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+#else
+#ifndef likely
+#define likely(x) (x)
+#endif
+#ifndef unlikely
+#define unlikely(x) (x)
+#endif
+#endif
+
+typedef enum {
+    NOTMUCH_VALUE_TIMESTAMP = 0,
+    NOTMUCH_VALUE_MESSAGE_ID,
+    NOTMUCH_VALUE_FROM,
+    NOTMUCH_VALUE_SUBJECT,
+    NOTMUCH_VALUE_LAST_MOD,
+} notmuch_value_t;
+
+/* Xapian (with flint backend) complains if we provide a term longer
+ * than this, but I haven't yet found a way to query the limit
+ * programmatically. */
+#define NOTMUCH_TERM_MAX 245
+
+#define NOTMUCH_METADATA_THREAD_ID_PREFIX "thread_id_"
+
+/* For message IDs we have to be even more restrictive. Beyond fitting
+ * into the term limit, we also use message IDs to construct
+ * metadata-key values. And the documentation says that these should
+ * be restricted to about 200 characters. (The actual limit for the
+ * chert backend at least is 252.)
+ */
+#define NOTMUCH_MESSAGE_ID_MAX (200 - sizeof (NOTMUCH_METADATA_THREAD_ID_PREFIX))
+
+typedef enum _notmuch_private_status {
+    /* First, copy all the public status values. */
+    NOTMUCH_PRIVATE_STATUS_SUCCESS = NOTMUCH_STATUS_SUCCESS,
+    NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY = NOTMUCH_STATUS_OUT_OF_MEMORY,
+    NOTMUCH_PRIVATE_STATUS_READ_ONLY_DATABASE = NOTMUCH_STATUS_READ_ONLY_DATABASE,
+    NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION = NOTMUCH_STATUS_XAPIAN_EXCEPTION,
+    NOTMUCH_PRIVATE_STATUS_FILE_NOT_EMAIL = NOTMUCH_STATUS_FILE_NOT_EMAIL,
+    NOTMUCH_PRIVATE_STATUS_NULL_POINTER = NOTMUCH_STATUS_NULL_POINTER,
+    NOTMUCH_PRIVATE_STATUS_TAG_TOO_LONG = NOTMUCH_STATUS_TAG_TOO_LONG,
+    NOTMUCH_PRIVATE_STATUS_UNBALANCED_FREEZE_THAW = NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
+
+    /* Then add our own private values. */
+    NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG = NOTMUCH_STATUS_LAST_STATUS,
+    NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND,
+
+    NOTMUCH_PRIVATE_STATUS_LAST_STATUS
+} notmuch_private_status_t;
+
+/* Coerce a notmuch_private_status_t value to a notmuch_status_t
+ * value, generating an internal error if the private value is equal
+ * to or greater than NOTMUCH_STATUS_LAST_STATUS. (The idea here is
+ * that the caller has previously handled any expected
+ * notmuch_private_status_t values.)
+ *
+ * Note that the function _internal_error does not return. Evaluating
+ * to NOTMUCH_STATUS_SUCCESS is done purely to appease the compiler.
+ */
+#define COERCE_STATUS(private_status, format, ...)			\
+    ((private_status >= (notmuch_private_status_t) NOTMUCH_STATUS_LAST_STATUS)\
+     ?									\
+     _internal_error (format " (%s).\n",				\
+		      ##__VA_ARGS__,					\
+		      __location__),					\
+     (notmuch_status_t) NOTMUCH_PRIVATE_STATUS_SUCCESS			\
+     :									\
+     (notmuch_status_t) private_status)
+
+/* Flags shared by various lookup functions. */
+typedef enum _notmuch_find_flags {
+    /* Lookup without creating any documents.  This is the default
+     * behavior. */
+    NOTMUCH_FIND_LOOKUP = 0,
+    /* If set, create the necessary document (or documents) if they
+     * are missing.  Requires a read/write database. */
+    NOTMUCH_FIND_CREATE = 1<<0,
+} notmuch_find_flags_t;
+
+typedef struct _notmuch_doc_id_set notmuch_doc_id_set_t;
+
+/* database.cc */
+
+/* Lookup a prefix value by name.
+ *
+ * XXX: This should really be static inside of message.cc, and we can
+ * do that once we convert database.cc to use the
+ * _notmuch_message_add/remove_term functions. */
+const char *
+_find_prefix (const char *name);
+
+char *
+_notmuch_message_id_compressed (void *ctx, const char *message_id);
+
+notmuch_status_t
+_notmuch_database_ensure_writable (notmuch_database_t *notmuch);
+
+notmuch_status_t
+_notmuch_database_reopen (notmuch_database_t *notmuch);
+
+void
+_notmuch_database_log (notmuch_database_t *notmuch,
+		       const char *format, ...);
+
+void
+_notmuch_database_log_append (notmuch_database_t *notmuch,
+			      const char *format, ...);
+
+unsigned long
+_notmuch_database_new_revision (notmuch_database_t *notmuch);
+
+const char *
+_notmuch_database_relative_path (notmuch_database_t *notmuch,
+				 const char *path);
+
+notmuch_status_t
+_notmuch_database_split_path (void *ctx,
+			      const char *path,
+			      const char **directory,
+			      const char **basename);
+
+const char *
+_notmuch_database_get_directory_db_path (const char *path);
+
+unsigned int
+_notmuch_database_generate_doc_id (notmuch_database_t *notmuch);
+
+notmuch_private_status_t
+_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
+				      const char *prefix_name,
+				      const char *value,
+				      unsigned int *doc_id);
+
+notmuch_status_t
+_notmuch_database_find_directory_id (notmuch_database_t *database,
+				     const char *path,
+				     notmuch_find_flags_t flags,
+				     unsigned int *directory_id);
+
+const char *
+_notmuch_database_get_directory_path (void *ctx,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id);
+
+notmuch_status_t
+_notmuch_database_filename_to_direntry (void *ctx,
+					notmuch_database_t *notmuch,
+					const char *filename,
+					notmuch_find_flags_t flags,
+					char **direntry);
+
+/* directory.cc */
+
+notmuch_directory_t *
+_notmuch_directory_create (notmuch_database_t *notmuch,
+			   const char *path,
+			   notmuch_find_flags_t flags,
+			   notmuch_status_t *status_ret);
+
+unsigned int
+_notmuch_directory_get_document_id (notmuch_directory_t *directory);
+
+/* message.cc */
+
+notmuch_message_t *
+_notmuch_message_create (const void *talloc_owner,
+			 notmuch_database_t *notmuch,
+			 unsigned int doc_id,
+			 notmuch_private_status_t *status);
+
+notmuch_message_t *
+_notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
+					const char *message_id,
+					notmuch_private_status_t *status);
+
+unsigned int
+_notmuch_message_get_doc_id (notmuch_message_t *message);
+
+const char *
+_notmuch_message_get_in_reply_to (notmuch_message_t *message);
+
+notmuch_private_status_t
+_notmuch_message_add_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value);
+
+notmuch_private_status_t
+_notmuch_message_remove_term (notmuch_message_t *message,
+			      const char *prefix_name,
+			      const char *value);
+
+notmuch_private_status_t
+_notmuch_message_has_term (notmuch_message_t *message,
+			   const char *prefix_name,
+			   const char *value,
+			   bool *result);
+
+notmuch_private_status_t
+_notmuch_message_gen_terms (notmuch_message_t *message,
+			    const char *prefix_name,
+			    const char *text);
+
+void
+_notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
+
+void
+_notmuch_message_upgrade_folder (notmuch_message_t *message);
+
+notmuch_status_t
+_notmuch_message_add_filename (notmuch_message_t *message,
+			       const char *filename);
+
+notmuch_status_t
+_notmuch_message_remove_filename (notmuch_message_t *message,
+				  const char *filename);
+
+notmuch_status_t
+_notmuch_message_rename (notmuch_message_t *message,
+			 const char *new_filename);
+
+void
+_notmuch_message_ensure_thread_id (notmuch_message_t *message);
+
+void
+_notmuch_message_set_header_values (notmuch_message_t *message,
+				    const char *date,
+				    const char *from,
+				    const char *subject);
+
+void
+_notmuch_message_upgrade_last_mod (notmuch_message_t *message);
+
+void
+_notmuch_message_sync (notmuch_message_t *message);
+
+notmuch_status_t
+_notmuch_message_delete (notmuch_message_t *message);
+
+notmuch_private_status_t
+_notmuch_message_initialize_ghost (notmuch_message_t *message,
+				   const char *thread_id);
+
+void
+_notmuch_message_close (notmuch_message_t *message);
+
+/* Get a copy of the data in this message document.
+ *
+ * Caller should talloc_free the result when done.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message);
+
+/* Clear the data in this message document.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+void
+_notmuch_message_clear_data (notmuch_message_t *message);
+
+/* Set the author member of 'message' - this is the representation used
+ * when displaying the message */
+void
+_notmuch_message_set_author (notmuch_message_t *message, const char *author);
+
+/* Get the author member of 'message' */
+const char *
+_notmuch_message_get_author (notmuch_message_t *message);
+
+/* message-file.c */
+
+/* XXX: I haven't decided yet whether these will actually get exported
+ * into the public interface in notmuch.h
+ */
+
+typedef struct _notmuch_message_file notmuch_message_file_t;
+
+/* Open a file containing a single email message.
+ *
+ * The caller should call notmuch_message_close when done with this.
+ *
+ * Returns NULL if any error occurs.
+ */
+notmuch_message_file_t *
+_notmuch_message_file_open (notmuch_database_t *notmuch, const char *filename);
+
+/* Like notmuch_message_file_open but with 'ctx' as the talloc owner. */
+notmuch_message_file_t *
+_notmuch_message_file_open_ctx (notmuch_database_t *notmuch,
+				void *ctx, const char *filename);
+
+/* Close a notmuch message previously opened with notmuch_message_open. */
+void
+_notmuch_message_file_close (notmuch_message_file_t *message);
+
+/* Parse the message.
+ *
+ * This will be done automatically as necessary on other calls
+ * depending on it, but an explicit call allows for better error
+ * status reporting.
+ */
+notmuch_status_t
+_notmuch_message_file_parse (notmuch_message_file_t *message);
+
+/* Get the gmime message of a message file.
+ *
+ * The message file is parsed as necessary.
+ *
+ * The GMimeMessage* is set to *mime_message on success (which the
+ * caller must not unref).
+ *
+ * XXX: Would be nice to not have to expose GMimeMessage here.
+ */
+notmuch_status_t
+_notmuch_message_file_get_mime_message (notmuch_message_file_t *message,
+					GMimeMessage **mime_message);
+
+/* Get the value of the specified header from the message as a UTF-8 string.
+ *
+ * The message file is parsed as necessary.
+ *
+ * The header name is case insensitive.
+ *
+ * The Received: header is special - for it all Received: headers in
+ * the message are concatenated
+ *
+ * The returned value is owned by the notmuch message and is valid
+ * only until the message is closed. The caller should copy it if
+ * needing to modify the value or to hold onto it for longer.
+ *
+ * Returns NULL on errors, empty string if the message does not
+ * contain a header line matching 'header'.
+ */
+const char *
+_notmuch_message_file_get_header (notmuch_message_file_t *message,
+				 const char *header);
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+				   const char **from_out,
+				   const char **subject_out,
+				   const char **to_out,
+				   const char **date_out,
+				   char **message_id_out);
+
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message);
+
+/* add-message.cc */
+notmuch_status_t
+_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
+					   notmuch_message_t *message,
+					   notmuch_message_file_t *message_file,
+					   const char **thread_id);
+/* index.cc */
+
+notmuch_status_t
+_notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_file_t *message_file);
+
+/* messages.c */
+
+typedef struct _notmuch_message_node {
+    notmuch_message_t *message;
+    struct _notmuch_message_node *next;
+} notmuch_message_node_t;
+
+typedef struct _notmuch_message_list {
+    notmuch_message_node_t *head;
+    notmuch_message_node_t **tail;
+} notmuch_message_list_t;
+
+/* There's a rumor that there's an alternate struct _notmuch_messages
+ * somewhere with some nasty C++ objects in it. We'll try to maintain
+ * ignorance of that here. (See notmuch_mset_messages_t in query.cc)
+ */
+struct _notmuch_messages {
+    bool is_of_list_type;
+    notmuch_doc_id_set_t *excluded_doc_ids;
+    notmuch_message_node_t *iterator;
+};
+
+notmuch_message_list_t *
+_notmuch_message_list_create (const void *ctx);
+
+bool
+_notmuch_message_list_empty (notmuch_message_list_t *list);
+
+void
+_notmuch_message_list_add_message (notmuch_message_list_t *list,
+				   notmuch_message_t *message);
+
+notmuch_messages_t *
+_notmuch_messages_create (notmuch_message_list_t *list);
+
+bool
+_notmuch_messages_has_next (notmuch_messages_t *messages);
+
+/* query.cc */
+
+bool
+_notmuch_mset_messages_valid (notmuch_messages_t *messages);
+
+notmuch_message_t *
+_notmuch_mset_messages_get (notmuch_messages_t *messages);
+
+void
+_notmuch_mset_messages_move_to_next (notmuch_messages_t *messages);
+
+bool
+_notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
+			      unsigned int doc_id);
+
+void
+_notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
+			    unsigned int doc_id);
+
+/* querying xapian documents by type (e.g. "mail" or "ghost"): */
+notmuch_status_t
+_notmuch_query_search_documents (notmuch_query_t *query,
+				 const char *type,
+				 notmuch_messages_t **out);
+
+notmuch_status_t
+_notmuch_query_count_documents (notmuch_query_t *query,
+				const char *type,
+				unsigned *count_out);
+/* message-id.c */
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimiters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+char *
+_notmuch_message_id_parse (void *ctx, const char *message_id, const char **next);
+
+/* Parse a message-id, discarding leading and trailing whitespace, and
+ * '<' and '>' delimiters.
+ *
+ * Apply a probably-stricter-than RFC definition of what is allowed in
+ * a message-id. In particular, forbid whitespace.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id.
+ */
+
+char *
+_notmuch_message_id_parse_strict (void *ctx, const char *message_id);
+
+
+/* message.cc */
+
+void
+_notmuch_message_add_reply (notmuch_message_t *message,
+			    notmuch_message_t *reply);
+
+void
+_notmuch_message_remove_unprefixed_terms (notmuch_message_t *message);
+
+const char *
+_notmuch_message_get_thread_id_only(notmuch_message_t *message);
+
+size_t _notmuch_message_get_thread_depth (notmuch_message_t *message);
+
+void
+_notmuch_message_label_depths (notmuch_message_t *message,
+			       size_t depth);
+
+notmuch_message_list_t *
+_notmuch_message_sort_subtrees (void *ctx, notmuch_message_list_t *list);
+
+/* sha1.c */
+
+char *
+_notmuch_sha1_of_string (const char *str);
+
+char *
+_notmuch_sha1_of_file (const char *filename);
+
+/* string-list.c */
+
+typedef struct _notmuch_string_node {
+    char *string;
+    struct _notmuch_string_node *next;
+} notmuch_string_node_t;
+
+typedef struct _notmuch_string_list {
+    int length;
+    notmuch_string_node_t *head;
+    notmuch_string_node_t **tail;
+} notmuch_string_list_t;
+
+notmuch_string_list_t *
+_notmuch_string_list_create (const void *ctx);
+
+/*
+ * return the number of strings in 'list'
+ */
+int
+_notmuch_string_list_length (notmuch_string_list_t *list);
+
+/* Add 'string' to 'list'.
+ *
+ * The list will create its own talloced copy of 'string'.
+ */
+void
+_notmuch_string_list_append (notmuch_string_list_t *list,
+			     const char *string);
+
+void
+_notmuch_string_list_sort (notmuch_string_list_t *list);
+
+const notmuch_string_list_t *
+_notmuch_message_get_references(notmuch_message_t *message);
+
+/* string-map.c */
+typedef struct _notmuch_string_map  notmuch_string_map_t;
+typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx);
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value);
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     bool exact);
+
+bool
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+
+/* tags.c */
+
+notmuch_tags_t *
+_notmuch_tags_create (const void *ctx, notmuch_string_list_t *list);
+
+/* filenames.c */
+
+/* The notmuch_filenames_t iterates over a notmuch_string_list_t of
+ * file names */
+notmuch_filenames_t *
+_notmuch_filenames_create (const void *ctx,
+			   notmuch_string_list_t *list);
+
+/* thread.cc */
+
+notmuch_thread_t *
+_notmuch_thread_create (void *ctx,
+			notmuch_database_t *notmuch,
+			unsigned int seed_doc_id,
+			notmuch_doc_id_set_t *match_set,
+			notmuch_string_list_t *excluded_terms,
+			notmuch_exclude_t omit_exclude,
+			notmuch_sort_t sort);
+
+/* indexopts.c */
+
+struct _notmuch_indexopts {
+    _notmuch_crypto_t crypto;
+};
+
+NOTMUCH_END_DECLS
+
+#ifdef __cplusplus
+/* Implicit typecast from 'void *' to 'T *' is okay in C, but not in
+ * C++. In talloc_steal, an explicit cast is provided for type safety
+ * in some GCC versions. Otherwise, a cast is required. Provide a
+ * template function for this to maintain type safety, and redefine
+ * talloc_steal to use it.
+ */
+#if !(__GNUC__ >= 3)
+template <class T> T *
+_notmuch_talloc_steal (const void *new_ctx, const T *ptr)
+{
+    return static_cast<T *> (talloc_steal (new_ctx, ptr));
+}
+#undef talloc_steal
+#define talloc_steal _notmuch_talloc_steal
+#endif
+#endif
+
+#endif
diff --git a/lib/notmuch.h b/lib/notmuch.h
new file mode 100644
index 00000000..247f6ad7
--- /dev/null
+++ b/lib/notmuch.h
@@ -0,0 +1,2317 @@
+/* notmuch - Not much of an email library, (just index and search)
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+/**
+ * @defgroup notmuch The notmuch API
+ *
+ * Not much of an email library, (just index and search)
+ *
+ * @{
+ */
+
+#ifndef NOTMUCH_H
+#define NOTMUCH_H
+
+#ifndef __DOXYGEN__
+
+#ifdef  __cplusplus
+# define NOTMUCH_BEGIN_DECLS  extern "C" {
+# define NOTMUCH_END_DECLS    }
+#else
+# define NOTMUCH_BEGIN_DECLS
+# define NOTMUCH_END_DECLS
+#endif
+
+NOTMUCH_BEGIN_DECLS
+
+#include <time.h>
+
+#pragma GCC visibility push(default)
+
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#ifndef TRUE
+#define TRUE 1
+#endif
+
+/*
+ * The library version number.  This must agree with the soname
+ * version in Makefile.local.
+ */
+#define LIBNOTMUCH_MAJOR_VERSION	5
+#define LIBNOTMUCH_MINOR_VERSION	2
+#define LIBNOTMUCH_MICRO_VERSION	0
+
+
+#if defined (__clang_major__) && __clang_major__ >= 3 \
+    || defined (__GNUC__) && __GNUC__ >= 5 \
+    || defined (__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ >= 5
+#define NOTMUCH_DEPRECATED(major,minor) \
+    __attribute__ ((deprecated ("function deprecated as of libnotmuch " #major "." #minor)))
+#else
+#define NOTMUCH_DEPRECATED(major,minor) __attribute__ ((deprecated))
+#endif
+
+
+#endif /* __DOXYGEN__ */
+
+/**
+ * Check the version of the notmuch library being compiled against.
+ *
+ * Return true if the library being compiled against is of the
+ * specified version or above. For example:
+ *
+ * @code
+ * #if LIBNOTMUCH_CHECK_VERSION(3, 1, 0)
+ *     (code requiring libnotmuch 3.1.0 or above)
+ * #endif
+ * @endcode
+ *
+ * LIBNOTMUCH_CHECK_VERSION has been defined since version 3.1.0; to
+ * check for versions prior to that, use:
+ *
+ * @code
+ * #if !defined(NOTMUCH_CHECK_VERSION)
+ *     (code requiring libnotmuch prior to 3.1.0)
+ * #endif
+ * @endcode
+ */
+#define LIBNOTMUCH_CHECK_VERSION(major, minor, micro)			\
+    (LIBNOTMUCH_MAJOR_VERSION > (major) ||					\
+     (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION > (minor)) || \
+     (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION == (minor) && \
+      LIBNOTMUCH_MICRO_VERSION >= (micro)))
+
+/**
+ * Notmuch boolean type.
+ */
+typedef int notmuch_bool_t;
+
+/**
+ * Status codes used for the return values of most functions.
+ *
+ * A zero value (NOTMUCH_STATUS_SUCCESS) indicates that the function
+ * completed without error. Any other value indicates an error.
+ */
+typedef enum _notmuch_status {
+    /**
+     * No error occurred.
+     */
+    NOTMUCH_STATUS_SUCCESS = 0,
+    /**
+     * Out of memory.
+     */
+    NOTMUCH_STATUS_OUT_OF_MEMORY,
+    /**
+     * An attempt was made to write to a database opened in read-only
+     * mode.
+     */
+    NOTMUCH_STATUS_READ_ONLY_DATABASE,
+    /**
+     * A Xapian exception occurred.
+     *
+     * @todo We don't really want to expose this lame XAPIAN_EXCEPTION
+     * value. Instead we should map to things like DATABASE_LOCKED or
+     * whatever.
+     */
+    NOTMUCH_STATUS_XAPIAN_EXCEPTION,
+    /**
+     * An error occurred trying to read or write to a file (this could
+     * be file not found, permission denied, etc.)
+     */
+    NOTMUCH_STATUS_FILE_ERROR,
+    /**
+     * A file was presented that doesn't appear to be an email
+     * message.
+     */
+    NOTMUCH_STATUS_FILE_NOT_EMAIL,
+    /**
+     * A file contains a message ID that is identical to a message
+     * already in the database.
+     */
+    NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID,
+    /**
+     * The user erroneously passed a NULL pointer to a notmuch
+     * function.
+     */
+    NOTMUCH_STATUS_NULL_POINTER,
+    /**
+     * A tag value is too long (exceeds NOTMUCH_TAG_MAX).
+     */
+    NOTMUCH_STATUS_TAG_TOO_LONG,
+    /**
+     * The notmuch_message_thaw function has been called more times
+     * than notmuch_message_freeze.
+     */
+    NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
+    /**
+     * notmuch_database_end_atomic has been called more times than
+     * notmuch_database_begin_atomic.
+     */
+    NOTMUCH_STATUS_UNBALANCED_ATOMIC,
+    /**
+     * The operation is not supported.
+     */
+    NOTMUCH_STATUS_UNSUPPORTED_OPERATION,
+    /**
+     * The operation requires a database upgrade.
+     */
+    NOTMUCH_STATUS_UPGRADE_REQUIRED,
+    /**
+     * There is a problem with the proposed path, e.g. a relative path
+     * passed to a function expecting an absolute path.
+     */
+    NOTMUCH_STATUS_PATH_ERROR,
+    /**
+     * The requested operation was ignored. Depending on the function,
+     * this may not be an actual error.
+     */
+    NOTMUCH_STATUS_IGNORED,
+    /**
+     * One of the arguments violates the preconditions for the
+     * function, in a way not covered by a more specific argument.
+     */
+    NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    /**
+     * A MIME object claimed to have cryptographic protection which
+     * notmuch tried to handle, but the protocol was not specified in
+     * an intelligible way.
+     */
+    NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL,
+    /**
+     * Notmuch attempted to do crypto processing, but could not
+     * initialize the engine needed to do so.
+     */
+    NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION,
+    /**
+     * A MIME object claimed to have cryptographic protection, and
+     * notmuch attempted to process it, but the specific protocol was
+     * something that notmuch doesn't know how to handle.
+     */
+    NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL,
+    /**
+     * Not an actual status value. Just a way to find out how many
+     * valid status values there are.
+     */
+    NOTMUCH_STATUS_LAST_STATUS
+} notmuch_status_t;
+
+/**
+ * Get a string representation of a notmuch_status_t value.
+ *
+ * The result is read-only.
+ */
+const char *
+notmuch_status_to_string (notmuch_status_t status);
+
+/* Various opaque data types. For each notmuch_<foo>_t see the various
+ * notmuch_<foo> functions below. */
+#ifndef __DOXYGEN__
+typedef struct _notmuch_database notmuch_database_t;
+typedef struct _notmuch_query notmuch_query_t;
+typedef struct _notmuch_threads notmuch_threads_t;
+typedef struct _notmuch_thread notmuch_thread_t;
+typedef struct _notmuch_messages notmuch_messages_t;
+typedef struct _notmuch_message notmuch_message_t;
+typedef struct _notmuch_tags notmuch_tags_t;
+typedef struct _notmuch_directory notmuch_directory_t;
+typedef struct _notmuch_filenames notmuch_filenames_t;
+typedef struct _notmuch_config_list notmuch_config_list_t;
+typedef struct _notmuch_indexopts notmuch_indexopts_t;
+#endif /* __DOXYGEN__ */
+
+/**
+ * Create a new, empty notmuch database located at 'path'.
+ *
+ * The path should be a top-level directory to a collection of
+ * plain-text email messages (one message per file). This call will
+ * create a new ".notmuch" directory within 'path' where notmuch will
+ * store its data.
+ *
+ * After a successful call to notmuch_database_create, the returned
+ * database will be open so the caller should call
+ * notmuch_database_destroy when finished with it.
+ *
+ * The database will not yet have any data in it
+ * (notmuch_database_create itself is a very cheap function). Messages
+ * contained within 'path' can be added to the database by calling
+ * notmuch_database_index_file.
+ *
+ * In case of any failure, this function returns an error status and
+ * sets *database to NULL (after printing an error message on stderr).
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully created the database.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'path' argument is NULL.
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
+ *
+ * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to create the
+ *	database file (such as permission denied, or file not found,
+ *	etc.), or the database already exists.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred.
+ */
+notmuch_status_t
+notmuch_database_create (const char *path, notmuch_database_t **database);
+
+/**
+ * Like notmuch_database_create, except optionally return an error
+ * message. This message is allocated by malloc and should be freed by
+ * the caller.
+ */
+notmuch_status_t
+notmuch_database_create_verbose (const char *path,
+				 notmuch_database_t **database,
+				 char **error_message);
+
+/**
+ * Database open mode for notmuch_database_open.
+ */
+typedef enum {
+    /**
+     * Open database for reading only.
+     */
+    NOTMUCH_DATABASE_MODE_READ_ONLY = 0,
+    /**
+     * Open database for reading and writing.
+     */
+    NOTMUCH_DATABASE_MODE_READ_WRITE
+} notmuch_database_mode_t;
+
+/**
+ * Open an existing notmuch database located at 'path'.
+ *
+ * The database should have been created at some time in the past,
+ * (not necessarily by this process), by calling
+ * notmuch_database_create with 'path'. By default the database should be
+ * opened for reading only. In order to write to the database you need to
+ * pass the NOTMUCH_DATABASE_MODE_READ_WRITE mode.
+ *
+ * An existing notmuch database can be identified by the presence of a
+ * directory named ".notmuch" below 'path'.
+ *
+ * The caller should call notmuch_database_destroy when finished with
+ * this database.
+ *
+ * In case of any failure, this function returns an error status and
+ * sets *database to NULL (after printing an error message on stderr).
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully opened the database.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'path' argument is NULL.
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
+ *
+ * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to open the
+ *	database file (such as permission denied, or file not found,
+ *	etc.), or the database version is unknown.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred.
+ */
+notmuch_status_t
+notmuch_database_open (const char *path,
+		       notmuch_database_mode_t mode,
+		       notmuch_database_t **database);
+/**
+ * Like notmuch_database_open, except optionally return an error
+ * message. This message is allocated by malloc and should be freed by
+ * the caller.
+ */
+
+notmuch_status_t
+notmuch_database_open_verbose (const char *path,
+			       notmuch_database_mode_t mode,
+			       notmuch_database_t **database,
+			       char **error_message);
+
+/**
+ * Retrieve last status string for given database.
+ *
+ */
+const char *
+notmuch_database_status_string (const notmuch_database_t *notmuch);
+
+/**
+ * Commit changes and close the given notmuch database.
+ *
+ * After notmuch_database_close has been called, calls to other
+ * functions on objects derived from this database may either behave
+ * as if the database had not been closed (e.g., if the required data
+ * has been cached) or may fail with a
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION. The only further operation
+ * permitted on the database itself is to call
+ * notmuch_database_destroy.
+ *
+ * notmuch_database_close can be called multiple times.  Later calls
+ * have no effect.
+ *
+ * For writable databases, notmuch_database_close commits all changes
+ * to disk before closing the database.  If the caller is currently in
+ * an atomic section (there was a notmuch_database_begin_atomic
+ * without a matching notmuch_database_end_atomic), this will discard
+ * changes made in that atomic section (but still commit changes made
+ * prior to entering the atomic section).
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully closed the database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred; the
+ *	database has been closed but there are no guarantees the
+ *	changes to the database, if any, have been flushed to disk.
+ */
+notmuch_status_t
+notmuch_database_close (notmuch_database_t *database);
+
+/**
+ * A callback invoked by notmuch_database_compact to notify the user
+ * of the progress of the compaction process.
+ */
+typedef void (*notmuch_compact_status_cb_t)(const char *message, void *closure);
+
+/**
+ * Compact a notmuch database, backing up the original database to the
+ * given path.
+ *
+ * The database will be opened with NOTMUCH_DATABASE_MODE_READ_WRITE
+ * during the compaction process to ensure no writes are made.
+ *
+ * If the optional callback function 'status_cb' is non-NULL, it will
+ * be called with diagnostic and informational messages. The argument
+ * 'closure' is passed verbatim to any callback invoked.
+ */
+notmuch_status_t
+notmuch_database_compact (const char* path,
+			  const char* backup_path,
+			  notmuch_compact_status_cb_t status_cb,
+			  void *closure);
+
+/**
+ * Destroy the notmuch database, closing it if necessary and freeing
+ * all associated resources.
+ *
+ * Return value as in notmuch_database_close if the database was open;
+ * notmuch_database_destroy itself has no failure modes.
+ */
+notmuch_status_t
+notmuch_database_destroy (notmuch_database_t *database);
+
+/**
+ * Return the database path of the given database.
+ *
+ * The return value is a string owned by notmuch so should not be
+ * modified nor freed by the caller.
+ */
+const char *
+notmuch_database_get_path (notmuch_database_t *database);
+
+/**
+ * Return the database format version of the given database.
+ */
+unsigned int
+notmuch_database_get_version (notmuch_database_t *database);
+
+/**
+ * Can the database be upgraded to a newer database version?
+ *
+ * If this function returns TRUE, then the caller may call
+ * notmuch_database_upgrade to upgrade the database.  If the caller
+ * does not upgrade an out-of-date database, then some functions may
+ * fail with NOTMUCH_STATUS_UPGRADE_REQUIRED.  This always returns
+ * FALSE for a read-only database because there's no way to upgrade a
+ * read-only database.
+ */
+notmuch_bool_t
+notmuch_database_needs_upgrade (notmuch_database_t *database);
+
+/**
+ * Upgrade the current database to the latest supported version.
+ *
+ * This ensures that all current notmuch functionality will be
+ * available on the database.  After opening a database in read-write
+ * mode, it is recommended that clients check if an upgrade is needed
+ * (notmuch_database_needs_upgrade) and if so, upgrade with this
+ * function before making any modifications.  If
+ * notmuch_database_needs_upgrade returns FALSE, this will be a no-op.
+ *
+ * The optional progress_notify callback can be used by the caller to
+ * provide progress indication to the user. If non-NULL it will be
+ * called periodically with 'progress' as a floating-point value in
+ * the range of [0.0 .. 1.0] indicating the progress made so far in
+ * the upgrade process.  The argument 'closure' is passed verbatim to
+ * any callback invoked.
+ */
+notmuch_status_t
+notmuch_database_upgrade (notmuch_database_t *database,
+			  void (*progress_notify) (void *closure,
+						   double progress),
+			  void *closure);
+
+/**
+ * Begin an atomic database operation.
+ *
+ * Any modifications performed between a successful begin and a
+ * notmuch_database_end_atomic will be applied to the database
+ * atomically.  Note that, unlike a typical database transaction, this
+ * only ensures atomicity, not durability; neither begin nor end
+ * necessarily flush modifications to disk.
+ *
+ * Atomic sections may be nested.  begin_atomic and end_atomic must
+ * always be called in pairs.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully entered atomic section.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	atomic section not entered.
+ */
+notmuch_status_t
+notmuch_database_begin_atomic (notmuch_database_t *notmuch);
+
+/**
+ * Indicate the end of an atomic database operation.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully completed atomic section.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	atomic section not ended.
+ *
+ * NOTMUCH_STATUS_UNBALANCED_ATOMIC: The database is not currently in
+ *	an atomic section.
+ */
+notmuch_status_t
+notmuch_database_end_atomic (notmuch_database_t *notmuch);
+
+/**
+ * Return the committed database revision and UUID.
+ *
+ * The database revision number increases monotonically with each
+ * commit to the database.  Hence, all messages and message changes
+ * committed to the database (that is, visible to readers) have a last
+ * modification revision <= the committed database revision.  Any
+ * messages committed in the future will be assigned a modification
+ * revision > the committed database revision.
+ *
+ * The UUID is a NUL-terminated opaque string that uniquely identifies
+ * this database.  Two revision numbers are only comparable if they
+ * have the same database UUID.
+ */
+unsigned long
+notmuch_database_get_revision (notmuch_database_t *notmuch,
+				const char **uuid);
+
+/**
+ * Retrieve a directory object from the database for 'path'.
+ *
+ * Here, 'path' should be a path relative to the path of 'database'
+ * (see notmuch_database_get_path), or else should be an absolute path
+ * with initial components that match the path of 'database'.
+ *
+ * If this directory object does not exist in the database, this
+ * returns NOTMUCH_STATUS_SUCCESS and sets *directory to NULL.
+ *
+ * Otherwise the returned directory object is owned by the database
+ * and as such, will only be valid until notmuch_database_destroy is
+ * called.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully retrieved directory.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'directory' argument is NULL.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	directory not retrieved.
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ * 	database to use this function.
+ */
+notmuch_status_t
+notmuch_database_get_directory (notmuch_database_t *database,
+				const char *path,
+				notmuch_directory_t **directory);
+
+/**
+ * Add a message file to a database, indexing it for retrieval by
+ * future searches.  If a message already exists with the same message
+ * ID as the specified file, their indexes will be merged, and this
+ * new filename will also be associated with the existing message.
+ *
+ * Here, 'filename' should be a path relative to the path of
+ * 'database' (see notmuch_database_get_path), or else should be an
+ * absolute filename with initial components that match the path of
+ * 'database'.
+ *
+ * The file should be a single mail message (not a multi-message mbox)
+ * that is expected to remain at its current location, (since the
+ * notmuch database will reference the filename, and will not copy the
+ * entire contents of the file.
+ *
+ * If another message with the same message ID already exists in the
+ * database, rather than creating a new message, this adds the search
+ * terms from the identified file to the existing message's index, and
+ * adds 'filename' to the list of filenames known for the message.
+ *
+ * The 'indexopts' parameter can be NULL (meaning, use the indexing
+ * defaults from the database), or can be an explicit choice of
+ * indexing options that should govern the indexing of this specific
+ * 'filename'.
+ *
+ * If 'message' is not NULL, then, on successful return
+ * (NOTMUCH_STATUS_SUCCESS or NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) '*message'
+ * will be initialized to a message object that can be used for things
+ * such as adding tags to the just-added message. The user should call
+ * notmuch_message_destroy when done with the message. On any failure
+ * '*message' will be set to NULL.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully added to database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred,
+ *	message not added.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
+ *	ID as another message already in the database. The new
+ *	filename was successfully added to the message in the database
+ *	(if not already present) and the existing message is returned.
+ *
+ * NOTMUCH_STATUS_FILE_ERROR: an error occurred trying to open the
+ *	file, (such as permission denied, or file not found,
+ *	etc.). Nothing added to the database.
+ *
+ * NOTMUCH_STATUS_FILE_NOT_EMAIL: the contents of filename don't look
+ *	like an email message. Nothing added to the database.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so no message can be added.
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ * 	database to use this function.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_database_index_file (notmuch_database_t *database,
+			     const char *filename,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_t **message);
+
+/**
+ * Deprecated alias for notmuch_database_index_file called with
+ * NULL indexopts.
+ *
+ * @deprecated Deprecated as of libnotmuch 5.1 (notmuch 0.26). Please
+ * use notmuch_database_index_file instead.
+ *
+ */
+NOTMUCH_DEPRECATED(5,1)
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *database,
+			      const char *filename,
+			      notmuch_message_t **message);
+
+/**
+ * Remove a message filename from the given notmuch database. If the
+ * message has no more filenames, remove the message.
+ *
+ * If the same message (as determined by the message ID) is still
+ * available via other filenames, then the message will persist in the
+ * database for those filenames. When the last filename is removed for
+ * a particular message, the database content for that message will be
+ * entirely removed.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: The last filename was removed and the
+ *	message was removed from the database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred,
+ *	message not removed.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: This filename was removed but
+ *	the message persists in the database with at least one other
+ *	filename.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so no message can be removed.
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ * 	database to use this function.
+ */
+notmuch_status_t
+notmuch_database_remove_message (notmuch_database_t *database,
+				 const char *filename);
+
+/**
+ * Find a message with the given message_id.
+ *
+ * If a message with the given message_id is found then, on successful return
+ * (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to a message
+ * object.  The caller should call notmuch_message_destroy when done with the
+ * message.
+ *
+ * On any failure or when the message is not found, this function initializes
+ * '*message' to NULL. This means, when NOTMUCH_STATUS_SUCCESS is returned, the
+ * caller is supposed to check '*message' for NULL to find out whether the
+ * message with the given message_id was found.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successful return, check '*message'.
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'message' argument is NULL
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory, creating message object
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
+ */
+notmuch_status_t
+notmuch_database_find_message (notmuch_database_t *database,
+			       const char *message_id,
+			       notmuch_message_t **message);
+
+/**
+ * Find a message with the given filename.
+ *
+ * If the database contains a message with the given filename then, on
+ * successful return (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to
+ * a message object. The caller should call notmuch_message_destroy when done
+ * with the message.
+ *
+ * On any failure or when the message is not found, this function initializes
+ * '*message' to NULL. This means, when NOTMUCH_STATUS_SUCCESS is returned, the
+ * caller is supposed to check '*message' for NULL to find out whether the
+ * message with the given filename is found.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successful return, check '*message'
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'message' argument is NULL
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory, creating the message object
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
+ *
+ * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
+ * 	database to use this function.
+ */
+notmuch_status_t
+notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
+					   const char *filename,
+					   notmuch_message_t **message);
+
+/**
+ * Return a list of all tags found in the database.
+ *
+ * This function creates a list of all tags found in the database. The
+ * resulting list contains all tags from all messages found in the database.
+ *
+ * On error this function returns NULL.
+ */
+notmuch_tags_t *
+notmuch_database_get_all_tags (notmuch_database_t *db);
+
+/**
+ * Create a new query for 'database'.
+ *
+ * Here, 'database' should be an open database, (see
+ * notmuch_database_open and notmuch_database_create).
+ *
+ * For the query string, we'll document the syntax here more
+ * completely in the future, but it's likely to be a specialized
+ * version of the general Xapian query syntax:
+ *
+ * https://xapian.org/docs/queryparser.html
+ *
+ * As a special case, passing either a length-zero string, (that is ""),
+ * or a string consisting of a single asterisk (that is "*"), will
+ * result in a query that returns all messages in the database.
+ *
+ * See notmuch_query_set_sort for controlling the order of results.
+ * See notmuch_query_search_messages and notmuch_query_search_threads
+ * to actually execute the query.
+ *
+ * User should call notmuch_query_destroy when finished with this
+ * query.
+ *
+ * Will return NULL if insufficient memory is available.
+ */
+notmuch_query_t *
+notmuch_query_create (notmuch_database_t *database,
+		      const char *query_string);
+
+/**
+ * Sort values for notmuch_query_set_sort.
+ */
+typedef enum {
+    /**
+     * Oldest first.
+     */
+    NOTMUCH_SORT_OLDEST_FIRST,
+    /**
+     * Newest first.
+     */
+    NOTMUCH_SORT_NEWEST_FIRST,
+    /**
+     * Sort by message-id.
+     */
+    NOTMUCH_SORT_MESSAGE_ID,
+    /**
+     * Do not sort.
+     */
+    NOTMUCH_SORT_UNSORTED
+} notmuch_sort_t;
+
+/**
+ * Return the query_string of this query. See notmuch_query_create.
+ */
+const char *
+notmuch_query_get_query_string (const notmuch_query_t *query);
+
+/**
+ * Return the notmuch database of this query. See notmuch_query_create.
+ */
+notmuch_database_t *
+notmuch_query_get_database (const notmuch_query_t *query);
+
+/**
+ * Exclude values for notmuch_query_set_omit_excluded. The strange
+ * order is to maintain backward compatibility: the old FALSE/TRUE
+ * options correspond to the new
+ * NOTMUCH_EXCLUDE_FLAG/NOTMUCH_EXCLUDE_TRUE options.
+ */
+typedef enum {
+    NOTMUCH_EXCLUDE_FLAG,
+    NOTMUCH_EXCLUDE_TRUE,
+    NOTMUCH_EXCLUDE_FALSE,
+    NOTMUCH_EXCLUDE_ALL
+} notmuch_exclude_t;
+
+/**
+ * Specify whether to omit excluded results or simply flag them.  By
+ * default, this is set to TRUE.
+ *
+ * If set to TRUE or ALL, notmuch_query_search_messages will omit excluded
+ * messages from the results, and notmuch_query_search_threads will omit
+ * threads that match only in excluded messages.  If set to TRUE,
+ * notmuch_query_search_threads will include all messages in threads that
+ * match in at least one non-excluded message.  Otherwise, if set to ALL,
+ * notmuch_query_search_threads will omit excluded messages from all threads.
+ *
+ * If set to FALSE or FLAG then both notmuch_query_search_messages and
+ * notmuch_query_search_threads will return all matching
+ * messages/threads regardless of exclude status. If set to FLAG then
+ * the exclude flag will be set for any excluded message that is
+ * returned by notmuch_query_search_messages, and the thread counts
+ * for threads returned by notmuch_query_search_threads will be the
+ * number of non-excluded messages/matches. Otherwise, if set to
+ * FALSE, then the exclude status is completely ignored.
+ *
+ * The performance difference when calling
+ * notmuch_query_search_messages should be relatively small (and both
+ * should be very fast).  However, in some cases,
+ * notmuch_query_search_threads is very much faster when omitting
+ * excluded messages as it does not need to construct the threads that
+ * only match in excluded messages.
+ */
+void
+notmuch_query_set_omit_excluded (notmuch_query_t *query,
+				 notmuch_exclude_t omit_excluded);
+
+/**
+ * Specify the sorting desired for this query.
+ */
+void
+notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort);
+
+/**
+ * Return the sort specified for this query. See
+ * notmuch_query_set_sort.
+ */
+notmuch_sort_t
+notmuch_query_get_sort (const notmuch_query_t *query);
+
+/**
+ * Add a tag that will be excluded from the query results by default.
+ * This exclusion will be ignored if this tag appears explicitly in
+ * the query.
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_SUCCESS: excluded was added successfully.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: a Xapian exception occurred.
+ *      Most likely a problem lazily parsing the query string.
+ *
+ * NOTMUCH_STATUS_IGNORED: tag is explicitly present in the query, so
+ *		not excluded.
+ */
+notmuch_status_t
+notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag);
+
+/**
+ * Execute a query for threads, returning a notmuch_threads_t object
+ * which can be used to iterate over the results. The returned threads
+ * object is owned by the query and as such, will only be valid until
+ * notmuch_query_destroy.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_query_t *query;
+ *     notmuch_threads_t *threads;
+ *     notmuch_thread_t *thread;
+ *
+ *     query = notmuch_query_create (database, query_string);
+ *
+ *     for (threads = notmuch_query_search_threads (query);
+ *          notmuch_threads_valid (threads);
+ *          notmuch_threads_move_to_next (threads))
+ *     {
+ *         thread = notmuch_threads_get (threads);
+ *         ....
+ *         notmuch_thread_destroy (thread);
+ *     }
+ *
+ *     notmuch_query_destroy (query);
+ *
+ * Note: If you are finished with a thread before its containing
+ * query, you can call notmuch_thread_destroy to clean up some memory
+ * sooner (as in the above example). Otherwise, if your thread objects
+ * are long-lived, then you don't need to call notmuch_thread_destroy
+ * and all the memory will still be reclaimed when the query is
+ * destroyed.
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_threads_t object. (For consistency, we do provide a
+ * notmuch_threads_destroy function, but there's no good reason
+ * to call it if the query is about to be destroyed).
+ *
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_search_threads (notmuch_query_t *query,
+			      notmuch_threads_t **out);
+
+/**
+ * Deprecated alias for notmuch_query_search_threads.
+ *
+ * @deprecated Deprecated as of libnotmuch 5 (notmuch 0.25). Please
+ * use notmuch_query_search_threads instead.
+ *
+ */
+NOTMUCH_DEPRECATED(5,0)
+notmuch_status_t
+notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out);
+
+/**
+ * Execute a query for messages, returning a notmuch_messages_t object
+ * which can be used to iterate over the results. The returned
+ * messages object is owned by the query and as such, will only be
+ * valid until notmuch_query_destroy.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_query_t *query;
+ *     notmuch_messages_t *messages;
+ *     notmuch_message_t *message;
+ *
+ *     query = notmuch_query_create (database, query_string);
+ *
+ *     for (messages = notmuch_query_search_messages (query);
+ *          notmuch_messages_valid (messages);
+ *          notmuch_messages_move_to_next (messages))
+ *     {
+ *         message = notmuch_messages_get (messages);
+ *         ....
+ *         notmuch_message_destroy (message);
+ *     }
+ *
+ *     notmuch_query_destroy (query);
+ *
+ * Note: If you are finished with a message before its containing
+ * query, you can call notmuch_message_destroy to clean up some memory
+ * sooner (as in the above example). Otherwise, if your message
+ * objects are long-lived, then you don't need to call
+ * notmuch_message_destroy and all the memory will still be reclaimed
+ * when the query is destroyed.
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_messages_t object. (For consistency, we do provide a
+ * notmuch_messages_destroy function, but there's no good
+ * reason to call it if the query is about to be destroyed).
+ *
+ * If a Xapian exception occurs this function will return NULL.
+ *
+ * @since libnotmuch 5 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_search_messages (notmuch_query_t *query,
+			       notmuch_messages_t **out);
+/**
+ * Deprecated alias for notmuch_query_search_messages
+ *
+ * @deprecated Deprecated as of libnotmuch 5 (notmuch 0.25). Please use
+ * notmuch_query_search_messages instead.
+ *
+ */
+
+NOTMUCH_DEPRECATED(5,0)
+notmuch_status_t
+notmuch_query_search_messages_st (notmuch_query_t *query,
+				  notmuch_messages_t **out);
+
+/**
+ * Destroy a notmuch_query_t along with any associated resources.
+ *
+ * This will in turn destroy any notmuch_threads_t and
+ * notmuch_messages_t objects generated by this query, (and in
+ * turn any notmuch_thread_t and notmuch_message_t objects generated
+ * from those results, etc.), if such objects haven't already been
+ * destroyed.
+ */
+void
+notmuch_query_destroy (notmuch_query_t *query);
+
+/**
+ * Is the given 'threads' iterator pointing at a valid thread.
+ *
+ * When this function returns TRUE, notmuch_threads_get will return a
+ * valid object. Whereas when this function returns FALSE,
+ * notmuch_threads_get will return NULL.
+ *
+ * If passed a NULL pointer, this function returns FALSE
+ *
+ * See the documentation of notmuch_query_search_threads for example
+ * code showing how to iterate over a notmuch_threads_t object.
+ */
+notmuch_bool_t
+notmuch_threads_valid (notmuch_threads_t *threads);
+
+/**
+ * Get the current thread from 'threads' as a notmuch_thread_t.
+ *
+ * Note: The returned thread belongs to 'threads' and has a lifetime
+ * identical to it (and the query to which it belongs).
+ *
+ * See the documentation of notmuch_query_search_threads for example
+ * code showing how to iterate over a notmuch_threads_t object.
+ *
+ * If an out-of-memory situation occurs, this function will return
+ * NULL.
+ */
+notmuch_thread_t *
+notmuch_threads_get (notmuch_threads_t *threads);
+
+/**
+ * Move the 'threads' iterator to the next thread.
+ *
+ * If 'threads' is already pointing at the last thread then the
+ * iterator will be moved to a point just beyond that last thread,
+ * (where notmuch_threads_valid will return FALSE and
+ * notmuch_threads_get will return NULL).
+ *
+ * See the documentation of notmuch_query_search_threads for example
+ * code showing how to iterate over a notmuch_threads_t object.
+ */
+void
+notmuch_threads_move_to_next (notmuch_threads_t *threads);
+
+/**
+ * Destroy a notmuch_threads_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_threads_t object will be reclaimed when the
+ * containing query object is destroyed.
+ */
+void
+notmuch_threads_destroy (notmuch_threads_t *threads);
+
+/**
+ * Return the number of messages matching a search.
+ *
+ * This function performs a search and returns the number of matching
+ * messages.
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_SUCCESS: query completed successfully.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: a Xapian exception occurred. The
+ *      value of *count is not defined.
+ *
+ * @since libnotmuch 5 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_count_messages (notmuch_query_t *query, unsigned int *count);
+
+/**
+ * Deprecated alias for notmuch_query_count_messages
+ *
+ *
+ * @deprecated Deprecated since libnotmuch 5.0 (notmuch 0.25). Please
+ * use notmuch_query_count_messages instead.
+ */
+NOTMUCH_DEPRECATED(5,0)
+notmuch_status_t
+notmuch_query_count_messages_st (notmuch_query_t *query, unsigned int *count);
+
+/**
+ * Return the number of threads matching a search.
+ *
+ * This function performs a search and returns the number of unique thread IDs
+ * in the matching messages. This is the same as number of threads matching a
+ * search.
+ *
+ * Note that this is a significantly heavier operation than
+ * notmuch_query_count_messages{_st}().
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Memory allocation failed. The value
+ *      of *count is not defined
+
+ * NOTMUCH_STATUS_SUCCESS: query completed successfully.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: a Xapian exception occurred. The
+ *      value of *count is not defined.
+ *
+ * @since libnotmuch 5 (notmuch 0.25)
+ */
+notmuch_status_t
+notmuch_query_count_threads (notmuch_query_t *query, unsigned *count);
+
+/**
+ * Deprecated alias for notmuch_query_count_threads
+ *
+ * @deprecated Deprecated as of libnotmuch 5.0 (notmuch 0.25). Please
+ * use notmuch_query_count_threads_st instead.
+ */
+NOTMUCH_DEPRECATED(5,0)
+notmuch_status_t
+notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count);
+
+/**
+ * Get the thread ID of 'thread'.
+ *
+ * The returned string belongs to 'thread' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * thread is valid, (which is until notmuch_thread_destroy or until
+ * the query from which it derived is destroyed).
+ */
+const char *
+notmuch_thread_get_thread_id (notmuch_thread_t *thread);
+
+/**
+ * Get the total number of messages in 'thread'.
+ *
+ * This count consists of all messages in the database belonging to
+ * this thread. Contrast with notmuch_thread_get_matched_messages() .
+ */
+int
+notmuch_thread_get_total_messages (notmuch_thread_t *thread);
+
+/**
+ * Get the total number of files in 'thread'.
+ *
+ * This sums notmuch_message_count_files over all messages in the
+ * thread
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread);
+
+/**
+ * Get a notmuch_messages_t iterator for the top-level messages in
+ * 'thread' in oldest-first order.
+ *
+ * This iterator will not necessarily iterate over all of the messages
+ * in the thread. It will only iterate over the messages in the thread
+ * which are not replies to other messages in the thread.
+ *
+ * The returned list will be destroyed when the thread is destroyed.
+ */
+notmuch_messages_t *
+notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread);
+
+/**
+ * Get a notmuch_thread_t iterator for all messages in 'thread' in
+ * oldest-first order.
+ *
+ * The returned list will be destroyed when the thread is destroyed.
+ */
+notmuch_messages_t *
+notmuch_thread_get_messages (notmuch_thread_t *thread);
+
+/**
+ * Get the number of messages in 'thread' that matched the search.
+ *
+ * This count includes only the messages in this thread that were
+ * matched by the search from which the thread was created and were
+ * not excluded by any exclude tags passed in with the query (see
+ * notmuch_query_add_tag_exclude). Contrast with
+ * notmuch_thread_get_total_messages() .
+ */
+int
+notmuch_thread_get_matched_messages (notmuch_thread_t *thread);
+
+/**
+ * Get the authors of 'thread' as a UTF-8 string.
+ *
+ * The returned string is a comma-separated list of the names of the
+ * authors of mail messages in the query results that belong to this
+ * thread.
+ *
+ * The string contains authors of messages matching the query first, then
+ * non-matched authors (with the two groups separated by '|'). Within
+ * each group, authors are ordered by date.
+ *
+ * The returned string belongs to 'thread' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * thread is valid, (which is until notmuch_thread_destroy or until
+ * the query from which it derived is destroyed).
+ */
+const char *
+notmuch_thread_get_authors (notmuch_thread_t *thread);
+
+/**
+ * Get the subject of 'thread' as a UTF-8 string.
+ *
+ * The subject is taken from the first message (according to the query
+ * order---see notmuch_query_set_sort) in the query results that
+ * belongs to this thread.
+ *
+ * The returned string belongs to 'thread' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * thread is valid, (which is until notmuch_thread_destroy or until
+ * the query from which it derived is destroyed).
+ */
+const char *
+notmuch_thread_get_subject (notmuch_thread_t *thread);
+
+/**
+ * Get the date of the oldest message in 'thread' as a time_t value.
+ */
+time_t
+notmuch_thread_get_oldest_date (notmuch_thread_t *thread);
+
+/**
+ * Get the date of the newest message in 'thread' as a time_t value.
+ */
+time_t
+notmuch_thread_get_newest_date (notmuch_thread_t *thread);
+
+/**
+ * Get the tags for 'thread', returning a notmuch_tags_t object which
+ * can be used to iterate over all tags.
+ *
+ * Note: In the Notmuch database, tags are stored on individual
+ * messages, not on threads. So the tags returned here will be all
+ * tags of the messages which matched the search and which belong to
+ * this thread.
+ *
+ * The tags object is owned by the thread and as such, will only be
+ * valid for as long as the thread is valid, (for example, until
+ * notmuch_thread_destroy or until the query from which it derived is
+ * destroyed).
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_thread_t *thread;
+ *     notmuch_tags_t *tags;
+ *     const char *tag;
+ *
+ *     thread = notmuch_threads_get (threads);
+ *
+ *     for (tags = notmuch_thread_get_tags (thread);
+ *          notmuch_tags_valid (tags);
+ *          notmuch_tags_move_to_next (tags))
+ *     {
+ *         tag = notmuch_tags_get (tags);
+ *         ....
+ *     }
+ *
+ *     notmuch_thread_destroy (thread);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_tags_t object. (For consistency, we do provide a
+ * notmuch_tags_destroy function, but there's no good reason to call
+ * it if the message is about to be destroyed).
+ */
+notmuch_tags_t *
+notmuch_thread_get_tags (notmuch_thread_t *thread);
+
+/**
+ * Destroy a notmuch_thread_t object.
+ */
+void
+notmuch_thread_destroy (notmuch_thread_t *thread);
+
+/**
+ * Is the given 'messages' iterator pointing at a valid message.
+ *
+ * When this function returns TRUE, notmuch_messages_get will return a
+ * valid object. Whereas when this function returns FALSE,
+ * notmuch_messages_get will return NULL.
+ *
+ * See the documentation of notmuch_query_search_messages for example
+ * code showing how to iterate over a notmuch_messages_t object.
+ */
+notmuch_bool_t
+notmuch_messages_valid (notmuch_messages_t *messages);
+
+/**
+ * Get the current message from 'messages' as a notmuch_message_t.
+ *
+ * Note: The returned message belongs to 'messages' and has a lifetime
+ * identical to it (and the query to which it belongs).
+ *
+ * See the documentation of notmuch_query_search_messages for example
+ * code showing how to iterate over a notmuch_messages_t object.
+ *
+ * If an out-of-memory situation occurs, this function will return
+ * NULL.
+ */
+notmuch_message_t *
+notmuch_messages_get (notmuch_messages_t *messages);
+
+/**
+ * Move the 'messages' iterator to the next message.
+ *
+ * If 'messages' is already pointing at the last message then the
+ * iterator will be moved to a point just beyond that last message,
+ * (where notmuch_messages_valid will return FALSE and
+ * notmuch_messages_get will return NULL).
+ *
+ * See the documentation of notmuch_query_search_messages for example
+ * code showing how to iterate over a notmuch_messages_t object.
+ */
+void
+notmuch_messages_move_to_next (notmuch_messages_t *messages);
+
+/**
+ * Destroy a notmuch_messages_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_messages_t object will be reclaimed when the containing
+ * query object is destroyed.
+ */
+void
+notmuch_messages_destroy (notmuch_messages_t *messages);
+
+/**
+ * Return a list of tags from all messages.
+ *
+ * The resulting list is guaranteed not to contain duplicated tags.
+ *
+ * WARNING: You can no longer iterate over messages after calling this
+ * function, because the iterator will point at the end of the list.
+ * We do not have a function to reset the iterator yet and the only
+ * way how you can iterate over the list again is to recreate the
+ * message list.
+ *
+ * The function returns NULL on error.
+ */
+notmuch_tags_t *
+notmuch_messages_collect_tags (notmuch_messages_t *messages);
+
+/**
+ * Get the database associated with this message.
+ *
+ * @since libnotmuch 5.2 (notmuch 0.27)
+ */
+notmuch_database_t *
+notmuch_message_get_database (const notmuch_message_t *message);
+
+/**
+ * Get the message ID of 'message'.
+ *
+ * The returned string belongs to 'message' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * message is valid, (which is until the query from which it derived
+ * is destroyed).
+ *
+ * This function will not return NULL since Notmuch ensures that every
+ * message has a unique message ID, (Notmuch will generate an ID for a
+ * message if the original file does not contain one).
+ */
+const char *
+notmuch_message_get_message_id (notmuch_message_t *message);
+
+/**
+ * Get the thread ID of 'message'.
+ *
+ * The returned string belongs to 'message' and as such, should not be
+ * modified by the caller and will only be valid for as long as the
+ * message is valid, (for example, until the user calls
+ * notmuch_message_destroy on 'message' or until a query from which it
+ * derived is destroyed).
+ *
+ * This function will not return NULL since Notmuch ensures that every
+ * message belongs to a single thread.
+ */
+const char *
+notmuch_message_get_thread_id (notmuch_message_t *message);
+
+/**
+ * Get a notmuch_messages_t iterator for all of the replies to
+ * 'message'.
+ *
+ * Note: This call only makes sense if 'message' was ultimately
+ * obtained from a notmuch_thread_t object, (such as by coming
+ * directly from the result of calling notmuch_thread_get_
+ * toplevel_messages or by any number of subsequent
+ * calls to notmuch_message_get_replies).
+ *
+ * If 'message' was obtained through some non-thread means, (such as
+ * by a call to notmuch_query_search_messages), then this function
+ * will return NULL.
+ *
+ * If there are no replies to 'message', this function will return
+ * NULL. (Note that notmuch_messages_valid will accept that NULL
+ * value as legitimate, and simply return FALSE for it.)
+ */
+notmuch_messages_t *
+notmuch_message_get_replies (notmuch_message_t *message);
+
+/**
+ * Get the total number of files associated with a message.
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+int
+notmuch_message_count_files (notmuch_message_t *message);
+
+/**
+ * Get a filename for the email corresponding to 'message'.
+ *
+ * The returned filename is an absolute filename, (the initial
+ * component will match notmuch_database_get_path() ).
+ *
+ * The returned string belongs to the message so should not be
+ * modified or freed by the caller (nor should it be referenced after
+ * the message is destroyed).
+ *
+ * Note: If this message corresponds to multiple files in the mail
+ * store, (that is, multiple files contain identical message IDs),
+ * this function will arbitrarily return a single one of those
+ * filenames. See notmuch_message_get_filenames for returning the
+ * complete list of filenames.
+ */
+const char *
+notmuch_message_get_filename (notmuch_message_t *message);
+
+/**
+ * Get all filenames for the email corresponding to 'message'.
+ *
+ * Returns a notmuch_filenames_t iterator listing all the filenames
+ * associated with 'message'. These files may not have identical
+ * content, but each will have the identical Message-ID.
+ *
+ * Each filename in the iterator is an absolute filename, (the initial
+ * component will match notmuch_database_get_path() ).
+ */
+notmuch_filenames_t *
+notmuch_message_get_filenames (notmuch_message_t *message);
+
+/**
+ * Re-index the e-mail corresponding to 'message' using the supplied index options
+ *
+ * Returns the status of the re-index operation.  (see the return
+ * codes documented in notmuch_database_index_file)
+ *
+ * After reindexing, the user should discard the message object passed
+ * in here by calling notmuch_message_destroy, since it refers to the
+ * original message, not to the reindexed message.
+ */
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+			 notmuch_indexopts_t *indexopts);
+
+/**
+ * Message flags.
+ */
+typedef enum _notmuch_message_flag {
+    NOTMUCH_MESSAGE_FLAG_MATCH,
+    NOTMUCH_MESSAGE_FLAG_EXCLUDED,
+
+    /* This message is a "ghost message", meaning it has no filenames
+     * or content, but we know it exists because it was referenced by
+     * some other message.  A ghost message has only a message ID and
+     * thread ID.
+     */
+    NOTMUCH_MESSAGE_FLAG_GHOST,
+} notmuch_message_flag_t;
+
+/**
+ * Get a value of a flag for the email corresponding to 'message'.
+ */
+notmuch_bool_t
+notmuch_message_get_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag);
+
+/**
+ * Set a value of a flag for the email corresponding to 'message'.
+ */
+void
+notmuch_message_set_flag (notmuch_message_t *message,
+			  notmuch_message_flag_t flag, notmuch_bool_t value);
+
+/**
+ * Get the date of 'message' as a time_t value.
+ *
+ * For the original textual representation of the Date header from the
+ * message call notmuch_message_get_header() with a header value of
+ * "date".
+ */
+time_t
+notmuch_message_get_date  (notmuch_message_t *message);
+
+/**
+ * Get the value of the specified header from 'message' as a UTF-8 string.
+ *
+ * Common headers are stored in the database when the message is
+ * indexed and will be returned from the database.  Other headers will
+ * be read from the actual message file.
+ *
+ * The header name is case insensitive.
+ *
+ * The returned string belongs to the message so should not be
+ * modified or freed by the caller (nor should it be referenced after
+ * the message is destroyed).
+ *
+ * Returns an empty string ("") if the message does not contain a
+ * header line matching 'header'. Returns NULL if any error occurs.
+ */
+const char *
+notmuch_message_get_header (notmuch_message_t *message, const char *header);
+
+/**
+ * Get the tags for 'message', returning a notmuch_tags_t object which
+ * can be used to iterate over all tags.
+ *
+ * The tags object is owned by the message and as such, will only be
+ * valid for as long as the message is valid, (which is until the
+ * query from which it derived is destroyed).
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_t *message;
+ *     notmuch_tags_t *tags;
+ *     const char *tag;
+ *
+ *     message = notmuch_database_find_message (database, message_id);
+ *
+ *     for (tags = notmuch_message_get_tags (message);
+ *          notmuch_tags_valid (tags);
+ *          notmuch_tags_move_to_next (tags))
+ *     {
+ *         tag = notmuch_tags_get (tags);
+ *         ....
+ *     }
+ *
+ *     notmuch_message_destroy (message);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_tags_t object. (For consistency, we do provide a
+ * notmuch_tags_destroy function, but there's no good reason to call
+ * it if the message is about to be destroyed).
+ */
+notmuch_tags_t *
+notmuch_message_get_tags (notmuch_message_t *message);
+
+/**
+ * The longest possible tag value.
+ */
+#define NOTMUCH_TAG_MAX 200
+
+/**
+ * Add a tag to the given message.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Tag successfully added to message
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The 'tag' argument is NULL
+ *
+ * NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
+ *	(exceeds NOTMUCH_TAG_MAX)
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
+
+/**
+ * Remove a tag from the given message.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Tag successfully removed from message
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The 'tag' argument is NULL
+ *
+ * NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
+ *	(exceeds NOTMUCH_TAG_MAX)
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
+
+/**
+ * Remove all tags from the given message.
+ *
+ * See notmuch_message_freeze for an example showing how to safely
+ * replace tag values.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_remove_all_tags (notmuch_message_t *message);
+
+/**
+ * Add/remove tags according to maildir flags in the message filename(s).
+ *
+ * This function examines the filenames of 'message' for maildir
+ * flags, and adds or removes tags on 'message' as follows when these
+ * flags are present:
+ *
+ *	Flag	Action if present
+ *	----	-----------------
+ *	'D'	Adds the "draft" tag to the message
+ *	'F'	Adds the "flagged" tag to the message
+ *	'P'	Adds the "passed" tag to the message
+ *	'R'	Adds the "replied" tag to the message
+ *	'S'	Removes the "unread" tag from the message
+ *
+ * For each flag that is not present, the opposite action (add/remove)
+ * is performed for the corresponding tags.
+ *
+ * Flags are identified as trailing components of the filename after a
+ * sequence of ":2,".
+ *
+ * If there are multiple filenames associated with this message, the
+ * flag is considered present if it appears in one or more
+ * filenames. (That is, the flags from the multiple filenames are
+ * combined with the logical OR operator.)
+ *
+ * A client can ensure that notmuch database tags remain synchronized
+ * with maildir flags by calling this function after each call to
+ * notmuch_database_index_file. See also
+ * notmuch_message_tags_to_maildir_flags for synchronizing tag changes
+ * back to maildir flags.
+ */
+notmuch_status_t
+notmuch_message_maildir_flags_to_tags (notmuch_message_t *message);
+
+/**
+ * return TRUE if any filename of 'message' has maildir flag 'flag',
+ * FALSE otherwise.
+ *
+ */
+notmuch_bool_t
+notmuch_message_has_maildir_flag (notmuch_message_t *message, char flag);
+
+/**
+ * Rename message filename(s) to encode tags as maildir flags.
+ *
+ * Specifically, for each filename corresponding to this message:
+ *
+ * If the filename is not in a maildir directory, do nothing.  (A
+ * maildir directory is determined as a directory named "new" or
+ * "cur".) Similarly, if the filename has invalid maildir info,
+ * (repeated or outof-ASCII-order flag characters after ":2,"), then
+ * do nothing.
+ *
+ * If the filename is in a maildir directory, rename the file so that
+ * its filename ends with the sequence ":2," followed by zero or more
+ * of the following single-character flags (in ASCII order):
+ *
+ *   * flag 'D' iff the message has the "draft" tag
+ *   * flag 'F' iff the message has the "flagged" tag
+ *   * flag 'P' iff the message has the "passed" tag
+ *   * flag 'R' iff the message has the "replied" tag
+ *   * flag 'S' iff the message does not have the "unread" tag
+ *
+ * Any existing flags unmentioned in the list above will be preserved
+ * in the renaming.
+ *
+ * Also, if this filename is in a directory named "new", rename it to
+ * be within the neighboring directory named "cur".
+ *
+ * A client can ensure that maildir filename flags remain synchronized
+ * with notmuch database tags by calling this function after changing
+ * tags, (after calls to notmuch_message_add_tag,
+ * notmuch_message_remove_tag, or notmuch_message_freeze/
+ * notmuch_message_thaw). See also notmuch_message_maildir_flags_to_tags
+ * for synchronizing maildir flag changes back to tags.
+ */
+notmuch_status_t
+notmuch_message_tags_to_maildir_flags (notmuch_message_t *message);
+
+/**
+ * Freeze the current state of 'message' within the database.
+ *
+ * This means that changes to the message state, (via
+ * notmuch_message_add_tag, notmuch_message_remove_tag, and
+ * notmuch_message_remove_all_tags), will not be committed to the
+ * database until the message is thawed with notmuch_message_thaw.
+ *
+ * Multiple calls to freeze/thaw are valid and these calls will
+ * "stack". That is there must be as many calls to thaw as to freeze
+ * before a message is actually thawed.
+ *
+ * The ability to do freeze/thaw allows for safe transactions to
+ * change tag values. For example, explicitly setting a message to
+ * have a given set of tags might look like this:
+ *
+ *    notmuch_message_freeze (message);
+ *
+ *    notmuch_message_remove_all_tags (message);
+ *
+ *    for (i = 0; i < NUM_TAGS; i++)
+ *        notmuch_message_add_tag (message, tags[i]);
+ *
+ *    notmuch_message_thaw (message);
+ *
+ * With freeze/thaw used like this, the message in the database is
+ * guaranteed to have either the full set of original tag values, or
+ * the full set of new tag values, but nothing in between.
+ *
+ * Imagine the example above without freeze/thaw and the operation
+ * somehow getting interrupted. This could result in the message being
+ * left with no tags if the interruption happened after
+ * notmuch_message_remove_all_tags but before notmuch_message_add_tag.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully frozen.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
+ */
+notmuch_status_t
+notmuch_message_freeze (notmuch_message_t *message);
+
+/**
+ * Thaw the current 'message', synchronizing any changes that may have
+ * occurred while 'message' was frozen into the notmuch database.
+ *
+ * See notmuch_message_freeze for an example of how to use this
+ * function to safely provide tag changes.
+ *
+ * Multiple calls to freeze/thaw are valid and these calls with
+ * "stack". That is there must be as many calls to thaw as to freeze
+ * before a message is actually thawed.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully thawed, (or at least
+ *	its frozen count has successfully been reduced by 1).
+ *
+ * NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: An attempt was made to thaw
+ *	an unfrozen message. That is, there have been an unbalanced
+ *	number of calls to notmuch_message_freeze and
+ *	notmuch_message_thaw.
+ */
+notmuch_status_t
+notmuch_message_thaw (notmuch_message_t *message);
+
+/**
+ * Destroy a notmuch_message_t object.
+ *
+ * It can be useful to call this function in the case of a single
+ * query object with many messages in the result, (such as iterating
+ * over the entire database). Otherwise, it's fine to never call this
+ * function and there will still be no memory leaks. (The memory from
+ * the messages get reclaimed when the containing query is destroyed.)
+ */
+void
+notmuch_message_destroy (notmuch_message_t *message);
+
+/**
+ * @name Message Properties
+ *
+ * This interface provides the ability to attach arbitrary (key,value)
+ * string pairs to a message, to remove such pairs, and to iterate
+ * over them.  The caller should take some care as to what keys they
+ * add or delete values for, as other subsystems or extensions may
+ * depend on these properties.
+ *
+ * Please see notmuch-properties(7) for more details about specific
+ * properties and conventions around their use.
+ *
+ */
+/**@{*/
+/**
+ * Retrieve the value for a single property key
+ *
+ * *value* is set to a string owned by the message or NULL if there is
+ * no such key. In the case of multiple values for the given key, the
+ * first one is retrieved.
+ *
+ * @returns
+ * - NOTMUCH_STATUS_NULL_POINTER: *value* may not be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value);
+
+/**
+ * Add a (key,value) pair to a message
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove a (key,value) pair from a message.
+ *
+ * It is not an error to remove a non-existant (key,value) pair
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove all (key,value) pairs from the given message.
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     key      key to delete properties for. If NULL, delete
+ *			   properties for all keys
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
+
+/**
+ * Remove all (prefix*,value) pairs from the given message
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     prefix   delete properties with keys that start with prefix.
+ *			   If NULL, delete all properties
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occurred.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix);
+
+/**
+ * Opaque message property iterator
+ */
+typedef struct _notmuch_string_map_iterator notmuch_message_properties_t;
+
+/**
+ * Get the properties for *message*, returning a
+ * notmuch_message_properties_t object which can be used to iterate
+ * over all properties.
+ *
+ * The notmuch_message_properties_t object is owned by the message and
+ * as such, will only be valid for as long as the message is valid,
+ * (which is until the query from which it derived is destroyed).
+ *
+ * @param[in] message  The message to examine
+ * @param[in] key      key or key prefix
+ * @param[in] exact    if TRUE, require exact match with key. Otherwise
+ *		       treat as prefix.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_properties_t *list;
+ *
+ *     for (list = notmuch_message_get_properties (message, "testkey1", TRUE);
+ *          notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ *        printf("%s\n", notmuch_message_properties_value(list));
+ *     }
+ *
+ *     notmuch_message_properties_destroy (list);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_message_properties_t object. (For consistency, we do
+ * provide a notmuch_message_properities_destroy function, but there's
+ * no good reason to call it if the message is about to be destroyed).
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact);
+
+/**
+ * Return the number of properties named "key" belonging to the specific message.
+ *
+ * @param[in] message  The message to examine
+ * @param[in] key      key to count
+ * @param[out] count   The number of matching properties associated with this message.
+ *
+ * @returns
+ *
+ * NOTMUCH_STATUS_SUCCESS: successful count, possibly some other error.
+ *
+ * @since libnotmuch 5.2 (notmuch 0.27)
+ */
+notmuch_status_t
+notmuch_message_count_properties (notmuch_message_t *message, const char *key, unsigned int *count);
+
+/**
+ * Is the given *properties* iterator pointing at a valid (key,value)
+ * pair.
+ *
+ * When this function returns TRUE,
+ * notmuch_message_properties_{key,value} will return a valid string,
+ * and notmuch_message_properties_move_to_next will do what it
+ * says. Whereas when this function returns FALSE, calling any of
+ * these functions results in undefined behaviour.
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t
+ * object.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties);
+
+/**
+ * Move the *properties* iterator to the next (key,value) pair
+ *
+ * If *properties* is already pointing at the last pair then the iterator
+ * will be moved to a point just beyond that last pair, (where
+ * notmuch_message_properties_valid will return FALSE).
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t object.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * this could be useful if iterating for a prefix
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties);
+
+/**
+ * Return the value from the current (key,value) pair.
+ *
+ * This could be useful if iterating for a prefix.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties);
+
+
+/**
+ * Destroy a notmuch_message_properties_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_message_properties_t object will be reclaimed when the
+ * containing message object is destroyed.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties);
+
+/**@}*/
+
+/**
+ * Is the given 'tags' iterator pointing at a valid tag.
+ *
+ * When this function returns TRUE, notmuch_tags_get will return a
+ * valid string. Whereas when this function returns FALSE,
+ * notmuch_tags_get will return NULL.
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+notmuch_bool_t
+notmuch_tags_valid (notmuch_tags_t *tags);
+
+/**
+ * Get the current tag from 'tags' as a string.
+ *
+ * Note: The returned string belongs to 'tags' and has a lifetime
+ * identical to it (and the query to which it ultimately belongs).
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+const char *
+notmuch_tags_get (notmuch_tags_t *tags);
+
+/**
+ * Move the 'tags' iterator to the next tag.
+ *
+ * If 'tags' is already pointing at the last tag then the iterator
+ * will be moved to a point just beyond that last tag, (where
+ * notmuch_tags_valid will return FALSE and notmuch_tags_get will
+ * return NULL).
+ *
+ * See the documentation of notmuch_message_get_tags for example code
+ * showing how to iterate over a notmuch_tags_t object.
+ */
+void
+notmuch_tags_move_to_next (notmuch_tags_t *tags);
+
+/**
+ * Destroy a notmuch_tags_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_tags_t object will be reclaimed when the containing
+ * message or query objects are destroyed.
+ */
+void
+notmuch_tags_destroy (notmuch_tags_t *tags);
+
+/**
+ * Store an mtime within the database for 'directory'.
+ *
+ * The 'directory' should be an object retrieved from the database
+ * with notmuch_database_get_directory for a particular path.
+ *
+ * The intention is for the caller to use the mtime to allow efficient
+ * identification of new messages to be added to the database. The
+ * recommended usage is as follows:
+ *
+ *   o Read the mtime of a directory from the filesystem
+ *
+ *   o Call index_file for all mail files in the directory
+ *
+ *   o Call notmuch_directory_set_mtime with the mtime read from the
+ *     filesystem.
+ *
+ * Then, when wanting to check for updates to the directory in the
+ * future, the client can call notmuch_directory_get_mtime and know
+ * that it only needs to add files if the mtime of the directory and
+ * files are newer than the stored timestamp.
+ *
+ * Note: The notmuch_directory_get_mtime function does not allow the
+ * caller to distinguish a timestamp of 0 from a non-existent
+ * timestamp. So don't store a timestamp of 0 unless you are
+ * comfortable with that.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: mtime successfully stored in database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception
+ *	occurred, mtime not stored.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so directory mtime cannot be modified.
+ */
+notmuch_status_t
+notmuch_directory_set_mtime (notmuch_directory_t *directory,
+			     time_t mtime);
+
+/**
+ * Get the mtime of a directory, (as previously stored with
+ * notmuch_directory_set_mtime).
+ *
+ * Returns 0 if no mtime has previously been stored for this
+ * directory.
+ */
+time_t
+notmuch_directory_get_mtime (notmuch_directory_t *directory);
+
+/**
+ * Get a notmuch_filenames_t iterator listing all the filenames of
+ * messages in the database within the given directory.
+ *
+ * The returned filenames will be the basename-entries only (not
+ * complete paths).
+ */
+notmuch_filenames_t *
+notmuch_directory_get_child_files (notmuch_directory_t *directory);
+
+/**
+ * Get a notmuch_filenames_t iterator listing all the filenames of
+ * sub-directories in the database within the given directory.
+ *
+ * The returned filenames will be the basename-entries only (not
+ * complete paths).
+ */
+notmuch_filenames_t *
+notmuch_directory_get_child_directories (notmuch_directory_t *directory);
+
+/**
+ * Delete directory document from the database, and destroy the
+ * notmuch_directory_t object. Assumes any child directories and files
+ * have been deleted by the caller.
+ *
+ * @since libnotmuch 4.3 (notmuch 0.21)
+ */
+notmuch_status_t
+notmuch_directory_delete (notmuch_directory_t *directory);
+
+/**
+ * Destroy a notmuch_directory_t object.
+ */
+void
+notmuch_directory_destroy (notmuch_directory_t *directory);
+
+/**
+ * Is the given 'filenames' iterator pointing at a valid filename.
+ *
+ * When this function returns TRUE, notmuch_filenames_get will return
+ * a valid string. Whereas when this function returns FALSE,
+ * notmuch_filenames_get will return NULL.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will always return FALSE.
+ */
+notmuch_bool_t
+notmuch_filenames_valid (notmuch_filenames_t *filenames);
+
+/**
+ * Get the current filename from 'filenames' as a string.
+ *
+ * Note: The returned string belongs to 'filenames' and has a lifetime
+ * identical to it (and the directory to which it ultimately belongs).
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will always return NULL.
+ */
+const char *
+notmuch_filenames_get (notmuch_filenames_t *filenames);
+
+/**
+ * Move the 'filenames' iterator to the next filename.
+ *
+ * If 'filenames' is already pointing at the last filename then the
+ * iterator will be moved to a point just beyond that last filename,
+ * (where notmuch_filenames_valid will return FALSE and
+ * notmuch_filenames_get will return NULL).
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will do nothing.
+ */
+void
+notmuch_filenames_move_to_next (notmuch_filenames_t *filenames);
+
+/**
+ * Destroy a notmuch_filenames_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_filenames_t object will be reclaimed when the
+ * containing directory object is destroyed.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will do nothing.
+ */
+void
+notmuch_filenames_destroy (notmuch_filenames_t *filenames);
+
+
+/**
+ * set config 'key' to 'value'
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_set_config (notmuch_database_t *db, const char *key, const char *value);
+
+/**
+ * retrieve config item 'key', assign to  'value'
+ *
+ * keys which have not been previously set with n_d_set_config will
+ * return an empty string.
+ *
+ * return value is allocated by malloc and should be freed by the
+ * caller.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_get_config (notmuch_database_t *db, const char *key, char **value);
+
+/**
+ * Create an iterator for all config items with keys matching a given prefix
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_get_config_list (notmuch_database_t *db, const char *prefix, notmuch_config_list_t **out);
+
+/**
+ * Is 'config_list' iterator valid (i.e. _key, _value, _move_to_next can be called).
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_config_list_valid (notmuch_config_list_t *config_list);
+
+/**
+ * return key for current config pair
+ *
+ * return value is owned by the iterator, and will be destroyed by the
+ * next call to notmuch_config_list_key or notmuch_config_list_destroy.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_config_list_key (notmuch_config_list_t *config_list);
+
+/**
+ * return 'value' for current config pair
+ *
+ * return value is owned by the iterator, and will be destroyed by the
+ * next call to notmuch_config_list_value or notmuch config_list_destroy
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_config_list_value (notmuch_config_list_t *config_list);
+
+
+/**
+ * move 'config_list' iterator to the next pair
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_config_list_move_to_next (notmuch_config_list_t *config_list);
+
+/**
+ * free any resources held by 'config_list'
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_config_list_destroy (notmuch_config_list_t *config_list);
+
+
+/**
+ * get the current default indexing options for a given database.
+ *
+ * This object will survive until the database itself is destroyed,
+ * but the caller may also release it earlier with
+ * notmuch_indexopts_destroy.
+ *
+ * This object represents a set of options on how a message can be
+ * added to the index.  At the moment it is a featureless stub.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_indexopts_t *
+notmuch_database_get_default_indexopts (notmuch_database_t *db);
+
+/**
+ * Stating a policy about how to decrypt messages.
+ *
+ * See index.decrypt in notmuch-config(1) for more details.
+ */
+typedef enum {
+    NOTMUCH_DECRYPT_FALSE,
+    NOTMUCH_DECRYPT_TRUE,
+    NOTMUCH_DECRYPT_AUTO,
+    NOTMUCH_DECRYPT_NOSTASH,
+} notmuch_decryption_policy_t;
+
+/**
+ * Specify whether to decrypt encrypted parts while indexing.
+ *
+ * Be aware that the index is likely sufficient to reconstruct the
+ * cleartext of the message itself, so please ensure that the notmuch
+ * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
+ * without considering the security of your index.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_indexopts_set_decrypt_policy (notmuch_indexopts_t *indexopts,
+				      notmuch_decryption_policy_t decrypt_policy);
+
+/**
+ * Return whether to decrypt encrypted parts while indexing.
+ * see notmuch_indexopts_set_decrypt_policy.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_decryption_policy_t
+notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts);
+
+/**
+ * Destroy a notmuch_indexopts_t object.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+void
+notmuch_indexopts_destroy (notmuch_indexopts_t *options);
+
+
+/**
+ * interrogate the library for compile time features
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_built_with (const char *name);
+/* @} */
+
+#pragma GCC visibility pop
+
+NOTMUCH_END_DECLS
+
+#endif
diff --git a/lib/notmuch.sym b/lib/notmuch.sym
new file mode 100644
index 00000000..7d0c0af4
--- /dev/null
+++ b/lib/notmuch.sym
@@ -0,0 +1,7 @@
+{
+global:
+	_ZTI*;
+	_ZTS*;
+	notmuch_*;
+local: *;
+};
diff --git a/lib/parse-time-vrp.cc b/lib/parse-time-vrp.cc
new file mode 100644
index 00000000..dd691494
--- /dev/null
+++ b/lib/parse-time-vrp.cc
@@ -0,0 +1,87 @@
+/* parse-time-vrp.cc - date range query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2012 Jani Nikula
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Jani Nikula <jani@nikula.org>
+ */
+
+#include "database-private.h"
+#include "parse-time-vrp.h"
+#include "parse-time-string.h"
+
+#define PREFIX "date:"
+
+/* See *ValueRangeProcessor in xapian-core/api/valuerangeproc.cc */
+Xapian::valueno
+ParseTimeValueRangeProcessor::operator() (std::string &begin, std::string &end)
+{
+    time_t t, now;
+    std::string b;
+
+    /* Require date: prefix in start of the range... */
+    if (STRNCMP_LITERAL (begin.c_str (), PREFIX))
+	return Xapian::BAD_VALUENO;
+
+    /* ...and remove it. */
+    begin.erase (0, sizeof (PREFIX) - 1);
+    b = begin;
+
+    /* Use the same 'now' for begin and end. */
+    if (time (&now) == (time_t) -1)
+	return Xapian::BAD_VALUENO;
+
+    if (!begin.empty ()) {
+	if (parse_time_string (begin.c_str (), &t, &now, PARSE_TIME_ROUND_DOWN))
+	    return Xapian::BAD_VALUENO;
+
+	begin.assign (Xapian::sortable_serialise ((double) t));
+    }
+
+    if (!end.empty ()) {
+	if (end == "!" && ! b.empty ())
+	    end = b;
+
+	if (parse_time_string (end.c_str (), &t, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+	    return Xapian::BAD_VALUENO;
+
+	end.assign (Xapian::sortable_serialise ((double) t));
+    }
+
+    return valno;
+}
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+/* XXX TODO: is throwing an exception the right thing to do here? */
+Xapian::Query DateFieldProcessor::operator()(const std::string & str) {
+    time_t from, to, now;
+
+    /* Use the same 'now' for begin and end. */
+    if (time (&now) == (time_t) -1)
+	throw Xapian::QueryParserError("Unable to get current time");
+
+    if (parse_time_string (str.c_str (), &from, &now, PARSE_TIME_ROUND_DOWN))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+
+    if (parse_time_string (str.c_str (), &to, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+
+    return Xapian::Query(Xapian::Query::OP_AND,
+			 Xapian::Query(Xapian::Query::OP_VALUE_GE, 0, Xapian::sortable_serialise ((double) from)),
+			 Xapian::Query(Xapian::Query::OP_VALUE_LE, 0, Xapian::sortable_serialise ((double) to)));
+}
+#endif
diff --git a/lib/parse-time-vrp.h b/lib/parse-time-vrp.h
new file mode 100644
index 00000000..c024dba2
--- /dev/null
+++ b/lib/parse-time-vrp.h
@@ -0,0 +1,45 @@
+/* parse-time-vrp.h - date range query glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2012 Jani Nikula
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Jani Nikula <jani@nikula.org>
+ */
+
+#ifndef NOTMUCH_PARSE_TIME_VRP_H
+#define NOTMUCH_PARSE_TIME_VRP_H
+
+#include <xapian.h>
+
+/* see *ValueRangeProcessor in xapian-core/include/xapian/queryparser.h */
+class ParseTimeValueRangeProcessor : public Xapian::ValueRangeProcessor {
+protected:
+    Xapian::valueno valno;
+
+public:
+    ParseTimeValueRangeProcessor (Xapian::valueno slot_)
+	: valno(slot_) { }
+
+    Xapian::valueno operator() (std::string &begin, std::string &end);
+};
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+class DateFieldProcessor : public Xapian::FieldProcessor {
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
+#endif /* NOTMUCH_PARSE_TIME_VRP_H */
diff --git a/lib/query-fp.cc b/lib/query-fp.cc
new file mode 100644
index 00000000..c39f5915
--- /dev/null
+++ b/lib/query-fp.cc
@@ -0,0 +1,43 @@
+/* query-fp.cc - "query:" field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "query-fp.h"
+#include <iostream>
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+
+Xapian::Query
+QueryFieldProcessor::operator() (const std::string & name)
+{
+    std::string key = "query." + name;
+    char *expansion;
+    notmuch_status_t status;
+
+    status = notmuch_database_get_config (notmuch, key.c_str (), &expansion);
+    if (status) {
+	throw Xapian::QueryParserError ("error looking up key" + name);
+    }
+
+    return parser.parse_query (expansion, NOTMUCH_QUERY_PARSER_FLAGS);
+}
+#endif
diff --git a/lib/query-fp.h b/lib/query-fp.h
new file mode 100644
index 00000000..d6e4b313
--- /dev/null
+++ b/lib/query-fp.h
@@ -0,0 +1,42 @@
+/* query-fp.h - query field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_QUERY_FP_H
+#define NOTMUCH_QUERY_FP_H
+
+#include <xapian.h>
+#include "notmuch.h"
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+class QueryFieldProcessor : public Xapian::FieldProcessor {
+ protected:
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+ public:
+    QueryFieldProcessor (Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)
+	: parser(parser_), notmuch(notmuch_) { };
+
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
+#endif /* NOTMUCH_QUERY_FP_H */
diff --git a/lib/query.cc b/lib/query.cc
new file mode 100644
index 00000000..7fdf992d
--- /dev/null
+++ b/lib/query.cc
@@ -0,0 +1,732 @@
+/* query.cc - Support for searching a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <glib.h> /* GHashTable, GPtrArray */
+
+struct _notmuch_query {
+    notmuch_database_t *notmuch;
+    const char *query_string;
+    notmuch_sort_t sort;
+    notmuch_string_list_t *exclude_terms;
+    notmuch_exclude_t omit_excluded;
+    bool parsed;
+    Xapian::Query xapian_query;
+    std::set<std::string> terms;
+};
+
+typedef struct _notmuch_mset_messages {
+    notmuch_messages_t base;
+    notmuch_database_t *notmuch;
+    Xapian::MSetIterator iterator;
+    Xapian::MSetIterator iterator_end;
+} notmuch_mset_messages_t;
+
+struct _notmuch_doc_id_set {
+    unsigned char *bitmap;
+    unsigned int bound;
+};
+
+#define DOCIDSET_WORD(bit) ((bit) / CHAR_BIT)
+#define DOCIDSET_BIT(bit) ((bit) % CHAR_BIT)
+
+struct _notmuch_threads {
+    notmuch_query_t *query;
+
+    /* The ordered list of doc ids matched by the query. */
+    GArray *doc_ids;
+    /* Our iterator's current position in doc_ids. */
+    unsigned int doc_id_pos;
+    /* The set of matched docid's that have not been assigned to a
+     * thread. Initially, this contains every docid in doc_ids. */
+    notmuch_doc_id_set_t match_set;
+};
+
+/* We need this in the message functions so forward declare. */
+static bool
+_notmuch_doc_id_set_init (void *ctx,
+			  notmuch_doc_id_set_t *doc_ids,
+			  GArray *arr);
+
+static bool
+_debug_query (void)
+{
+    char *env = getenv ("NOTMUCH_DEBUG_QUERY");
+    return (env && strcmp (env, "") != 0);
+}
+
+/* Explicit destructor call for placement new */
+static int
+_notmuch_query_destructor (notmuch_query_t *query) {
+    query->xapian_query.~Query();
+    query->terms.~set<std::string>();
+    return 0;
+}
+
+notmuch_query_t *
+notmuch_query_create (notmuch_database_t *notmuch,
+		      const char *query_string)
+{
+    notmuch_query_t *query;
+
+    if (_debug_query ())
+	fprintf (stderr, "Query string is:\n%s\n", query_string);
+
+    query = talloc (notmuch, notmuch_query_t);
+    if (unlikely (query == NULL))
+	return NULL;
+
+    new (&query->xapian_query) Xapian::Query ();
+    new (&query->terms) std::set<std::string> ();
+    query->parsed = false;
+
+    talloc_set_destructor (query, _notmuch_query_destructor);
+
+    query->notmuch = notmuch;
+
+    query->query_string = talloc_strdup (query, query_string);
+
+    query->sort = NOTMUCH_SORT_NEWEST_FIRST;
+
+    query->exclude_terms = _notmuch_string_list_create (query);
+
+    query->omit_excluded = NOTMUCH_EXCLUDE_TRUE;
+
+    return query;
+}
+
+static notmuch_status_t
+_notmuch_query_ensure_parsed (notmuch_query_t *query)
+{
+    if (query->parsed)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    try {
+	query->xapian_query =
+	    query->notmuch->query_parser->
+		parse_query (query->query_string, NOTMUCH_QUERY_PARSER_FLAGS);
+
+       /* Xapian doesn't support skip_to on terms from a query since
+	*  they are unordered, so cache a copy of all terms in
+	*  something searchable.
+	*/
+
+	for (Xapian::TermIterator t = query->xapian_query.get_terms_begin ();
+	     t != query->xapian_query.get_terms_end (); ++t)
+	    query->terms.insert (*t);
+
+	query->parsed = true;
+
+    } catch (const Xapian::Error &error) {
+	if (!query->notmuch->exception_reported) {
+	    _notmuch_database_log (query->notmuch,
+				   "A Xapian exception occurred parsing query: %s\n",
+				   error.get_msg ().c_str ());
+	    _notmuch_database_log_append (query->notmuch,
+					  "Query string was: %s\n",
+					  query->query_string);
+	    query->notmuch->exception_reported = true;
+	}
+
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+const char *
+notmuch_query_get_query_string (const notmuch_query_t *query)
+{
+    return query->query_string;
+}
+
+void
+notmuch_query_set_omit_excluded (notmuch_query_t *query,
+				 notmuch_exclude_t omit_excluded)
+{
+    query->omit_excluded = omit_excluded;
+}
+
+void
+notmuch_query_set_sort (notmuch_query_t *query, notmuch_sort_t sort)
+{
+    query->sort = sort;
+}
+
+notmuch_sort_t
+notmuch_query_get_sort (const notmuch_query_t *query)
+{
+    return query->sort;
+}
+
+notmuch_status_t
+notmuch_query_add_tag_exclude (notmuch_query_t *query, const char *tag)
+{
+    notmuch_status_t status;
+    char *term;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
+
+    term = talloc_asprintf (query, "%s%s", _find_prefix ("tag"), tag);
+    if (query->terms.count(term) != 0)
+	return NOTMUCH_STATUS_IGNORED;
+
+    _notmuch_string_list_append (query->exclude_terms, term);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* We end up having to call the destructors explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_messages_destructor (notmuch_mset_messages_t *messages)
+{
+    messages->iterator.~MSetIterator ();
+    messages->iterator_end.~MSetIterator ();
+
+    return 0;
+}
+
+/* Return a query that matches messages with the excluded tags
+ * registered with query. The caller of this function has to combine the returned
+ * query appropriately.*/
+static Xapian::Query
+_notmuch_exclude_tags (notmuch_query_t *query)
+{
+    Xapian::Query exclude_query = Xapian::Query::MatchNothing;
+
+    for (notmuch_string_node_t *term = query->exclude_terms->head; term;
+	 term = term->next) {
+	exclude_query = Xapian::Query (Xapian::Query::OP_OR,
+				       exclude_query, Xapian::Query (term->string));
+    }
+    return exclude_query;
+}
+
+
+notmuch_status_t
+notmuch_query_search_messages_st (notmuch_query_t *query,
+				  notmuch_messages_t **out)
+{
+    return notmuch_query_search_messages (query, out);
+}
+
+notmuch_status_t
+notmuch_query_search_messages (notmuch_query_t *query,
+				  notmuch_messages_t **out)
+{
+    return _notmuch_query_search_documents (query, "mail", out);
+}
+
+notmuch_status_t
+_notmuch_query_search_documents (notmuch_query_t *query,
+				 const char *type,
+				 notmuch_messages_t **out)
+{
+    notmuch_database_t *notmuch = query->notmuch;
+    const char *query_string = query->query_string;
+    notmuch_mset_messages_t *messages;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
+
+    messages = talloc (query, notmuch_mset_messages_t);
+    if (unlikely (messages == NULL))
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+
+    try {
+
+	messages->base.is_of_list_type = false;
+	messages->base.iterator = NULL;
+	messages->notmuch = notmuch;
+	new (&messages->iterator) Xapian::MSetIterator ();
+	new (&messages->iterator_end) Xapian::MSetIterator ();
+
+	talloc_set_destructor (messages, _notmuch_messages_destructor);
+
+	Xapian::Enquire enquire (*notmuch->xapian_db);
+	Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
+						   _find_prefix ("type"),
+						   type));
+	Xapian::Query final_query, exclude_query;
+	Xapian::MSet mset;
+	Xapian::MSetIterator iterator;
+
+	if (strcmp (query_string, "") == 0 ||
+	    strcmp (query_string, "*") == 0)
+	{
+	    final_query = mail_query;
+	} else {
+	    final_query = Xapian::Query (Xapian::Query::OP_AND,
+					 mail_query, query->xapian_query);
+	}
+	messages->base.excluded_doc_ids = NULL;
+
+	if ((query->omit_excluded != NOTMUCH_EXCLUDE_FALSE) && (query->exclude_terms)) {
+	    exclude_query = _notmuch_exclude_tags (query);
+
+	    if (query->omit_excluded == NOTMUCH_EXCLUDE_TRUE ||
+		query->omit_excluded == NOTMUCH_EXCLUDE_ALL)
+	    {
+		final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
+					     final_query, exclude_query);
+	    } else { /* NOTMUCH_EXCLUDE_FLAG */
+		exclude_query = Xapian::Query (Xapian::Query::OP_AND,
+					   exclude_query, final_query);
+
+		enquire.set_weighting_scheme (Xapian::BoolWeight());
+		enquire.set_query (exclude_query);
+
+		mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
+
+		GArray *excluded_doc_ids = g_array_new (false, false, sizeof (unsigned int));
+
+		for (iterator = mset.begin (); iterator != mset.end (); iterator++) {
+		    unsigned int doc_id = *iterator;
+		    g_array_append_val (excluded_doc_ids, doc_id);
+		}
+		messages->base.excluded_doc_ids = talloc (messages, _notmuch_doc_id_set);
+		_notmuch_doc_id_set_init (query, messages->base.excluded_doc_ids,
+					  excluded_doc_ids);
+		g_array_unref (excluded_doc_ids);
+	    }
+	}
+
+
+	enquire.set_weighting_scheme (Xapian::BoolWeight());
+
+	switch (query->sort) {
+	case NOTMUCH_SORT_OLDEST_FIRST:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, false);
+	    break;
+	case NOTMUCH_SORT_NEWEST_FIRST:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, true);
+	    break;
+	case NOTMUCH_SORT_MESSAGE_ID:
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, false);
+	    break;
+	case NOTMUCH_SORT_UNSORTED:
+	    break;
+	}
+
+	if (_debug_query ()) {
+	    fprintf (stderr, "Exclude query is:\n%s\n",
+		     exclude_query.get_description ().c_str ());
+	    fprintf (stderr, "Final query is:\n%s\n",
+		     final_query.get_description ().c_str ());
+	}
+
+	enquire.set_query (final_query);
+
+	mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
+
+	messages->iterator = mset.begin ();
+	messages->iterator_end = mset.end ();
+
+	*out = &messages->base;
+	return NOTMUCH_STATUS_SUCCESS;
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred performing query: %s\n",
+			       error.get_msg().c_str());
+	_notmuch_database_log_append (notmuch,
+			       "Query string was: %s\n",
+			       query->query_string);
+
+	notmuch->exception_reported = true;
+	talloc_free (messages);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+}
+
+bool
+_notmuch_mset_messages_valid (notmuch_messages_t *messages)
+{
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    return (mset_messages->iterator != mset_messages->iterator_end);
+}
+
+static Xapian::docid
+_notmuch_mset_messages_get_doc_id (notmuch_messages_t *messages)
+{
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    if (! _notmuch_mset_messages_valid (&mset_messages->base))
+	return 0;
+
+    return *mset_messages->iterator;
+}
+
+notmuch_message_t *
+_notmuch_mset_messages_get (notmuch_messages_t *messages)
+{
+    notmuch_message_t *message;
+    Xapian::docid doc_id;
+    notmuch_private_status_t status;
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    if (! _notmuch_mset_messages_valid (&mset_messages->base))
+	return NULL;
+
+    doc_id = *mset_messages->iterator;
+
+    message = _notmuch_message_create (mset_messages,
+				       mset_messages->notmuch, doc_id,
+				       &status);
+
+    if (message == NULL &&
+       status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
+    {
+	INTERNAL_ERROR ("a messages iterator contains a non-existent document ID.\n");
+    }
+
+    if (messages->excluded_doc_ids &&
+	_notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id))
+	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
+
+    return message;
+}
+
+void
+_notmuch_mset_messages_move_to_next (notmuch_messages_t *messages)
+{
+    notmuch_mset_messages_t *mset_messages;
+
+    mset_messages = (notmuch_mset_messages_t *) messages;
+
+    mset_messages->iterator++;
+}
+
+static bool
+_notmuch_doc_id_set_init (void *ctx,
+			  notmuch_doc_id_set_t *doc_ids,
+			  GArray *arr)
+{
+    unsigned int max = 0;
+    unsigned char *bitmap;
+
+    for (unsigned int i = 0; i < arr->len; i++)
+	max = MAX(max, g_array_index (arr, unsigned int, i));
+    bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD(max) + 1);
+
+    if (bitmap == NULL)
+	return false;
+
+    doc_ids->bitmap = bitmap;
+    doc_ids->bound = max + 1;
+
+    for (unsigned int i = 0; i < arr->len; i++) {
+	unsigned int doc_id = g_array_index (arr, unsigned int, i);
+	bitmap[DOCIDSET_WORD(doc_id)] |= 1 << DOCIDSET_BIT(doc_id);
+    }
+
+    return true;
+}
+
+bool
+_notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
+			      unsigned int doc_id)
+{
+    if (doc_id >= doc_ids->bound)
+	return false;
+    return doc_ids->bitmap[DOCIDSET_WORD(doc_id)] & (1 << DOCIDSET_BIT(doc_id));
+}
+
+void
+_notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
+			    unsigned int doc_id)
+{
+    if (doc_id < doc_ids->bound)
+	doc_ids->bitmap[DOCIDSET_WORD(doc_id)] &= ~(1 << DOCIDSET_BIT(doc_id));
+}
+
+/* Glib objects force use to use a talloc destructor as well, (but not
+ * nearly as ugly as the for messages due to C++ objects). At
+ * this point, I'd really like to have some talloc-friendly
+ * equivalents for the few pieces of glib that I'm using. */
+static int
+_notmuch_threads_destructor (notmuch_threads_t *threads)
+{
+    if (threads->doc_ids)
+	g_array_unref (threads->doc_ids);
+
+    return 0;
+}
+
+notmuch_status_t
+notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out)
+{
+    return notmuch_query_search_threads(query, out);
+}
+
+notmuch_status_t
+notmuch_query_search_threads (notmuch_query_t *query,
+			      notmuch_threads_t **out)
+{
+    notmuch_threads_t *threads;
+    notmuch_messages_t *messages;
+    notmuch_status_t status;
+
+    threads = talloc (query, notmuch_threads_t);
+    if (threads == NULL)
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    threads->doc_ids = NULL;
+    talloc_set_destructor (threads, _notmuch_threads_destructor);
+
+    threads->query = query;
+
+    status = notmuch_query_search_messages (query, &messages);
+    if (status) {
+	talloc_free (threads);
+	return status;
+    }
+
+    threads->doc_ids = g_array_new (false, false, sizeof (unsigned int));
+    while (notmuch_messages_valid (messages)) {
+	unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages);
+	g_array_append_val (threads->doc_ids, doc_id);
+	notmuch_messages_move_to_next (messages);
+    }
+    threads->doc_id_pos = 0;
+
+    talloc_free (messages);
+
+    if (! _notmuch_doc_id_set_init (threads, &threads->match_set,
+				    threads->doc_ids)) {
+	talloc_free (threads);
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    }
+
+    *out = threads;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+void
+notmuch_query_destroy (notmuch_query_t *query)
+{
+    talloc_free (query);
+}
+
+notmuch_bool_t
+notmuch_threads_valid (notmuch_threads_t *threads)
+{
+    unsigned int doc_id;
+
+    if (! threads)
+	return false;
+
+    while (threads->doc_id_pos < threads->doc_ids->len) {
+	doc_id = g_array_index (threads->doc_ids, unsigned int,
+				threads->doc_id_pos);
+	if (_notmuch_doc_id_set_contains (&threads->match_set, doc_id))
+	    break;
+
+	threads->doc_id_pos++;
+    }
+
+    return threads->doc_id_pos < threads->doc_ids->len;
+}
+
+notmuch_thread_t *
+notmuch_threads_get (notmuch_threads_t *threads)
+{
+    unsigned int doc_id;
+
+    if (! notmuch_threads_valid (threads))
+	return NULL;
+
+    doc_id = g_array_index (threads->doc_ids, unsigned int,
+			    threads->doc_id_pos);
+    return _notmuch_thread_create (threads->query,
+				   threads->query->notmuch,
+				   doc_id,
+				   &threads->match_set,
+				   threads->query->exclude_terms,
+				   threads->query->omit_excluded,
+				   threads->query->sort);
+}
+
+void
+notmuch_threads_move_to_next (notmuch_threads_t *threads)
+{
+    threads->doc_id_pos++;
+}
+
+void
+notmuch_threads_destroy (notmuch_threads_t *threads)
+{
+    talloc_free (threads);
+}
+
+notmuch_status_t
+notmuch_query_count_messages_st (notmuch_query_t *query, unsigned *count_out)
+{
+    return notmuch_query_count_messages (query, count_out);
+}
+
+notmuch_status_t
+notmuch_query_count_messages (notmuch_query_t *query, unsigned *count_out)
+{
+    return _notmuch_query_count_documents (query, "mail", count_out);
+}
+
+notmuch_status_t
+_notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsigned *count_out)
+{
+    notmuch_database_t *notmuch = query->notmuch;
+    const char *query_string = query->query_string;
+    Xapian::doccount count = 0;
+    notmuch_status_t status;
+
+    status = _notmuch_query_ensure_parsed (query);
+    if (status)
+	return status;
+
+    try {
+	Xapian::Enquire enquire (*notmuch->xapian_db);
+	Xapian::Query mail_query (talloc_asprintf (query, "%s%s",
+						   _find_prefix ("type"),
+						   type));
+	Xapian::Query final_query, exclude_query;
+	Xapian::MSet mset;
+
+	if (strcmp (query_string, "") == 0 ||
+	    strcmp (query_string, "*") == 0)
+	{
+	    final_query = mail_query;
+	} else {
+	    final_query = Xapian::Query (Xapian::Query::OP_AND,
+					 mail_query, query->xapian_query);
+	}
+
+	exclude_query = _notmuch_exclude_tags (query);
+
+	final_query = Xapian::Query (Xapian::Query::OP_AND_NOT,
+					 final_query, exclude_query);
+
+	enquire.set_weighting_scheme(Xapian::BoolWeight());
+	enquire.set_docid_order(Xapian::Enquire::ASCENDING);
+
+	if (_debug_query ()) {
+	    fprintf (stderr, "Exclude query is:\n%s\n",
+		     exclude_query.get_description ().c_str ());
+	    fprintf (stderr, "Final query is:\n%s\n",
+		     final_query.get_description ().c_str ());
+	}
+
+	enquire.set_query (final_query);
+
+	/*
+	 * Set the checkatleast parameter to the number of documents
+	 * in the database to make get_matches_estimated() exact.
+	 * Set the max parameter to 1 to avoid fetching documents we will discard.
+	 */
+	mset = enquire.get_mset (0, 1,
+				 notmuch->xapian_db->get_doccount ());
+
+	count = mset.get_matches_estimated();
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch,
+			       "A Xapian exception occurred performing query: %s\n",
+			       error.get_msg().c_str());
+	_notmuch_database_log_append (notmuch,
+				      "Query string was: %s\n",
+				      query->query_string);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    *count_out = count;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_query_count_threads_st (notmuch_query_t *query, unsigned *count)
+{
+    return notmuch_query_count_threads (query, count);
+}
+
+notmuch_status_t
+notmuch_query_count_threads (notmuch_query_t *query, unsigned *count)
+{
+    notmuch_messages_t *messages;
+    GHashTable *hash;
+    notmuch_sort_t sort;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    sort = query->sort;
+    query->sort = NOTMUCH_SORT_UNSORTED;
+    ret = notmuch_query_search_messages (query, &messages);
+    if (ret)
+	return ret;
+    query->sort = sort;
+    if (messages == NULL)
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+
+    hash = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
+    if (hash == NULL) {
+	talloc_free (messages);
+	return NOTMUCH_STATUS_OUT_OF_MEMORY;
+    }
+
+    while (notmuch_messages_valid (messages)) {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	const char *thread_id = notmuch_message_get_thread_id (message);
+	char *thread_id_copy = talloc_strdup (messages, thread_id);
+	if (unlikely (thread_id_copy == NULL)) {
+	    notmuch_message_destroy (message);
+	    ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	    goto DONE;
+	}
+	g_hash_table_insert (hash, thread_id_copy, NULL);
+	notmuch_message_destroy (message);
+	notmuch_messages_move_to_next (messages);
+    }
+
+    *count = g_hash_table_size (hash);
+
+  DONE:
+    g_hash_table_unref (hash);
+    talloc_free (messages);
+
+    return ret;
+}
+
+notmuch_database_t *
+notmuch_query_get_database (const notmuch_query_t *query)
+{
+    return query->notmuch;
+}
diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc
new file mode 100644
index 00000000..084bc8c0
--- /dev/null
+++ b/lib/regexp-fields.cc
@@ -0,0 +1,210 @@
+/* regexp-fields.cc - field processor glue for regex supporting fields
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2015 Austin Clements
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Austin Clements <aclements@csail.mit.edu>
+ *                David Bremner <david@tethera.net>
+ */
+
+#include "regexp-fields.h"
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+static void
+compile_regex (regex_t &regexp, const char *str)
+{
+    int err = regcomp (&regexp, str, REG_EXTENDED | REG_NOSUB);
+
+    if (err != 0) {
+	size_t len = regerror (err, &regexp, NULL, 0);
+	char *buffer = new char[len];
+	std::string msg;
+	(void) regerror (err, &regexp, buffer, len);
+	msg.assign (buffer, len);
+	delete[] buffer;
+
+	throw Xapian::QueryParserError (msg);
+    }
+}
+
+RegexpPostingSource::RegexpPostingSource (Xapian::valueno slot, const std::string &regexp)
+    : slot_ (slot)
+{
+    compile_regex (regexp_, regexp.c_str ());
+}
+
+RegexpPostingSource::~RegexpPostingSource ()
+{
+    regfree (&regexp_);
+}
+
+void
+RegexpPostingSource::init (const Xapian::Database &db)
+{
+    db_ = db;
+    it_ = db_.valuestream_begin (slot_);
+    end_ = db.valuestream_end (slot_);
+    started_ = false;
+}
+
+Xapian::doccount
+RegexpPostingSource::get_termfreq_min () const
+{
+    return 0;
+}
+
+Xapian::doccount
+RegexpPostingSource::get_termfreq_est () const
+{
+    return get_termfreq_max () / 2;
+}
+
+Xapian::doccount
+RegexpPostingSource::get_termfreq_max () const
+{
+    return db_.get_value_freq (slot_);
+}
+
+Xapian::docid
+RegexpPostingSource::get_docid () const
+{
+    return it_.get_docid ();
+}
+
+bool
+RegexpPostingSource::at_end () const
+{
+    return it_ == end_;
+}
+
+void
+RegexpPostingSource::next (unused (double min_wt))
+{
+    if (started_ && ! at_end ())
+	++it_;
+    started_ = true;
+
+    for (; ! at_end (); ++it_) {
+	std::string value = *it_;
+	if (regexec (&regexp_, value.c_str (), 0, NULL, 0) == 0)
+	    break;
+    }
+}
+
+void
+RegexpPostingSource::skip_to (Xapian::docid did, unused (double min_wt))
+{
+    started_ = true;
+    it_.skip_to (did);
+    for (; ! at_end (); ++it_) {
+	std::string value = *it_;
+	if (regexec (&regexp_, value.c_str (), 0, NULL, 0) == 0)
+	    break;
+    }
+}
+
+bool
+RegexpPostingSource::check (Xapian::docid did, unused (double min_wt))
+{
+    started_ = true;
+    if (!it_.check (did) || at_end ())
+	return false;
+    return (regexec (&regexp_, (*it_).c_str (), 0, NULL, 0) == 0);
+}
+
+static inline Xapian::valueno _find_slot (std::string prefix)
+{
+    if (prefix == "from")
+	return NOTMUCH_VALUE_FROM;
+    else if (prefix == "subject")
+	return NOTMUCH_VALUE_SUBJECT;
+    else if (prefix == "mid")
+	return NOTMUCH_VALUE_MESSAGE_ID;
+    else
+	return Xapian::BAD_VALUENO;
+}
+
+RegexpFieldProcessor::RegexpFieldProcessor (std::string prefix,
+					    notmuch_field_flag_t options_,
+					    Xapian::QueryParser &parser_,
+					    notmuch_database_t *notmuch_)
+	: slot (_find_slot (prefix)),
+	  term_prefix (_find_prefix (prefix.c_str ())),
+	  options (options_),
+	  parser (parser_),
+	  notmuch (notmuch_)
+{
+};
+
+Xapian::Query
+RegexpFieldProcessor::operator() (const std::string & str)
+{
+    if (str.empty ()) {
+	if (options & NOTMUCH_FIELD_PROBABILISTIC) {
+	    return Xapian::Query(Xapian::Query::OP_AND_NOT,
+			     Xapian::Query::MatchAll,
+			     Xapian::Query (Xapian::Query::OP_WILDCARD, term_prefix));
+	} else {
+	    return Xapian::Query (term_prefix);
+	}
+    }
+
+    if (str.at (0) == '/') {
+	if (str.length() > 1 && str.at (str.size () - 1) == '/'){
+	    std::string regexp_str = str.substr(1,str.size () - 2);
+	    if (slot != Xapian::BAD_VALUENO) {
+		RegexpPostingSource *postings = new RegexpPostingSource (slot, regexp_str);
+		return Xapian::Query (postings->release ());
+	    } else {
+		std::vector<std::string> terms;
+		regex_t regexp;
+
+		compile_regex(regexp, regexp_str.c_str ());
+		for (Xapian::TermIterator it = notmuch->xapian_db->allterms_begin (term_prefix);
+		     it != notmuch->xapian_db->allterms_end (); ++it) {
+		    if (regexec (&regexp, (*it).c_str () + term_prefix.size(),
+				 0, NULL, 0) == 0)
+			terms.push_back(*it);
+		}
+		return Xapian::Query (Xapian::Query::OP_OR, terms.begin(), terms.end());
+	    }
+	} else {
+	    throw Xapian::QueryParserError ("unmatched regex delimiter in '" + str + "'");
+	}
+    } else {
+	if (options & NOTMUCH_FIELD_PROBABILISTIC) {
+	    /* TODO replace this with a nicer API level triggering of
+	     * phrase parsing, when possible */
+	    std::string query_str;
+
+	    if (str.find (' ') != std::string::npos)
+		query_str = '"' + str + '"';
+	    else
+		query_str = str;
+
+	    return parser.parse_query (query_str, NOTMUCH_QUERY_PARSER_FLAGS, term_prefix);
+	} else {
+	    /* Boolean prefix */
+	    std::string term = term_prefix + str;
+	    return Xapian::Query (term);
+	}
+    }
+}
+#endif
diff --git a/lib/regexp-fields.h b/lib/regexp-fields.h
new file mode 100644
index 00000000..d5f93445
--- /dev/null
+++ b/lib/regexp-fields.h
@@ -0,0 +1,81 @@
+/* regex-fields.h - xapian glue for semi-bruteforce regexp search
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2015 Austin Clements
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Austin Clements <aclements@csail.mit.edu>
+ *                David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_REGEXP_FIELDS_H
+#define NOTMUCH_REGEXP_FIELDS_H
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+#include <sys/types.h>
+#include <regex.h>
+#include "database-private.h"
+#include "notmuch-private.h"
+
+/* A posting source that returns documents where a value matches a
+ * regexp.
+ */
+class RegexpPostingSource : public Xapian::PostingSource
+{
+ protected:
+    const Xapian::valueno slot_;
+    regex_t regexp_;
+    Xapian::Database db_;
+    bool started_;
+    Xapian::ValueIterator it_, end_;
+
+/* No copying */
+    RegexpPostingSource (const RegexpPostingSource &);
+    RegexpPostingSource &operator= (const RegexpPostingSource &);
+
+ public:
+    RegexpPostingSource (Xapian::valueno slot, const std::string &regexp);
+    ~RegexpPostingSource ();
+    void init (const Xapian::Database &db);
+    Xapian::doccount get_termfreq_min () const;
+    Xapian::doccount get_termfreq_est () const;
+    Xapian::doccount get_termfreq_max () const;
+    Xapian::docid get_docid () const;
+    bool at_end () const;
+    void next (unused (double min_wt));
+    void skip_to (Xapian::docid did, unused (double min_wt));
+    bool check (Xapian::docid did, unused (double min_wt));
+};
+
+
+class RegexpFieldProcessor : public Xapian::FieldProcessor {
+ protected:
+    Xapian::valueno slot;
+    std::string term_prefix;
+    notmuch_field_flag_t options;
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+ public:
+    RegexpFieldProcessor (std::string prefix, notmuch_field_flag_t options,
+			  Xapian::QueryParser &parser_, notmuch_database_t *notmuch_);
+
+    ~RegexpFieldProcessor () { };
+
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
+#endif /* NOTMUCH_REGEXP_FIELDS_H */
diff --git a/lib/sha1.c b/lib/sha1.c
new file mode 100644
index 00000000..cb55b49a
--- /dev/null
+++ b/lib/sha1.c
@@ -0,0 +1,93 @@
+/* sha1.c - Interfaces to SHA-1 hash for the notmuch mail system
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+#include <glib.h>
+
+/* Create a hexadecimal string version of the SHA-1 digest of 'str'
+ * (including its null terminating character).
+ *
+ * This function returns a newly allocated string which the caller
+ * should free() when finished.
+ */
+char *
+_notmuch_sha1_of_string (const char *str)
+{
+    GChecksum *sha1;
+    char *digest;
+
+    sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+    g_checksum_update (sha1, (const guchar *) str, strlen (str) + 1);
+    digest = xstrdup (g_checksum_get_string (sha1));
+    g_checksum_free (sha1);
+
+    return digest;
+}
+
+/* Create a hexadecimal string version of the SHA-1 digest of the
+ * contents of the named file.
+ *
+ * This function returns a newly allocated string which the caller
+ * should free() when finished.
+ *
+ * If any error occurs while reading the file, (permission denied,
+ * file not found, etc.), this function returns NULL.
+ */
+char *
+_notmuch_sha1_of_file (const char *filename)
+{
+    FILE *file;
+#define BLOCK_SIZE 4096
+    unsigned char block[BLOCK_SIZE];
+    size_t bytes_read;
+    GChecksum *sha1;
+    char *digest = NULL;
+
+    file = fopen (filename, "r");
+    if (file == NULL)
+	return NULL;
+
+    sha1 = g_checksum_new (G_CHECKSUM_SHA1);
+    if (sha1 == NULL)
+	goto DONE;
+
+    while (1) {
+	bytes_read = fread (block, 1, 4096, file);
+	if (bytes_read == 0) {
+	    if (feof (file))
+		break;
+	    else if (ferror (file))
+		goto DONE;
+	} else {
+	    g_checksum_update (sha1, block, bytes_read);
+	}
+    }
+
+    digest = xstrdup (g_checksum_get_string (sha1));
+
+  DONE:
+    if (sha1)
+	g_checksum_free (sha1);
+    if (file)
+	fclose (file);
+
+    return digest;
+}
diff --git a/lib/string-list.c b/lib/string-list.c
new file mode 100644
index 00000000..9c3ae7ef
--- /dev/null
+++ b/lib/string-list.c
@@ -0,0 +1,101 @@
+/* strings.c - Iterator for a list of strings
+ *
+ * Copyright © 2010 Intel Corporation
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ *         Austin Clements <aclements@csail.mit.edu>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_list_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+notmuch_string_list_t *
+_notmuch_string_list_create (const void *ctx)
+{
+    notmuch_string_list_t *list;
+
+    list = talloc (ctx, notmuch_string_list_t);
+    if (unlikely (list == NULL))
+	return NULL;
+
+    list->length = 0;
+    list->head = NULL;
+    list->tail = &list->head;
+
+    return list;
+}
+
+int
+_notmuch_string_list_length (notmuch_string_list_t *list)
+{
+    return list->length;
+}
+
+void
+_notmuch_string_list_append (notmuch_string_list_t *list,
+			     const char *string)
+{
+    /* Create and initialize new node. */
+    notmuch_string_node_t *node = talloc (list, notmuch_string_node_t);
+
+    node->string = talloc_strdup (node, string);
+    node->next = NULL;
+
+    /* Append the node to the list. */
+    *(list->tail) = node;
+    list->tail = &node->next;
+    list->length++;
+}
+
+static int
+cmpnode (const void *pa, const void *pb)
+{
+    notmuch_string_node_t *a = *(notmuch_string_node_t * const *)pa;
+    notmuch_string_node_t *b = *(notmuch_string_node_t * const *)pb;
+
+    return strcmp (a->string, b->string);
+}
+
+void
+_notmuch_string_list_sort (notmuch_string_list_t *list)
+{
+    notmuch_string_node_t **nodes, *node;
+    int i;
+
+    if (list->length == 0)
+	return;
+
+    nodes = talloc_array (list, notmuch_string_node_t *, list->length);
+    if (unlikely (nodes == NULL))
+	INTERNAL_ERROR ("Could not allocate memory for list sort");
+
+    for (i = 0, node = list->head; node; i++, node = node->next)
+	nodes[i] = node;
+
+    qsort (nodes, list->length, sizeof (*nodes), cmpnode);
+
+    for (i = 0; i < list->length - 1; ++i)
+	nodes[i]->next = nodes[i+1];
+    nodes[i]->next = NULL;
+    list->head = nodes[0];
+    list->tail = &nodes[i]->next;
+
+    talloc_free (nodes);
+}
diff --git a/lib/string-map.c b/lib/string-map.c
new file mode 100644
index 00000000..ad818207
--- /dev/null
+++ b/lib/string-map.c
@@ -0,0 +1,228 @@
+/* string-map.c - associative arrays of strings
+ *
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_map_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+
+typedef struct _notmuch_string_pair_t {
+    char *key;
+    char *value;
+} notmuch_string_pair_t;
+
+struct _notmuch_string_map {
+    bool sorted;
+    size_t length;
+    notmuch_string_pair_t *pairs;
+};
+
+struct _notmuch_string_map_iterator {
+    notmuch_string_pair_t *current;
+    bool exact;
+    const char *key;
+};
+
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx)
+{
+    notmuch_string_map_t *map;
+
+    map = talloc (ctx, notmuch_string_map_t);
+    if (unlikely (map == NULL))
+	return NULL;
+
+    map->length = 0;
+    map->pairs = NULL;
+    map->sorted = true;
+
+    return map;
+}
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value)
+{
+
+    map->length++;
+    map->sorted = false;
+
+    if (map->pairs)
+	map->pairs = talloc_realloc (map, map->pairs, notmuch_string_pair_t, map->length + 1);
+    else
+	map->pairs = talloc_array (map, notmuch_string_pair_t, map->length + 1);
+
+    map->pairs[map->length - 1].key = talloc_strdup (map, key);
+    map->pairs[map->length - 1].value = talloc_strdup (map, value);
+
+    /* Add sentinel */
+    map->pairs[map->length].key = NULL;
+    map->pairs[map->length].value = NULL;
+
+}
+
+static int
+cmppair (const void *pa, const void *pb)
+{
+    notmuch_string_pair_t *a = (notmuch_string_pair_t *) pa;
+    notmuch_string_pair_t *b = (notmuch_string_pair_t *) pb;
+
+    return strcmp (a->key, b->key);
+}
+
+static void
+_notmuch_string_map_sort (notmuch_string_map_t *map)
+{
+    if (map->length == 0)
+	return;
+
+    if (map->sorted)
+	return;
+
+    qsort (map->pairs, map->length, sizeof (notmuch_string_pair_t), cmppair);
+
+    map->sorted = true;
+}
+
+static bool
+string_cmp (const char *a, const char *b, bool exact)
+{
+    if (exact)
+	return (strcmp (a, b));
+    else
+	return (strncmp (a, b, strlen (a)));
+}
+
+static notmuch_string_pair_t *
+bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, bool exact)
+{
+    size_t first = 0;
+    size_t last = len - 1;
+    size_t mid;
+
+    if (len <= 0)
+	return NULL;
+
+    while (last > first) {
+	mid = (first + last) / 2;
+	int sign = string_cmp (key, array[mid].key, exact);
+
+	if (sign <= 0)
+	    last = mid;
+	else
+	    first = mid + 1;
+    }
+
+
+    if (string_cmp (key, array[first].key, exact) == 0)
+	return array + first;
+    else
+	return NULL;
+
+}
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
+{
+    notmuch_string_pair_t *pair;
+
+    /* this means that calling append invalidates iterators */
+    _notmuch_string_map_sort (map);
+
+    pair = bsearch_first (map->pairs, map->length, key, true);
+    if (! pair)
+	return NULL;
+
+    return pair->value;
+}
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     bool exact)
+{
+    notmuch_string_map_iterator_t *iter;
+
+    _notmuch_string_map_sort (map);
+
+    iter = talloc (map, notmuch_string_map_iterator_t);
+    if (unlikely (iter == NULL))
+	return NULL;
+
+    if (unlikely (talloc_reference (iter, map) == NULL))
+	return NULL;
+
+    iter->key = talloc_strdup (iter, key);
+    iter->exact = exact;
+    iter->current = bsearch_first (map->pairs, map->length, key, exact);
+    return iter;
+}
+
+bool
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iterator)
+{
+    if (iterator->current == NULL)
+	return false;
+
+    /* sentinel */
+    if (iterator->current->key == NULL)
+	return false;
+
+    return (0 == string_cmp (iterator->key, iterator->current->key, iterator->exact));
+
+}
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iterator)
+{
+
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return;
+
+    (iterator->current)++;
+}
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->key;
+}
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->value;
+}
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator)
+{
+    talloc_free (iterator);
+}
diff --git a/lib/tags.c b/lib/tags.c
new file mode 100644
index 00000000..c7d3f66f
--- /dev/null
+++ b/lib/tags.c
@@ -0,0 +1,76 @@
+/* tags.c - Iterator for tags returned from message or thread
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+
+struct _notmuch_tags {
+    notmuch_string_node_t *iterator;
+};
+
+/* Create a new notmuch_tags_t object, with 'ctx' as its talloc owner.
+ * The returned iterator will talloc_steal the 'list', since the list
+ * is almost always transient.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+notmuch_tags_t *
+_notmuch_tags_create (const void *ctx, notmuch_string_list_t *list)
+{
+    notmuch_tags_t *tags;
+
+    tags = talloc (ctx, notmuch_tags_t);
+    if (unlikely (tags == NULL))
+	return NULL;
+
+    tags->iterator = list->head;
+    (void) talloc_steal (tags, list);
+
+    return tags;
+}
+
+notmuch_bool_t
+notmuch_tags_valid (notmuch_tags_t *tags)
+{
+    return tags->iterator != NULL;
+}
+
+const char *
+notmuch_tags_get (notmuch_tags_t *tags)
+{
+    if (tags->iterator == NULL)
+	return NULL;
+
+    return (char *) tags->iterator->string;
+}
+
+void
+notmuch_tags_move_to_next (notmuch_tags_t *tags)
+{
+    if (tags->iterator == NULL)
+	return;
+
+    tags->iterator = tags->iterator->next;
+}
+
+void
+notmuch_tags_destroy (notmuch_tags_t *tags)
+{
+    talloc_free (tags);
+}
diff --git a/lib/thread-fp.cc b/lib/thread-fp.cc
new file mode 100644
index 00000000..73277006
--- /dev/null
+++ b/lib/thread-fp.cc
@@ -0,0 +1,67 @@
+/* thread-fp.cc - "thread:" field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2018 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "thread-fp.h"
+#include <iostream>
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+
+Xapian::Query
+ThreadFieldProcessor::operator() (const std::string & str)
+{
+    notmuch_status_t status;
+    const char *thread_prefix = _find_prefix ("thread");
+
+    if (str.at (0) == '{') {
+	if (str.size () <= 1 || str.at (str.size () - 1) != '}') {
+	    throw Xapian::QueryParserError ("missing } in '" + str + "'");
+	} else {
+	    std::string subquery_str = str.substr (1, str.size () - 2);
+	    notmuch_query_t *subquery = notmuch_query_create (notmuch, subquery_str.c_str ());
+	    notmuch_messages_t *messages;
+	    std::set<std::string> terms;
+
+	    if (! subquery)
+		throw Xapian::QueryParserError ("failed to create subquery for '" + subquery_str + "'");
+
+	    status = notmuch_query_search_messages (subquery, &messages);
+	    if (status)
+		throw Xapian::QueryParserError ("failed to search messages for '" + subquery_str + "'");
+
+	    for (; notmuch_messages_valid (messages); notmuch_messages_move_to_next (messages)) {
+		std::string term = thread_prefix;
+		notmuch_message_t *message;
+		message = notmuch_messages_get (messages);
+		term += _notmuch_message_get_thread_id_only (message);
+		terms.insert (term);
+	    }
+	    return Xapian::Query (Xapian::Query::OP_OR, terms.begin (), terms.end ());
+	}
+    } else {
+	/* literal thread id */
+	std::string term = thread_prefix + str;
+	return Xapian::Query (term);
+    }
+
+}
+#endif
diff --git a/lib/thread-fp.h b/lib/thread-fp.h
new file mode 100644
index 00000000..47c066c1
--- /dev/null
+++ b/lib/thread-fp.h
@@ -0,0 +1,42 @@
+/* thread-fp.h - thread field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2018 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_THREAD_FP_H
+#define NOTMUCH_THREAD_FP_H
+
+#include <xapian.h>
+#include "notmuch.h"
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+class ThreadFieldProcessor : public Xapian::FieldProcessor {
+ protected:
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+ public:
+    ThreadFieldProcessor (Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)
+	: parser(parser_), notmuch(notmuch_) { };
+
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
+#endif /* NOTMUCH_THREAD_FP_H */
diff --git a/lib/thread.cc b/lib/thread.cc
new file mode 100644
index 00000000..47c90664
--- /dev/null
+++ b/lib/thread.cc
@@ -0,0 +1,738 @@
+/* thread.cc - Results of thread-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <gmime/gmime.h>
+#include <glib.h> /* GHashTable */
+
+#ifdef DEBUG_THREADING
+#define THREAD_DEBUG(format, ...) fprintf(stderr, format " (%s).\n", ##__VA_ARGS__, __location__)
+#else
+#define THREAD_DEBUG(format, ...) do {} while (0) /* ignored */
+#endif
+
+#define EMPTY_STRING(s) ((s)[0] == '\0')
+
+struct _notmuch_thread {
+    notmuch_database_t *notmuch;
+    char *thread_id;
+    char *subject;
+    GHashTable *authors_hash;
+    GPtrArray *authors_array;
+    GHashTable *matched_authors_hash;
+    GPtrArray *matched_authors_array;
+    char *authors;
+    GHashTable *tags;
+
+    /* All messages, oldest first. */
+    notmuch_message_list_t *message_list;
+    /* Top-level messages, oldest first. */
+    notmuch_message_list_t *toplevel_list;
+
+    GHashTable *message_hash;
+    int total_messages;
+    int total_files;
+    int matched_messages;
+    time_t oldest;
+    time_t newest;
+};
+
+static int
+_notmuch_thread_destructor (notmuch_thread_t *thread)
+{
+    g_hash_table_unref (thread->authors_hash);
+    g_hash_table_unref (thread->matched_authors_hash);
+    g_hash_table_unref (thread->tags);
+    g_hash_table_unref (thread->message_hash);
+
+    if (thread->authors_array) {
+	g_ptr_array_free (thread->authors_array, true);
+	thread->authors_array = NULL;
+    }
+
+    if (thread->matched_authors_array) {
+	g_ptr_array_free (thread->matched_authors_array, true);
+	thread->matched_authors_array = NULL;
+    }
+
+    return 0;
+}
+
+/* Add each author of the thread to the thread's authors_hash and to
+ * the thread's authors_array. */
+static void
+_thread_add_author (notmuch_thread_t *thread,
+		    const char *author)
+{
+    char *author_copy;
+
+    if (author == NULL)
+	return;
+
+    if (g_hash_table_lookup_extended (thread->authors_hash,
+				      author, NULL, NULL))
+	return;
+
+    author_copy = talloc_strdup (thread, author);
+
+    g_hash_table_insert (thread->authors_hash, author_copy, NULL);
+
+    g_ptr_array_add (thread->authors_array, author_copy);
+}
+
+/* Add each matched author of the thread to the thread's
+ * matched_authors_hash and to the thread's matched_authors_array. */
+static void
+_thread_add_matched_author (notmuch_thread_t *thread,
+			    const char *author)
+{
+    char *author_copy;
+
+    if (author == NULL)
+	return;
+
+    if (g_hash_table_lookup_extended (thread->matched_authors_hash,
+				      author, NULL, NULL))
+	return;
+
+    author_copy = talloc_strdup (thread, author);
+
+    g_hash_table_insert (thread->matched_authors_hash, author_copy, NULL);
+
+    g_ptr_array_add (thread->matched_authors_array, author_copy);
+}
+
+/* Construct an authors string from matched_authors_array and
+ * authors_array. The string contains matched authors first, then
+ * non-matched authors (with the two groups separated by '|'). Within
+ * each group, authors are listed in date order. */
+static void
+_resolve_thread_authors_string (notmuch_thread_t *thread)
+{
+    unsigned int i;
+    char *author;
+    int first_non_matched_author = 1;
+
+    /* First, list all matched authors in date order. */
+    for (i = 0; i < thread->matched_authors_array->len; i++) {
+	author = (char *) g_ptr_array_index (thread->matched_authors_array, i);
+	if (thread->authors)
+	    thread->authors = talloc_asprintf (thread, "%s, %s",
+					       thread->authors,
+					       author);
+	else
+	    thread->authors = author;
+    }
+
+    /* Next, append any non-matched authors that haven't already appeared. */
+    for (i = 0; i < thread->authors_array->len; i++) {
+	author = (char *) g_ptr_array_index (thread->authors_array, i);
+	if (g_hash_table_lookup_extended (thread->matched_authors_hash,
+					  author, NULL, NULL))
+	    continue;
+	if (first_non_matched_author) {
+	    thread->authors = talloc_asprintf (thread, "%s| %s",
+					       thread->authors,
+					       author);
+	} else {
+	    thread->authors = talloc_asprintf (thread, "%s, %s",
+					       thread->authors,
+					       author);
+	}
+
+	first_non_matched_author = 0;
+    }
+
+    g_ptr_array_free (thread->authors_array, true);
+    thread->authors_array = NULL;
+    g_ptr_array_free (thread->matched_authors_array, true);
+    thread->matched_authors_array = NULL;
+
+    if (!thread->authors)
+	thread->authors = talloc_strdup(thread, "");
+}
+
+/* clean up the ugly "Lastname, Firstname" format that some mail systems
+ * (most notably, Exchange) are creating to be "Firstname Lastname"
+ * To make sure that we don't change other potential situations where a
+ * comma is in the name, we check that we match one of these patterns
+ * "Last, First" <first.last@company.com>
+ * "Last, First MI" <first.mi.last@company.com>
+ */
+static char *
+_thread_cleanup_author (notmuch_thread_t *thread,
+			const char *author, const char *from)
+{
+    char *clean_author,*test_author;
+    const char *comma;
+    char *blank;
+    int fname,lname;
+
+    if (author == NULL)
+	return NULL;
+    clean_author = talloc_strdup(thread, author);
+    if (clean_author == NULL)
+	return NULL;
+    /* check if there's a comma in the name and that there's a
+     * component of the name behind it (so the name doesn't end with
+     * the comma - in which case the string that strchr finds is just
+     * one character long ",\0").
+     * Otherwise just return the copy of the original author name that
+     * we just made*/
+    comma = strchr(author,',');
+    if (comma && strlen(comma) > 1) {
+	/* let's assemble what we think is the correct name */
+	lname = comma - author;
+
+	/* Skip all the spaces after the comma */
+	fname = strlen(author) - lname - 1;
+	comma += 1;
+	while (*comma == ' ') {
+	    fname -= 1;
+	    comma += 1;
+	}
+	strncpy(clean_author, comma, fname);
+
+	*(clean_author+fname) = ' ';
+	strncpy(clean_author + fname + 1, author, lname);
+	*(clean_author+fname+1+lname) = '\0';
+	/* make a temporary copy and see if it matches the email */
+	test_author = talloc_strdup(thread,clean_author);
+
+	blank=strchr(test_author,' ');
+	while (blank != NULL) {
+	    *blank = '.';
+	    blank=strchr(test_author,' ');
+	}
+	if (strcasestr(from, test_author) == NULL)
+	    /* we didn't identify this as part of the email address
+	    * so let's punt and return the original author */
+	    strcpy (clean_author, author);
+    }
+    return clean_author;
+}
+
+/* Add 'message' as a message that belongs to 'thread'.
+ *
+ * The 'thread' will talloc_steal the 'message' and hold onto a
+ * reference to it.
+ */
+static void
+_thread_add_message (notmuch_thread_t *thread,
+		     notmuch_message_t *message,
+		     notmuch_string_list_t *exclude_terms,
+		     notmuch_exclude_t omit_exclude)
+{
+    notmuch_tags_t *tags;
+    const char *tag;
+    InternetAddressList *list = NULL;
+    InternetAddress *address;
+    const char *from, *author;
+    char *clean_author;
+    bool message_excluded = false;
+
+    if (omit_exclude != NOTMUCH_EXCLUDE_FALSE) {
+	for (tags = notmuch_message_get_tags (message);
+	     notmuch_tags_valid (tags);
+	     notmuch_tags_move_to_next (tags))
+	{
+	    tag = notmuch_tags_get (tags);
+	    /* Is message excluded? */
+	    for (notmuch_string_node_t *term = exclude_terms->head;
+		 term != NULL;
+		 term = term->next)
+	    {
+		/* Check for an empty string, and then ignore initial 'K'. */
+		if (*(term->string) && strcmp(tag, (term->string + 1)) == 0) {
+		    message_excluded = true;
+		    break;
+		}
+	    }
+	}
+    }
+
+    if (message_excluded && omit_exclude == NOTMUCH_EXCLUDE_ALL)
+	return;
+
+    _notmuch_message_list_add_message (thread->message_list,
+				       talloc_steal (thread, message));
+    thread->total_messages++;
+    thread->total_files += notmuch_message_count_files (message);
+
+    g_hash_table_insert (thread->message_hash,
+			 xstrdup (notmuch_message_get_message_id (message)),
+			 message);
+
+    from = notmuch_message_get_header (message, "from");
+    if (from)
+	list = internet_address_list_parse_string (from);
+
+    if (list) {
+	address = internet_address_list_get_address (list, 0);
+	if (address) {
+	    author = internet_address_get_name (address);
+	    /* We treat quoted empty names as if they were empty. */
+	    if (author == NULL || author[0] == '\0') {
+		InternetAddressMailbox *mailbox;
+		mailbox = INTERNET_ADDRESS_MAILBOX (address);
+		author = internet_address_mailbox_get_addr (mailbox);
+	    }
+	    clean_author = _thread_cleanup_author (thread, author, from);
+	    _thread_add_author (thread, clean_author);
+	    _notmuch_message_set_author (message, clean_author);
+	}
+	g_object_unref (G_OBJECT (list));
+    }
+
+    if (! thread->subject) {
+	const char *subject;
+	subject = notmuch_message_get_header (message, "subject");
+	thread->subject = talloc_strdup (thread, subject ? subject : "");
+    }
+
+    for (tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags))
+    {
+	tag = notmuch_tags_get (tags);
+	g_hash_table_insert (thread->tags, xstrdup (tag), NULL);
+    }
+
+    /* Mark excluded messages. */
+    if (message_excluded)
+	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
+}
+
+static void
+_thread_set_subject_from_message (notmuch_thread_t *thread,
+				  notmuch_message_t *message)
+{
+    const char *subject;
+    const char *cleaned_subject;
+
+    subject = notmuch_message_get_header (message, "subject");
+    if (! subject)
+	return;
+
+    if ((strncasecmp (subject, "Re: ", 4) == 0) ||
+	(strncasecmp (subject, "Aw: ", 4) == 0) ||
+	(strncasecmp (subject, "Vs: ", 4) == 0) ||
+	(strncasecmp (subject, "Sv: ", 4) == 0)) {
+
+	cleaned_subject = talloc_strndup (thread,
+					  subject + 4,
+					  strlen(subject) - 4);
+    } else {
+	cleaned_subject = talloc_strdup (thread, subject);
+    }
+
+    if (! EMPTY_STRING(cleaned_subject)) {
+	if (thread->subject)
+	    talloc_free (thread->subject);
+
+	thread->subject = talloc_strdup (thread, cleaned_subject);
+    }
+}
+
+/* Add a message to this thread which is known to match the original
+ * search specification. The 'sort' parameter controls whether the
+ * oldest or newest matching subject is applied to the thread as a
+ * whole. */
+static void
+_thread_add_matched_message (notmuch_thread_t *thread,
+			     notmuch_message_t *message,
+			     notmuch_sort_t sort)
+{
+    time_t date;
+    notmuch_message_t *hashed_message;
+
+    date = notmuch_message_get_date (message);
+
+    if (date < thread->oldest || ! thread->matched_messages) {
+	thread->oldest = date;
+	if (sort == NOTMUCH_SORT_OLDEST_FIRST)
+	    _thread_set_subject_from_message (thread, message);
+    }
+
+    if (date > thread->newest || ! thread->matched_messages) {
+	thread->newest = date;
+	const char *cur_subject = notmuch_thread_get_subject(thread);
+	if (sort != NOTMUCH_SORT_OLDEST_FIRST || EMPTY_STRING(cur_subject))
+	    _thread_set_subject_from_message (thread, message);
+    }
+
+    if (!notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED))
+	thread->matched_messages++;
+
+    if (g_hash_table_lookup_extended (thread->message_hash,
+			    notmuch_message_get_message_id (message), NULL,
+			    (void **) &hashed_message)) {
+	notmuch_message_set_flag (hashed_message,
+				  NOTMUCH_MESSAGE_FLAG_MATCH, 1);
+    }
+
+    _thread_add_matched_author (thread, _notmuch_message_get_author (hashed_message));
+}
+
+static bool
+_parent_via_in_reply_to (notmuch_thread_t *thread, notmuch_message_t *message) {
+    notmuch_message_t *parent;
+    const char *in_reply_to;
+
+    in_reply_to = _notmuch_message_get_in_reply_to (message);
+    THREAD_DEBUG("checking message = %s in_reply_to=%s\n",
+		 notmuch_message_get_message_id (message), in_reply_to);
+
+    if (in_reply_to && (! EMPTY_STRING(in_reply_to)) &&
+	g_hash_table_lookup_extended (thread->message_hash,
+				      in_reply_to, NULL,
+				      (void **) &parent)) {
+	_notmuch_message_add_reply (parent, message);
+	return true;
+    } else {
+	return false;
+    }
+}
+
+static void
+_parent_or_toplevel (notmuch_thread_t *thread, notmuch_message_t *message)
+{
+    size_t max_depth = 0;
+    notmuch_message_t *new_parent;
+    notmuch_message_t *parent = NULL;
+    const notmuch_string_list_t *references =
+	_notmuch_message_get_references (message);
+
+    THREAD_DEBUG("trying to reparent via references: %s\n",
+		     notmuch_message_get_message_id (message));
+
+    for (notmuch_string_node_t *ref_node = references->head;
+	 ref_node; ref_node = ref_node->next) {
+	THREAD_DEBUG("checking reference=%s\n", ref_node->string);
+	if ((g_hash_table_lookup_extended (thread->message_hash,
+					   ref_node->string, NULL,
+					   (void **) &new_parent))) {
+	    size_t new_depth = _notmuch_message_get_thread_depth (new_parent);
+	    THREAD_DEBUG("got depth %lu\n", new_depth);
+	    if (new_depth > max_depth || !parent) {
+		THREAD_DEBUG("adding at depth %lu parent=%s\n", new_depth, ref_node->string);
+		max_depth = new_depth;
+		parent = new_parent;
+	    }
+	}
+    }
+    if (parent) {
+	THREAD_DEBUG("adding reply %s to parent=%s\n",
+		 notmuch_message_get_message_id (message),
+		 notmuch_message_get_message_id (parent));
+	_notmuch_message_add_reply (parent, message);
+    } else {
+	THREAD_DEBUG("adding as toplevel %s\n",
+		 notmuch_message_get_message_id (message));
+	_notmuch_message_list_add_message (thread->toplevel_list, message);
+    }
+}
+
+static void
+_resolve_thread_relationships (notmuch_thread_t *thread)
+{
+    notmuch_message_node_t *node, *first_node;
+    notmuch_message_t *message;
+    void *local;
+    notmuch_message_list_t *maybe_toplevel_list;
+
+    first_node = thread->message_list->head;
+    if (! first_node)
+	return;
+
+    local = talloc_new (thread);
+    maybe_toplevel_list = _notmuch_message_list_create (local);
+
+    for (node = first_node->next; node; node = node->next) {
+	message = node->message;
+	if (! _parent_via_in_reply_to (thread, message))
+	    _notmuch_message_list_add_message (maybe_toplevel_list, message);
+    }
+
+    /*
+     * if we reach the end of the list without finding a top-level
+     * message, that means the thread is a cycle (or set of cycles)
+     * and any message can be considered top-level.  Choose the oldest
+     * message, which happens to be first in our list.
+     */
+    if (first_node) {
+	message = first_node->message;
+	THREAD_DEBUG("checking first message  %s\n",
+		     notmuch_message_get_message_id (message));
+
+        if (_notmuch_message_list_empty (maybe_toplevel_list) ||
+	    ! _parent_via_in_reply_to (thread, message)) {
+
+	    THREAD_DEBUG("adding first message as toplevel = %s\n",
+			 notmuch_message_get_message_id (message));
+	    _notmuch_message_list_add_message (maybe_toplevel_list, message);
+	}
+    }
+
+    for (notmuch_messages_t *messages = _notmuch_messages_create (maybe_toplevel_list);
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages))
+    {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	_notmuch_message_label_depths (message, 0);
+    }
+
+    for (notmuch_messages_t *roots = _notmuch_messages_create (maybe_toplevel_list);
+	 notmuch_messages_valid (roots);
+	 notmuch_messages_move_to_next (roots)) {
+	notmuch_message_t *message = notmuch_messages_get (roots);
+	if (_notmuch_messages_has_next (roots) || ! _notmuch_message_list_empty (thread->toplevel_list))
+	    _parent_or_toplevel (thread, message);
+	else
+	    _notmuch_message_list_add_message (thread->toplevel_list, message);
+    }
+
+    /* XXX this could be made conditional on messages being inserted
+     * (out of order) in later passes
+     */
+    thread->toplevel_list = _notmuch_message_sort_subtrees (thread, thread->toplevel_list);
+
+    talloc_free (local);
+}
+
+/* Create a new notmuch_thread_t object by finding the thread
+ * containing the message with the given doc ID, treating any messages
+ * contained in match_set as "matched".  Remove all messages in the
+ * thread from match_set.
+ *
+ * Creating the thread will perform a database search to get all
+ * messages belonging to the thread and will get the first subject
+ * line, the total count of messages, and all authors in the thread.
+ * Each message in the thread is checked against match_set to allow
+ * for a separate count of matched messages, and to allow a viewer to
+ * display these messages differently.
+ *
+ * Here, 'ctx' is talloc context for the resulting thread object.
+ *
+ * This function returns NULL in the case of any error.
+ */
+notmuch_thread_t *
+_notmuch_thread_create (void *ctx,
+			notmuch_database_t *notmuch,
+			unsigned int seed_doc_id,
+			notmuch_doc_id_set_t *match_set,
+			notmuch_string_list_t *exclude_terms,
+			notmuch_exclude_t omit_excluded,
+			notmuch_sort_t sort)
+{
+    void *local = talloc_new (ctx);
+    notmuch_thread_t *thread = NULL;
+    notmuch_message_t *seed_message;
+    const char *thread_id;
+    char *thread_id_query_string;
+    notmuch_query_t *thread_id_query;
+
+    notmuch_messages_t *messages;
+    notmuch_message_t *message;
+    notmuch_status_t status;
+
+    seed_message = _notmuch_message_create (local, notmuch, seed_doc_id, NULL);
+    if (! seed_message)
+	INTERNAL_ERROR ("Thread seed message %u does not exist", seed_doc_id);
+
+    thread_id = notmuch_message_get_thread_id (seed_message);
+    thread_id_query_string = talloc_asprintf (local, "thread:%s", thread_id);
+    if (unlikely (thread_id_query_string == NULL))
+	goto DONE;
+
+    thread_id_query = talloc_steal (
+	local, notmuch_query_create (notmuch, thread_id_query_string));
+    if (unlikely (thread_id_query == NULL))
+	goto DONE;
+
+    thread = talloc (local, notmuch_thread_t);
+    if (unlikely (thread == NULL))
+	goto DONE;
+
+    talloc_set_destructor (thread, _notmuch_thread_destructor);
+
+    thread->notmuch = notmuch;
+    thread->thread_id = talloc_strdup (thread, thread_id);
+    thread->subject = NULL;
+    thread->authors_hash = g_hash_table_new_full (g_str_hash, g_str_equal,
+						  NULL, NULL);
+    thread->authors_array = g_ptr_array_new ();
+    thread->matched_authors_hash = g_hash_table_new_full (g_str_hash,
+							  g_str_equal,
+							  NULL, NULL);
+    thread->matched_authors_array = g_ptr_array_new ();
+    thread->authors = NULL;
+    thread->tags = g_hash_table_new_full (g_str_hash, g_str_equal,
+					  free, NULL);
+
+    thread->message_list = _notmuch_message_list_create (thread);
+    thread->toplevel_list = _notmuch_message_list_create (thread);
+    if (unlikely (thread->message_list == NULL ||
+		  thread->toplevel_list == NULL)) {
+	thread = NULL;
+	goto DONE;
+    }
+
+    thread->message_hash = g_hash_table_new_full (g_str_hash, g_str_equal,
+						  free, NULL);
+
+    thread->total_messages = 0;
+    thread->total_files = 0;
+    thread->matched_messages = 0;
+    thread->oldest = 0;
+    thread->newest = 0;
+
+    /* We use oldest-first order unconditionally here to obtain the
+     * proper author ordering for the thread. The 'sort' parameter
+     * passed to this function is used only to indicate whether the
+     * oldest or newest subject is desired. */
+    notmuch_query_set_sort (thread_id_query, NOTMUCH_SORT_OLDEST_FIRST);
+
+    status = notmuch_query_search_messages (thread_id_query, &messages);
+    if (status)
+	goto DONE;
+
+    for (;
+	 notmuch_messages_valid (messages);
+	 notmuch_messages_move_to_next (messages))
+    {
+	unsigned int doc_id;
+
+	message = notmuch_messages_get (messages);
+	doc_id = _notmuch_message_get_doc_id (message);
+	if (doc_id == seed_doc_id)
+	    message = seed_message;
+
+	_thread_add_message (thread, message, exclude_terms, omit_excluded);
+
+	if ( _notmuch_doc_id_set_contains (match_set, doc_id)) {
+	    _notmuch_doc_id_set_remove (match_set, doc_id);
+	    _thread_add_matched_message (thread, message, sort);
+	}
+
+	_notmuch_message_close (message);
+    }
+
+    _resolve_thread_authors_string (thread);
+
+    _resolve_thread_relationships (thread);
+
+    /* Commit to returning thread. */
+    (void) talloc_steal (ctx, thread);
+
+  DONE:
+    talloc_free (local);
+    return thread;
+}
+
+notmuch_messages_t *
+notmuch_thread_get_toplevel_messages (notmuch_thread_t *thread)
+{
+    return _notmuch_messages_create (thread->toplevel_list);
+}
+
+notmuch_messages_t *
+notmuch_thread_get_messages (notmuch_thread_t *thread)
+{
+    return _notmuch_messages_create (thread->message_list);
+}
+
+const char *
+notmuch_thread_get_thread_id (notmuch_thread_t *thread)
+{
+    return thread->thread_id;
+}
+
+int
+notmuch_thread_get_total_messages (notmuch_thread_t *thread)
+{
+    return thread->total_messages;
+}
+
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread)
+{
+    return thread->total_files;
+}
+
+int
+notmuch_thread_get_matched_messages (notmuch_thread_t *thread)
+{
+    return thread->matched_messages;
+}
+
+const char *
+notmuch_thread_get_authors (notmuch_thread_t *thread)
+{
+    return thread->authors;
+}
+
+const char *
+notmuch_thread_get_subject (notmuch_thread_t *thread)
+{
+    return thread->subject;
+}
+
+time_t
+notmuch_thread_get_oldest_date (notmuch_thread_t *thread)
+{
+    return thread->oldest;
+}
+
+time_t
+notmuch_thread_get_newest_date (notmuch_thread_t *thread)
+{
+    return thread->newest;
+}
+
+notmuch_tags_t *
+notmuch_thread_get_tags (notmuch_thread_t *thread)
+{
+    notmuch_string_list_t *tags;
+    GList *keys, *l;
+
+    tags = _notmuch_string_list_create (thread);
+    if (unlikely (tags == NULL))
+	return NULL;
+
+    keys = g_hash_table_get_keys (thread->tags);
+
+    for (l = keys; l; l = l->next)
+	_notmuch_string_list_append (tags, (char *) l->data);
+
+    g_list_free (keys);
+
+    _notmuch_string_list_sort (tags);
+
+    return _notmuch_tags_create (thread, tags);
+}
+
+void
+notmuch_thread_destroy (notmuch_thread_t *thread)
+{
+    talloc_free (thread);
+}