Merge tag 'debian/0.26-1' into debian/stretch-backports

notmuch Debian 0.26-1 upload (same as 0.26)

22 files changed, 1589 insertions, 880 deletions

diff --git a/lib/Makefile.local b/lib/Makefile.local
index bf6e0649..8aa03891 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -19,7 +19,7 @@ LIBRARY_SUFFIX = so
 LINKER_NAME = libnotmuch.$(LIBRARY_SUFFIX)
 SONAME = $(LINKER_NAME).$(LIBNOTMUCH_VERSION_MAJOR)
 LIBNAME = $(SONAME).$(LIBNOTMUCH_VERSION_MINOR).$(LIBNOTMUCH_VERSION_RELEASE)
-LIBRARY_LINK_FLAG = -shared -Wl,--version-script=$(lib)/notmuch.sym,-soname=$(SONAME) $(NO_UNDEFINED_LDFLAGS)
+LIBRARY_LINK_FLAG = -shared -Wl,--version-script=$(srcdir)/$(lib)/notmuch.sym,-soname=$(SONAME) $(NO_UNDEFINED_LDFLAGS)
 ifeq ($(PLATFORM),OPENBSD)
 LIBRARY_LINK_FLAG += -lc
 endif
@@ -38,10 +38,12 @@ libnotmuch_c_srcs =		\
 	$(dir)/filenames.c	\
 	$(dir)/string-list.c	\
 	$(dir)/message-file.c	\
+	$(dir)/message-id.c	\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
 	$(dir)/built-with.c	\
 	$(dir)/string-map.c	\
+	$(dir)/indexopts.c	\
 	$(dir)/tags.c
 
 libnotmuch_cxx_srcs =		\
@@ -50,6 +52,7 @@ libnotmuch_cxx_srcs =		\
 	$(dir)/directory.cc	\
 	$(dir)/index.cc		\
 	$(dir)/message.cc	\
+	$(dir)/add-message.cc	\
 	$(dir)/message-property.cc \
 	$(dir)/query.cc		\
 	$(dir)/query-fp.cc      \
diff --git a/lib/add-message.cc b/lib/add-message.cc
new file mode 100644
index 00000000..f5fac8be
--- /dev/null
+++ b/lib/add-message.cc
@@ -0,0 +1,598 @@
+#include "database-private.h"
+
+/* Parse a References header value, putting a (talloc'ed under 'ctx')
+ * copy of each referenced message-id into 'hash'.
+ *
+ * We explicitly avoid including any reference identical to
+ * 'message_id' in the result (to avoid mass confusion when a single
+ * message references itself cyclically---and yes, mail messages are
+ * not infrequent in the wild that do this---don't ask me why).
+ *
+ * Return the last reference parsed, if it is not equal to message_id.
+ */
+static char *
+parse_references (void *ctx,
+		  const char *message_id,
+		  GHashTable *hash,
+		  const char *refs)
+{
+    char *ref, *last_ref = NULL;
+
+    if (refs == NULL || *refs == '\0')
+	return NULL;
+
+    while (*refs) {
+	ref = _notmuch_message_id_parse (ctx, refs, &refs);
+
+	if (ref && strcmp (ref, message_id)) {
+	    g_hash_table_add (hash, ref);
+	    last_ref = ref;
+	}
+    }
+
+    /* The return value of this function is used to add a parent
+     * reference to the database.  We should avoid making a message
+     * its own parent, thus the above check.
+     */
+    return talloc_strdup(ctx, last_ref);
+}
+
+static const char *
+_notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
+{
+    /* 16 bytes (+ terminator) for hexadecimal representation of
+     * a 64-bit integer. */
+    static char thread_id[17];
+    Xapian::WritableDatabase *db;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    notmuch->last_thread_id++;
+
+    sprintf (thread_id, "%016" PRIx64, notmuch->last_thread_id);
+
+    db->set_metadata ("last_thread_id", thread_id);
+
+    return thread_id;
+}
+
+static char *
+_get_metadata_thread_id_key (void *ctx, const char *message_id)
+{
+    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
+	message_id = _notmuch_message_id_compressed (ctx, message_id);
+
+    return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
+			    message_id);
+}
+
+
+static notmuch_status_t
+_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
+				      void *ctx,
+				      const char *message_id,
+				      const char **thread_id_ret);
+
+
+/* Find the thread ID to which the message with 'message_id' belongs.
+ *
+ * Note: 'thread_id_ret' must not be NULL!
+ * On success '*thread_id_ret' is set to a newly talloced string belonging to
+ * 'ctx'.
+ *
+ * Note: If there is no message in the database with the given
+ * 'message_id' then a new thread_id will be allocated for this
+ * message ID and stored in the database metadata so that the
+ * thread ID can be looked up if the message is added to the database
+ * later.
+ */
+static notmuch_status_t
+_resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
+				  void *ctx,
+				  const char *message_id,
+				  const char **thread_id_ret)
+{
+    notmuch_private_status_t status;
+    notmuch_message_t *message;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
+	return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
+						     thread_id_ret);
+
+    /* Look for this message (regular or ghost) */
+    message = _notmuch_message_create_for_message_id (
+	notmuch, message_id, &status);
+    if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
+	/* Message exists */
+	*thread_id_ret = talloc_steal (
+	    ctx, notmuch_message_get_thread_id (message));
+    } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	/* Message did not exist.  Give it a fresh thread ID and
+	 * populate this message as a ghost message. */
+	*thread_id_ret = talloc_strdup (
+	    ctx, _notmuch_database_generate_thread_id (notmuch));
+	if (! *thread_id_ret) {
+	    status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
+	} else {
+	    status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
+	    if (status == 0)
+		/* Commit the new ghost message */
+		_notmuch_message_sync (message);
+	}
+    } else {
+	/* Create failed. Fall through. */
+    }
+
+    notmuch_message_destroy (message);
+
+    return COERCE_STATUS (status, "Error creating ghost message");
+}
+
+/* Pre-ghost messages _resolve_message_id_to_thread_id */
+static notmuch_status_t
+_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
+				      void *ctx,
+				      const char *message_id,
+				      const char **thread_id_ret)
+{
+    notmuch_status_t status;
+    notmuch_message_t *message;
+    std::string thread_id_string;
+    char *metadata_key;
+    Xapian::WritableDatabase *db;
+
+    status = notmuch_database_find_message (notmuch, message_id, &message);
+
+    if (status)
+	return status;
+
+    if (message) {
+	*thread_id_ret = talloc_steal (ctx,
+				       notmuch_message_get_thread_id (message));
+
+	notmuch_message_destroy (message);
+
+	return NOTMUCH_STATUS_SUCCESS;
+    }
+
+    /* Message has not been seen yet.
+     *
+     * We may have seen a reference to it already, in which case, we
+     * can return the thread ID stored in the metadata. Otherwise, we
+     * generate a new thread ID and store it there.
+     */
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
+    thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
+
+    if (thread_id_string.empty()) {
+	*thread_id_ret = talloc_strdup (ctx,
+					_notmuch_database_generate_thread_id (notmuch));
+	db->set_metadata (metadata_key, *thread_id_ret);
+    } else {
+	*thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str());
+    }
+
+    talloc_free (metadata_key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_merge_threads (notmuch_database_t *notmuch,
+		const char *winner_thread_id,
+		const char *loser_thread_id)
+{
+    Xapian::PostingIterator loser, loser_end;
+    notmuch_message_t *message = NULL;
+    notmuch_private_status_t private_status;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    _notmuch_database_find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);
+
+    for ( ; loser != loser_end; loser++) {
+	message = _notmuch_message_create (notmuch, notmuch,
+					   *loser, &private_status);
+	if (message == NULL) {
+	    ret = COERCE_STATUS (private_status,
+				 "Cannot find document for doc_id from query");
+	    goto DONE;
+	}
+
+	_notmuch_message_remove_term (message, "thread", loser_thread_id);
+	_notmuch_message_add_term (message, "thread", winner_thread_id);
+	_notmuch_message_sync (message);
+
+	notmuch_message_destroy (message);
+	message = NULL;
+    }
+
+  DONE:
+    if (message)
+	notmuch_message_destroy (message);
+
+    return ret;
+}
+
+static void
+_my_talloc_free_for_g_hash (void *ptr)
+{
+    talloc_free (ptr);
+}
+
+notmuch_status_t
+_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
+					   notmuch_message_t *message,
+					   notmuch_message_file_t *message_file,
+					   const char **thread_id)
+{
+    GHashTable *parents = NULL;
+    const char *refs, *in_reply_to, *in_reply_to_message_id;
+    const char *last_ref_message_id, *this_message_id;
+    GList *l, *keys = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+
+    parents = g_hash_table_new_full (g_str_hash, g_str_equal,
+				     _my_talloc_free_for_g_hash, NULL);
+    this_message_id = notmuch_message_get_message_id (message);
+
+    refs = _notmuch_message_file_get_header (message_file, "references");
+    last_ref_message_id = parse_references (message,
+					    this_message_id,
+					    parents, refs);
+
+    in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
+    in_reply_to_message_id = parse_references (message,
+					       this_message_id,
+					       parents, in_reply_to);
+
+    /* For the parent of this message, use the last message ID of the
+     * References header, if available.  If not, fall back to the
+     * first message ID in the In-Reply-To header. */
+    if (last_ref_message_id) {
+	_notmuch_message_add_term (message, "replyto",
+				   last_ref_message_id);
+    } else if (in_reply_to_message_id) {
+	_notmuch_message_add_term (message, "replyto",
+			     in_reply_to_message_id);
+    }
+
+    keys = g_hash_table_get_keys (parents);
+    for (l = keys; l; l = l->next) {
+	char *parent_message_id;
+	const char *parent_thread_id = NULL;
+
+	parent_message_id = (char *) l->data;
+
+	_notmuch_message_add_term (message, "reference",
+				   parent_message_id);
+
+	ret = _resolve_message_id_to_thread_id (notmuch,
+						message,
+						parent_message_id,
+						&parent_thread_id);
+	if (ret)
+	    goto DONE;
+
+	if (*thread_id == NULL) {
+	    *thread_id = talloc_strdup (message, parent_thread_id);
+	    _notmuch_message_add_term (message, "thread", *thread_id);
+	} else if (strcmp (*thread_id, parent_thread_id)) {
+	    ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
+	    if (ret)
+		goto DONE;
+	}
+    }
+
+  DONE:
+    if (keys)
+	g_list_free (keys);
+    if (parents)
+	g_hash_table_unref (parents);
+
+    return ret;
+}
+
+static notmuch_status_t
+_notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
+					    notmuch_message_t *message,
+					    const char **thread_id)
+{
+    const char *message_id = notmuch_message_get_message_id (message);
+    Xapian::PostingIterator child, children_end;
+    notmuch_message_t *child_message = NULL;
+    const char *child_thread_id;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_private_status_t private_status;
+
+    _notmuch_database_find_doc_ids (notmuch, "reference", message_id, &child, &children_end);
+
+    for ( ; child != children_end; child++) {
+
+	child_message = _notmuch_message_create (message, notmuch,
+						 *child, &private_status);
+	if (child_message == NULL) {
+	    ret = COERCE_STATUS (private_status,
+				 "Cannot find document for doc_id from query");
+	    goto DONE;
+	}
+
+	child_thread_id = notmuch_message_get_thread_id (child_message);
+	if (*thread_id == NULL) {
+	    *thread_id = talloc_strdup (message, child_thread_id);
+	    _notmuch_message_add_term (message, "thread", *thread_id);
+	} else if (strcmp (*thread_id, child_thread_id)) {
+	    _notmuch_message_remove_term (child_message, "reference",
+					  message_id);
+	    _notmuch_message_sync (child_message);
+	    ret = _merge_threads (notmuch, *thread_id, child_thread_id);
+	    if (ret)
+		goto DONE;
+	}
+
+	notmuch_message_destroy (child_message);
+	child_message = NULL;
+    }
+
+  DONE:
+    if (child_message)
+	notmuch_message_destroy (child_message);
+
+    return ret;
+}
+
+/* Fetch and clear the stored thread_id for message, or NULL if none. */
+static char *
+_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
+			     notmuch_message_t *message)
+{
+    const char *message_id;
+    std::string stored_id;
+    char *metadata_key;
+
+    message_id = notmuch_message_get_message_id (message);
+    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
+
+    /* Check if we have already seen related messages to this one.
+     * If we have then use the thread_id that we stored at that time.
+     */
+    stored_id = notmuch->xapian_db->get_metadata (metadata_key);
+    if (stored_id.empty ()) {
+	return NULL;
+    } else {
+	Xapian::WritableDatabase *db;
+
+	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+	/* Clear the metadata for this message ID. We don't need it
+	 * anymore. */
+	db->set_metadata (metadata_key, "");
+
+	return talloc_strdup (ctx, stored_id.c_str ());
+    }
+}
+
+/* Given a blank or ghost 'message' and its corresponding
+ * 'message_file' link it to existing threads in the database.
+ *
+ * First, if is_ghost, this retrieves the thread ID already stored in
+ * the message (which will be the case if a message was previously
+ * added that referenced this one).  If the message is blank
+ * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
+ * later in this function).  If the database does not support ghost
+ * messages, this checks for a thread ID stored in database metadata
+ * for this message ID.
+ *
+ * Second, we look at 'message_file' and its link-relevant headers
+ * (References and In-Reply-To) for message IDs.
+ *
+ * Finally, we look in the database for existing message that
+ * reference 'message'.
+ *
+ * In all cases, we assign to the current message the first thread ID
+ * found. We will also merge any existing, distinct threads where this
+ * message belongs to both, (which is not uncommon when messages are
+ * processed out of order).
+ *
+ * Finally, if no thread ID has been found through referenced messages, we
+ * call _notmuch_message_generate_thread_id to generate a new thread
+ * ID. This should only happen for new, top-level messages, (no
+ * References or In-Reply-To header in this message, and no previously
+ * added message refers to this message).
+ */
+static notmuch_status_t
+_notmuch_database_link_message (notmuch_database_t *notmuch,
+				notmuch_message_t *message,
+				notmuch_message_file_t *message_file,
+				bool is_ghost)
+{
+    void *local = talloc_new (NULL);
+    notmuch_status_t status;
+    const char *thread_id = NULL;
+
+    /* Check if the message already had a thread ID */
+    if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
+	if (is_ghost)
+	    thread_id = notmuch_message_get_thread_id (message);
+    } else {
+	thread_id = _consume_metadata_thread_id (local, notmuch, message);
+	if (thread_id)
+	    _notmuch_message_add_term (message, "thread", thread_id);
+    }
+
+    status = _notmuch_database_link_message_to_parents (notmuch, message,
+							message_file,
+							&thread_id);
+    if (status)
+	goto DONE;
+
+    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
+	/* In general, it shouldn't be necessary to link children,
+	 * since the earlier indexing of those children will have
+	 * stored a thread ID for the missing parent.  However, prior
+	 * to ghost messages, these stored thread IDs were NOT
+	 * rewritten during thread merging (and there was no
+	 * performant way to do so), so if indexed children were
+	 * pulled into a different thread ID by a merge, it was
+	 * necessary to pull them *back* into the stored thread ID of
+	 * the parent.  With ghost messages, we just rewrite the
+	 * stored thread IDs during merging, so this workaround isn't
+	 * necessary. */
+	status = _notmuch_database_link_message_to_children (notmuch, message,
+							     &thread_id);
+	if (status)
+	    goto DONE;
+    }
+
+    /* If not part of any existing thread, generate a new thread ID. */
+    if (thread_id == NULL) {
+	thread_id = _notmuch_database_generate_thread_id (notmuch);
+
+	_notmuch_message_add_term (message, "thread", thread_id);
+    }
+
+ DONE:
+    talloc_free (local);
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_index_file (notmuch_database_t *notmuch,
+			     const char *filename,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_t **message_ret)
+{
+    notmuch_message_file_t *message_file;
+    notmuch_message_t *message = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
+    notmuch_private_status_t private_status;
+    bool is_ghost = false, is_new = false;
+    notmuch_indexopts_t *def_indexopts = NULL;
+
+    const char *date;
+    const char *from, *to, *subject;
+    char *message_id = NULL;
+
+    if (message_ret)
+	*message_ret = NULL;
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    message_file = _notmuch_message_file_open (notmuch, filename);
+    if (message_file == NULL)
+	return NOTMUCH_STATUS_FILE_ERROR;
+
+    /* Adding a message may change many documents.  Do this all
+     * atomically. */
+    ret = notmuch_database_begin_atomic (notmuch);
+    if (ret)
+	goto DONE;
+
+    ret = _notmuch_message_file_get_headers (message_file,
+					     &from, &subject, &to, &date,
+					     &message_id);
+    if (ret)
+	goto DONE;
+
+    try {
+	/* Now that we have a message ID, we get a message object,
+	 * (which may or may not reference an existing document in the
+	 * database). */
+
+	message = _notmuch_message_create_for_message_id (notmuch,
+							  message_id,
+							  &private_status);
+
+	talloc_free (message_id);
+
+	/* We cannot call notmuch_message_get_flag for a new message */
+	switch (private_status) {
+	case NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND:
+	    is_ghost = false;
+	    is_new = true;
+	    break;
+	case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+	    is_ghost = notmuch_message_get_flag (message, NOTMUCH_MESSAGE_FLAG_GHOST);
+	    is_new = false;
+	    break;
+	default:
+	    ret = COERCE_STATUS (private_status,
+				 "Unexpected status value from _notmuch_message_create_for_message_id");
+	    goto DONE;
+	}
+
+	_notmuch_message_add_filename (message, filename);
+
+	if (is_new || is_ghost) {
+	    _notmuch_message_add_term (message, "type", "mail");
+	    if (is_ghost)
+		/* Convert ghost message to a regular message */
+		_notmuch_message_remove_term (message, "type", "ghost");
+	}
+
+	ret = _notmuch_database_link_message (notmuch, message,
+						  message_file, is_ghost);
+	if (ret)
+	    goto DONE;
+
+	if (is_new || is_ghost)
+	    _notmuch_message_set_header_values (message, date, from, subject);
+
+	if (!indexopts) {
+	    def_indexopts = notmuch_database_get_default_indexopts (notmuch);
+	    indexopts = def_indexopts;
+	}
+
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
+	if (ret)
+	    goto DONE;
+
+	if (! is_new && !is_ghost)
+	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
+	_notmuch_message_sync (message);
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = true;
+	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	goto DONE;
+    }
+
+  DONE:
+    if (def_indexopts)
+	notmuch_indexopts_destroy (def_indexopts);
+
+    if (message) {
+	if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
+	    *message_ret = message;
+	else
+	    notmuch_message_destroy (message);
+    }
+
+    if (message_file)
+	_notmuch_message_file_close (message_file);
+
+    ret2 = notmuch_database_end_atomic (notmuch);
+    if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
+	ret2 != NOTMUCH_STATUS_SUCCESS)
+	ret = ret2;
+
+    return ret;
+}
+
+notmuch_status_t
+notmuch_database_add_message (notmuch_database_t *notmuch,
+			      const char *filename,
+			      notmuch_message_t **message_ret)
+{
+    return notmuch_database_index_file (notmuch, filename,
+					NULL,
+					message_ret);
+
+}
diff --git a/lib/built-with.c b/lib/built-with.c
index 2f1f0b5c..9cffd9f9 100644
--- a/lib/built-with.c
+++ b/lib/built-with.c
@@ -30,7 +30,9 @@ notmuch_built_with (const char *name)
 	return HAVE_XAPIAN_FIELD_PROCESSOR;
     } else if (STRNCMP_LITERAL (name, "retry_lock") == 0) {
 	return HAVE_XAPIAN_DB_RETRY_LOCK;
+    } else if (STRNCMP_LITERAL (name, "session_key") == 0) {
+	return HAVE_GMIME_SESSION_KEYS;
     } else {
-	return FALSE;
+	return false;
     }
 }
diff --git a/lib/config.cc b/lib/config.cc
index 0703b9bb..da71c16e 100644
--- a/lib/config.cc
+++ b/lib/config.cc
@@ -56,7 +56,7 @@ notmuch_database_set_config (notmuch_database_t *notmuch,
 	db->set_metadata (CONFIG_PREFIX + key, value);
     } catch (const Xapian::Error &error) {
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred setting metadata: %s\n",
 			       error.get_msg().c_str());
     }
@@ -74,7 +74,7 @@ _metadata_value (notmuch_database_t *notmuch,
 	value = notmuch->xapian_db->get_metadata (CONFIG_PREFIX + key);
     } catch (const Xapian::Error &error) {
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred getting metadata: %s\n",
 			       error.get_msg().c_str());
     }
@@ -128,7 +128,7 @@ notmuch_database_get_config_list (notmuch_database_t *notmuch,
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "A Xapian exception occurred getting metadata iterator: %s.\n",
 			       error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
@@ -145,9 +145,9 @@ notmuch_bool_t
 notmuch_config_list_valid (notmuch_config_list_t *metadata)
 {
     if (metadata->iterator == metadata->notmuch->xapian_db->metadata_keys_end ())
-	return FALSE;
+	return false;
 
-    return TRUE;
+    return true;
 }
 
 const char *
diff --git a/lib/database-private.h b/lib/database-private.h
index 727b1d61..a499b259 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -179,14 +179,14 @@ operator&(notmuch_field_flag_t a, notmuch_field_flag_t b)
 				    Xapian::QueryParser::FLAG_PURE_NOT)
 
 struct _notmuch_database {
-    notmuch_bool_t exception_reported;
+    bool exception_reported;
 
     char *path;
 
     notmuch_database_mode_t mode;
     int atomic_nesting;
-    /* TRUE if changes have been made in this atomic section */
-    notmuch_bool_t atomic_dirty;
+    /* true if changes have been made in this atomic section */
+    bool atomic_dirty;
     Xapian::Database *xapian_db;
 
     /* Bit mask of features used by this database.  This is a
@@ -246,4 +246,10 @@ _notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
 					 Xapian::TermIterator &end,
 					 const char *prefix);
 
+void
+_notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
+				const char *prefix_name,
+				const char *value,
+				Xapian::PostingIterator *begin,
+				Xapian::PostingIterator *end);
 #endif
diff --git a/lib/database.cc b/lib/database.cc
index 5b13f541..02444e09 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -413,6 +413,12 @@ notmuch_status_to_string (notmuch_status_t status)
 	return "Operation requires a database upgrade";
     case NOTMUCH_STATUS_PATH_ERROR:
 	return "Path supplied is illegal for this function";
+    case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
+	return "Crypto protocol missing, malformed, or unintelligible";
+    case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
+	return "Crypto engine initialization failure";
+    case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
+	return "Unknown crypto protocol";
     default:
     case NOTMUCH_STATUS_LAST_STATUS:
 	return "Unknown error status value";
@@ -463,12 +469,12 @@ find_doc_ids_for_term (notmuch_database_t *notmuch,
     *end = notmuch->xapian_db->postlist_end (term);
 }
 
-static void
-find_doc_ids (notmuch_database_t *notmuch,
-	      const char *prefix_name,
-	      const char *value,
-	      Xapian::PostingIterator *begin,
-	      Xapian::PostingIterator *end)
+void
+_notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
+				const char *prefix_name,
+				const char *value,
+				Xapian::PostingIterator *begin,
+				Xapian::PostingIterator *end)
 {
     char *term;
 
@@ -488,7 +494,7 @@ _notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
 {
     Xapian::PostingIterator i, end;
 
-    find_doc_ids (notmuch, prefix_name, value, &i, &end);
+    _notmuch_database_find_doc_ids (notmuch, prefix_name, value, &i, &end);
 
     if (i == end) {
 	*doc_id = 0;
@@ -562,153 +568,12 @@ notmuch_database_find_message (notmuch_database_t *notmuch,
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "A Xapian exception occurred finding message: %s.\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	*message_ret = NULL;
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 }
 
-/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
- * a (potentially nested) parenthesized sequence with '\' used to
- * escape any character (including parentheses).
- *
- * If the sequence to be skipped continues to the end of the string,
- * then 'str' will be left pointing at the final terminating '\0'
- * character.
- */
-static void
-skip_space_and_comments (const char **str)
-{
-    const char *s;
-
-    s = *str;
-    while (*s && (isspace (*s) || *s == '(')) {
-	while (*s && isspace (*s))
-	    s++;
-	if (*s == '(') {
-	    int nesting = 1;
-	    s++;
-	    while (*s && nesting) {
-		if (*s == '(') {
-		    nesting++;
-		} else if (*s == ')') {
-		    nesting--;
-		} else if (*s == '\\') {
-		    if (*(s+1))
-			s++;
-		}
-		s++;
-	    }
-	}
-    }
-
-    *str = s;
-}
-
-/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
- * comments, and the '<' and '>' delimiters.
- *
- * If not NULL, then *next will be made to point to the first character
- * not parsed, (possibly pointing to the final '\0' terminator.
- *
- * Returns a newly talloc'ed string belonging to 'ctx'.
- *
- * Returns NULL if there is any error parsing the message-id. */
-static char *
-_parse_message_id (void *ctx, const char *message_id, const char **next)
-{
-    const char *s, *end;
-    char *result;
-
-    if (message_id == NULL || *message_id == '\0')
-	return NULL;
-
-    s = message_id;
-
-    skip_space_and_comments (&s);
-
-    /* Skip any unstructured text as well. */
-    while (*s && *s != '<')
-	s++;
-
-    if (*s == '<') {
-	s++;
-    } else {
-	if (next)
-	    *next = s;
-	return NULL;
-    }
-
-    skip_space_and_comments (&s);
-
-    end = s;
-    while (*end && *end != '>')
-	end++;
-    if (next) {
-	if (*end)
-	    *next = end + 1;
-	else
-	    *next = end;
-    }
-
-    if (end > s && *end == '>')
-	end--;
-    if (end <= s)
-	return NULL;
-
-    result = talloc_strndup (ctx, s, end - s + 1);
-
-    /* Finally, collapse any whitespace that is within the message-id
-     * itself. */
-    {
-	char *r;
-	int len;
-
-	for (r = result, len = strlen (r); *r; r++, len--)
-	    if (*r == ' ' || *r == '\t')
-		memmove (r, r+1, len);
-    }
-
-    return result;
-}
-
-/* Parse a References header value, putting a (talloc'ed under 'ctx')
- * copy of each referenced message-id into 'hash'.
- *
- * We explicitly avoid including any reference identical to
- * 'message_id' in the result (to avoid mass confusion when a single
- * message references itself cyclically---and yes, mail messages are
- * not infrequent in the wild that do this---don't ask me why).
- *
- * Return the last reference parsed, if it is not equal to message_id.
- */
-static char *
-parse_references (void *ctx,
-		  const char *message_id,
-		  GHashTable *hash,
-		  const char *refs)
-{
-    char *ref, *last_ref = NULL;
-
-    if (refs == NULL || *refs == '\0')
-	return NULL;
-
-    while (*refs) {
-	ref = _parse_message_id (ctx, refs, &refs);
-
-	if (ref && strcmp (ref, message_id)) {
-	    g_hash_table_add (hash, ref);
-	    last_ref = ref;
-	}
-    }
-
-    /* The return value of this function is used to add a parent
-     * reference to the database.  We should avoid making a message
-     * its own parent, thus the above check.
-     */
-    return talloc_strdup(ctx, last_ref);
-}
-
 notmuch_status_t
 notmuch_database_create (const char *path, notmuch_database_t **database)
 {
@@ -832,7 +697,7 @@ _notmuch_database_new_revision (notmuch_database_t *notmuch)
      * committed revision number until we commit the atomic section.
      */
     if (notmuch->atomic_nesting)
-	notmuch->atomic_dirty = TRUE;
+	notmuch->atomic_dirty = true;
     else
 	notmuch->revision = new_revision;
 
@@ -995,12 +860,11 @@ notmuch_database_open_verbose (const char *path,
     }
 
     notmuch = talloc_zero (NULL, notmuch_database_t);
-    notmuch->exception_reported = FALSE;
+    notmuch->exception_reported = false;
     notmuch->status_string = NULL;
     notmuch->path = talloc_strdup (notmuch, path);
 
-    if (notmuch->path[strlen (notmuch->path) - 1] == '/')
-	notmuch->path[strlen (notmuch->path) - 1] = '\0';
+    strip_trailing(notmuch->path, '/');
 
     notmuch->mode = mode;
     notmuch->atomic_nesting = 0;
@@ -1182,7 +1046,7 @@ _notmuch_database_reopen (notmuch_database_t *notmuch)
 	if (! notmuch->exception_reported) {
 	    _notmuch_database_log (notmuch, "Error: A Xapian exception reopening database: %s\n",
 				   error.get_msg ().c_str ());
-	    notmuch->exception_reported = TRUE;
+	    notmuch->exception_reported = true;
 	}
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
@@ -1260,7 +1124,7 @@ notmuch_database_compact (const char *path,
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
     notmuch_database_t *notmuch = NULL;
     struct stat statbuf;
-    notmuch_bool_t keep_backup;
+    bool keep_backup;
     char *message = NULL;
 
     local = talloc_new (NULL);
@@ -1296,10 +1160,10 @@ notmuch_database_compact (const char *path,
 	    ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
 	    goto DONE;
 	}
-	keep_backup = FALSE;
+	keep_backup = false;
     }
     else {
-	keep_backup = TRUE;
+	keep_backup = true;
     }
 
     if (stat (backup_path, &statbuf) != -1) {
@@ -1455,7 +1319,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
     Xapian::WritableDatabase *db;
     struct sigaction action;
     struct itimerval timerval;
-    notmuch_bool_t timer_is_active = FALSE;
+    bool timer_is_active = false;
     enum _notmuch_features target_features, new_features;
     notmuch_status_t status;
     notmuch_private_status_t private_status;
@@ -1489,7 +1353,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
 	timerval.it_value.tv_usec = 0;
 	setitimer (ITIMER_REAL, &timerval, NULL);
 
-	timer_is_active = TRUE;
+	timer_is_active = true;
     }
 
     /* Figure out how much total work we need to do. */
@@ -1730,7 +1594,7 @@ notmuch_database_begin_atomic (notmuch_database_t *notmuch)
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "A Xapian exception occurred beginning transaction: %s.\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
@@ -1764,13 +1628,13 @@ notmuch_database_end_atomic (notmuch_database_t *notmuch)
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "A Xapian exception occurred committing transaction: %s.\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
     if (notmuch->atomic_dirty) {
 	++notmuch->revision;
-	notmuch->atomic_dirty = FALSE;
+	notmuch->atomic_dirty = false;
     }
 
 DONE:
@@ -2013,7 +1877,7 @@ notmuch_database_get_directory (notmuch_database_t *notmuch,
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "A Xapian exception occurred getting directory: %s.\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
     return status;
@@ -2044,583 +1908,6 @@ _notmuch_database_generate_doc_id (notmuch_database_t *notmuch)
     return notmuch->last_doc_id;
 }
 
-static const char *
-_notmuch_database_generate_thread_id (notmuch_database_t *notmuch)
-{
-    /* 16 bytes (+ terminator) for hexadecimal representation of
-     * a 64-bit integer. */
-    static char thread_id[17];
-    Xapian::WritableDatabase *db;
-
-    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
-
-    notmuch->last_thread_id++;
-
-    sprintf (thread_id, "%016" PRIx64, notmuch->last_thread_id);
-
-    db->set_metadata ("last_thread_id", thread_id);
-
-    return thread_id;
-}
-
-static char *
-_get_metadata_thread_id_key (void *ctx, const char *message_id)
-{
-    if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
-	message_id = _notmuch_message_id_compressed (ctx, message_id);
-
-    return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s",
-			    message_id);
-}
-
-static notmuch_status_t
-_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
-				      void *ctx,
-				      const char *message_id,
-				      const char **thread_id_ret);
-
-/* Find the thread ID to which the message with 'message_id' belongs.
- *
- * Note: 'thread_id_ret' must not be NULL!
- * On success '*thread_id_ret' is set to a newly talloced string belonging to
- * 'ctx'.
- *
- * Note: If there is no message in the database with the given
- * 'message_id' then a new thread_id will be allocated for this
- * message ID and stored in the database metadata so that the
- * thread ID can be looked up if the message is added to the database
- * later.
- */
-static notmuch_status_t
-_resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
-				  void *ctx,
-				  const char *message_id,
-				  const char **thread_id_ret)
-{
-    notmuch_private_status_t status;
-    notmuch_message_t *message;
-
-    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS))
-	return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id,
-						     thread_id_ret);
-
-    /* Look for this message (regular or ghost) */
-    message = _notmuch_message_create_for_message_id (
-	notmuch, message_id, &status);
-    if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
-	/* Message exists */
-	*thread_id_ret = talloc_steal (
-	    ctx, notmuch_message_get_thread_id (message));
-    } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
-	/* Message did not exist.  Give it a fresh thread ID and
-	 * populate this message as a ghost message. */
-	*thread_id_ret = talloc_strdup (
-	    ctx, _notmuch_database_generate_thread_id (notmuch));
-	if (! *thread_id_ret) {
-	    status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY;
-	} else {
-	    status = _notmuch_message_initialize_ghost (message, *thread_id_ret);
-	    if (status == 0)
-		/* Commit the new ghost message */
-		_notmuch_message_sync (message);
-	}
-    } else {
-	/* Create failed. Fall through. */
-    }
-
-    notmuch_message_destroy (message);
-
-    return COERCE_STATUS (status, "Error creating ghost message");
-}
-
-/* Pre-ghost messages _resolve_message_id_to_thread_id */
-static notmuch_status_t
-_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch,
-				      void *ctx,
-				      const char *message_id,
-				      const char **thread_id_ret)
-{
-    notmuch_status_t status;
-    notmuch_message_t *message;
-    string thread_id_string;
-    char *metadata_key;
-    Xapian::WritableDatabase *db;
-
-    status = notmuch_database_find_message (notmuch, message_id, &message);
-
-    if (status)
-	return status;
-
-    if (message) {
-	*thread_id_ret = talloc_steal (ctx,
-				       notmuch_message_get_thread_id (message));
-
-	notmuch_message_destroy (message);
-
-	return NOTMUCH_STATUS_SUCCESS;
-    }
-
-    /* Message has not been seen yet.
-     *
-     * We may have seen a reference to it already, in which case, we
-     * can return the thread ID stored in the metadata. Otherwise, we
-     * generate a new thread ID and store it there.
-     */
-    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
-    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
-    thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
-
-    if (thread_id_string.empty()) {
-	*thread_id_ret = talloc_strdup (ctx,
-					_notmuch_database_generate_thread_id (notmuch));
-	db->set_metadata (metadata_key, *thread_id_ret);
-    } else {
-	*thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str());
-    }
-
-    talloc_free (metadata_key);
-
-    return NOTMUCH_STATUS_SUCCESS;
-}
-
-static notmuch_status_t
-_merge_threads (notmuch_database_t *notmuch,
-		const char *winner_thread_id,
-		const char *loser_thread_id)
-{
-    Xapian::PostingIterator loser, loser_end;
-    notmuch_message_t *message = NULL;
-    notmuch_private_status_t private_status;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
-
-    find_doc_ids (notmuch, "thread", loser_thread_id, &loser, &loser_end);
-
-    for ( ; loser != loser_end; loser++) {
-	message = _notmuch_message_create (notmuch, notmuch,
-					   *loser, &private_status);
-	if (message == NULL) {
-	    ret = COERCE_STATUS (private_status,
-				 "Cannot find document for doc_id from query");
-	    goto DONE;
-	}
-
-	_notmuch_message_remove_term (message, "thread", loser_thread_id);
-	_notmuch_message_add_term (message, "thread", winner_thread_id);
-	_notmuch_message_sync (message);
-
-	notmuch_message_destroy (message);
-	message = NULL;
-    }
-
-  DONE:
-    if (message)
-	notmuch_message_destroy (message);
-
-    return ret;
-}
-
-static void
-_my_talloc_free_for_g_hash (void *ptr)
-{
-    talloc_free (ptr);
-}
-
-static notmuch_status_t
-_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
-					   notmuch_message_t *message,
-					   notmuch_message_file_t *message_file,
-					   const char **thread_id)
-{
-    GHashTable *parents = NULL;
-    const char *refs, *in_reply_to, *in_reply_to_message_id;
-    const char *last_ref_message_id, *this_message_id;
-    GList *l, *keys = NULL;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
-
-    parents = g_hash_table_new_full (g_str_hash, g_str_equal,
-				     _my_talloc_free_for_g_hash, NULL);
-    this_message_id = notmuch_message_get_message_id (message);
-
-    refs = _notmuch_message_file_get_header (message_file, "references");
-    last_ref_message_id = parse_references (message,
-					    this_message_id,
-					    parents, refs);
-
-    in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
-    in_reply_to_message_id = parse_references (message,
-					       this_message_id,
-					       parents, in_reply_to);
-
-    /* For the parent of this message, use the last message ID of the
-     * References header, if available.  If not, fall back to the
-     * first message ID in the In-Reply-To header. */
-    if (last_ref_message_id) {
-	_notmuch_message_add_term (message, "replyto",
-				   last_ref_message_id);
-    } else if (in_reply_to_message_id) {
-	_notmuch_message_add_term (message, "replyto",
-			     in_reply_to_message_id);
-    }
-
-    keys = g_hash_table_get_keys (parents);
-    for (l = keys; l; l = l->next) {
-	char *parent_message_id;
-	const char *parent_thread_id = NULL;
-
-	parent_message_id = (char *) l->data;
-
-	_notmuch_message_add_term (message, "reference",
-				   parent_message_id);
-
-	ret = _resolve_message_id_to_thread_id (notmuch,
-						message,
-						parent_message_id,
-						&parent_thread_id);
-	if (ret)
-	    goto DONE;
-
-	if (*thread_id == NULL) {
-	    *thread_id = talloc_strdup (message, parent_thread_id);
-	    _notmuch_message_add_term (message, "thread", *thread_id);
-	} else if (strcmp (*thread_id, parent_thread_id)) {
-	    ret = _merge_threads (notmuch, *thread_id, parent_thread_id);
-	    if (ret)
-		goto DONE;
-	}
-    }
-
-  DONE:
-    if (keys)
-	g_list_free (keys);
-    if (parents)
-	g_hash_table_unref (parents);
-
-    return ret;
-}
-
-static notmuch_status_t
-_notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
-					    notmuch_message_t *message,
-					    const char **thread_id)
-{
-    const char *message_id = notmuch_message_get_message_id (message);
-    Xapian::PostingIterator child, children_end;
-    notmuch_message_t *child_message = NULL;
-    const char *child_thread_id;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
-    notmuch_private_status_t private_status;
-
-    find_doc_ids (notmuch, "reference", message_id, &child, &children_end);
-
-    for ( ; child != children_end; child++) {
-
-	child_message = _notmuch_message_create (message, notmuch,
-						 *child, &private_status);
-	if (child_message == NULL) {
-	    ret = COERCE_STATUS (private_status,
-				 "Cannot find document for doc_id from query");
-	    goto DONE;
-	}
-
-	child_thread_id = notmuch_message_get_thread_id (child_message);
-	if (*thread_id == NULL) {
-	    *thread_id = talloc_strdup (message, child_thread_id);
-	    _notmuch_message_add_term (message, "thread", *thread_id);
-	} else if (strcmp (*thread_id, child_thread_id)) {
-	    _notmuch_message_remove_term (child_message, "reference",
-					  message_id);
-	    _notmuch_message_sync (child_message);
-	    ret = _merge_threads (notmuch, *thread_id, child_thread_id);
-	    if (ret)
-		goto DONE;
-	}
-
-	notmuch_message_destroy (child_message);
-	child_message = NULL;
-    }
-
-  DONE:
-    if (child_message)
-	notmuch_message_destroy (child_message);
-
-    return ret;
-}
-
-/* Fetch and clear the stored thread_id for message, or NULL if none. */
-static char *
-_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
-			     notmuch_message_t *message)
-{
-    const char *message_id;
-    string stored_id;
-    char *metadata_key;
-
-    message_id = notmuch_message_get_message_id (message);
-    metadata_key = _get_metadata_thread_id_key (ctx, message_id);
-
-    /* Check if we have already seen related messages to this one.
-     * If we have then use the thread_id that we stored at that time.
-     */
-    stored_id = notmuch->xapian_db->get_metadata (metadata_key);
-    if (stored_id.empty ()) {
-	return NULL;
-    } else {
-	Xapian::WritableDatabase *db;
-
-	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
-
-	/* Clear the metadata for this message ID. We don't need it
-	 * anymore. */
-	db->set_metadata (metadata_key, "");
-
-	return talloc_strdup (ctx, stored_id.c_str ());
-    }
-}
-
-/* Given a blank or ghost 'message' and its corresponding
- * 'message_file' link it to existing threads in the database.
- *
- * First, if is_ghost, this retrieves the thread ID already stored in
- * the message (which will be the case if a message was previously
- * added that referenced this one).  If the message is blank
- * (!is_ghost), it doesn't have a thread ID yet (we'll generate one
- * later in this function).  If the database does not support ghost
- * messages, this checks for a thread ID stored in database metadata
- * for this message ID.
- *
- * Second, we look at 'message_file' and its link-relevant headers
- * (References and In-Reply-To) for message IDs.
- *
- * Finally, we look in the database for existing message that
- * reference 'message'.
- *
- * In all cases, we assign to the current message the first thread ID
- * found. We will also merge any existing, distinct threads where this
- * message belongs to both, (which is not uncommon when messages are
- * processed out of order).
- *
- * Finally, if no thread ID has been found through referenced messages, we
- * call _notmuch_message_generate_thread_id to generate a new thread
- * ID. This should only happen for new, top-level messages, (no
- * References or In-Reply-To header in this message, and no previously
- * added message refers to this message).
- */
-static notmuch_status_t
-_notmuch_database_link_message (notmuch_database_t *notmuch,
-				notmuch_message_t *message,
-				notmuch_message_file_t *message_file,
-				notmuch_bool_t is_ghost)
-{
-    void *local = talloc_new (NULL);
-    notmuch_status_t status;
-    const char *thread_id = NULL;
-
-    /* Check if the message already had a thread ID */
-    if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) {
-	if (is_ghost)
-	    thread_id = notmuch_message_get_thread_id (message);
-    } else {
-	thread_id = _consume_metadata_thread_id (local, notmuch, message);
-	if (thread_id)
-	    _notmuch_message_add_term (message, "thread", thread_id);
-    }
-
-    status = _notmuch_database_link_message_to_parents (notmuch, message,
-							message_file,
-							&thread_id);
-    if (status)
-	goto DONE;
-
-    if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) {
-	/* In general, it shouldn't be necessary to link children,
-	 * since the earlier indexing of those children will have
-	 * stored a thread ID for the missing parent.  However, prior
-	 * to ghost messages, these stored thread IDs were NOT
-	 * rewritten during thread merging (and there was no
-	 * performant way to do so), so if indexed children were
-	 * pulled into a different thread ID by a merge, it was
-	 * necessary to pull them *back* into the stored thread ID of
-	 * the parent.  With ghost messages, we just rewrite the
-	 * stored thread IDs during merging, so this workaround isn't
-	 * necessary. */
-	status = _notmuch_database_link_message_to_children (notmuch, message,
-							     &thread_id);
-	if (status)
-	    goto DONE;
-    }
-
-    /* If not part of any existing thread, generate a new thread ID. */
-    if (thread_id == NULL) {
-	thread_id = _notmuch_database_generate_thread_id (notmuch);
-
-	_notmuch_message_add_term (message, "thread", thread_id);
-    }
-
- DONE:
-    talloc_free (local);
-
-    return status;
-}
-
-notmuch_status_t
-notmuch_database_add_message (notmuch_database_t *notmuch,
-			      const char *filename,
-			      notmuch_message_t **message_ret)
-{
-    notmuch_message_file_t *message_file;
-    notmuch_message_t *message = NULL;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
-    notmuch_private_status_t private_status;
-    notmuch_bool_t is_ghost = false;
-
-    const char *date, *header;
-    const char *from, *to, *subject;
-    char *message_id = NULL;
-
-    if (message_ret)
-	*message_ret = NULL;
-
-    ret = _notmuch_database_ensure_writable (notmuch);
-    if (ret)
-	return ret;
-
-    message_file = _notmuch_message_file_open (notmuch, filename);
-    if (message_file == NULL)
-	return NOTMUCH_STATUS_FILE_ERROR;
-
-    /* Adding a message may change many documents.  Do this all
-     * atomically. */
-    ret = notmuch_database_begin_atomic (notmuch);
-    if (ret)
-	goto DONE;
-
-    /* Parse message up front to get better error status. */
-    ret = _notmuch_message_file_parse (message_file);
-    if (ret)
-	goto DONE;
-
-    /* Before we do any real work, (especially before doing a
-     * potential SHA-1 computation on the entire file's contents),
-     * let's make sure that what we're looking at looks like an
-     * actual email message.
-     */
-    from = _notmuch_message_file_get_header (message_file, "from");
-    subject = _notmuch_message_file_get_header (message_file, "subject");
-    to = _notmuch_message_file_get_header (message_file, "to");
-
-    if ((from == NULL || *from == '\0') &&
-	(subject == NULL || *subject == '\0') &&
-	(to == NULL || *to == '\0')) {
-	ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
-	goto DONE;
-    }
-
-    /* Now that we're sure it's mail, the first order of business
-     * is to find a message ID (or else create one ourselves).
-     */
-    header = _notmuch_message_file_get_header (message_file, "message-id");
-    if (header && *header != '\0') {
-	message_id = _parse_message_id (message_file, header, NULL);
-
-	/* So the header value isn't RFC-compliant, but it's
-	 * better than no message-id at all.
-	 */
-	if (message_id == NULL)
-	    message_id = talloc_strdup (message_file, header);
-    }
-
-    if (message_id == NULL ) {
-	/* No message-id at all, let's generate one by taking a
-	 * hash over the file's contents.
-	 */
-	char *sha1 = _notmuch_sha1_of_file (filename);
-
-	/* If that failed too, something is really wrong. Give up. */
-	if (sha1 == NULL) {
-	    ret = NOTMUCH_STATUS_FILE_ERROR;
-	    goto DONE;
-	}
-
-	message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
-	free (sha1);
-    }
-
-    try {
-	/* Now that we have a message ID, we get a message object,
-	 * (which may or may not reference an existing document in the
-	 * database). */
-
-	message = _notmuch_message_create_for_message_id (notmuch,
-							  message_id,
-							  &private_status);
-
-	talloc_free (message_id);
-
-	if (message == NULL) {
-	    ret = COERCE_STATUS (private_status,
-				 "Unexpected status value from _notmuch_message_create_for_message_id");
-	    goto DONE;
-	}
-
-	_notmuch_message_add_filename (message, filename);
-
-	/* Is this a newly created message object or a ghost
-	 * message?  We have to be slightly careful: if this is a
-	 * blank message, it's not safe to call
-	 * notmuch_message_get_flag yet. */
-	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND ||
-	    (is_ghost = notmuch_message_get_flag (
-		message, NOTMUCH_MESSAGE_FLAG_GHOST))) {
-	    _notmuch_message_add_term (message, "type", "mail");
-	    if (is_ghost)
-		/* Convert ghost message to a regular message */
-		_notmuch_message_remove_term (message, "type", "ghost");
-
-	    ret = _notmuch_database_link_message (notmuch, message,
-						  message_file, is_ghost);
-	    if (ret)
-		goto DONE;
-
-	    date = _notmuch_message_file_get_header (message_file, "date");
-	    _notmuch_message_set_header_values (message, date, from, subject);
-
-	    ret = _notmuch_message_index_file (message, message_file);
-	    if (ret)
-		goto DONE;
-	} else {
-	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-	}
-
-	_notmuch_message_sync (message);
-    } catch (const Xapian::Error &error) {
-	_notmuch_database_log (notmuch, "A Xapian exception occurred adding message: %s.\n",
-		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
-	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
-	goto DONE;
-    }
-
-  DONE:
-    if (message) {
-	if ((ret == NOTMUCH_STATUS_SUCCESS ||
-	     ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) && message_ret)
-	    *message_ret = message;
-	else
-	    notmuch_message_destroy (message);
-    }
-
-    if (message_file)
-	_notmuch_message_file_close (message_file);
-
-    ret2 = notmuch_database_end_atomic (notmuch);
-    if ((ret == NOTMUCH_STATUS_SUCCESS ||
-	 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
-	ret2 != NOTMUCH_STATUS_SUCCESS)
-	ret = ret2;
-
-    return ret;
-}
-
 notmuch_status_t
 notmuch_database_remove_message (notmuch_database_t *notmuch,
 				 const char *filename)
@@ -2687,7 +1974,7 @@ notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred finding message by filename: %s\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
@@ -2740,7 +2027,7 @@ notmuch_database_get_all_tags (notmuch_database_t *db)
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (db, "A Xapian exception occurred getting tags: %s.\n",
 		 error.get_msg().c_str());
-	db->exception_reported = TRUE;
+	db->exception_reported = true;
 	return NULL;
     }
 }
diff --git a/lib/directory.cc b/lib/directory.cc
index 5de3319c..4fcb0177 100644
--- a/lib/directory.cc
+++ b/lib/directory.cc
@@ -103,7 +103,7 @@ _notmuch_directory_create (notmuch_database_t *notmuch,
     notmuch_directory_t *directory;
     notmuch_private_status_t private_status;
     const char *db_path;
-    notmuch_bool_t create = (flags & NOTMUCH_FIND_CREATE);
+    bool create = (flags & NOTMUCH_FIND_CREATE);
 
     if (! (notmuch->features & NOTMUCH_FEATURE_DIRECTORY_DOCS)) {
 	*status_ret = NOTMUCH_STATUS_UPGRADE_REQUIRED;
@@ -189,7 +189,7 @@ _notmuch_directory_create (notmuch_database_t *notmuch,
 	_notmuch_database_log (notmuch,
 		 "A Xapian exception occurred creating a directory: %s.\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	notmuch_directory_destroy (directory);
 	directory = NULL;
 	*status_ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
@@ -234,7 +234,7 @@ notmuch_directory_set_mtime (notmuch_directory_t *directory,
 	_notmuch_database_log (notmuch,
 		 "A Xapian exception occurred setting directory mtime: %s.\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
@@ -301,7 +301,7 @@ notmuch_directory_delete (notmuch_directory_t *directory)
 	_notmuch_database_log (directory->notmuch,
 			       "A Xapian exception occurred deleting directory entry: %s.\n",
 			       error.get_msg().c_str());
-	directory->notmuch->exception_reported = TRUE;
+	directory->notmuch->exception_reported = true;
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
     notmuch_directory_destroy (directory);
diff --git a/lib/filenames.c b/lib/filenames.c
index 63e737dd..37d631d6 100644
--- a/lib/filenames.c
+++ b/lib/filenames.c
@@ -46,7 +46,7 @@ notmuch_bool_t
 notmuch_filenames_valid (notmuch_filenames_t *filenames)
 {
     if (filenames == NULL)
-	return FALSE;
+	return false;
 
     return (filenames->iterator != NULL);
 }
diff --git a/lib/index.cc b/lib/index.cc
index 10420d84..0ad683fa 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -184,7 +184,7 @@ filter_filter (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t pres
 
     int next;
 
-    g_mime_filter_set_size (gmime_filter, inlen, FALSE);
+    g_mime_filter_set_size (gmime_filter, inlen, false);
     outptr = gmime_filter->outbuf;
 
     next = filter->state;
@@ -261,7 +261,7 @@ notmuch_filter_discard_non_term_new (GMimeContentType *content_type)
 	type = g_type_register_static (GMIME_TYPE_FILTER, "NotmuchFilterDiscardNonTerm", &info, (GTypeFlags) 0);
     }
 
-    filter = (NotmuchFilterDiscardNonTerm *) g_object_newv (type, 0, NULL);
+    filter = (NotmuchFilterDiscardNonTerm *) g_object_new (type, NULL);
     filter->content_type = content_type;
     filter->state = 0;
     if (g_mime_content_type_is_type (content_type, "text", "html")) {
@@ -351,9 +351,28 @@ _index_address_list (notmuch_message_t *message,
     }
 }
 
+static void
+_index_content_type (notmuch_message_t *message, GMimeObject *part)
+{
+    GMimeContentType *content_type = g_mime_object_get_content_type (part);
+    if (content_type) {
+	char *mime_string = g_mime_content_type_to_string (content_type);
+	if (mime_string) {
+	    _notmuch_message_gen_terms (message, "mimetype", mime_string);
+	    g_free (mime_string);
+	}
+    }
+}
+
+static void
+_index_encrypted_mime_part (notmuch_message_t *message, notmuch_indexopts_t *indexopts,
+			    GMimeContentType *content_type,
+			    GMimeMultipartEncrypted *part);
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
+		  notmuch_indexopts_t *indexopts,
 		  GMimeObject *part)
 {
     GMimeStream *stream, *filter;
@@ -361,6 +380,7 @@ _index_mime_part (notmuch_message_t *message,
     GMimeDataWrapper *wrapper;
     GByteArray *byte_array;
     GMimeContentDisposition *disposition;
+    GMimeContentType *content_type;
     char *body;
     const char *charset;
 
@@ -370,15 +390,8 @@ _index_mime_part (notmuch_message_t *message,
 	return;
     }
 
-    GMimeContentType *content_type = g_mime_object_get_content_type(part);
-    if (content_type) {
-	char *mime_string = g_mime_content_type_to_string(content_type);
-	if (mime_string)
-	{
-	    _notmuch_message_gen_terms (message, "mimetype", mime_string);
-	    g_free(mime_string);
-	}
-    }
+    _index_content_type (message, part);
+    content_type = g_mime_object_get_content_type (part);
 
     if (GMIME_IS_MULTIPART (part)) {
 	GMimeMultipart *multipart = GMIME_MULTIPART (part);
@@ -392,18 +405,32 @@ _index_mime_part (notmuch_message_t *message,
 
 	for (i = 0; i < g_mime_multipart_get_count (multipart); i++) {
 	    if (GMIME_IS_MULTIPART_SIGNED (multipart)) {
-		/* Don't index the signature. */
-		if (i == 1)
+		/* Don't index the signature, but index its content type. */
+		if (i == GMIME_MULTIPART_SIGNED_SIGNATURE) {
+		    _index_content_type (message,
+					 g_mime_multipart_get_part (multipart, i));
 		    continue;
-		if (i > 1)
+		} else if (i != GMIME_MULTIPART_SIGNED_CONTENT) {
 		    _notmuch_database_log (_notmuch_message_database (message),
-					  "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+					   "Warning: Unexpected extra parts of multipart/signed. Indexing anyway.\n");
+		}
 	    }
 	    if (GMIME_IS_MULTIPART_ENCRYPTED (multipart)) {
-		/* Don't index encrypted parts. */
+		_index_content_type (message,
+				     g_mime_multipart_get_part (multipart, i));
+		if (i == GMIME_MULTIPART_ENCRYPTED_CONTENT) {
+		    _index_encrypted_mime_part(message, indexopts,
+					       content_type,
+					       GMIME_MULTIPART_ENCRYPTED (part));
+		} else {
+		    if (i != GMIME_MULTIPART_ENCRYPTED_VERSION) {
+			_notmuch_database_log (_notmuch_message_database (message),
+					       "Warning: Unexpected extra parts of multipart/encrypted.\n");
+		    }
+		}
 		continue;
 	    }
-	    _index_mime_part (message,
+	    _index_mime_part (message, indexopts,
 			      g_mime_multipart_get_part (multipart, i));
 	}
 	return;
@@ -414,7 +441,7 @@ _index_mime_part (notmuch_message_t *message,
 
 	mime_message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part));
 
-	_index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+	_index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
 	return;
     }
@@ -444,9 +471,10 @@ _index_mime_part (notmuch_message_t *message,
     byte_array = g_byte_array_new ();
 
     stream = g_mime_stream_mem_new_with_byte_array (byte_array);
-    g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE);
+    g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), false);
 
     filter = g_mime_stream_filter_new (stream);
+
     discard_non_term_filter = notmuch_filter_discard_non_term_new (content_type);
 
     g_mime_stream_filter_add (GMIME_STREAM_FILTER (filter),
@@ -475,7 +503,7 @@ _index_mime_part (notmuch_message_t *message,
     g_object_unref (discard_non_term_filter);
 
     g_byte_array_append (byte_array, (guint8 *) "\0", 1);
-    body = (char *) g_byte_array_free (byte_array, FALSE);
+    body = (char *) g_byte_array_free (byte_array, false);
 
     if (body) {
 	_notmuch_message_gen_terms (message, NULL, body);
@@ -484,8 +512,91 @@ _index_mime_part (notmuch_message_t *message,
     }
 }
 
+/* descend (if desired) into the cleartext part of an encrypted MIME
+ * part while indexing. */
+static void
+_index_encrypted_mime_part (notmuch_message_t *message,
+			    notmuch_indexopts_t *indexopts,
+			    g_mime_3_unused(GMimeContentType *content_type),
+			    GMimeMultipartEncrypted *encrypted_data)
+{
+    notmuch_status_t status;
+    GError *err = NULL;
+    notmuch_database_t * notmuch = NULL;
+    GMimeObject *clear = NULL;
+
+    if (!indexopts || (notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_FALSE))
+	return;
+
+    notmuch = _notmuch_message_database (message);
+
+    GMimeCryptoContext* crypto_ctx = NULL;
+#if (GMIME_MAJOR_VERSION < 3)
+    {
+	const char *protocol = NULL;
+	protocol = g_mime_content_type_get_parameter (content_type, "protocol");
+	status = _notmuch_crypto_get_gmime_ctx_for_protocol (&(indexopts->crypto),
+							 protocol, &crypto_ctx);
+	if (status) {
+	    _notmuch_database_log (notmuch, "Warning: setup failed for decrypting "
+				   "during indexing. (%d)\n", status);
+	    status = notmuch_message_add_property (message, "index.decryption", "failure");
+	    if (status)
+		_notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					      "property (%d)\n", status);
+	    return;
+	}
+    }
+#endif
+    bool attempted = false;
+    GMimeDecryptResult *decrypt_result = NULL;
+    bool get_sk = (HAVE_GMIME_SESSION_KEYS && notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_TRUE);
+    clear = _notmuch_crypto_decrypt (&attempted, notmuch_indexopts_get_decrypt_policy (indexopts),
+				     message, crypto_ctx, encrypted_data, get_sk ? &decrypt_result : NULL, &err);
+    if (!attempted)
+	return;
+    if (err || !clear) {
+	if (decrypt_result)
+	    g_object_unref (decrypt_result);
+	if (err) {
+	    _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (%d:%d) [%s]\n",
+				   err->domain, err->code, err->message);
+	    g_error_free(err);
+	} else {
+	    _notmuch_database_log (notmuch, "Failed to decrypt during indexing. (unknown error)\n");
+	}
+	/* Indicate that we failed to decrypt during indexing */
+	status = notmuch_message_add_property (message, "index.decryption", "failure");
+	if (status)
+	    _notmuch_database_log_append (notmuch, "failed to add index.decryption "
+					  "property (%d)\n", status);
+	return;
+    }
+    if (decrypt_result) {
+#if HAVE_GMIME_SESSION_KEYS
+	if (get_sk) {
+	    status = notmuch_message_add_property (message, "session-key",
+						   g_mime_decrypt_result_get_session_key (decrypt_result));
+	    if (status)
+		_notmuch_database_log (notmuch, "failed to add session-key "
+				       "property (%d)\n", status);
+	}
+#endif
+	g_object_unref (decrypt_result);
+    }
+    _index_mime_part (message, indexopts, clear);
+    g_object_unref (clear);
+
+    status = notmuch_message_add_property (message, "index.decryption", "success");
+    if (status)
+	_notmuch_database_log (notmuch, "failed to add index.decryption "
+			       "property (%d)\n", status);
+
+}
+
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file)
 {
     GMimeMessage *mime_message;
@@ -513,7 +624,7 @@ _notmuch_message_index_file (notmuch_message_t *message,
     subject = g_mime_message_get_subject (mime_message);
     _notmuch_message_gen_terms (message, "subject", subject);
 
-    _index_mime_part (message, g_mime_message_get_mime_part (mime_message));
+    _index_mime_part (message, indexopts, g_mime_message_get_mime_part (mime_message));
 
     return NOTMUCH_STATUS_SUCCESS;
 }
diff --git a/lib/indexopts.c b/lib/indexopts.c
new file mode 100644
index 00000000..b78a57b6
--- /dev/null
+++ b/lib/indexopts.c
@@ -0,0 +1,75 @@
+/* indexopts.c - options for indexing messages (currently a stub)
+ *
+ * Copyright © 2017 Daniel Kahn Gillmor
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
+ */
+
+#include "notmuch-private.h"
+
+notmuch_indexopts_t *
+notmuch_database_get_default_indexopts (notmuch_database_t *db)
+{
+    notmuch_indexopts_t *ret = talloc_zero (db, notmuch_indexopts_t);
+    if (!ret)
+	return ret;
+    ret->crypto.decrypt = NOTMUCH_DECRYPT_AUTO;
+
+    char * decrypt_policy;
+    notmuch_status_t err = notmuch_database_get_config (db, "index.decrypt", &decrypt_policy);
+    if (err)
+	return ret;
+
+    if (decrypt_policy) {
+	if ((!(strcasecmp(decrypt_policy, "true"))) ||
+	    (!(strcasecmp(decrypt_policy, "yes"))) ||
+	    (!(strcasecmp(decrypt_policy, "1"))))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_TRUE);
+	else if ((!(strcasecmp(decrypt_policy, "false"))) ||
+		 (!(strcasecmp(decrypt_policy, "no"))) ||
+		 (!(strcasecmp(decrypt_policy, "0"))))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_FALSE);
+	else if (!strcasecmp(decrypt_policy, "nostash"))
+	    notmuch_indexopts_set_decrypt_policy (ret, NOTMUCH_DECRYPT_NOSTASH);
+    }
+
+    free (decrypt_policy);
+    return ret;
+}
+
+notmuch_status_t
+notmuch_indexopts_set_decrypt_policy (notmuch_indexopts_t *indexopts,
+				      notmuch_decryption_policy_t decrypt_policy)
+{
+    if (!indexopts)
+	return NOTMUCH_STATUS_NULL_POINTER;
+    indexopts->crypto.decrypt = decrypt_policy;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_decryption_policy_t
+notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts)
+{
+    if (!indexopts)
+	return false;
+    return indexopts->crypto.decrypt;
+}
+
+void
+notmuch_indexopts_destroy (notmuch_indexopts_t *indexopts)
+{
+    talloc_free (indexopts);
+}
diff --git a/lib/message-file.c b/lib/message-file.c
index d7acf0d5..8f0dbbda 100644
--- a/lib/message-file.c
+++ b/lib/message-file.c
@@ -92,22 +92,28 @@ _notmuch_message_file_open (notmuch_database_t *notmuch,
     return _notmuch_message_file_open_ctx (notmuch, NULL, filename);
 }
 
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message_file)
+{
+    return message_file->filename;
+}
+
 void
 _notmuch_message_file_close (notmuch_message_file_t *message)
 {
     talloc_free (message);
 }
 
-static notmuch_bool_t
+static bool
 _is_mbox (FILE *file)
 {
     char from_buf[5];
-    notmuch_bool_t ret = FALSE;
+    bool ret = false;
 
     /* Is this mbox? */
     if (fread (from_buf, sizeof (from_buf), 1, file) == 1 &&
 	strncmp (from_buf, "From ", 5) == 0)
-	ret = TRUE;
+	ret = true;
 
     rewind (file);
 
@@ -121,7 +127,7 @@ _notmuch_message_file_parse (notmuch_message_file_t *message)
     GMimeParser *parser;
     notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
     static int initialized = 0;
-    notmuch_bool_t is_mbox;
+    bool is_mbox;
 
     if (message->message)
 	return NOTMUCH_STATUS_SUCCESS;
@@ -141,7 +147,7 @@ _notmuch_message_file_parse (notmuch_message_file_t *message)
     stream = g_mime_stream_file_new (message->file);
 
     /* We'll own and fclose the FILE* ourselves. */
-    g_mime_stream_file_set_owner (GMIME_STREAM_FILE (stream), FALSE);
+    g_mime_stream_file_set_owner (GMIME_STREAM_FILE (stream), false);
 
     parser = g_mime_parser_new_with_stream (stream);
     g_mime_parser_set_scan_from (parser, is_mbox);
@@ -345,3 +351,83 @@ _notmuch_message_file_get_header (notmuch_message_file_t *message,
 
     return decoded;
 }
+
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+				   const char **from_out,
+				   const char **subject_out,
+				   const char **to_out,
+				   const char **date_out,
+				   char **message_id_out)
+{
+    notmuch_status_t ret;
+    const char *header;
+    const char *from, *to, *subject, *date;
+    char *message_id = NULL;
+
+    /* Parse message up front to get better error status. */
+    ret = _notmuch_message_file_parse (message_file);
+    if (ret)
+	goto DONE;
+
+    /* Before we do any real work, (especially before doing a
+     * potential SHA-1 computation on the entire file's contents),
+     * let's make sure that what we're looking at looks like an
+     * actual email message.
+     */
+    from = _notmuch_message_file_get_header (message_file, "from");
+    subject = _notmuch_message_file_get_header (message_file, "subject");
+    to = _notmuch_message_file_get_header (message_file, "to");
+    date = _notmuch_message_file_get_header (message_file, "date");
+
+    if ((from == NULL || *from == '\0') &&
+	(subject == NULL || *subject == '\0') &&
+	(to == NULL || *to == '\0')) {
+	ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+	goto DONE;
+    }
+
+    /* Now that we're sure it's mail, the first order of business
+     * is to find a message ID (or else create one ourselves).
+     */
+    header = _notmuch_message_file_get_header (message_file, "message-id");
+    if (header && *header != '\0') {
+	message_id = _notmuch_message_id_parse (message_file, header, NULL);
+
+	/* So the header value isn't RFC-compliant, but it's
+	 * better than no message-id at all.
+	 */
+	if (message_id == NULL)
+	    message_id = talloc_strdup (message_file, header);
+    }
+
+    if (message_id == NULL ) {
+	/* No message-id at all, let's generate one by taking a
+	 * hash over the file's contents.
+	 */
+	char *sha1 = _notmuch_sha1_of_file (_notmuch_message_file_get_filename (message_file));
+
+	/* If that failed too, something is really wrong. Give up. */
+	if (sha1 == NULL) {
+	    ret = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+
+	message_id = talloc_asprintf (message_file, "notmuch-sha1-%s", sha1);
+	free (sha1);
+    }
+ DONE:
+    if (ret == NOTMUCH_STATUS_SUCCESS) {
+	if (from_out)
+	    *from_out = from;
+	if (subject_out)
+	    *subject_out = subject;
+	if (to_out)
+	    *to_out = to;
+	if (date_out)
+	    *date_out = date;
+	if (message_id_out)
+	    *message_id_out = message_id;
+    }
+    return ret;
+}
diff --git a/lib/message-id.c b/lib/message-id.c
new file mode 100644
index 00000000..d7541d50
--- /dev/null
+++ b/lib/message-id.c
@@ -0,0 +1,96 @@
+#include "notmuch-private.h"
+
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+    const char *s;
+
+    s = *str;
+    while (*s && (isspace (*s) || *s == '(')) {
+	while (*s && isspace (*s))
+	    s++;
+	if (*s == '(') {
+	    int nesting = 1;
+	    s++;
+	    while (*s && nesting) {
+		if (*s == '(') {
+		    nesting++;
+		} else if (*s == ')') {
+		    nesting--;
+		} else if (*s == '\\') {
+		    if (*(s+1))
+			s++;
+		}
+		s++;
+	    }
+	}
+    }
+
+    *str = s;
+}
+
+char *
+_notmuch_message_id_parse (void *ctx, const char *message_id, const char **next)
+{
+    const char *s, *end;
+    char *result;
+
+    if (message_id == NULL || *message_id == '\0')
+	return NULL;
+
+    s = message_id;
+
+    skip_space_and_comments (&s);
+
+    /* Skip any unstructured text as well. */
+    while (*s && *s != '<')
+	s++;
+
+    if (*s == '<') {
+	s++;
+    } else {
+	if (next)
+	    *next = s;
+	return NULL;
+    }
+
+    skip_space_and_comments (&s);
+
+    end = s;
+    while (*end && *end != '>')
+	end++;
+    if (next) {
+	if (*end)
+	    *next = end + 1;
+	else
+	    *next = end;
+    }
+
+    if (end > s && *end == '>')
+	end--;
+    if (end <= s)
+	return NULL;
+
+    result = talloc_strndup (ctx, s, end - s + 1);
+
+    /* Finally, collapse any whitespace that is within the message-id
+     * itself. */
+    {
+	char *r;
+	int len;
+
+	for (r = result, len = strlen (r); *r; r++, len--)
+	    if (*r == ' ' || *r == '\t')
+		memmove (r, r+1, len);
+    }
+
+    return result;
+}
diff --git a/lib/message-private.h b/lib/message-private.h
index 74199256..73b080e4 100644
--- a/lib/message-private.h
+++ b/lib/message-private.h
@@ -4,7 +4,7 @@
 notmuch_string_map_t *
 _notmuch_message_property_map (notmuch_message_t *message);
 
-notmuch_bool_t
+bool
 _notmuch_message_frozen (notmuch_message_t *message);
 
 void
diff --git a/lib/message-property.cc b/lib/message-property.cc
index f32d5550..35eaf3c6 100644
--- a/lib/message-property.cc
+++ b/lib/message-property.cc
@@ -38,7 +38,7 @@ notmuch_message_get_property (notmuch_message_t *message, const char *key, const
 
 static notmuch_status_t
 _notmuch_message_modify_property (notmuch_message_t *message, const char *key, const char *value,
-				  notmuch_bool_t delete_it)
+				  bool delete_it)
 {
     notmuch_private_status_t private_status;
     notmuch_status_t status;
@@ -76,17 +76,18 @@ _notmuch_message_modify_property (notmuch_message_t *message, const char *key, c
 notmuch_status_t
 notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value)
 {
-    return _notmuch_message_modify_property (message, key, value, FALSE);
+    return _notmuch_message_modify_property (message, key, value, false);
 }
 
 notmuch_status_t
 notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value)
 {
-    return _notmuch_message_modify_property (message, key, value, TRUE);
+    return _notmuch_message_modify_property (message, key, value, true);
 }
 
+static
 notmuch_status_t
-notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+_notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key, bool prefix)
 {
     notmuch_status_t status;
     const char * term_prefix;
@@ -97,7 +98,8 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
 
     _notmuch_message_invalidate_metadata (message, "property");
     if (key)
-	term_prefix = talloc_asprintf (message, "%s%s=", _find_prefix ("property"), key);
+	term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key,
+				       prefix ? "" : "=");
     else
 	term_prefix = _find_prefix ("property");
 
@@ -107,6 +109,18 @@ notmuch_message_remove_all_properties (notmuch_message_t *message, const char *k
     return NOTMUCH_STATUS_SUCCESS;
 }
 
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    return _notmuch_message_remove_all_properties (message, key, false);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix)
+{
+    return _notmuch_message_remove_all_properties (message, prefix, true);
+}
+
 notmuch_message_properties_t *
 notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
 {
diff --git a/lib/message.cc b/lib/message.cc
index f78e5a9d..d5db89b6 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -36,6 +36,7 @@ struct _notmuch_message {
     notmuch_string_list_t *tag_list;
     notmuch_string_list_t *filename_term_list;
     notmuch_string_list_t *filename_list;
+    char *maildir_flags;
     char *author;
     notmuch_message_file_t *message_file;
     notmuch_string_list_t *property_term_list;
@@ -47,7 +48,7 @@ struct _notmuch_message {
     unsigned long lazy_flags;
 
     /* Message document modified since last sync */
-    notmuch_bool_t modified;
+    bool modified;
 
     /* last view of database the struct is synced with */
     unsigned long last_view;
@@ -61,16 +62,16 @@ struct _notmuch_message {
 struct maildir_flag_tag {
     char flag;
     const char *tag;
-    notmuch_bool_t inverse;
+    bool inverse;
 };
 
 /* ASCII ordered table of Maildir flags and associated tags */
 static struct maildir_flag_tag flag2tag[] = {
-    { 'D', "draft",   FALSE},
-    { 'F', "flagged", FALSE},
-    { 'P', "passed",  FALSE},
-    { 'R', "replied", FALSE},
-    { 'S', "unread",  TRUE }
+    { 'D', "draft",   false},
+    { 'F', "flagged", false},
+    { 'P', "passed",  false},
+    { 'R', "replied", false},
+    { 'S', "unread",  true }
 };
 
 /* We end up having to call the destructor explicitly because we had
@@ -123,6 +124,7 @@ _notmuch_message_create_for_document (const void *talloc_owner,
     message->tag_list = NULL;
     message->filename_term_list = NULL;
     message->filename_list = NULL;
+    message->maildir_flags = NULL;
     message->message_file = NULL;
     message->author = NULL;
     message->property_term_list = NULL;
@@ -268,7 +270,7 @@ _notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log(_notmuch_message_database (message), "A Xapian exception occurred creating message: %s\n",
 		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	*status_ret = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
 	return NULL;
     }
@@ -525,7 +527,7 @@ notmuch_message_get_header (notmuch_message_t *message, const char *header)
 	} catch (Xapian::Error &error) {
 	    _notmuch_database_log(_notmuch_message_database (message), "A Xapian exception occurred when reading header: %s\n",
 		     error.get_msg().c_str());
-	    message->notmuch->exception_reported = TRUE;
+	    message->notmuch->exception_reported = true;
 	    return NULL;
 	}
     }
@@ -579,7 +581,9 @@ void
 _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
     Xapian::TermIterator i;
-    size_t prefix_len = strlen (prefix);
+    size_t prefix_len = 0;
+
+    prefix_len = strlen (prefix);
 
     while (1) {
 	i = message->doc.termlist_begin ();
@@ -592,11 +596,64 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 
 	try {
 	    message->doc.remove_term ((*i));
-	    message->modified = TRUE;
+	    message->modified = true;
+	} catch (const Xapian::InvalidArgumentError) {
+	    /* Ignore failure to remove non-existent term. */
+	}
+    }
+}
+
+
+/* Remove all terms generated by indexing, i.e. not tags or
+ * properties, along with any automatic tags*/
+notmuch_private_status_t
+_notmuch_message_remove_indexed_terms (notmuch_message_t *message)
+{
+    Xapian::TermIterator i;
+
+    const std::string
+	id_prefix = _find_prefix ("id"),
+	property_prefix = _find_prefix ("property"),
+	tag_prefix = _find_prefix ("tag"),
+	type_prefix = _find_prefix ("type");
+
+    for (i = message->doc.termlist_begin ();
+	 i != message->doc.termlist_end (); i++) {
+
+	const std::string term = *i;
+
+	if (term.compare (0, type_prefix.size (), type_prefix) == 0)
+	    continue;
+
+	if (term.compare (0, id_prefix.size (), id_prefix) == 0)
+	    continue;
+
+	if (term.compare (0, property_prefix.size (), property_prefix) == 0)
+	    continue;
+
+	if (term.compare (0, tag_prefix.size (), tag_prefix) == 0 &&
+	    term.compare (1, strlen("encrypted"), "encrypted") != 0 &&
+	    term.compare (1, strlen("signed"), "signed") != 0 &&
+	    term.compare (1, strlen("attachment"), "attachment") != 0)
+	    continue;
+
+	try {
+	    message->doc.remove_term ((*i));
+	    message->modified = true;
 	} catch (const Xapian::InvalidArgumentError) {
 	    /* Ignore failure to remove non-existent term. */
+	} catch (const Xapian::Error &error) {
+	    notmuch_database_t *notmuch = message->notmuch;
+
+	    if (!notmuch->exception_reported) {
+		_notmuch_database_log(_notmuch_message_database (message), "A Xapian exception occurred creating message: %s\n",
+				      error.get_msg().c_str());
+		notmuch->exception_reported = true;
+	    }
+	    return NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
 	}
     }
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
 }
 
 /* Return true if p points at "new" or "cur". */
@@ -646,6 +703,7 @@ _notmuch_message_add_folder_terms (notmuch_message_t *message,
 
     talloc_free (folder);
 
+    message->modified = true;
     return NOTMUCH_STATUS_SUCCESS;
 }
 
@@ -847,7 +905,7 @@ void
 _notmuch_message_clear_data (notmuch_message_t *message)
 {
     message->doc.set_data ("");
-    message->modified = TRUE;
+    message->modified = true;
 }
 
 static void
@@ -946,6 +1004,14 @@ notmuch_message_get_filenames (notmuch_message_t *message)
     return _notmuch_filenames_create (message, message->filename_list);
 }
 
+int
+notmuch_message_count_files (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_filename_list (message);
+
+    return _notmuch_string_list_length (message->filename_list);
+}
+
 notmuch_bool_t
 notmuch_message_get_flag (notmuch_message_t *message,
 			  notmuch_message_flag_t flag)
@@ -978,7 +1044,7 @@ notmuch_message_get_date (notmuch_message_t *message)
     } catch (Xapian::Error &error) {
 	_notmuch_database_log(_notmuch_message_database (message), "A Xapian exception occurred when reading date: %s\n",
 		 error.get_msg().c_str());
-	message->notmuch->exception_reported = TRUE;
+	message->notmuch->exception_reported = true;
 	return 0;
     }
 
@@ -1049,7 +1115,7 @@ _notmuch_message_set_header_values (notmuch_message_t *message,
 			    Xapian::sortable_serialise (time_value));
     message->doc.add_value (NOTMUCH_VALUE_FROM, from);
     message->doc.add_value (NOTMUCH_VALUE_SUBJECT, subject);
-    message->modified = TRUE;
+    message->modified = true;
 }
 
 /* Upgrade a message to support NOTMUCH_FEATURE_LAST_MOD.  The caller
@@ -1059,7 +1125,7 @@ _notmuch_message_upgrade_last_mod (notmuch_message_t *message)
 {
     /* _notmuch_message_sync will update the last modification
      * revision; we just have to ask it to. */
-    message->modified = TRUE;
+    message->modified = true;
 }
 
 /* Synchronize changes made to message->doc out into the database. */
@@ -1088,7 +1154,7 @@ _notmuch_message_sync (notmuch_message_t *message)
 
     db = static_cast <Xapian::WritableDatabase *> (message->notmuch->xapian_db);
     db->replace_document (message->doc_id, message->doc);
-    message->modified = FALSE;
+    message->modified = false;
 }
 
 /* Delete a message document from the database, leaving a ghost
@@ -1104,7 +1170,7 @@ _notmuch_message_delete (notmuch_message_t *message)
     notmuch_database_t *notmuch;
     notmuch_query_t *query;
     unsigned int count = 0;
-    notmuch_bool_t is_ghost;
+    bool is_ghost;
 
     mid = notmuch_message_get_message_id (message);
     tid = notmuch_message_get_thread_id (message);
@@ -1233,7 +1299,7 @@ _notmuch_message_add_term (notmuch_message_t *message,
 	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
 
     message->doc.add_term (term, 0);
-    message->modified = TRUE;
+    message->modified = true;
 
     talloc_free (term);
 
@@ -1302,7 +1368,7 @@ _notmuch_message_remove_term (notmuch_message_t *message,
 
     try {
 	message->doc.remove_term (term);
-	message->modified = TRUE;
+	message->modified = true;
     } catch (const Xapian::InvalidArgumentError) {
 	/* We'll let the philosophers try to wrestle with the
 	 * question of whether failing to remove that which was not
@@ -1321,10 +1387,10 @@ notmuch_private_status_t
 _notmuch_message_has_term (notmuch_message_t *message,
 			   const char *prefix_name,
 			   const char *value,
-			   notmuch_bool_t *result)
+			   bool *result)
 {
     char *term;
-    notmuch_bool_t out = FALSE;
+    bool out = false;
     notmuch_private_status_t status = NOTMUCH_PRIVATE_STATUS_SUCCESS;
 
     if (value == NULL)
@@ -1342,7 +1408,7 @@ _notmuch_message_has_term (notmuch_message_t *message,
 	i.skip_to (term);
 	if (i != message->doc.termlist_end () &&
 	    !strcmp ((*i).c_str (), term))
-	    out = TRUE;
+	    out = true;
     } catch (Xapian::Error &error) {
 	status = NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
     }
@@ -1449,17 +1515,22 @@ _filename_is_in_maildir (const char *filename)
     return NULL;
 }
 
-notmuch_status_t
-notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
+static void
+_ensure_maildir_flags (notmuch_message_t *message, bool force)
 {
     const char *flags;
-    notmuch_status_t status;
     notmuch_filenames_t *filenames;
     const char *filename, *dir;
     char *combined_flags = talloc_strdup (message, "");
-    unsigned i;
     int seen_maildir_info = 0;
 
+    if (message->maildir_flags) {
+	if (force) {
+	    talloc_free (message->maildir_flags);
+	    message->maildir_flags = NULL;
+	}
+    }
+
     for (filenames = notmuch_message_get_filenames (message);
 	 notmuch_filenames_valid (filenames);
 	 notmuch_filenames_move_to_next (filenames))
@@ -1485,11 +1556,28 @@ notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
 	    seen_maildir_info = 1;
 	}
     }
+    if (seen_maildir_info)
+	message->maildir_flags = combined_flags;
+}
+
+notmuch_bool_t
+notmuch_message_has_maildir_flag (notmuch_message_t *message, char flag)
+{
+    _ensure_maildir_flags (message, false);
+    return message->maildir_flags && (strchr (message->maildir_flags, flag) != NULL);
+}
 
+notmuch_status_t
+notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+    unsigned i;
+
+    _ensure_maildir_flags (message, true);
     /* If none of the filenames have any maildir info field (not even
      * an empty info with no flags set) then there's no information to
      * go on, so do nothing. */
-    if (! seen_maildir_info)
+    if (! message->maildir_flags)
 	return NOTMUCH_STATUS_SUCCESS;
 
     status = notmuch_message_freeze (message);
@@ -1497,7 +1585,7 @@ notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
 	return status;
 
     for (i = 0; i < ARRAY_SIZE(flag2tag); i++) {
-	if ((strchr (combined_flags, flag2tag[i].flag) != NULL)
+	if ((strchr (message->maildir_flags, flag2tag[i].flag) != NULL)
 	    ^
 	    flag2tag[i].inverse)
 	{
@@ -1510,8 +1598,6 @@ notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
     }
     status = notmuch_message_thaw (message);
 
-    talloc_free (combined_flags);
-
     return status;
 }
 
@@ -1599,7 +1685,7 @@ _new_maildir_filename (void *ctx,
     char *filename_new, *dir;
     char flag_map[128];
     int flags_in_map = 0;
-    notmuch_bool_t flags_changed = FALSE;
+    bool flags_changed = false;
     unsigned int i;
     char *s;
 
@@ -1640,7 +1726,7 @@ _new_maildir_filename (void *ctx,
 	if (flag_map[flag] == 0) {
 	    flag_map[flag] = 1;
 	    flags_in_map++;
-	    flags_changed = TRUE;
+	    flags_changed = true;
 	}
     }
 
@@ -1649,7 +1735,7 @@ _new_maildir_filename (void *ctx,
 	if (flag_map[flag]) {
 	    flag_map[flag] = 0;
 	    flags_in_map--;
-	    flags_changed = TRUE;
+	    flags_changed = true;
 	}
     }
 
@@ -1867,8 +1953,123 @@ _notmuch_message_property_map (notmuch_message_t *message)
     return message->property_map;
 }
 
-notmuch_bool_t
+bool
 _notmuch_message_frozen (notmuch_message_t *message)
 {
     return message->frozen;
 }
+
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+			 notmuch_indexopts_t *indexopts)
+{
+    notmuch_database_t *notmuch = NULL;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_private_status_t private_status;
+    notmuch_filenames_t *orig_filenames = NULL;
+    const char *orig_thread_id = NULL;
+    notmuch_message_file_t *message_file = NULL;
+
+    int found = 0;
+
+    if (message == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    /* Save in case we need to delete message */
+    orig_thread_id = notmuch_message_get_thread_id (message);
+    if (!orig_thread_id) {
+	/* XXX TODO: make up new error return? */
+	INTERNAL_ERROR ("message without thread-id");
+    }
+
+    /* strdup it because the metadata may be invalidated */
+    orig_thread_id = talloc_strdup (message, orig_thread_id);
+
+    notmuch = _notmuch_message_database (message);
+
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
+    orig_filenames = notmuch_message_get_filenames (message);
+
+    private_status = _notmuch_message_remove_indexed_terms (message);
+    if (private_status) {
+	ret = COERCE_STATUS(private_status, "error removing terms");
+	goto DONE;
+    }
+
+    ret = notmuch_message_remove_all_properties_with_prefix (message, "index.");
+    if (ret)
+	goto DONE; /* XXX TODO: distinguish from other error returns above? */
+    if (indexopts && notmuch_indexopts_get_decrypt_policy (indexopts) == NOTMUCH_DECRYPT_FALSE) {
+	ret = notmuch_message_remove_all_properties (message, "session-key");
+	if (ret)
+	    goto DONE;
+    }
+
+    /* re-add the filenames with the associated indexopts */
+    for (; notmuch_filenames_valid (orig_filenames);
+	 notmuch_filenames_move_to_next (orig_filenames)) {
+
+	const char *date;
+	const char *from, *to, *subject;
+	char *message_id = NULL;
+	const char *thread_id = NULL;
+
+	const char *filename = notmuch_filenames_get (orig_filenames);
+
+	message_file = _notmuch_message_file_open (notmuch, filename);
+	if (message_file == NULL)
+	    continue;
+
+	ret = _notmuch_message_file_get_headers (message_file,
+						 &from, &subject, &to, &date,
+						 &message_id);
+	if (ret)
+	    goto DONE;
+
+	/* XXX TODO: deal with changing message id? */
+
+	_notmuch_message_add_filename (message, filename);
+
+	ret = _notmuch_database_link_message_to_parents (notmuch, message,
+							 message_file,
+							 &thread_id);
+	if (ret)
+	    goto DONE;
+
+	if (thread_id == NULL)
+	    thread_id = orig_thread_id;
+
+	_notmuch_message_add_term (message, "thread", thread_id);
+	/* Take header values only from first filename */
+	if (found == 0)
+	    _notmuch_message_set_header_values (message, date, from, subject);
+
+	ret = _notmuch_message_index_file (message, indexopts, message_file);
+
+	if (ret == NOTMUCH_STATUS_FILE_ERROR)
+	    continue;
+	if (ret)
+	    goto DONE;
+
+	found++;
+	_notmuch_message_file_close (message_file);
+	message_file = NULL;
+    }
+    if (found == 0) {
+	/* put back thread id to help cleanup */
+	_notmuch_message_add_term (message, "thread", orig_thread_id);
+	ret = _notmuch_message_delete (message);
+    } else {
+	_notmuch_message_sync (message);
+    }
+
+ DONE:
+    if (message_file)
+	_notmuch_message_file_close (message_file);
+
+    /* XXX TODO destroy orig_filenames? */
+    return ret;
+}
diff --git a/lib/messages.c b/lib/messages.c
index b5363bb8..a88f974f 100644
--- a/lib/messages.c
+++ b/lib/messages.c
@@ -68,7 +68,7 @@ _notmuch_messages_create (notmuch_message_list_t *list)
     if (unlikely (messages == NULL))
 	return NULL;
 
-    messages->is_of_list_type = TRUE;
+    messages->is_of_list_type = true;
     messages->iterator = list->head;
 
     return messages;
@@ -93,7 +93,7 @@ notmuch_bool_t
 notmuch_messages_valid (notmuch_messages_t *messages)
 {
     if (messages == NULL)
-	return FALSE;
+	return false;
 
     if (! messages->is_of_list_type)
 	return _notmuch_mset_messages_valid (messages);
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 5dfebf5d..1093429c 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -24,6 +24,7 @@
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE /* For getline and asprintf */
 #endif
+#include <stdbool.h>
 #include <stdio.h>
 
 #include "compat.h"
@@ -51,6 +52,7 @@ NOTMUCH_BEGIN_DECLS
 #include "xutil.h"
 #include "error_util.h"
 #include "string-util.h"
+#include "crypto.h"
 
 #ifdef DEBUG
 # define DEBUG_DATABASE_SANITY 1
@@ -282,7 +284,7 @@ notmuch_private_status_t
 _notmuch_message_has_term (notmuch_message_t *message,
 			   const char *prefix_name,
 			   const char *value,
-			   notmuch_bool_t *result);
+			   bool *result);
 
 notmuch_private_status_t
 _notmuch_message_gen_terms (notmuch_message_t *message,
@@ -425,10 +427,28 @@ const char *
 _notmuch_message_file_get_header (notmuch_message_file_t *message,
 				 const char *header);
 
+notmuch_status_t
+_notmuch_message_file_get_headers (notmuch_message_file_t *message_file,
+				   const char **from_out,
+				   const char **subject_out,
+				   const char **to_out,
+				   const char **date_out,
+				   char **message_id_out);
+
+const char *
+_notmuch_message_file_get_filename (notmuch_message_file_t *message);
+
+/* add-message.cc */
+notmuch_status_t
+_notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
+					   notmuch_message_t *message,
+					   notmuch_message_file_t *message_file,
+					   const char **thread_id);
 /* index.cc */
 
 notmuch_status_t
 _notmuch_message_index_file (notmuch_message_t *message,
+			     notmuch_indexopts_t *indexopts,
 			     notmuch_message_file_t *message_file);
 
 /* messages.c */
@@ -448,7 +468,7 @@ typedef struct _notmuch_message_list {
  * ignorance of that here. (See notmuch_mset_messages_t in query.cc)
  */
 struct _notmuch_messages {
-    notmuch_bool_t is_of_list_type;
+    bool is_of_list_type;
     notmuch_doc_id_set_t *excluded_doc_ids;
     notmuch_message_node_t *iterator;
 };
@@ -465,7 +485,7 @@ _notmuch_messages_create (notmuch_message_list_t *list);
 
 /* query.cc */
 
-notmuch_bool_t
+bool
 _notmuch_mset_messages_valid (notmuch_messages_t *messages);
 
 notmuch_message_t *
@@ -474,7 +494,7 @@ _notmuch_mset_messages_get (notmuch_messages_t *messages);
 void
 _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages);
 
-notmuch_bool_t
+bool
 _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
 			      unsigned int doc_id);
 
@@ -492,6 +512,20 @@ notmuch_status_t
 _notmuch_query_count_documents (notmuch_query_t *query,
 				const char *type,
 				unsigned *count_out);
+/* message-id.c */
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimiters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly talloc'ed string belonging to 'ctx'.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+char *
+_notmuch_message_id_parse (void *ctx, const char *message_id, const char **next);
+
 
 /* message.cc */
 
@@ -501,6 +535,8 @@ _notmuch_message_add_reply (notmuch_message_t *message,
 notmuch_database_t *
 _notmuch_message_database (notmuch_message_t *message);
 
+void
+_notmuch_message_remove_unprefixed_terms (notmuch_message_t *message);
 /* sha1.c */
 
 char *
@@ -525,6 +561,12 @@ typedef struct _notmuch_string_list {
 notmuch_string_list_t *
 _notmuch_string_list_create (const void *ctx);
 
+/*
+ * return the number of strings in 'list'
+ */
+int
+_notmuch_string_list_length (notmuch_string_list_t *list);
+
 /* Add 'string' to 'list'.
  *
  * The list will create its own talloced copy of 'string'.
@@ -552,9 +594,9 @@ _notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
 
 notmuch_string_map_iterator_t *
 _notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
-				     notmuch_bool_t exact);
+				     bool exact);
 
-notmuch_bool_t
+bool
 _notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
 
 void
@@ -593,6 +635,12 @@ _notmuch_thread_create (void *ctx,
 			notmuch_exclude_t omit_exclude,
 			notmuch_sort_t sort);
 
+/* indexopts.c */
+
+struct _notmuch_indexopts {
+    _notmuch_crypto_t crypto;
+};
+
 NOTMUCH_END_DECLS
 
 #ifdef __cplusplus
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 17f0872e..39759b7a 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -192,6 +192,23 @@ typedef enum _notmuch_status {
      */
     NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
     /**
+     * A MIME object claimed to have cryptographic protection which
+     * notmuch tried to handle, but the protocol was not specified in
+     * an intelligible way.
+     */
+    NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL,
+    /**
+     * Notmuch attempted to do crypto processing, but could not
+     * initialize the engine needed to do so.
+     */
+    NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION,
+    /**
+     * A MIME object claimed to have cryptographic protection, and
+     * notmuch attempted to process it, but the specific protocol was
+     * something that notmuch doesn't know how to handle.
+     */
+    NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL,
+    /**
      * Not an actual status value. Just a way to find out how many
      * valid status values there are.
      */
@@ -219,6 +236,7 @@ typedef struct _notmuch_tags notmuch_tags_t;
 typedef struct _notmuch_directory notmuch_directory_t;
 typedef struct _notmuch_filenames notmuch_filenames_t;
 typedef struct _notmuch_config_list notmuch_config_list_t;
+typedef struct _notmuch_indexopts notmuch_indexopts_t;
 #endif /* __DOXYGEN__ */
 
 /**
@@ -236,7 +254,7 @@ typedef struct _notmuch_config_list notmuch_config_list_t;
  * The database will not yet have any data in it
  * (notmuch_database_create itself is a very cheap function). Messages
  * contained within 'path' can be added to the database by calling
- * notmuch_database_add_message.
+ * notmuch_database_index_file.
  *
  * In case of any failure, this function returns an error status and
  * sets *database to NULL (after printing an error message on stderr).
@@ -541,8 +559,10 @@ notmuch_database_get_directory (notmuch_database_t *database,
 				notmuch_directory_t **directory);
 
 /**
- * Add a new message to the given notmuch database or associate an
- * additional filename with an existing message.
+ * Add a message file to a database, indexing it for retrieval by
+ * future searches.  If a message already exists with the same message
+ * ID as the specified file, their indexes will be merged, and this
+ * new filename will also be associated with the existing message.
  *
  * Here, 'filename' should be a path relative to the path of
  * 'database' (see notmuch_database_get_path), or else should be an
@@ -555,8 +575,14 @@ notmuch_database_get_directory (notmuch_database_t *database,
  * entire contents of the file.
  *
  * If another message with the same message ID already exists in the
- * database, rather than creating a new message, this adds 'filename'
- * to the list of the filenames for the existing message.
+ * database, rather than creating a new message, this adds the search
+ * terms from the identified file to the existing message's index, and
+ * adds 'filename' to the list of filenames known for the message.
+ *
+ * The 'indexopts' parameter can be NULL (meaning, use the indexing
+ * defaults from the database), or can be an explicit choice of
+ * indexing options that should govern the indexing of this specific
+ * 'filename'.
  *
  * If 'message' is not NULL, then, on successful return
  * (NOTMUCH_STATUS_SUCCESS or NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) '*message'
@@ -589,7 +615,24 @@ notmuch_database_get_directory (notmuch_database_t *database,
  *
  * NOTMUCH_STATUS_UPGRADE_REQUIRED: The caller must upgrade the
  * 	database to use this function.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_database_index_file (notmuch_database_t *database,
+			     const char *filename,
+			     notmuch_indexopts_t *indexopts,
+			     notmuch_message_t **message);
+
+/**
+ * Deprecated alias for notmuch_database_index_file called with
+ * NULL indexopts.
+ *
+ * @deprecated Deprecated as of libnotmuch 5.1 (notmuch 0.26). Please
+ * use notmuch_database_index_file instead.
+ *
  */
+NOTMUCH_DEPRECATED(5,1)
 notmuch_status_t
 notmuch_database_add_message (notmuch_database_t *database,
 			      const char *filename,
@@ -1097,6 +1140,18 @@ int
 notmuch_thread_get_total_messages (notmuch_thread_t *thread);
 
 /**
+ * Get the total number of files in 'thread'.
+ *
+ * This sums notmuch_message_count_files over all messages in the
+ * thread
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+
+int
+notmuch_thread_get_total_files (notmuch_thread_t *thread);
+
+/**
  * Get a notmuch_messages_t iterator for the top-level messages in
  * 'thread' in oldest-first order.
  *
@@ -1342,6 +1397,14 @@ notmuch_messages_t *
 notmuch_message_get_replies (notmuch_message_t *message);
 
 /**
+ * Get the total number of files associated with a message.
+ * @returns Non-negative integer
+ * @since libnotmuch 5.0 (notmuch 0.25)
+ */
+int
+notmuch_message_count_files (notmuch_message_t *message);
+
+/**
  * Get a filename for the email corresponding to 'message'.
  *
  * The returned filename is an absolute filename, (the initial
@@ -1374,6 +1437,20 @@ notmuch_filenames_t *
 notmuch_message_get_filenames (notmuch_message_t *message);
 
 /**
+ * Re-index the e-mail corresponding to 'message' using the supplied index options
+ *
+ * Returns the status of the re-index operation.  (see the return
+ * codes documented in notmuch_database_index_file)
+ *
+ * After reindexing, the user should discard the message object passed
+ * in here by calling notmuch_message_destroy, since it refers to the
+ * original message, not to the reindexed message.
+ */
+notmuch_status_t
+notmuch_message_reindex (notmuch_message_t *message,
+			 notmuch_indexopts_t *indexopts);
+
+/**
  * Message flags.
  */
 typedef enum _notmuch_message_flag {
@@ -1546,7 +1623,7 @@ notmuch_message_remove_all_tags (notmuch_message_t *message);
  *
  * A client can ensure that notmuch database tags remain synchronized
  * with maildir flags by calling this function after each call to
- * notmuch_database_add_message. See also
+ * notmuch_database_index_file. See also
  * notmuch_message_tags_to_maildir_flags for synchronizing tag changes
  * back to maildir flags.
  */
@@ -1554,6 +1631,14 @@ notmuch_status_t
 notmuch_message_maildir_flags_to_tags (notmuch_message_t *message);
 
 /**
+ * return TRUE if any filename of 'message' has maildir flag 'flag',
+ * FALSE otherwise.
+ *
+ */
+notmuch_bool_t
+notmuch_message_has_maildir_flag (notmuch_message_t *message, char flag);
+
+/**
  * Rename message filename(s) to encode tags as maildir flags.
  *
  * Specifically, for each filename corresponding to this message:
@@ -1679,6 +1764,9 @@ notmuch_message_destroy (notmuch_message_t *message);
  * add or delete values for, as other subsystems or extensions may
  * depend on these properties.
  *
+ * Please see notmuch-properties(7) for more details about specific
+ * properties and conventions around their use.
+ *
  */
 /**@{*/
 /**
@@ -1739,6 +1827,22 @@ notmuch_status_t
 notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
 
 /**
+ * Remove all (prefix*,value) pairs from the given message
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     prefix   delete properties with keys that start with prefix.
+ *			   If NULL, delete all properties
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties_with_prefix (notmuch_message_t *message, const char *prefix);
+
+/**
  * Opaque message property iterator
  */
 typedef struct _notmuch_string_map_iterator notmuch_message_properties_t;
@@ -1788,7 +1892,7 @@ notmuch_message_get_properties (notmuch_message_t *message, const char *key, not
  * says. Whereas when this function returns FALSE, calling any of
  * these functions results in undefined behaviour.
  *
- * See the documentation of notmuch_message_properties_get for example
+ * See the documentation of notmuch_message_get_properties for example
  * code showing how to iterate over a notmuch_message_properties_t
  * object.
  *
@@ -1908,7 +2012,7 @@ notmuch_tags_destroy (notmuch_tags_t *tags);
  *
  *   o Read the mtime of a directory from the filesystem
  *
- *   o Call add_message for all mail files in the directory
+ *   o Call index_file for all mail files in the directory
  *
  *   o Call notmuch_directory_set_mtime with the mtime read from the
  *     filesystem.
@@ -2113,6 +2217,66 @@ notmuch_config_list_move_to_next (notmuch_config_list_t *config_list);
 void
 notmuch_config_list_destroy (notmuch_config_list_t *config_list);
 
+
+/**
+ * get the current default indexing options for a given database.
+ *
+ * This object will survive until the database itself is destroyed,
+ * but the caller may also release it earlier with
+ * notmuch_indexopts_destroy.
+ *
+ * This object represents a set of options on how a message can be
+ * added to the index.  At the moment it is a featureless stub.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_indexopts_t *
+notmuch_database_get_default_indexopts (notmuch_database_t *db);
+
+/**
+ * Stating a policy about how to decrypt messages.
+ *
+ * See index.decrypt in notmuch-config(1) for more details.
+ */
+typedef enum {
+    NOTMUCH_DECRYPT_FALSE,
+    NOTMUCH_DECRYPT_TRUE,
+    NOTMUCH_DECRYPT_AUTO,
+    NOTMUCH_DECRYPT_NOSTASH,
+} notmuch_decryption_policy_t;
+
+/**
+ * Specify whether to decrypt encrypted parts while indexing.
+ *
+ * Be aware that the index is likely sufficient to reconstruct the
+ * cleartext of the message itself, so please ensure that the notmuch
+ * message index is adequately protected. DO NOT SET THIS FLAG TO TRUE
+ * without considering the security of your index.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_status_t
+notmuch_indexopts_set_decrypt_policy (notmuch_indexopts_t *indexopts,
+				      notmuch_decryption_policy_t decrypt_policy);
+
+/**
+ * Return whether to decrypt encrypted parts while indexing.
+ * see notmuch_indexopts_set_decrypt_policy.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+notmuch_decryption_policy_t
+notmuch_indexopts_get_decrypt_policy (const notmuch_indexopts_t *indexopts);
+
+/**
+ * Destroy a notmuch_indexopts_t object.
+ *
+ * @since libnotmuch 5.1 (notmuch 0.26)
+ */
+void
+notmuch_indexopts_destroy (notmuch_indexopts_t *options);
+
+
 /**
  * interrogate the library for compile time features
  *
diff --git a/lib/query.cc b/lib/query.cc
index 9c6ecc8d..d633fa3d 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -29,7 +29,7 @@ struct _notmuch_query {
     notmuch_sort_t sort;
     notmuch_string_list_t *exclude_terms;
     notmuch_exclude_t omit_excluded;
-    notmuch_bool_t parsed;
+    bool parsed;
     Xapian::Query xapian_query;
     std::set<std::string> terms;
 };
@@ -62,12 +62,12 @@ struct _notmuch_threads {
 };
 
 /* We need this in the message functions so forward declare. */
-static notmuch_bool_t
+static bool
 _notmuch_doc_id_set_init (void *ctx,
 			  notmuch_doc_id_set_t *doc_ids,
 			  GArray *arr);
 
-static notmuch_bool_t
+static bool
 _debug_query (void)
 {
     char *env = getenv ("NOTMUCH_DEBUG_QUERY");
@@ -97,7 +97,7 @@ notmuch_query_create (notmuch_database_t *notmuch,
 
     new (&query->xapian_query) Xapian::Query ();
     new (&query->terms) std::set<std::string> ();
-    query->parsed = FALSE;
+    query->parsed = false;
 
     talloc_set_destructor (query, _notmuch_query_destructor);
 
@@ -134,7 +134,7 @@ _notmuch_query_ensure_parsed (notmuch_query_t *query)
 	     t != query->xapian_query.get_terms_end (); ++t)
 	    query->terms.insert (*t);
 
-	query->parsed = TRUE;
+	query->parsed = true;
 
     } catch (const Xapian::Error &error) {
 	if (!query->notmuch->exception_reported) {
@@ -144,7 +144,7 @@ _notmuch_query_ensure_parsed (notmuch_query_t *query)
 	    _notmuch_database_log_append (query->notmuch,
 					  "Query string was: %s\n",
 					  query->query_string);
-	    query->notmuch->exception_reported = TRUE;
+	    query->notmuch->exception_reported = true;
 	}
 
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
@@ -261,7 +261,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
     try {
 
-	messages->base.is_of_list_type = FALSE;
+	messages->base.is_of_list_type = false;
 	messages->base.iterator = NULL;
 	messages->notmuch = notmuch;
 	new (&messages->iterator) Xapian::MSetIterator ();
@@ -304,7 +304,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
 		mset = enquire.get_mset (0, notmuch->xapian_db->get_doccount ());
 
-		GArray *excluded_doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int));
+		GArray *excluded_doc_ids = g_array_new (false, false, sizeof (unsigned int));
 
 		for (iterator = mset.begin (); iterator != mset.end (); iterator++) {
 		    unsigned int doc_id = *iterator;
@@ -322,13 +322,13 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
 	switch (query->sort) {
 	case NOTMUCH_SORT_OLDEST_FIRST:
-	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, FALSE);
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, false);
 	    break;
 	case NOTMUCH_SORT_NEWEST_FIRST:
-	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, TRUE);
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, true);
 	    break;
 	case NOTMUCH_SORT_MESSAGE_ID:
-	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
+	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, false);
 	    break;
 	case NOTMUCH_SORT_UNSORTED:
 	    break;
@@ -359,13 +359,13 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 			       "Query string was: %s\n",
 			       query->query_string);
 
-	notmuch->exception_reported = TRUE;
+	notmuch->exception_reported = true;
 	talloc_free (messages);
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 }
 
-notmuch_bool_t
+bool
 _notmuch_mset_messages_valid (notmuch_messages_t *messages)
 {
     notmuch_mset_messages_t *mset_messages;
@@ -415,7 +415,7 @@ _notmuch_mset_messages_get (notmuch_messages_t *messages)
 
     if (messages->excluded_doc_ids &&
 	_notmuch_doc_id_set_contains (messages->excluded_doc_ids, doc_id))
-	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, TRUE);
+	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
 
     return message;
 }
@@ -430,7 +430,7 @@ _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages)
     mset_messages->iterator++;
 }
 
-static notmuch_bool_t
+static bool
 _notmuch_doc_id_set_init (void *ctx,
 			  notmuch_doc_id_set_t *doc_ids,
 			  GArray *arr)
@@ -443,7 +443,7 @@ _notmuch_doc_id_set_init (void *ctx,
     bitmap = talloc_zero_array (ctx, unsigned char, DOCIDSET_WORD(max) + 1);
 
     if (bitmap == NULL)
-	return FALSE;
+	return false;
 
     doc_ids->bitmap = bitmap;
     doc_ids->bound = max + 1;
@@ -453,15 +453,15 @@ _notmuch_doc_id_set_init (void *ctx,
 	bitmap[DOCIDSET_WORD(doc_id)] |= 1 << DOCIDSET_BIT(doc_id);
     }
 
-    return TRUE;
+    return true;
 }
 
-notmuch_bool_t
+bool
 _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
 			      unsigned int doc_id)
 {
     if (doc_id >= doc_ids->bound)
-	return FALSE;
+	return false;
     return doc_ids->bitmap[DOCIDSET_WORD(doc_id)] & (1 << DOCIDSET_BIT(doc_id));
 }
 
@@ -514,7 +514,7 @@ notmuch_query_search_threads (notmuch_query_t *query,
 	return status;
     }
 
-    threads->doc_ids = g_array_new (FALSE, FALSE, sizeof (unsigned int));
+    threads->doc_ids = g_array_new (false, false, sizeof (unsigned int));
     while (notmuch_messages_valid (messages)) {
 	unsigned int doc_id = _notmuch_mset_messages_get_doc_id (messages);
 	g_array_append_val (threads->doc_ids, doc_id);
@@ -546,7 +546,7 @@ notmuch_threads_valid (notmuch_threads_t *threads)
     unsigned int doc_id;
 
     if (! threads)
-	return FALSE;
+	return false;
 
     while (threads->doc_id_pos < threads->doc_ids->len) {
 	doc_id = g_array_index (threads->doc_ids, unsigned int,
diff --git a/lib/string-list.c b/lib/string-list.c
index 43ebe499..9c3ae7ef 100644
--- a/lib/string-list.c
+++ b/lib/string-list.c
@@ -42,6 +42,12 @@ _notmuch_string_list_create (const void *ctx)
     return list;
 }
 
+int
+_notmuch_string_list_length (notmuch_string_list_t *list)
+{
+    return list->length;
+}
+
 void
 _notmuch_string_list_append (notmuch_string_list_t *list,
 			     const char *string)
diff --git a/lib/string-map.c b/lib/string-map.c
index 0bb77e93..5aac8bcc 100644
--- a/lib/string-map.c
+++ b/lib/string-map.c
@@ -33,14 +33,14 @@ typedef struct _notmuch_string_pair_t {
 } notmuch_string_pair_t;
 
 struct _notmuch_string_map {
-    notmuch_bool_t sorted;
+    bool sorted;
     size_t length;
     notmuch_string_pair_t *pairs;
 };
 
 struct _notmuch_string_map_iterator {
     notmuch_string_pair_t *current;
-    notmuch_bool_t exact;
+    bool exact;
     const char *key;
 };
 
@@ -55,7 +55,7 @@ _notmuch_string_map_create (const void *ctx)
 
     map->length = 0;
     map->pairs = NULL;
-    map->sorted = TRUE;
+    map->sorted = true;
 
     return map;
 }
@@ -67,7 +67,7 @@ _notmuch_string_map_append (notmuch_string_map_t *map,
 {
 
     map->length++;
-    map->sorted = FALSE;
+    map->sorted = false;
 
     if (map->pairs)
 	map->pairs = talloc_realloc (map, map->pairs, notmuch_string_pair_t, map->length + 1);
@@ -103,11 +103,11 @@ _notmuch_string_map_sort (notmuch_string_map_t *map)
 
     qsort (map->pairs, map->length, sizeof (notmuch_string_pair_t), cmppair);
 
-    map->sorted = TRUE;
+    map->sorted = true;
 }
 
-static notmuch_bool_t
-string_cmp (const char *a, const char *b, notmuch_bool_t exact)
+static bool
+string_cmp (const char *a, const char *b, bool exact)
 {
     if (exact)
 	return (strcmp (a, b));
@@ -116,7 +116,7 @@ string_cmp (const char *a, const char *b, notmuch_bool_t exact)
 }
 
 static notmuch_string_pair_t *
-bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, notmuch_bool_t exact)
+bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, bool exact)
 {
     size_t first = 0;
     size_t last = len - 1;
@@ -151,7 +151,7 @@ _notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
     /* this means that calling append invalidates iterators */
     _notmuch_string_map_sort (map);
 
-    pair = bsearch_first (map->pairs, map->length, key, TRUE);
+    pair = bsearch_first (map->pairs, map->length, key, true);
     if (! pair)
 	return NULL;
 
@@ -160,7 +160,7 @@ _notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
 
 notmuch_string_map_iterator_t *
 _notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
-				     notmuch_bool_t exact)
+				     bool exact)
 {
     notmuch_string_map_iterator_t *iter;
 
@@ -179,15 +179,15 @@ _notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
     return iter;
 }
 
-notmuch_bool_t
+bool
 _notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iterator)
 {
     if (iterator->current == NULL)
-	return FALSE;
+	return false;
 
     /* sentinel */
     if (iterator->current->key == NULL)
-	return FALSE;
+	return false;
 
     return (0 == string_cmp (iterator->key, iterator->current->key, iterator->exact));
 
diff --git a/lib/thread.cc b/lib/thread.cc
index 1a1ecfa5..3561b27f 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -44,6 +44,7 @@ struct _notmuch_thread {
 
     GHashTable *message_hash;
     int total_messages;
+    int total_files;
     int matched_messages;
     time_t oldest;
     time_t newest;
@@ -58,12 +59,12 @@ _notmuch_thread_destructor (notmuch_thread_t *thread)
     g_hash_table_unref (thread->message_hash);
 
     if (thread->authors_array) {
-	g_ptr_array_free (thread->authors_array, TRUE);
+	g_ptr_array_free (thread->authors_array, true);
 	thread->authors_array = NULL;
     }
 
     if (thread->matched_authors_array) {
-	g_ptr_array_free (thread->matched_authors_array, TRUE);
+	g_ptr_array_free (thread->matched_authors_array, true);
 	thread->matched_authors_array = NULL;
     }
 
@@ -155,10 +156,13 @@ _resolve_thread_authors_string (notmuch_thread_t *thread)
 	first_non_matched_author = 0;
     }
 
-    g_ptr_array_free (thread->authors_array, TRUE);
+    g_ptr_array_free (thread->authors_array, true);
     thread->authors_array = NULL;
-    g_ptr_array_free (thread->matched_authors_array, TRUE);
+    g_ptr_array_free (thread->matched_authors_array, true);
     thread->matched_authors_array = NULL;
+
+    if (!thread->authors)
+	thread->authors = talloc_strdup(thread, "");
 }
 
 /* clean up the ugly "Lastname, Firstname" format that some mail systems
@@ -238,7 +242,7 @@ _thread_add_message (notmuch_thread_t *thread,
     InternetAddress *address;
     const char *from, *author;
     char *clean_author;
-    notmuch_bool_t message_excluded = FALSE;
+    bool message_excluded = false;
 
     if (omit_exclude != NOTMUCH_EXCLUDE_FALSE) {
 	for (tags = notmuch_message_get_tags (message);
@@ -253,7 +257,7 @@ _thread_add_message (notmuch_thread_t *thread,
 	    {
 		/* Check for an empty string, and then ignore initial 'K'. */
 		if (*(term->string) && strcmp(tag, (term->string + 1)) == 0) {
-		    message_excluded = TRUE;
+		    message_excluded = true;
 		    break;
 		}
 	    }
@@ -266,6 +270,7 @@ _thread_add_message (notmuch_thread_t *thread,
     _notmuch_message_list_add_message (thread->message_list,
 				       talloc_steal (thread, message));
     thread->total_messages++;
+    thread->total_files += notmuch_message_count_files (message);
 
     g_hash_table_insert (thread->message_hash,
 			 xstrdup (notmuch_message_get_message_id (message)),
@@ -308,7 +313,7 @@ _thread_add_message (notmuch_thread_t *thread,
 
     /* Mark excluded messages. */
     if (message_excluded)
-	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, TRUE);
+	notmuch_message_set_flag (message, NOTMUCH_MESSAGE_FLAG_EXCLUDED, true);
 }
 
 static void
@@ -495,6 +500,7 @@ _notmuch_thread_create (void *ctx,
 						  free, NULL);
 
     thread->total_messages = 0;
+    thread->total_files = 0;
     thread->matched_messages = 0;
     thread->oldest = 0;
     thread->newest = 0;
@@ -567,6 +573,12 @@ notmuch_thread_get_total_messages (notmuch_thread_t *thread)
 }
 
 int
+notmuch_thread_get_total_files (notmuch_thread_t *thread)
+{
+    return thread->total_files;
+}
+
+int
 notmuch_thread_get_matched_messages (notmuch_thread_t *thread)
 {
     return thread->matched_messages;