Merge branch 'upstream'

10 files changed, 1389 insertions, 303 deletions

diff --git a/lib/Makefile b/lib/Makefile
index 9a29ffcf..b6859eac 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,5 +1,7 @@
+# See Makfefile.local for the list of files to be compiled in this
+# directory.
 all:
 	$(MAKE) -C .. all
 
-clean:
-	$(MAKE) -C .. clean
+.DEFAULT:
+	$(MAKE) -C .. $@
diff --git a/lib/Makefile.local b/lib/Makefile.local
index a7562c9b..70489e17 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -11,6 +11,7 @@ libnotmuch_c_srcs =		\
 
 libnotmuch_cxx_srcs =		\
 	$(dir)/database.cc	\
+	$(dir)/directory.cc	\
 	$(dir)/index.cc		\
 	$(dir)/message.cc	\
 	$(dir)/query.cc		\
diff --git a/lib/database-private.h b/lib/database-private.h
index 643b0507..5891584e 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -33,6 +33,8 @@ struct _notmuch_database {
     Xapian::QueryParser *query_parser;
     Xapian::TermGenerator *term_gen;
     Xapian::ValueRangeProcessor *value_range_processor;
+
+    notmuch_bool_t needs_upgrade;
 };
 
 /* Convert tags from Xapian internal format to notmuch format.
diff --git a/lib/database.cc b/lib/database.cc
index b6c4d07b..cce78478 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -22,6 +22,8 @@
 
 #include <iostream>
 
+#include <sys/time.h>
+#include <signal.h>
 #include <xapian.h>
 
 #include <glib.h> /* g_free, GPtrArray, GHashTable */
@@ -35,9 +37,14 @@ typedef struct {
     const char *prefix;
 } prefix_t;
 
-/* Here's the current schema for our database:
+#define NOTMUCH_DATABASE_VERSION 1
+
+#define STRINGIFY(s) _SUB_STRINGIFY(s)
+#define _SUB_STRINGIFY(s) #s
+
+/* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
  *
- * We currently have two different types of documents: mail and timestamps.
+ * We currently have two different types of documents: mail and directory.
  *
  * Mail document
  * -------------
@@ -63,6 +70,12 @@ typedef struct {
  *
  *	tag:	   Any tags associated with this message by the user.
  *
+ *	file-direntry:  A colon-separated pair of values
+ *		        (INTEGER:STRING), where INTEGER is the
+ *		        document ID of a directory document, and
+ *		        STRING is the name of a file within that
+ *		        directory for this mail message.
+ *
  *    A mail document also has two values:
  *
  *	TIMESTAMP:	The time_t value corresponding to the message's
@@ -75,21 +88,36 @@ typedef struct {
  * user in searching. But the database doesn't really care itself
  * about any of these.
  *
- * Timestamp document
+ * The data portion of a mail document is empty.
+ *
+ * Directory document
  * ------------------
- * A timestamp document is used by a client of the notmuch library to
+ * A directory document is used by a client of the notmuch library to
  * maintain data necessary to allow for efficient polling of mail
- * directories. The notmuch library does no interpretation of
- * timestamps, but merely allows the user to store and retrieve
- * timestamps as name/value pairs.
+ * directories.
+ *
+ * All directory documents contain one term:
  *
- * The timestamp document is indexed with a single prefixed term:
+ *	directory:	The directory path (relative to the database path)
+ *			Or the SHA1 sum of the directory path (if the
+ *			path itself is too long to fit in a Xapian
+ *			term).
  *
- *	timestamp:	The user's key value (likely a directory name)
+ * And all directory documents for directories other than top-level
+ * directories also contain the following term:
  *
- * and has a single value:
+ *	directory-direntry: A colon-separated pair of values
+ *		            (INTEGER:STRING), where INTEGER is the
+ *		            document ID of the parent directory
+ *		            document, and STRING is the name of this
+ *		            directory within that parent.
  *
- *	TIMESTAMP:	The time_t value from the user.
+ * All directory documents have a single value:
+ *
+ *	TIMESTAMP:	The mtime of the directory (at last scan)
+ *
+ * The data portion of a directory document contains the path of the
+ * directory (relative to the database path).
  */
 
 /* With these prefix values we follow the conventions published here:
@@ -108,23 +136,25 @@ typedef struct {
  */
 
 prefix_t BOOLEAN_PREFIX_INTERNAL[] = {
-    { "type", "T" },
-    { "reference", "XREFERENCE" },
-    { "replyto", "XREPLYTO" },
-    { "timestamp", "XTIMESTAMP" },
+    { "type",			"T" },
+    { "reference",		"XREFERENCE" },
+    { "replyto",		"XREPLYTO" },
+    { "directory",		"XDIRECTORY" },
+    { "file-direntry",		"XFDIRENTRY" },
+    { "directory-direntry",	"XDDIRENTRY" },
 };
 
 prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
-    { "thread", "G" },
-    { "tag", "K" },
-    { "id", "Q" }
+    { "thread",			"G" },
+    { "tag",			"K" },
+    { "id",			"Q" }
 };
 
 prefix_t PROBABILISTIC_PREFIX[]= {
-    { "from", "XFROM" },
-    { "to", "XTO" },
-    { "attachment", "XATTACHMENT" },
-    { "subject", "XSUBJECT"}
+    { "from",			"XFROM" },
+    { "to",			"XTO" },
+    { "attachment",		"XATTACHMENT" },
+    { "subject",		"XSUBJECT"}
 };
 
 int
@@ -175,8 +205,8 @@ notmuch_status_to_string (notmuch_status_t status)
 	return "No error occurred";
     case NOTMUCH_STATUS_OUT_OF_MEMORY:
 	return "Out of memory";
-    case NOTMUCH_STATUS_READONLY_DATABASE:
-	return "The database is read-only";
+    case NOTMUCH_STATUS_READ_ONLY_DATABASE:
+	return "Attempt to write to a read-only database";
     case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
 	return "A Xapian exception occurred";
     case NOTMUCH_STATUS_FILE_ERROR:
@@ -198,30 +228,38 @@ notmuch_status_to_string (notmuch_status_t status)
 }
 
 static void
+find_doc_ids_for_term (notmuch_database_t *notmuch,
+		       const char *term,
+		       Xapian::PostingIterator *begin,
+		       Xapian::PostingIterator *end)
+{
+    *begin = notmuch->xapian_db->postlist_begin (term);
+
+    *end = notmuch->xapian_db->postlist_end (term);
+}
+
+static void
 find_doc_ids (notmuch_database_t *notmuch,
 	      const char *prefix_name,
 	      const char *value,
 	      Xapian::PostingIterator *begin,
 	      Xapian::PostingIterator *end)
 {
-    Xapian::PostingIterator i;
     char *term;
 
     term = talloc_asprintf (notmuch, "%s%s",
 			    _find_prefix (prefix_name), value);
 
-    *begin = notmuch->xapian_db->postlist_begin (term);
-
-    *end = notmuch->xapian_db->postlist_end (term);
+    find_doc_ids_for_term (notmuch, term, begin, end);
 
     talloc_free (term);
 }
 
-static notmuch_private_status_t
-find_unique_doc_id (notmuch_database_t *notmuch,
-		    const char *prefix_name,
-		    const char *value,
-		    unsigned int *doc_id)
+notmuch_private_status_t
+_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
+				      const char *prefix_name,
+				      const char *value,
+				      unsigned int *doc_id)
 {
     Xapian::PostingIterator i, end;
 
@@ -230,10 +268,19 @@ find_unique_doc_id (notmuch_database_t *notmuch,
     if (i == end) {
 	*doc_id = 0;
 	return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
-    } else {
-	*doc_id = *i;
-	return NOTMUCH_PRIVATE_STATUS_SUCCESS;
     }
+
+    *doc_id = *i;
+
+#if DEBUG_DATABASE_SANITY
+    i++;
+
+    if (i != end)
+	INTERNAL_ERROR ("Term %s:%s is not unique as expected.\n",
+			prefix_name, value);
+#endif
+
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
 }
 
 static Xapian::Document
@@ -242,26 +289,6 @@ find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
     return notmuch->xapian_db->get_document (doc_id);
 }
 
-static notmuch_private_status_t
-find_unique_document (notmuch_database_t *notmuch,
-		      const char *prefix_name,
-		      const char *value,
-		      Xapian::Document *document,
-		      unsigned int *doc_id)
-{
-    notmuch_private_status_t status;
-
-    status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
-
-    if (status) {
-	*document = Xapian::Document ();
-	return status;
-    }
-
-    *document = find_document_for_doc_id (notmuch, *doc_id);
-    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
-}
-
 notmuch_message_t *
 notmuch_database_find_message (notmuch_database_t *notmuch,
 			       const char *message_id)
@@ -269,7 +296,8 @@ notmuch_database_find_message (notmuch_database_t *notmuch,
     notmuch_private_status_t status;
     unsigned int doc_id;
 
-    status = find_unique_doc_id (notmuch, "id", message_id, &doc_id);
+    status = _notmuch_database_find_unique_doc_id (notmuch, "id",
+						   message_id, &doc_id);
 
     if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
 	return NULL;
@@ -446,6 +474,7 @@ notmuch_database_create (const char *path)
 
     notmuch = notmuch_database_open (path,
 				     NOTMUCH_DATABASE_MODE_READ_WRITE);
+    notmuch_database_upgrade (notmuch, NULL, NULL);
 
   DONE:
     if (notmuch_path)
@@ -454,6 +483,17 @@ notmuch_database_create (const char *path)
     return notmuch;
 }
 
+notmuch_status_t
+_notmuch_database_ensure_writable (notmuch_database_t *notmuch)
+{
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
+	fprintf (stderr, "Cannot write to a read-only database.\n");
+	return NOTMUCH_STATUS_READ_ONLY_DATABASE;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 notmuch_database_t *
 notmuch_database_open (const char *path,
 		       notmuch_database_mode_t mode)
@@ -462,7 +502,7 @@ notmuch_database_open (const char *path,
     char *notmuch_path = NULL, *xapian_path = NULL;
     struct stat st;
     int err;
-    unsigned int i;
+    unsigned int i, version;
 
     if (asprintf (&notmuch_path, "%s/%s", path, ".notmuch") == -1) {
 	notmuch_path = NULL;
@@ -490,13 +530,42 @@ notmuch_database_open (const char *path,
     if (notmuch->path[strlen (notmuch->path) - 1] == '/')
 	notmuch->path[strlen (notmuch->path) - 1] = '\0';
 
+    notmuch->needs_upgrade = FALSE;
     notmuch->mode = mode;
     try {
 	if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
 	    notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
 							       Xapian::DB_CREATE_OR_OPEN);
+	    version = notmuch_database_get_version (notmuch);
+
+	    if (version > NOTMUCH_DATABASE_VERSION) {
+		fprintf (stderr,
+			 "Error: Notmuch database at %s\n"
+			 "       has a newer database format version (%u) than supported by this\n"
+			 "       version of notmuch (%u). Refusing to open this database in\n"
+			 "       read-write mode.\n",
+			 notmuch_path, version, NOTMUCH_DATABASE_VERSION);
+		notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY;
+		notmuch_database_close (notmuch);
+		notmuch = NULL;
+		goto DONE;
+	    }
+
+	    if (version < NOTMUCH_DATABASE_VERSION)
+		notmuch->needs_upgrade = TRUE;
 	} else {
 	    notmuch->xapian_db = new Xapian::Database (xapian_path);
+	    version = notmuch_database_get_version (notmuch);
+	    if (version > NOTMUCH_DATABASE_VERSION)
+	    {
+		fprintf (stderr,
+			 "Warning: Notmuch database at %s\n"
+			 "         has a newer database format version (%u) than supported by this\n"
+			 "         version of notmuch (%u). Some operations may behave incorrectly,\n"
+			 "         (but the database will not be harmed since it is being opened\n"
+			 "         in read-only mode).\n",
+			 notmuch_path, version, NOTMUCH_DATABASE_VERSION);
+	    }
 	}
 	notmuch->query_parser = new Xapian::QueryParser;
 	notmuch->term_gen = new Xapian::TermGenerator;
@@ -560,110 +629,469 @@ notmuch_database_get_path (notmuch_database_t *notmuch)
     return notmuch->path;
 }
 
-static notmuch_private_status_t
-find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
-			 Xapian::Document *doc, unsigned int *doc_id)
+unsigned int
+notmuch_database_get_version (notmuch_database_t *notmuch)
 {
-    return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
+    unsigned int version;
+    string version_string;
+    const char *str;
+    char *end;
+
+    version_string = notmuch->xapian_db->get_metadata ("version");
+    if (version_string.empty ())
+	return 0;
+
+    str = version_string.c_str ();
+    if (str == NULL || *str == '\0')
+	return 0;
+
+    version = strtoul (str, &end, 10);
+    if (*end != '\0')
+	INTERNAL_ERROR ("Malformed database version: %s", str);
+
+    return version;
+}
+
+notmuch_bool_t
+notmuch_database_needs_upgrade (notmuch_database_t *notmuch)
+{
+    return notmuch->needs_upgrade;
+}
+
+static volatile sig_atomic_t do_progress_notify = 0;
+
+static void
+handle_sigalrm (unused (int signal))
+{
+    do_progress_notify = 1;
 }
 
-/* We allow the user to use arbitrarily long keys for timestamps,
- * (they're for filesystem paths after all, which have no limit we
- * know about). But we have a term-length limit. So if we exceed that,
- * we'll use the SHA-1 of the user's key as the actual key for
- * constructing a database term.
+/* Upgrade the current database.
  *
- * Caution: This function returns a newly allocated string which the
- * caller should free() when finished.
+ * After opening a database in read-write mode, the client should
+ * check if an upgrade is needed (notmuch_database_needs_upgrade) and
+ * if so, upgrade with this function before making any modifications.
+ *
+ * The optional progress_notify callback can be used by the caller to
+ * provide progress indication to the user. If non-NULL it will be
+ * called periodically with 'count' as the number of messages upgraded
+ * so far and 'total' the overall number of messages that will be
+ * converted.
  */
-static char *
-timestamp_db_key (const char *key)
+notmuch_status_t
+notmuch_database_upgrade (notmuch_database_t *notmuch,
+			  void (*progress_notify) (void *closure,
+						   double progress),
+			  void *closure)
 {
-    int term_len = strlen (_find_prefix ("timestamp")) + strlen (key);
+    Xapian::WritableDatabase *db;
+    struct sigaction action;
+    struct itimerval timerval;
+    notmuch_bool_t timer_is_active = FALSE;
+    unsigned int version;
+    notmuch_status_t status;
+    unsigned int count = 0, total = 0;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    version = notmuch_database_get_version (notmuch);
+
+    if (version >= NOTMUCH_DATABASE_VERSION)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    if (progress_notify) {
+	/* Setup our handler for SIGALRM */
+	memset (&action, 0, sizeof (struct sigaction));
+	action.sa_handler = handle_sigalrm;
+	sigemptyset (&action.sa_mask);
+	action.sa_flags = SA_RESTART;
+	sigaction (SIGALRM, &action, NULL);
+
+	/* Then start a timer to send SIGALRM once per second. */
+	timerval.it_interval.tv_sec = 1;
+	timerval.it_interval.tv_usec = 0;
+	timerval.it_value.tv_sec = 1;
+	timerval.it_value.tv_usec = 0;
+	setitimer (ITIMER_REAL, &timerval, NULL);
+
+	timer_is_active = TRUE;
+    }
+
+    /* Before version 1, each message document had its filename in the
+     * data field. Copy that into the new format by calling
+     * notmuch_message_add_filename.
+     */
+    if (version < 1) {
+	notmuch_query_t *query = notmuch_query_create (notmuch, "");
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+	char *filename;
+	Xapian::TermIterator t, t_end;
+
+	total = notmuch_query_count_messages (query);
+
+	for (messages = notmuch_query_search_messages (query);
+	     notmuch_messages_has_more (messages);
+	     notmuch_messages_advance (messages))
+	{
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    filename = _notmuch_message_talloc_copy_data (message);
+	    if (filename && *filename != '\0') {
+		_notmuch_message_add_filename (message, filename);
+		_notmuch_message_sync (message);
+	    }
+	    talloc_free (filename);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+
+	/* Also, before version 1 we stored directory timestamps in
+	 * XTIMESTAMP documents instead of the current XDIRECTORY
+	 * documents. So copy those as well. */
+
+	t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
+
+	for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
+	     t != t_end;
+	     t++)
+	{
+	    Xapian::PostingIterator p, p_end;
+	    std::string term = *t;
+
+	    p_end = notmuch->xapian_db->postlist_end (term);
+
+	    for (p = notmuch->xapian_db->postlist_begin (term);
+		 p != p_end;
+		 p++)
+	    {
+		Xapian::Document document;
+		time_t mtime;
+		notmuch_directory_t *directory;
+
+		if (do_progress_notify) {
+		    progress_notify (closure, (double) count / total);
+		    do_progress_notify = 0;
+		}
+
+		document = find_document_for_doc_id (notmuch, *p);
+		mtime = Xapian::sortable_unserialise (
+		    document.get_value (NOTMUCH_VALUE_TIMESTAMP));
+
+		directory = notmuch_database_get_directory (notmuch,
+							    term.c_str() + 10);
+		notmuch_directory_set_mtime (directory, mtime);
+		notmuch_directory_destroy (directory);
+	    }
+	}
+    }
+
+    db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
+    db->flush ();
+
+    /* Now that the upgrade is complete we can remove the old data
+     * and documents that are no longer needed. */
+    if (version < 1) {
+	notmuch_query_t *query = notmuch_query_create (notmuch, "");
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+	char *filename;
+
+	for (messages = notmuch_query_search_messages (query);
+	     notmuch_messages_has_more (messages);
+	     notmuch_messages_advance (messages))
+	{
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    filename = _notmuch_message_talloc_copy_data (message);
+	    if (filename && *filename != '\0') {
+		_notmuch_message_clear_data (message);
+		_notmuch_message_sync (message);
+	    }
+	    talloc_free (filename);
+
+	    notmuch_message_destroy (message);
+	}
+
+	notmuch_query_destroy (query);
+    }
+
+    if (version < 1) {
+	Xapian::TermIterator t, t_end;
+
+	t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
+
+	for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
+	     t != t_end;
+	     t++)
+	{
+	    Xapian::PostingIterator p, p_end;
+	    std::string term = *t;
+
+	    p_end = notmuch->xapian_db->postlist_end (term);
+
+	    for (p = notmuch->xapian_db->postlist_begin (term);
+		 p != p_end;
+		 p++)
+	    {
+		if (do_progress_notify) {
+		    progress_notify (closure, (double) count / total);
+		    do_progress_notify = 0;
+		}
+
+		db->delete_document (*p);
+	    }
+	}
+    }
+
+    if (timer_is_active) {
+	/* Now stop the timer. */
+	timerval.it_interval.tv_sec = 0;
+	timerval.it_interval.tv_usec = 0;
+	timerval.it_value.tv_sec = 0;
+	timerval.it_value.tv_usec = 0;
+	setitimer (ITIMER_REAL, &timerval, NULL);
+
+	/* And disable the signal handler. */
+	action.sa_handler = SIG_IGN;
+	sigaction (SIGALRM, &action, NULL);
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* We allow the user to use arbitrarily long paths for directories. But
+ * we have a term-length limit. So if we exceed that, we'll use the
+ * SHA-1 of the path for the database term.
+ *
+ * Note: This function may return the original value of 'path'. If it
+ * does not, then the caller is responsible to free() the returned
+ * value.
+ */
+const char *
+_notmuch_database_get_directory_db_path (const char *path)
+{
+    int term_len = strlen (_find_prefix ("directory")) + strlen (path);
 
     if (term_len > NOTMUCH_TERM_MAX)
-	return notmuch_sha1_of_string (key);
+	return notmuch_sha1_of_string (path);
     else
-	return strdup (key);
+	return path;
 }
 
+/* Given a path, split it into two parts: the directory part is all
+ * components except for the last, and the basename is that last
+ * component. Getting the return-value for either part is optional
+ * (the caller can pass NULL).
+ *
+ * The original 'path' can represent either a regular file or a
+ * directory---the splitting will be carried out in the same way in
+ * either case. Trailing slashes on 'path' will be ignored, and any
+ * cases of multiple '/' characters appearing in series will be
+ * treated as a single '/'.
+ *
+ * Allocation (if any) will have 'ctx' as the talloc owner. But
+ * pointers will be returned within the original path string whenever
+ * possible.
+ *
+ * Note: If 'path' is non-empty and contains no non-trailing slash,
+ * (that is, consists of a filename with no parent directory), then
+ * the directory returned will be an empty string. However, if 'path'
+ * is an empty string, then both directory and basename will be
+ * returned as NULL.
+ */
 notmuch_status_t
-notmuch_database_set_timestamp (notmuch_database_t *notmuch,
-				const char *key, time_t timestamp)
+_notmuch_database_split_path (void *ctx,
+			      const char *path,
+			      const char **directory,
+			      const char **basename)
 {
-    Xapian::Document doc;
-    Xapian::WritableDatabase *db;
-    unsigned int doc_id;
-    notmuch_private_status_t status;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
-    char *db_key = NULL;
+    const char *slash;
 
-    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
-	fprintf (stderr, "Attempted to update a read-only database.\n");
-	return NOTMUCH_STATUS_READONLY_DATABASE;
+    if (path == NULL || *path == '\0') {
+	if (directory)
+	    *directory = NULL;
+	if (basename)
+	    *basename = NULL;
+	return NOTMUCH_STATUS_SUCCESS;
     }
 
-    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
-    db_key = timestamp_db_key (key);
+    /* Find the last slash (not counting a trailing slash), if any. */
 
-    try {
-	status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
+    slash = path + strlen (path) - 1;
 
-	doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
-		       Xapian::sortable_serialise (timestamp));
+    /* First, skip trailing slashes. */
+    while (slash != path) {
+	if (*slash != '/')
+	    break;
 
-	if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
-	    char *term = talloc_asprintf (NULL, "%s%s",
-					  _find_prefix ("timestamp"), db_key);
-	    doc.add_term (term);
-	    talloc_free (term);
+	--slash;
+    }
 
-	    db->add_document (doc);
-	} else {
-	    db->replace_document (doc_id, doc);
-	}
+    /* Then, find a slash. */
+    while (slash != path) {
+	if (*slash == '/')
+	    break;
 
-    } catch (const Xapian::Error &error) {
-	fprintf (stderr, "A Xapian exception occurred setting timestamp: %s.\n",
-		 error.get_msg().c_str());
-	notmuch->exception_reported = TRUE;
-	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	if (basename)
+	    *basename = slash;
+
+	--slash;
     }
 
-    if (db_key)
-	free (db_key);
+    /* Finally, skip multiple slashes. */
+    while (slash != path) {
+	if (*slash != '/')
+	    break;
 
-    return ret;
+	--slash;
+    }
+
+    if (slash == path) {
+	if (directory)
+	    *directory = talloc_strdup (ctx, "");
+	if (basename)
+	    *basename = path;
+    } else {
+	if (directory)
+	    *directory = talloc_strndup (ctx, path, slash - path + 1);
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
-time_t
-notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
+notmuch_status_t
+_notmuch_database_find_directory_id (notmuch_database_t *notmuch,
+				     const char *path,
+				     unsigned int *directory_id)
 {
-    Xapian::Document doc;
-    unsigned int doc_id;
-    notmuch_private_status_t status;
-    char *db_key = NULL;
-    time_t ret = 0;
+    notmuch_directory_t *directory;
+    notmuch_status_t status;
 
-    db_key = timestamp_db_key (key);
+    if (path == NULL) {
+	*directory_id = 0;
+	return NOTMUCH_STATUS_SUCCESS;
+    }
 
-    try {
-	status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
+    directory = _notmuch_directory_create (notmuch, path, &status);
+    if (status) {
+	*directory_id = -1;
+	return status;
+    }
 
-	if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
-	    goto DONE;
+    *directory_id = _notmuch_directory_get_document_id (directory);
 
-	ret =  Xapian::sortable_unserialise (doc.get_value (NOTMUCH_VALUE_TIMESTAMP));
-    } catch (Xapian::Error &error) {
-	ret = 0;
-	goto DONE;
+    notmuch_directory_destroy (directory);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+const char *
+_notmuch_database_get_directory_path (void *ctx,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id)
+{
+    Xapian::Document document;
+
+    document = find_document_for_doc_id (notmuch, doc_id);
+
+    return talloc_strdup (ctx, document.get_data ().c_str ());
+}
+
+/* Given a legal 'filename' for the database, (either relative to
+ * database path or absolute with initial components identical to
+ * database path), return a new string (with 'ctx' as the talloc
+ * owner) suitable for use as a direntry term value.
+ *
+ * The necessary directory documents will be created in the database
+ * as needed.
+ */
+notmuch_status_t
+_notmuch_database_filename_to_direntry (void *ctx,
+					notmuch_database_t *notmuch,
+					const char *filename,
+					char **direntry)
+{
+    const char *relative, *directory, *basename;
+    Xapian::docid directory_id;
+    notmuch_status_t status;
+
+    relative = _notmuch_database_relative_path (notmuch, filename);
+
+    status = _notmuch_database_split_path (ctx, relative,
+					   &directory, &basename);
+    if (status)
+	return status;
+
+    status = _notmuch_database_find_directory_id (notmuch, directory,
+						  &directory_id);
+    if (status)
+	return status;
+
+    *direntry = talloc_asprintf (ctx, "%u:%s", directory_id, basename);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+/* Given a legal 'path' for the database, return the relative path.
+ *
+ * The return value will be a pointer to the originl path contents,
+ * and will be either the original string (if 'path' was relative) or
+ * a portion of the string (if path was absolute and begins with the
+ * database path).
+ */
+const char *
+_notmuch_database_relative_path (notmuch_database_t *notmuch,
+				 const char *path)
+{
+    const char *db_path, *relative;
+    unsigned int db_path_len;
+
+    db_path = notmuch_database_get_path (notmuch);
+    db_path_len = strlen (db_path);
+
+    relative = path;
+
+    if (*relative == '/') {
+	while (*relative == '/' && *(relative+1) == '/')
+	    relative++;
+
+	if (strncmp (relative, db_path, db_path_len) == 0)
+	{
+	    relative += db_path_len;
+	    while (*relative == '/')
+		relative++;
+	}
     }
 
-  DONE:
-    if (db_key)
-	free (db_key);
+    return relative;
+}
 
-    return ret;
+notmuch_directory_t *
+notmuch_database_get_directory (notmuch_database_t *notmuch,
+				const char *path)
+{
+    notmuch_status_t status;
+
+    return _notmuch_directory_create (notmuch, path, &status);
 }
 
 /* Find the thread ID to which the message with 'message_id' belongs.
@@ -903,11 +1331,13 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     if (message_ret)
 	*message_ret = NULL;
 
+    ret = _notmuch_database_ensure_writable (notmuch);
+    if (ret)
+	return ret;
+
     message_file = notmuch_message_file_open (filename);
-    if (message_file == NULL) {
-	ret = NOTMUCH_STATUS_FILE_ERROR;
-	goto DONE;
-    }
+    if (message_file == NULL)
+	return NOTMUCH_STATUS_FILE_ERROR;
 
     notmuch_message_file_restrict_headers (message_file,
 					   "date",
@@ -988,23 +1418,24 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	    goto DONE;
 	}
 
+	_notmuch_message_add_filename (message, filename);
+
 	/* Is this a newly created message object? */
 	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
-	    _notmuch_message_set_filename (message, filename);
 	    _notmuch_message_add_term (message, "type", "mail");
-	} else {
-	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-	    goto DONE;
-	}
 
-	ret = _notmuch_database_link_message (notmuch, message, message_file);
-	if (ret)
-	    goto DONE;
+	    ret = _notmuch_database_link_message (notmuch, message,
+						  message_file);
+	    if (ret)
+		goto DONE;
 
-	date = notmuch_message_file_get_header (message_file, "date");
-	_notmuch_message_set_date (message, date);
+	    date = notmuch_message_file_get_header (message_file, "date");
+	    _notmuch_message_set_date (message, date);
 
-	_notmuch_message_index_file (message, filename);
+	    _notmuch_message_index_file (message, filename);
+	} else {
+	    ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+	}
 
 	_notmuch_message_sync (message);
     } catch (const Xapian::Error &error) {
@@ -1029,6 +1460,60 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     return ret;
 }
 
+notmuch_status_t
+notmuch_database_remove_message (notmuch_database_t *notmuch,
+				 const char *filename)
+{
+    Xapian::WritableDatabase *db;
+    void *local = talloc_new (notmuch);
+    const char *prefix = _find_prefix ("file-direntry");
+    char *direntry, *term;
+    Xapian::PostingIterator i, end;
+    Xapian::Document document;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    status = _notmuch_database_filename_to_direntry (local, notmuch,
+						     filename, &direntry);
+    if (status)
+	return status;
+
+    term = talloc_asprintf (notmuch, "%s%s", prefix, direntry);
+
+    find_doc_ids_for_term (notmuch, term, &i, &end);
+
+    for ( ; i != end; i++) {
+	Xapian::TermIterator j;
+
+	document = find_document_for_doc_id (notmuch, *i);
+
+	document.remove_term (term);
+
+	j = document.termlist_begin ();
+	j.skip_to (prefix);
+
+	/* Was this the last file-direntry in the message? */
+	if (j == document.termlist_end () ||
+	    strncmp ((*j).c_str (), prefix, strlen (prefix)))
+	{
+	    db->delete_document (document.get_docid ());
+	    status = NOTMUCH_STATUS_SUCCESS;
+	} else {
+	    db->replace_document (document.get_docid (), document);
+	    status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+	}
+    }
+
+    talloc_free (local);
+
+    return status;
+}
+
 notmuch_tags_t *
 _notmuch_convert_tags (void *ctx, Xapian::TermIterator &i,
 		       Xapian::TermIterator &end)
diff --git a/lib/directory.cc b/lib/directory.cc
new file mode 100644
index 00000000..bb6314ad
--- /dev/null
+++ b/lib/directory.cc
@@ -0,0 +1,338 @@
+/* directory.cc - Results of directory-based searches from a notmuch database
+ *
+ * Copyright © 2009 Carl Worth
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: Carl Worth <cworth@cworth.org>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+
+#include <xapian.h>
+
+struct _notmuch_filenames {
+    Xapian::TermIterator iterator;
+    Xapian::TermIterator end;
+    int prefix_len;
+    char *filename;
+};
+
+/* We end up having to call the destructors explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_filenames_destructor (notmuch_filenames_t *filenames)
+{
+    filenames->iterator.~TermIterator ();
+    filenames->end.~TermIterator ();
+
+    return 0;
+}
+
+/* Create an iterator to iterate over the basenames of files (or
+ * directories) that all share a common parent directory.
+ *
+ * The code here is general enough to be reused for any case of
+ * iterating over the non-prefixed portion of terms sharing a common
+ * prefix.
+ */
+notmuch_filenames_t *
+_notmuch_filenames_create (void *ctx,
+			   notmuch_database_t *notmuch,
+			   const char *prefix)
+{
+    notmuch_filenames_t *filenames;
+
+    filenames = talloc (ctx, notmuch_filenames_t);
+    if (unlikely (filenames == NULL))
+	return NULL;
+
+    new (&filenames->iterator) Xapian::TermIterator ();
+    new (&filenames->end) Xapian::TermIterator ();
+
+    talloc_set_destructor (filenames, _notmuch_filenames_destructor);
+
+    filenames->iterator = notmuch->xapian_db->allterms_begin (prefix);
+    filenames->end = notmuch->xapian_db->allterms_end (prefix);
+
+    filenames->prefix_len = strlen (prefix);
+
+    filenames->filename = NULL;
+
+    return filenames;
+}
+
+notmuch_bool_t
+notmuch_filenames_has_more (notmuch_filenames_t *filenames)
+{
+    if (filenames == NULL)
+	return NULL;
+
+    return (filenames->iterator != filenames->end);
+}
+
+const char *
+notmuch_filenames_get (notmuch_filenames_t *filenames)
+{
+    if (filenames == NULL || filenames->iterator == filenames->end)
+	return NULL;
+
+    if (filenames->filename == NULL) {
+	std::string term = *filenames->iterator;
+
+	filenames->filename = talloc_strdup (filenames,
+					     term.c_str () +
+					     filenames->prefix_len);
+    }
+
+    return filenames->filename;
+}
+
+void
+notmuch_filenames_advance (notmuch_filenames_t *filenames)
+{
+    if (filenames == NULL)
+	return;
+
+    if (filenames->filename) {
+	talloc_free (filenames->filename);
+	filenames->filename = NULL;
+    }
+
+    if (filenames->iterator != filenames->end)
+	filenames->iterator++;
+}
+
+void
+notmuch_filenames_destroy (notmuch_filenames_t *filenames)
+{
+    if (filenames == NULL)
+	return;
+
+    talloc_free (filenames);
+}
+
+struct _notmuch_directory {
+    notmuch_database_t *notmuch;
+    Xapian::docid document_id;
+    Xapian::Document doc;
+    time_t mtime;
+};
+
+/* We end up having to call the destructor explicitly because we had
+ * to use "placement new" in order to initialize C++ objects within a
+ * block that we allocated with talloc. So C++ is making talloc
+ * slightly less simple to use, (we wouldn't need
+ * talloc_set_destructor at all otherwise).
+ */
+static int
+_notmuch_directory_destructor (notmuch_directory_t *directory)
+{
+    directory->doc.~Document ();
+
+    return 0;
+}
+
+static notmuch_private_status_t
+find_directory_document (notmuch_database_t *notmuch,
+			 const char *db_path,
+			 Xapian::Document *document)
+{
+    notmuch_private_status_t status;
+    Xapian::docid doc_id;
+
+    status = _notmuch_database_find_unique_doc_id (notmuch, "directory",
+						   db_path, &doc_id);
+    if (status) {
+	*document = Xapian::Document ();
+	return status;
+    }
+
+    *document = notmuch->xapian_db->get_document (doc_id);
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+}
+
+notmuch_directory_t *
+_notmuch_directory_create (notmuch_database_t *notmuch,
+			   const char *path,
+			   notmuch_status_t *status_ret)
+{
+    Xapian::WritableDatabase *db;
+    notmuch_directory_t *directory;
+    notmuch_private_status_t private_status;
+    const char *db_path;
+
+    *status_ret = NOTMUCH_STATUS_SUCCESS;
+
+    path = _notmuch_database_relative_path (notmuch, path);
+
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	INTERNAL_ERROR ("Failure to ensure database is writable");
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    directory = talloc (notmuch, notmuch_directory_t);
+    if (unlikely (directory == NULL))
+	return NULL;
+
+    directory->notmuch = notmuch;
+
+    /* "placement new"---not actually allocating memory */
+    new (&directory->doc) Xapian::Document;
+
+    talloc_set_destructor (directory, _notmuch_directory_destructor);
+
+    db_path = _notmuch_database_get_directory_db_path (path);
+
+    try {
+	Xapian::TermIterator i, end;
+
+	private_status = find_directory_document (notmuch, db_path,
+						  &directory->doc);
+	directory->document_id = directory->doc.get_docid ();
+
+	if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
+	    void *local = talloc_new (directory);
+	    const char *parent, *basename;
+	    Xapian::docid parent_id;
+	    char *term = talloc_asprintf (local, "%s%s",
+					  _find_prefix ("directory"), db_path);
+	    directory->doc.add_term (term, 0);
+
+	    directory->doc.set_data (path);
+
+	    _notmuch_database_split_path (local, path, &parent, &basename);
+
+	    _notmuch_database_find_directory_id (notmuch, parent, &parent_id);
+
+	    if (basename) {
+		term = talloc_asprintf (local, "%s%u:%s",
+					_find_prefix ("directory-direntry"),
+					parent_id, basename);
+		directory->doc.add_term (term, 0);
+	    }
+
+	    directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+				      Xapian::sortable_serialise (0));
+
+	    directory->document_id = db->add_document (directory->doc);
+	    talloc_free (local);
+	}
+
+	directory->mtime = Xapian::sortable_unserialise (
+	    directory->doc.get_value (NOTMUCH_VALUE_TIMESTAMP));
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr,
+		 "A Xapian exception occurred creating a directory: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = TRUE;
+	notmuch_directory_destroy (directory);
+	*status_ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	return NULL;
+    }
+
+    if (db_path != path)
+	free ((char *) db_path);
+
+    return directory;
+}
+
+unsigned int
+_notmuch_directory_get_document_id (notmuch_directory_t *directory)
+{
+    return directory->document_id;
+}
+
+notmuch_status_t
+notmuch_directory_set_mtime (notmuch_directory_t *directory,
+			     time_t mtime)
+{
+    notmuch_database_t *notmuch = directory->notmuch;
+    Xapian::WritableDatabase *db;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+
+    try {
+	directory->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
+				   Xapian::sortable_serialise (mtime));
+
+	db->replace_document (directory->document_id, directory->doc);
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr,
+		 "A Xapian exception occurred setting directory mtime: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = TRUE;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+time_t
+notmuch_directory_get_mtime (notmuch_directory_t *directory)
+{
+    return directory->mtime;
+}
+
+notmuch_filenames_t *
+notmuch_directory_get_child_files (notmuch_directory_t *directory)
+{
+    char *term;
+    notmuch_filenames_t *child_files;
+
+    term = talloc_asprintf (directory, "%s%u:",
+			    _find_prefix ("file-direntry"),
+			    directory->document_id);
+
+    child_files = _notmuch_filenames_create (directory,
+					     directory->notmuch, term);
+
+    talloc_free (term);
+
+    return child_files;
+}
+
+notmuch_filenames_t *
+notmuch_directory_get_child_directories (notmuch_directory_t *directory)
+{
+    char *term;
+    notmuch_filenames_t *child_directories;
+
+    term = talloc_asprintf (directory, "%s%u:",
+			    _find_prefix ("directory-direntry"),
+			    directory->document_id);
+
+    child_directories = _notmuch_filenames_create (directory,
+						   directory->notmuch, term);
+
+    talloc_free (term);
+
+    return child_directories;
+}
+
+void
+notmuch_directory_destroy (notmuch_directory_t *directory)
+{
+    talloc_free (directory);
+}
diff --git a/lib/index.cc b/lib/index.cc
index 125fa6c9..7e2da085 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -31,7 +31,7 @@ _index_address_mailbox (notmuch_message_t *message,
 {
     InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
     const char *name, *addr;
-    void *local = talloc_new (NULL);
+    void *local = talloc_new (message);
 
     name = internet_address_get_name (address);
     addr = internet_address_mailbox_get_addr (mailbox);
@@ -123,60 +123,6 @@ skip_re_in_subject (const char *subject)
     return s;
 }
 
-/* Given a string representing the body of a message, generate terms
- * for it, (skipping quoted portions and signatures).
- *
- * This function is evil in that it modifies the string passed to it,
- * (changing some newlines into '\0').
- */
-static void
-_index_body_text (notmuch_message_t *message, char *body)
-{
-    char *line, *line_end, *next_line;
-
-    if (body == NULL)
-	return;
-
-    next_line = body;
-
-    while (1) {
-	line = next_line;
-	if (*line == '\0')
-	    break;
-
-	next_line = strchr (line, '\n');
-	if (next_line == NULL) {
-	    next_line = line + strlen (line);
-	}
-	line_end = next_line - 1;
-
-	/* Get to the next non-blank line. */
-	while (*next_line == '\n')
-	    next_line++;
-
-	/* Skip blank lines. */
-	if (line_end < line)
-	    continue;
-
-	/* Skip lines that are quotes. */
-	if (*line == '>')
-	    continue;
-
-	/* Also skip lines introducing a quote on the next line. */
-	if (*line_end == ':' && *next_line == '>')
-	    continue;
-
-	/* Finally, bail as soon as we see a signature. */
-	/* XXX: Should only do this if "near" the end of the message. */
-	if (strncmp (line, "-- ", 3) == 0)
-	    break;
-
-	*(line_end + 1) = '\0';
-
-	_notmuch_message_gen_terms (message, NULL, line);
-    }
-}
-
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
@@ -249,9 +195,11 @@ _index_mime_part (notmuch_message_t *message,
     g_byte_array_append (byte_array, (guint8 *) "\0", 1);
     body = (char *) g_byte_array_free (byte_array, FALSE);
 
-    _index_body_text (message, body);
+    if (body) {
+	_notmuch_message_gen_terms (message, NULL, body);
 
-    free (body);
+	free (body);
+    }
 }
 
 notmuch_status_t
diff --git a/lib/message.cc b/lib/message.cc
index 49519f1e..f0e905b7 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -174,11 +174,6 @@ _notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
     unsigned int doc_id;
     char *term;
 
-    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY) {
-	*status_ret = NOTMUCH_PRIVATE_STATUS_READONLY_DATABASE;
-	return NULL;
-    }
-
     *status_ret = NOTMUCH_PRIVATE_STATUS_SUCCESS;
 
     message = notmuch_database_find_message (notmuch, message_id);
@@ -192,9 +187,12 @@ _notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
 	return NULL;
     }
 
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY)
+	INTERNAL_ERROR ("Failure to ensure database is writable.");
+
     db = static_cast<Xapian::WritableDatabase *> (notmuch->xapian_db);
     try {
-	doc.add_term (term);
+	doc.add_term (term, 0);
 	talloc_free (term);
 
 	doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
@@ -385,20 +383,17 @@ notmuch_message_get_replies (notmuch_message_t *message)
     return _notmuch_messages_create (message->replies);
 }
 
-/* Set the filename for 'message' to 'filename'.
- *
- * XXX: We should still figure out if we think it's important to store
- * multiple filenames for email messages with identical message IDs.
+/* Add an additional 'filename' for 'message'.
  *
  * This change will not be reflected in the database until the next
  * call to _notmuch_message_set_sync. */
-void
-_notmuch_message_set_filename (notmuch_message_t *message,
+notmuch_status_t
+_notmuch_message_add_filename (notmuch_message_t *message,
 			       const char *filename)
 {
-    const char *s;
-    const char *db_path;
-    unsigned int db_path_len;
+    notmuch_status_t status;
+    void *local = talloc_new (message);
+    char *direntry;
 
     if (message->filename) {
 	talloc_free (message->filename);
@@ -408,41 +403,98 @@ _notmuch_message_set_filename (notmuch_message_t *message,
     if (filename == NULL)
 	INTERNAL_ERROR ("Message filename cannot be NULL.");
 
-    s = filename;
+    status = _notmuch_database_filename_to_direntry (local,
+						     message->notmuch,
+						     filename, &direntry);
+    if (status)
+	return status;
 
-    db_path = notmuch_database_get_path (message->notmuch);
-    db_path_len = strlen (db_path);
+    _notmuch_message_add_term (message, "file-direntry", direntry);
 
-    if (*s == '/' && strlen (s) > db_path_len
-	&& strncmp (s, db_path, db_path_len) == 0)
-    {
-	s += db_path_len;
-	while (*s == '/') s++;
+    talloc_free (local);
 
-	if (!*s)
-		INTERNAL_ERROR ("Message filename was same as db prefix.");
-    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
 
-    message->doc.set_data (s);
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message)
+{
+    return talloc_strdup (message, message->doc.get_data ().c_str ());
+}
+
+void
+_notmuch_message_clear_data (notmuch_message_t *message)
+{
+    message->doc.set_data ("");
 }
 
 const char *
 notmuch_message_get_filename (notmuch_message_t *message)
 {
-    std::string filename_str;
-    const char *db_path;
+    const char *prefix = _find_prefix ("file-direntry");
+    int prefix_len = strlen (prefix);
+    Xapian::TermIterator i;
+    char *direntry, *colon;
+    const char *db_path, *directory, *basename;
+    unsigned int directory_id;
+    void *local = talloc_new (message);
 
     if (message->filename)
 	return message->filename;
 
-    filename_str = message->doc.get_data ();
+    i = message->doc.termlist_begin ();
+    i.skip_to (prefix);
+
+    if (i != message->doc.termlist_end ())
+	direntry = talloc_strdup (local, (*i).c_str ());
+
+    if (i == message->doc.termlist_end () ||
+	strncmp (direntry, prefix, prefix_len))
+    {
+	/* A message document created by an old version of notmuch
+	 * (prior to rename support) will have the filename in the
+	 * data of the document rather than as a file-direntry term.
+	 *
+	 * It would be nice to do the upgrade of the document directly
+	 * here, but the database is likely open in read-only mode. */
+	const char *data;
+
+	data = message->doc.get_data ().c_str ();
+
+	if (data == NULL)
+	    INTERNAL_ERROR ("message with no filename");
+
+	message->filename = talloc_strdup (message, data);
+
+	return message->filename;
+    }
+
+    direntry += prefix_len;
+
+    directory_id = strtol (direntry, &colon, 10);
+
+    if (colon == NULL || *colon != ':')
+	INTERNAL_ERROR ("malformed direntry");
+
+    basename = colon + 1;
+
+    *colon = '\0';
+
     db_path = notmuch_database_get_path (message->notmuch);
 
-    if (filename_str[0] != '/')
-	message->filename = talloc_asprintf (message, "%s/%s", db_path,
-					     filename_str.c_str ());
+    directory = _notmuch_database_get_directory_path (local,
+						      message->notmuch,
+						      directory_id);
+
+    if (strlen (directory))
+	message->filename = talloc_asprintf (message, "%s/%s/%s",
+					     db_path, directory, basename);
     else
-	message->filename = talloc_strdup (message, filename_str.c_str ());
+	message->filename = talloc_asprintf (message, "%s/%s",
+					     db_path, basename);
+    talloc_free ((void *) directory);
+
+    talloc_free (local);
 
     return message->filename;
 }
@@ -594,7 +646,7 @@ _notmuch_message_add_term (notmuch_message_t *message,
     if (strlen (term) > NOTMUCH_TERM_MAX)
 	return NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG;
 
-    message->doc.add_term (term);
+    message->doc.add_term (term, 0);
 
     talloc_free (term);
 
@@ -667,7 +719,12 @@ _notmuch_message_remove_term (notmuch_message_t *message,
 notmuch_status_t
 notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
 {
-    notmuch_private_status_t status;
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
 
     if (tag == NULL)
 	return NOTMUCH_STATUS_NULL_POINTER;
@@ -675,10 +732,10 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
     if (strlen (tag) > NOTMUCH_TAG_MAX)
 	return NOTMUCH_STATUS_TAG_TOO_LONG;
 
-    status = _notmuch_message_add_term (message, "tag", tag);
-    if (status) {
+    private_status = _notmuch_message_add_term (message, "tag", tag);
+    if (private_status) {
 	INTERNAL_ERROR ("_notmuch_message_add_term return unexpected value: %d\n",
-			status);
+			private_status);
     }
 
     if (! message->frozen)
@@ -690,7 +747,12 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag)
 notmuch_status_t
 notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
 {
-    notmuch_private_status_t status;
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
 
     if (tag == NULL)
 	return NOTMUCH_STATUS_NULL_POINTER;
@@ -698,10 +760,10 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
     if (strlen (tag) > NOTMUCH_TAG_MAX)
 	return NOTMUCH_STATUS_TAG_TOO_LONG;
 
-    status = _notmuch_message_remove_term (message, "tag", tag);
-    if (status) {
+    private_status = _notmuch_message_remove_term (message, "tag", tag);
+    if (private_status) {
 	INTERNAL_ERROR ("_notmuch_message_remove_term return unexpected value: %d\n",
-			status);
+			private_status);
     }
 
     if (! message->frozen)
@@ -710,39 +772,60 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag)
     return NOTMUCH_STATUS_SUCCESS;
 }
 
-void
+notmuch_status_t
 notmuch_message_remove_all_tags (notmuch_message_t *message)
 {
-    notmuch_private_status_t status;
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
     notmuch_tags_t *tags;
     const char *tag;
 
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
     for (tags = notmuch_message_get_tags (message);
 	 notmuch_tags_has_more (tags);
 	 notmuch_tags_advance (tags))
     {
 	tag = notmuch_tags_get (tags);
 
-	status = _notmuch_message_remove_term (message, "tag", tag);
-	if (status) {
+	private_status = _notmuch_message_remove_term (message, "tag", tag);
+	if (private_status) {
 	    INTERNAL_ERROR ("_notmuch_message_remove_term return unexpected value: %d\n",
-			    status);
+			    private_status);
 	}
     }
 
     if (! message->frozen)
 	_notmuch_message_sync (message);
+
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
-void
+notmuch_status_t
 notmuch_message_freeze (notmuch_message_t *message)
 {
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
     message->frozen++;
+
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
 notmuch_status_t
 notmuch_message_thaw (notmuch_message_t *message)
 {
+    notmuch_status_t status;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
     if (message->frozen > 0) {
 	message->frozen--;
 	if (message->frozen == 0)
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 116f63d6..c7fb0ef8 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -112,14 +112,15 @@ typedef enum _notmuch_private_status {
     /* First, copy all the public status values. */
     NOTMUCH_PRIVATE_STATUS_SUCCESS = NOTMUCH_STATUS_SUCCESS,
     NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY = NOTMUCH_STATUS_OUT_OF_MEMORY,
-    NOTMUCH_PRIVATE_STATUS_READONLY_DATABASE = NOTMUCH_STATUS_READONLY_DATABASE,
+    NOTMUCH_PRIVATE_STATUS_READ_ONLY_DATABASE = NOTMUCH_STATUS_READ_ONLY_DATABASE,
     NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION = NOTMUCH_STATUS_XAPIAN_EXCEPTION,
     NOTMUCH_PRIVATE_STATUS_FILE_NOT_EMAIL = NOTMUCH_STATUS_FILE_NOT_EMAIL,
     NOTMUCH_PRIVATE_STATUS_NULL_POINTER = NOTMUCH_STATUS_NULL_POINTER,
     NOTMUCH_PRIVATE_STATUS_TAG_TOO_LONG = NOTMUCH_STATUS_TAG_TOO_LONG,
+    NOTMUCH_PRIVATE_STATUS_UNBALANCED_FREEZE_THAW = NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
 
     /* Then add our own private values. */
-    NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG,
+    NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG = NOTMUCH_STATUS_LAST_STATUS,
     NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND,
 
     NOTMUCH_PRIVATE_STATUS_LAST_STATUS
@@ -150,6 +151,54 @@ typedef enum _notmuch_private_status {
 const char *
 _find_prefix (const char *name);
 
+notmuch_status_t
+_notmuch_database_ensure_writable (notmuch_database_t *notmuch);
+
+const char *
+_notmuch_database_relative_path (notmuch_database_t *notmuch,
+				 const char *path);
+
+notmuch_status_t
+_notmuch_database_split_path (void *ctx,
+			      const char *path,
+			      const char **directory,
+			      const char **basename);
+
+const char *
+_notmuch_database_get_directory_db_path (const char *path);
+
+notmuch_private_status_t
+_notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
+				      const char *prefix_name,
+				      const char *value,
+				      unsigned int *doc_id);
+
+notmuch_status_t
+_notmuch_database_find_directory_id (notmuch_database_t *database,
+				     const char *path,
+				     unsigned int *directory_id);
+
+const char *
+_notmuch_database_get_directory_path (void *ctx,
+				      notmuch_database_t *notmuch,
+				      unsigned int doc_id);
+
+notmuch_status_t
+_notmuch_database_filename_to_direntry (void *ctx,
+					notmuch_database_t *notmuch,
+					const char *filename,
+					char **direntry);
+
+/* directory.cc */
+
+notmuch_directory_t *
+_notmuch_directory_create (notmuch_database_t *notmuch,
+			   const char *path,
+			   notmuch_status_t *status_ret);
+
+unsigned int
+_notmuch_directory_get_document_id (notmuch_directory_t *directory);
+
 /* thread.cc */
 
 notmuch_thread_t *
@@ -190,7 +239,10 @@ _notmuch_message_gen_terms (notmuch_message_t *message,
 			    const char *text);
 
 void
-_notmuch_message_set_filename (notmuch_message_t *message,
+_notmuch_message_upgrade_filename_storage (notmuch_message_t *message);
+
+notmuch_status_t
+_notmuch_message_add_filename (notmuch_message_t *message,
 			       const char *filename);
 
 void
@@ -206,6 +258,22 @@ _notmuch_message_sync (notmuch_message_t *message);
 void
 _notmuch_message_close (notmuch_message_t *message);
 
+/* Get a copy of the data in this message document.
+ *
+ * Caller should talloc_free the result when done.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+char *
+_notmuch_message_talloc_copy_data (notmuch_message_t *message);
+
+/* Clear the data in this message document.
+ *
+ * This function is intended to support database upgrade and really
+ * shouldn't be used otherwise. */
+void
+_notmuch_message_clear_data (notmuch_message_t *message);
+
 /* index.cc */
 
 notmuch_status_t
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 60834fb5..15c9db40 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -57,6 +57,9 @@ typedef int notmuch_bool_t;
  * value. Instead we should map to things like DATABASE_LOCKED or
  * whatever.
  *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: An attempt was made to write to
+ *	a database opened in read-only mode.
+ *
  * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
  *
  * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to read or
@@ -86,7 +89,7 @@ typedef int notmuch_bool_t;
 typedef enum _notmuch_status {
     NOTMUCH_STATUS_SUCCESS = 0,
     NOTMUCH_STATUS_OUT_OF_MEMORY,
-    NOTMUCH_STATUS_READONLY_DATABASE,
+    NOTMUCH_STATUS_READ_ONLY_DATABASE,
     NOTMUCH_STATUS_XAPIAN_EXCEPTION,
     NOTMUCH_STATUS_FILE_ERROR,
     NOTMUCH_STATUS_FILE_NOT_EMAIL,
@@ -114,6 +117,8 @@ typedef struct _notmuch_thread notmuch_thread_t;
 typedef struct _notmuch_messages notmuch_messages_t;
 typedef struct _notmuch_message notmuch_message_t;
 typedef struct _notmuch_tags notmuch_tags_t;
+typedef struct _notmuch_directory notmuch_directory_t;
+typedef struct _notmuch_filenames notmuch_filenames_t;
 
 /* Create a new, empty notmuch database located at 'path'.
  *
@@ -178,56 +183,46 @@ notmuch_database_close (notmuch_database_t *database);
 const char *
 notmuch_database_get_path (notmuch_database_t *database);
 
-/* Store a timestamp within the database.
- *
- * The Notmuch database will not interpret this key nor the timestamp
- * values at all. It will merely store them together and return the
- * timestamp when notmuch_database_get_timestamp is called with the
- * same value for 'key'.
- *
- * The intention is for the caller to use the timestamp to allow
- * efficient identification of new messages to be added to the
- * database. The recommended usage is as follows:
- *
- *   o Read the mtime of a directory from the filesystem
- *
- *   o Call add_message for all mail files in the directory
- *
- *   o Call notmuch_database_set_timestamp with the path of the
- *     directory as 'key' and the originally read mtime as 'value'.
- *
- * Then, when wanting to check for updates to the directory in the
- * future, the client can call notmuch_database_get_timestamp and know
- * that it only needs to add files if the mtime of the directory and
- * files are newer than the stored timestamp.
- *
- * Note: The notmuch_database_get_timestamp function does not allow
- * the caller to distinguish a timestamp of 0 from a non-existent
- * timestamp. So don't store a timestamp of 0 unless you are
- * comfortable with that.
+/* Return the database format version of the given database. */
+unsigned int
+notmuch_database_get_version (notmuch_database_t *database);
+
+/* Does this database need to be upgraded before writing to it?
  *
- * Return value:
+ * If this function returns TRUE then no functions that modify the
+ * database (notmuch_database_add_message, notmuch_message_add_tag,
+ * notmuch_directory_set_mtime, etc.) will work unless the function
+ * notmuch_database_upgrade is called successfully first. */
+notmuch_bool_t
+notmuch_database_needs_upgrade (notmuch_database_t *database);
+
+/* Upgrade the current database.
  *
- * NOTMUCH_STATUS_SUCCESS: Timestamp successfully stored in database.
+ * After opening a database in read-write mode, the client should
+ * check if an upgrade is needed (notmuch_database_needs_upgrade) and
+ * if so, upgrade with this function before making any modifications.
  *
- * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception
- *	occurred. Timestamp not stored.
+ * The optional progress_notify callback can be used by the caller to
+ * provide progress indication to the user. If non-NULL it will be
+ * called periodically with 'progress' as a floating-point value in
+ * the range of [0.0 .. 1.0] indicating the progress made so far in
+ * the upgrade process.
  */
 notmuch_status_t
-notmuch_database_set_timestamp (notmuch_database_t *database,
-				const char *key, time_t timestamp);
+notmuch_database_upgrade (notmuch_database_t *database,
+			  void (*progress_notify) (void *closure,
+						   double progress),
+			  void *closure);
 
-/* Retrieve a timestamp from the database.
- *
- * Returns the timestamp value previously stored by calling
- * notmuch_database_set_timestamp with the same value for 'key'.
+/* Retrieve a directory object from the database for 'path'.
  *
- * Returns 0 if no timestamp is stored for 'key' or if any error
- * occurred querying the database.
+ * Here, 'path' should be a path relative to the path of 'database'
+ * (see notmuch_database_get_path), or else should be an absolute path
+ * with initial components that match the path of 'database'.
  */
-time_t
-notmuch_database_get_timestamp (notmuch_database_t *database,
-				const char *key);
+notmuch_directory_t *
+notmuch_database_get_directory (notmuch_database_t *database,
+				const char *path);
 
 /* Add a new message to the given notmuch database.
  *
@@ -252,8 +247,8 @@ notmuch_database_get_timestamp (notmuch_database_t *database,
  * NOTMUCH_STATUS_SUCCESS: Message successfully added to database.
  *
  * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
- *	ID as another message already in the database. Nothing added
- *	to the database.
+ *	ID as another message already in the database. The new filename
+ *	was successfully added to the message in the database.
  *
  * NOTMUCH_STATUS_FILE_ERROR: an error occurred trying to open the
  *	file, (such as permission denied, or file not found,
@@ -261,12 +256,40 @@ notmuch_database_get_timestamp (notmuch_database_t *database,
  *
  * NOTMUCH_STATUS_FILE_NOT_EMAIL: the contents of filename don't look
  *	like an email message. Nothing added to the database.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so no message can be added.
  */
 notmuch_status_t
 notmuch_database_add_message (notmuch_database_t *database,
 			      const char *filename,
 			      notmuch_message_t **message);
 
+/* Remove a message from the given notmuch database.
+ *
+ * Note that only this particular filename association is removed from
+ * the database. If the same message (as determined by the message ID)
+ * is still available via other filenames, then the message will
+ * persist in the database for those filenames. When the last filename
+ * is removed for a particular message, the database content for that
+ * message will be entirely removed.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: The last filename was removed and the
+ *	message was removed from the database.
+ *
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: This filename was removed but
+ *	the message persists in the database with at least one other
+ *	filename.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so no message can be removed.
+ */
+notmuch_status_t
+notmuch_database_remove_message (notmuch_database_t *database,
+				 const char *filename);
+
 /* Find a message with the given message_id.
  *
  * If the database contains a message with the given message_id, then
@@ -698,14 +721,20 @@ notmuch_message_get_thread_id (notmuch_message_t *message);
 notmuch_messages_t *
 notmuch_message_get_replies (notmuch_message_t *message);
 
-/* Get the filename for the email corresponding to 'message'.
+/* Get a filename for the email corresponding to 'message'.
  *
  * The returned filename is an absolute filename, (the initial
  * component will match notmuch_database_get_path() ).
  *
  * The returned string belongs to the message so should not be
  * modified or freed by the caller (nor should it be referenced after
- * the message is destroyed). */
+ * the message is destroyed).
+ *
+ * Note: If this message corresponds to multiple files in the mail
+ * store, (that is, multiple files contain identical message IDs),
+ * this function will arbitrarily return a single one of those
+ * filenames.
+ */
 const char *
 notmuch_message_get_filename (notmuch_message_t *message);
 
@@ -793,6 +822,9 @@ notmuch_message_get_tags (notmuch_message_t *message);
  *
  * NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
  *	(exceeds NOTMUCH_TAG_MAX)
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
  */
 notmuch_status_t
 notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
@@ -807,6 +839,9 @@ notmuch_message_add_tag (notmuch_message_t *message, const char *tag);
  *
  * NOTMUCH_STATUS_TAG_TOO_LONG: The length of 'tag' is too long
  *	(exceeds NOTMUCH_TAG_MAX)
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
  */
 notmuch_status_t
 notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
@@ -815,8 +850,11 @@ notmuch_message_remove_tag (notmuch_message_t *message, const char *tag);
  *
  * See notmuch_message_freeze for an example showing how to safely
  * replace tag values.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
  */
-void
+notmuch_status_t
 notmuch_message_remove_all_tags (notmuch_message_t *message);
 
 /* Freeze the current state of 'message' within the database.
@@ -851,8 +889,15 @@ notmuch_message_remove_all_tags (notmuch_message_t *message);
  * somehow getting interrupted. This could result in the message being
  * left with no tags if the interruption happened after
  * notmuch_message_remove_all_tags but before notmuch_message_add_tag.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Message successfully frozen.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so message cannot be modified.
  */
-void
+notmuch_status_t
 notmuch_message_freeze (notmuch_message_t *message);
 
 /* Thaw the current 'message', synchronizing any changes that may have
@@ -929,6 +974,118 @@ notmuch_tags_advance (notmuch_tags_t *tags);
 void
 notmuch_tags_destroy (notmuch_tags_t *tags);
 
+/* Store an mtime within the database for 'directory'.
+ *
+ * The 'directory' should be an object retrieved from the database
+ * with notmuch_database_get_directory for a particular path.
+ *
+ * The intention is for the caller to use the mtime to allow efficient
+ * identification of new messages to be added to the database. The
+ * recommended usage is as follows:
+ *
+ *   o Read the mtime of a directory from the filesystem
+ *
+ *   o Call add_message for all mail files in the directory
+ *
+ *   o Call notmuch_directory_set_mtime with the mtime read from the
+ *     filesystem.
+ *
+ * Then, when wanting to check for updates to the directory in the
+ * future, the client can call notmuch_directory_get_mtime and know
+ * that it only needs to add files if the mtime of the directory and
+ * files are newer than the stored timestamp.
+ *
+ * Note: The notmuch_directory_get_mtime function does not allow the
+ * caller to distinguish a timestamp of 0 from a non-existent
+ * timestamp. So don't store a timestamp of 0 unless you are
+ * comfortable with that.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: mtime successfully stored in database.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception
+ *	occurred, mtime not stored.
+ *
+ * NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in read-only
+ *	mode so directory mtime cannot be modified.
+ */
+notmuch_status_t
+notmuch_directory_set_mtime (notmuch_directory_t *directory,
+			     time_t mtime);
+
+/* Get the mtime of a directory, (as previously stored with
+ * notmuch_directory_set_mtime).
+ *
+ * Returns 0 if no mtime has previously been stored for this
+ * directory.*/
+time_t
+notmuch_directory_get_mtime (notmuch_directory_t *directory);
+
+/* Get a notmuch_filenames_t iterator listing all the filenames of
+ * messages in the database within the given directory.
+ *
+ * The returned filenames will be the basename-entries only (not
+ * complete paths). */
+notmuch_filenames_t *
+notmuch_directory_get_child_files (notmuch_directory_t *directory);
+
+/* Get a notmuch_filenams_t iterator listing all the filenames of
+ * sub-directories in the database within the given directory.
+ *
+ * The returned filenames will be the basename-entries only (not
+ * complete paths). */
+notmuch_filenames_t *
+notmuch_directory_get_child_directories (notmuch_directory_t *directory);
+
+/* Destroy a notmuch_directory_t object. */
+void
+notmuch_directory_destroy (notmuch_directory_t *directory);
+
+/* Does the given notmuch_filenames_t object contain any more
+ * filenames.
+ *
+ * When this function returns TRUE, notmuch_filenames_get will return
+ * a valid string. Whereas when this function returns FALSE,
+ * notmuch_filenames_get will return NULL.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will always return FALSE.
+ */
+notmuch_bool_t
+notmuch_filenames_has_more (notmuch_filenames_t *filenames);
+
+/* Get the current filename from 'filenames' as a string.
+ *
+ * Note: The returned string belongs to 'filenames' and has a lifetime
+ * identical to it (and the directory to which it ultimately belongs).
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will always return NULL.
+ */
+const char *
+notmuch_filenames_get (notmuch_filenames_t *filenames);
+
+/* Advance the 'filenames' iterator to the next filename.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will do nothing.
+ */
+void
+notmuch_filenames_advance (notmuch_filenames_t *filenames);
+
+/* Destroy a notmuch_filenames_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_filenames_t object will be reclaimed when the
+ * containing directory object is destroyed.
+ *
+ * It is acceptable to pass NULL for 'filenames', in which case this
+ * function will do nothing.
+ */
+void
+notmuch_filenames_destroy (notmuch_filenames_t *filenames);
+
 NOTMUCH_END_DECLS
 
 #endif
diff --git a/lib/query.cc b/lib/query.cc
index 9106b92d..2c8d1672 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -134,6 +134,8 @@ notmuch_query_search_messages (notmuch_query_t *query)
 					 mail_query, string_query);
 	}
 
+	enquire.set_weighting_scheme (Xapian::BoolWeight());
+
 	switch (query->sort) {
 	case NOTMUCH_SORT_OLDEST_FIRST:
 	    enquire.set_sort_by_value (NOTMUCH_VALUE_TIMESTAMP, FALSE);