Merge commit 'debian/0.10.2-1' into squeeze-backports

Conflicts:
	debian/changelog

13 files changed, 433 insertions, 342 deletions

diff --git a/lib/Makefile b/lib/Makefile
index b6859eac..de492a7c 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -1,4 +1,4 @@
-# See Makfefile.local for the list of files to be compiled in this
+# See Makefile.local for the list of files to be compiled in this
 # directory.
 all:
 	$(MAKE) -C .. all
diff --git a/lib/Makefile.local b/lib/Makefile.local
index 7e2bc87b..57dca702 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -5,19 +5,13 @@
 # the library interface, (such as the deletion of an API or a major
 # semantic change that breaks formerly functioning code).
 #
-# Note: We don't currently have plans to increment this at this time.
-# If we *do* want to make an incompatible change to the library
-# interface, we'll have to decide whether to increment this (creating
-# a new soname) or to introduce symbol versioning to be able to
-# provide support for both old and new interfaces without having to
-# increment this.
-LIBNOTMUCH_VERSION_MAJOR = 1
+LIBNOTMUCH_VERSION_MAJOR = 2
 
 # The minor version of the library interface. This should be incremented at
 # the time of release for any additions to the library interface,
 # (and when it is incremented, the release version of the library should
 #  be reset to 0).
-LIBNOTMUCH_VERSION_MINOR = 3
+LIBNOTMUCH_VERSION_MINOR = 0
 
 # The release version the library interface. This should be incremented at
 # the time of release if there have been no changes to the interface, (but
@@ -47,6 +41,11 @@ endif
 dir := lib
 extra_cflags += -I$(srcdir)/$(dir) -fPIC
 
+# The (often-reused) $dir works fine within targets/prerequisites,
+# but cannot be used reliably within commands, so copy its value to a
+# variable that is not reused.
+lib := $(dir)
+
 libnotmuch_c_srcs =		\
 	$(notmuch_compat_srcs)	\
 	$(dir)/filenames.c	\
@@ -55,8 +54,7 @@ libnotmuch_c_srcs =		\
 	$(dir)/message-file.c	\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
-	$(dir)/tags.c		\
-	$(dir)/xutil.c
+	$(dir)/tags.c
 
 libnotmuch_cxx_srcs =		\
 	$(dir)/database.cc	\
@@ -72,13 +70,10 @@ $(dir)/libnotmuch.a: $(libnotmuch_modules)
 	$(call quiet,AR) rcs $@ $^
 
 $(dir)/$(LIBNAME): $(libnotmuch_modules) notmuch.sym
-	echo $(libnotmuch_modules)
-	$(call quiet,CXX $(CXXFLAGS)) $(libnotmuch_modules) $(FINAL_LIBNOTMUCH_LDFLAGS) $(LIBRARY_LINK_FLAG) -o $@
+	$(call quiet,CXX $(CXXFLAGS)) $(libnotmuch_modules) $(FINAL_LIBNOTMUCH_LDFLAGS) $(LIBRARY_LINK_FLAG) -o $@ util/libutil.a
 
-notmuch.sym: lib/notmuch.h
-	printf "{\nglobal:\n" > notmuch.sym
-	sed  -n 's/^\s*\(notmuch_[a-z_]*\)\s*(.*/\t\1;/p' $< >> notmuch.sym
-	printf "local: *;\n};\n" >> notmuch.sym
+notmuch.sym: $(srcdir)/$(dir)/notmuch.h $(libnotmuch_modules)
+	sh $(srcdir)/$(lib)/gen-version-script.sh $< $(libnotmuch_modules) > $@
 
 $(dir)/$(SONAME): $(dir)/$(LIBNAME)
 	ln -sf $(LIBNAME) $@
@@ -88,10 +83,6 @@ $(dir)/$(LINKER_NAME): $(dir)/$(SONAME)
 
 install: install-$(dir)
 
-# The (often-reused) $dir works fine within targets/pre-requisites,
-# but cannot be used reliably within commands, so copy its value to a
-# variable that is not reused.
-lib := $(dir)
 install-$(dir): $(dir)/$(LIBNAME)
 	mkdir -p "$(DESTDIR)$(libdir)/"
 	install -m0644 "$(lib)/$(LIBNAME)" "$(DESTDIR)$(libdir)/"
@@ -102,4 +93,6 @@ install-$(dir): $(dir)/$(LIBNAME)
 	$(LIBRARY_INSTALL_POST_COMMAND)
 
 SRCS  := $(SRCS) $(libnotmuch_c_srcs) $(libnotmuch_cxx_srcs)
-CLEAN := $(CLEAN) $(libnotmuch_modules) $(dir)/$(SONAME) $(dir)/$(LINKER_NAME) $(dir)$(LIBNAME) libnotmuch.a notmuch.sym
+CLEAN += $(libnotmuch_modules) $(dir)/$(SONAME) $(dir)/$(LINKER_NAME)
+CLEAN += $(dir)/$(LIBNAME) $(dir)/libnotmuch.a notmuch.sym
+CLEAN += $(dir)/notmuch.h.gch
diff --git a/lib/database-private.h b/lib/database-private.h
index f7050097..88532d51 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -43,6 +43,7 @@ struct _notmuch_database {
 
     notmuch_bool_t needs_upgrade;
     notmuch_database_mode_t mode;
+    int atomic_nesting;
     Xapian::Database *xapian_db;
 
     unsigned int last_doc_id;
diff --git a/lib/database.cc b/lib/database.cc
index 7f79cf47..98f101e6 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -26,6 +26,7 @@
 #include <signal.h>
 
 #include <glib.h> /* g_free, GPtrArray, GHashTable */
+#include <glib-object.h> /* g_type_init */
 
 using namespace std;
 
@@ -69,7 +70,7 @@ typedef struct {
  *
  *    Multiple terms of given prefix:
  *
- *	reference: All message IDs from In-Reply-To and Re ferences
+ *	reference: All message IDs from In-Reply-To and References
  *		   headers in the message.
  *
  *	tag:	   Any tags associated with this message by the user.
@@ -137,7 +138,7 @@ typedef struct {
  *			ASCII integer. The initial database version
  *			was 1, (though a schema existed before that
  *			were no "version" database value existed at
- *			all). Succesive versions are allocated as
+ *			all). Successive versions are allocated as
  *			changes are made to the database (such as by
  *			indexing new fields).
  *
@@ -148,7 +149,7 @@ typedef struct {
  *			incremented for each thread ID.
  *
  *	thread_id_*	A pre-allocated thread ID for a particular
- *			message. This is actually an arbitarily large
+ *			message. This is actually an arbitrarily large
  *			family of metadata name. Any particular name is
  *			formed by concatenating "thread_id_" with a message
  *			ID (or the SHA1 sum of a message ID if it is very
@@ -209,21 +210,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= {
     { "folder",			"XFOLDER"}
 };
 
-int
-_internal_error (const char *format, ...)
-{
-    va_list va_args;
-
-    va_start (va_args, format);
-
-    fprintf (stderr, "Internal error: ");
-    vfprintf (stderr, format, va_args);
-
-    exit (1);
-
-    return 1;
-}
-
 const char *
 _find_prefix (const char *name)
 {
@@ -273,6 +259,8 @@ notmuch_status_to_string (notmuch_status_t status)
 	return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
     case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
 	return "Unbalanced number of calls to notmuch_message_freeze/thaw";
+    case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
+	return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic";
     default:
     case NOTMUCH_STATUS_LAST_STATUS:
 	return "Unknown error status value";
@@ -358,13 +346,17 @@ _message_id_compressed (void *ctx, const char *message_id)
     return compressed;
 }
 
-notmuch_message_t *
+notmuch_status_t
 notmuch_database_find_message (notmuch_database_t *notmuch,
-			       const char *message_id)
+			       const char *message_id,
+			       notmuch_message_t **message_ret)
 {
     notmuch_private_status_t status;
     unsigned int doc_id;
 
+    if (message_ret == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
     if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
 	message_id = _message_id_compressed (notmuch, message_id);
 
@@ -373,14 +365,21 @@ notmuch_database_find_message (notmuch_database_t *notmuch,
 						       message_id, &doc_id);
 
 	if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
-	    return NULL;
+	    *message_ret = NULL;
+	else {
+	    *message_ret = _notmuch_message_create (notmuch, notmuch, doc_id,
+						    NULL);
+	    if (*message_ret == NULL)
+		return NOTMUCH_STATUS_OUT_OF_MEMORY;
+	}
 
-	return _notmuch_message_create (notmuch, notmuch, doc_id, NULL);
+	return NOTMUCH_STATUS_SUCCESS;
     } catch (const Xapian::Error &error) {
 	fprintf (stderr, "A Xapian exception occurred finding message: %s.\n",
 		 error.get_msg().c_str());
 	notmuch->exception_reported = TRUE;
-	return NULL;
+	*message_ret = NULL;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 }
 
@@ -422,7 +421,7 @@ skip_space_and_comments (const char **str)
 }
 
 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
- * comments, and the '<' and '>' delimeters.
+ * comments, and the '<' and '>' delimiters.
  *
  * If not NULL, then *next will be made to point to the first character
  * not parsed, (possibly pointing to the final '\0' terminator.
@@ -602,6 +601,9 @@ notmuch_database_open (const char *path,
 	goto DONE;
     }
 
+    /* Initialize the GLib type system and threads */
+    g_type_init ();
+
     notmuch = talloc (NULL, notmuch_database_t);
     notmuch->exception_reported = FALSE;
     notmuch->path = talloc_strdup (notmuch, path);
@@ -611,6 +613,7 @@ notmuch_database_open (const char *path,
 
     notmuch->needs_upgrade = FALSE;
     notmuch->mode = mode;
+    notmuch->atomic_nesting = 0;
     try {
 	string last_thread_id;
 
@@ -974,6 +977,61 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
     return NOTMUCH_STATUS_SUCCESS;
 }
 
+notmuch_status_t
+notmuch_database_begin_atomic (notmuch_database_t *notmuch)
+{
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY ||
+	notmuch->atomic_nesting > 0)
+	goto DONE;
+
+    try {
+	(static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))->begin_transaction (false);
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr, "A Xapian exception occurred beginning transaction: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = TRUE;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+DONE:
+    notmuch->atomic_nesting++;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_end_atomic (notmuch_database_t *notmuch)
+{
+    Xapian::WritableDatabase *db;
+
+    if (notmuch->atomic_nesting == 0)
+	return NOTMUCH_STATUS_UNBALANCED_ATOMIC;
+
+    if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_ONLY ||
+	notmuch->atomic_nesting > 1)
+	goto DONE;
+
+    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+    try {
+	db->commit_transaction ();
+
+	/* This is a hack for testing.  Xapian never flushes on a
+	 * non-flushed commit, even if the flush threshold is 1.
+	 * However, we rely on flushing to test atomicity. */
+	const char *thresh = getenv ("XAPIAN_FLUSH_THRESHOLD");
+	if (thresh && atoi (thresh) == 1)
+	    db->flush ();
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr, "A Xapian exception occurred committing transaction: %s.\n",
+		 error.get_msg().c_str());
+	notmuch->exception_reported = TRUE;
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+DONE:
+    notmuch->atomic_nesting--;
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 /* We allow the user to use arbitrarily long paths for directories. But
  * we have a term-length limit. So if we exceed that, we'll use the
  * SHA-1 of the path for the database term.
@@ -1149,7 +1207,7 @@ _notmuch_database_filename_to_direntry (void *ctx,
 
 /* Given a legal 'path' for the database, return the relative path.
  *
- * The return value will be a pointer to the originl path contents,
+ * The return value will be a pointer to the original path contents,
  * and will be either the original string (if 'path' was relative) or
  * a portion of the string (if path was absolute and begins with the
  * database path).
@@ -1253,7 +1311,9 @@ _get_metadata_thread_id_key (void *ctx, const char *message_id)
 
 /* Find the thread ID to which the message with 'message_id' belongs.
  *
- * Always returns a newly talloced string belonging to 'ctx'.
+ * Note: 'thread_id_ret' must not be NULL!
+ * On success '*thread_id_ret' is set to a newly talloced string belonging to
+ * 'ctx'.
  *
  * Note: If there is no message in the database with the given
  * 'message_id' then a new thread_id will be allocated for this
@@ -1261,25 +1321,30 @@ _get_metadata_thread_id_key (void *ctx, const char *message_id)
  * thread ID can be looked up if the message is added to the database
  * later).
  */
-static const char *
+static notmuch_status_t
 _resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
 				  void *ctx,
-				  const char *message_id)
+				  const char *message_id,
+				  const char **thread_id_ret)
 {
+    notmuch_status_t status;
     notmuch_message_t *message;
     string thread_id_string;
-    const char *thread_id;
     char *metadata_key;
     Xapian::WritableDatabase *db;
 
-    message = notmuch_database_find_message (notmuch, message_id);
+    status = notmuch_database_find_message (notmuch, message_id, &message);
+
+    if (status)
+	return status;
 
     if (message) {
-	thread_id = talloc_steal (ctx, notmuch_message_get_thread_id (message));
+	*thread_id_ret = talloc_steal (ctx,
+				       notmuch_message_get_thread_id (message));
 
 	notmuch_message_destroy (message);
 
-	return thread_id;
+	return NOTMUCH_STATUS_SUCCESS;
     }
 
     /* Message has not been seen yet.
@@ -1293,15 +1358,16 @@ _resolve_message_id_to_thread_id (notmuch_database_t *notmuch,
     thread_id_string = notmuch->xapian_db->get_metadata (metadata_key);
 
     if (thread_id_string.empty()) {
-	thread_id = _notmuch_database_generate_thread_id (notmuch);
-	db->set_metadata (metadata_key, thread_id);
+	*thread_id_ret = talloc_strdup (ctx,
+					_notmuch_database_generate_thread_id (notmuch));
+	db->set_metadata (metadata_key, *thread_id_ret);
     } else {
-	thread_id = thread_id_string.c_str();
+	*thread_id_ret = talloc_strdup (ctx, thread_id_string.c_str());
     }
 
     talloc_free (metadata_key);
 
-    return talloc_strdup (ctx, thread_id);
+    return NOTMUCH_STATUS_SUCCESS;
 }
 
 static notmuch_status_t
@@ -1388,9 +1454,12 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
 	_notmuch_message_add_term (message, "reference",
 				   parent_message_id);
 
-	parent_thread_id = _resolve_message_id_to_thread_id (notmuch,
-							     message,
-							     parent_message_id);
+	ret = _resolve_message_id_to_thread_id (notmuch,
+						message,
+						parent_message_id,
+						&parent_thread_id);
+	if (ret)
+	    goto DONE;
 
 	if (*thread_id == NULL) {
 	    *thread_id = talloc_strdup (message, parent_thread_id);
@@ -1476,7 +1545,7 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch,
  * In all cases, we assign to the current message the first thread_id
  * found (through either parent or child). We will also merge any
  * existing, distinct threads where this message belongs to both,
- * (which is not uncommon when mesages are processed out of order).
+ * (which is not uncommon when messages are processed out of order).
  *
  * Finally, if no thread ID has been found through parent or child, we
  * call _notmuch_message_generate_thread_id to generate a new thread
@@ -1543,7 +1612,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 {
     notmuch_message_file_t *message_file;
     notmuch_message_t *message = NULL;
-    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2;
     notmuch_private_status_t private_status;
 
     const char *date, *header;
@@ -1561,6 +1630,12 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     if (message_file == NULL)
 	return NOTMUCH_STATUS_FILE_ERROR;
 
+    /* Adding a message may change many documents.  Do this all
+     * atomically. */
+    ret = notmuch_database_begin_atomic (notmuch);
+    if (ret)
+	goto DONE;
+
     notmuch_message_file_restrict_headers (message_file,
 					   "date",
 					   "from",
@@ -1654,7 +1729,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 		goto DONE;
 
 	    date = notmuch_message_file_get_header (message_file, "date");
-	    _notmuch_message_set_date (message, date);
+	    _notmuch_message_set_header_values (message, date, from, subject);
 
 	    _notmuch_message_index_file (message, filename);
 	} else {
@@ -1682,6 +1757,12 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     if (message_file)
 	notmuch_message_file_close (message_file);
 
+    ret2 = notmuch_database_end_atomic (notmuch);
+    if ((ret == NOTMUCH_STATUS_SUCCESS ||
+	 ret == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) &&
+	ret2 != NOTMUCH_STATUS_SUCCESS)
+	ret = ret2;
+
     return ret;
 }
 
@@ -1689,71 +1770,73 @@ notmuch_status_t
 notmuch_database_remove_message (notmuch_database_t *notmuch,
 				 const char *filename)
 {
-    Xapian::WritableDatabase *db;
+    notmuch_status_t status;
+    notmuch_message_t *message;
+
+    status = notmuch_database_find_message_by_filename (notmuch, filename,
+							&message);
+
+    if (status == NOTMUCH_STATUS_SUCCESS && message) {
+	    status = _notmuch_message_remove_filename (message, filename);
+	    if (status == NOTMUCH_STATUS_SUCCESS)
+		_notmuch_message_delete (message);
+	    else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
+		_notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+    }
+
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
+					   const char *filename,
+					   notmuch_message_t **message_ret)
+{
     void *local;
     const char *prefix = _find_prefix ("file-direntry");
     char *direntry, *term;
     Xapian::PostingIterator i, end;
-    Xapian::Document document;
     notmuch_status_t status;
 
-    status = _notmuch_database_ensure_writable (notmuch);
-    if (status)
-	return status;
+    if (message_ret == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
 
     local = talloc_new (notmuch);
 
-    db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
-
     try {
-
 	status = _notmuch_database_filename_to_direntry (local, notmuch,
 							 filename, &direntry);
 	if (status)
-	    return status;
+	    goto DONE;
 
 	term = talloc_asprintf (local, "%s%s", prefix, direntry);
 
 	find_doc_ids_for_term (notmuch, term, &i, &end);
 
-	for ( ; i != end; i++) {
-	    Xapian::TermIterator j;
-	    notmuch_message_t *message;
+	if (i != end) {
 	    notmuch_private_status_t private_status;
 
-	    message = _notmuch_message_create (local, notmuch,
-					       *i, &private_status);
-	    if (message == NULL)
-		return COERCE_STATUS (private_status,
-				      "Inconsistent document ID in datbase.");
-
-	    _notmuch_message_remove_filename (message, filename);
-	    _notmuch_message_sync (message);
-
-	    /* Take care to find document after sync'ing filename removal. */
-	    document = find_document_for_doc_id (notmuch, *i);
-	    j = document.termlist_begin ();
-	    j.skip_to (prefix);
-
-	    /* Was this the last file-direntry in the message? */
-	    if (j == document.termlist_end () ||
-		strncmp ((*j).c_str (), prefix, strlen (prefix)))
-	    {
-		db->delete_document (document.get_docid ());
-		status = NOTMUCH_STATUS_SUCCESS;
-	    } else {
-		status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
-	    }
+	    *message_ret = _notmuch_message_create (notmuch, notmuch, *i,
+						    &private_status);
+	    if (*message_ret == NULL)
+		status = NOTMUCH_STATUS_OUT_OF_MEMORY;
 	}
     } catch (const Xapian::Error &error) {
-	fprintf (stderr, "Error: A Xapian exception occurred removing message: %s\n",
+	fprintf (stderr, "Error: A Xapian exception occurred finding message by filename: %s\n",
 		 error.get_msg().c_str());
 	notmuch->exception_reported = TRUE;
 	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
+  DONE:
     talloc_free (local);
 
+    if (status && *message_ret) {
+	notmuch_message_destroy (*message_ret);
+	*message_ret = NULL;
+    }
     return status;
 }
 
diff --git a/lib/gen-version-script.sh b/lib/gen-version-script.sh
new file mode 100644
index 00000000..76670d57
--- /dev/null
+++ b/lib/gen-version-script.sh
@@ -0,0 +1,28 @@
+
+# we go through a bit of work to get the unmangled names of the
+# typeinfo symbols because of
+# http://sourceware.org/bugzilla/show_bug.cgi?id=10326
+
+if [ $# -lt 2 ]; then
+    echo Usage: $0 header obj1 obj2 obj3
+    exit 1;
+fi
+
+HEADER=$1
+shift
+
+printf '{\nglobal:\n'
+nm  $* | awk '$1 ~ "^[0-9a-fA-F][0-9a-fA-F]*$" && $3 ~ "Xapian.*Error" {print $3}' | sort | uniq | \
+while read sym; do
+    demangled=$(c++filt $sym)
+    case $demangled in
+	typeinfo*) 
+	    printf "\t$sym;\n"
+	    ;;
+	*)
+	    ;;
+    esac
+done
+nm $* | awk '$1 ~ "^[0-9a-fA-F][0-9a-fA-F]*$" && $2 == "T" && $3 ~ "^get(line|delim)$" {print $3 ";"}'
+sed  -n 's/^[[:space:]]*\(notmuch_[a-z_]*\)[[:space:]]*(.*/ \1;/p' $HEADER
+printf "local: *;\n};\n"
diff --git a/lib/libsha1.c b/lib/libsha1.c
index c39a5a17..5d16f6ab 100644
--- a/lib/libsha1.c
+++ b/lib/libsha1.c
@@ -174,7 +174,7 @@ void sha1_hash(const unsigned char data[], unsigned long len, sha1_ctx ctx[1])
     if((ctx->count[0] += len) < len)
         ++(ctx->count[1]);
 
-    while(len >= space)     /* tranfer whole blocks if possible  */
+    while(len >= space)     /* transfer whole blocks if possible  */
     {
         memcpy(((unsigned char*)ctx->wbuf) + pos, sp, space);
         sp += space; len -= space; space = SHA1_BLOCK_SIZE; pos = 0;
diff --git a/lib/libsha1.h b/lib/libsha1.h
index c1c848fc..56f445a9 100644
--- a/lib/libsha1.h
+++ b/lib/libsha1.h
@@ -38,7 +38,7 @@ extern "C"
 {
 #endif
 #if 0
-} /* Appleasing Emacs */
+} /* Appeasing Emacs */
 #endif
 
 #include <stdint.h>
diff --git a/lib/message.cc b/lib/message.cc
index 2a4954dd..ca7fbf21 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -216,11 +216,13 @@ _notmuch_message_create_for_message_id (notmuch_database_t *notmuch,
     unsigned int doc_id;
     char *term;
 
-    *status_ret = NOTMUCH_PRIVATE_STATUS_SUCCESS;
-
-    message = notmuch_database_find_message (notmuch, message_id);
+    *status_ret = (notmuch_private_status_t) notmuch_database_find_message (notmuch,
+									    message_id,
+									    &message);
     if (message)
 	return talloc_steal (notmuch, message);
+    else if (*status_ret)
+	return NULL;
 
     term = talloc_asprintf (NULL, "%s%s",
 			    _find_prefix ("id"), message_id);
@@ -410,6 +412,21 @@ _notmuch_message_ensure_message_file (notmuch_message_t *message)
 const char *
 notmuch_message_get_header (notmuch_message_t *message, const char *header)
 {
+    std::string value;
+
+    /* Fetch header from the appropriate xapian value field if
+     * available */
+    if (strcasecmp (header, "from") == 0)
+	value = message->doc.get_value (NOTMUCH_VALUE_FROM);
+    else if (strcasecmp (header, "subject") == 0)
+	value = message->doc.get_value (NOTMUCH_VALUE_SUBJECT);
+    else if (strcasecmp (header, "message-id") == 0)
+	value = message->doc.get_value (NOTMUCH_VALUE_MESSAGE_ID);
+
+    if (!value.empty())
+	return talloc_strdup (message, value.c_str ());
+
+    /* Otherwise fall back to parsing the file */
     _notmuch_message_ensure_message_file (message);
     if (message->message_file == NULL)
 	return NULL;
@@ -501,6 +518,9 @@ _notmuch_message_add_filename (notmuch_message_t *message,
  * This change will not be reflected in the database until the next
  * call to _notmuch_message_sync.
  *
+ * If this message still has other filenames, returns
+ * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID.
+ *
  * Note: This function does not remove a document from the database,
  * even if the specified filename is the only filename for this
  * message. For that functionality, see
@@ -531,6 +551,8 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
 						   "file-direntry", direntry);
     status = COERCE_STATUS (private_status,
 			    "Unexpected error from _notmuch_message_remove_term");
+    if (status)
+	return status;
 
     /* Re-synchronize "folder:" terms for this message. This requires:
      *  1. removing all "folder:" terms
@@ -588,6 +610,9 @@ _notmuch_message_remove_filename (notmuch_message_t *message,
 	if (strncmp ((*i).c_str (), direntry_prefix, direntry_prefix_len))
 	    break;
 
+	/* Indicate that there are filenames remaining. */
+	status = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
+
 	direntry = (*i).c_str ();
 	direntry += direntry_prefix_len;
 
@@ -785,8 +810,10 @@ notmuch_message_set_author (notmuch_message_t *message,
 }
 
 void
-_notmuch_message_set_date (notmuch_message_t *message,
-			   const char *date)
+_notmuch_message_set_header_values (notmuch_message_t *message,
+				    const char *date,
+				    const char *from,
+				    const char *subject)
 {
     time_t time_value;
 
@@ -799,6 +826,8 @@ _notmuch_message_set_date (notmuch_message_t *message,
 
     message->doc.add_value (NOTMUCH_VALUE_TIMESTAMP,
 			    Xapian::sortable_serialise (time_value));
+    message->doc.add_value (NOTMUCH_VALUE_FROM, from);
+    message->doc.add_value (NOTMUCH_VALUE_SUBJECT, subject);
 }
 
 /* Synchronize changes made to message->doc out into the database. */
@@ -814,6 +843,22 @@ _notmuch_message_sync (notmuch_message_t *message)
     db->replace_document (message->doc_id, message->doc);
 }
 
+/* Delete a message document from the database. */
+notmuch_status_t
+_notmuch_message_delete (notmuch_message_t *message)
+{
+    notmuch_status_t status;
+    Xapian::WritableDatabase *db;
+
+    status = _notmuch_database_ensure_writable (message->notmuch);
+    if (status)
+	return status;
+
+    db = static_cast <Xapian::WritableDatabase *> (message->notmuch->xapian_db);
+    db->delete_document (message->doc_id);
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 /* Ensure that 'message' is not holding any file object open. Future
  * calls to various functions will still automatically open the
  * message file as needed.
@@ -862,7 +907,7 @@ _notmuch_message_add_term (notmuch_message_t *message,
 
 /* Parse 'text' and add a term to 'message' for each parsed word. Each
  * term will be added both prefixed (if prefix_name is not NULL) and
- * also unprefixed). */
+ * also non-prefixed). */
 notmuch_private_status_t
 _notmuch_message_gen_terms (notmuch_message_t *message,
 			    const char *prefix_name,
@@ -1280,7 +1325,8 @@ notmuch_message_tags_to_maildir_flags (notmuch_message_t *message)
 	    new_status = _notmuch_message_remove_filename (message,
 							   filename);
 	    /* Hold on to only the first error. */
-	    if (! status && new_status) {
+	    if (! status && new_status
+		&& new_status != NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) {
 		status = new_status;
 		continue;
 	    }
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 02e24ee8..60a932fc 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -47,6 +47,7 @@ NOTMUCH_BEGIN_DECLS
 #include <talloc.h>
 
 #include "xutil.h"
+#include "error_util.h"
 
 #pragma GCC visibility push(hidden)
 
@@ -60,25 +61,6 @@ NOTMUCH_BEGIN_DECLS
 #define STRNCMP_LITERAL(var, literal) \
     strncmp ((var), (literal), sizeof (literal) - 1)
 
-/* There's no point in continuing when we've detected that we've done
- * something wrong internally (as opposed to the user passing in a
- * bogus value).
- *
- * Note that PRINTF_ATTRIBUTE comes from talloc.h
- */
-int
-_internal_error (const char *format, ...) PRINTF_ATTRIBUTE (1, 2);
-
-/* There's no point in continuing when we've detected that we've done
- * something wrong internally (as opposed to the user passing in a
- * bogus value).
- *
- * Note that __location__ comes from talloc.h.
- */
-#define INTERNAL_ERROR(format, ...)			\
-    _internal_error (format " (%s).\n",			\
-		     ##__VA_ARGS__, __location__)
-
 #define unused(x) x __attribute__ ((unused))
 
 #ifdef __cplusplus
@@ -111,7 +93,9 @@ _internal_error (const char *format, ...) PRINTF_ATTRIBUTE (1, 2);
 
 typedef enum {
     NOTMUCH_VALUE_TIMESTAMP = 0,
-    NOTMUCH_VALUE_MESSAGE_ID
+    NOTMUCH_VALUE_MESSAGE_ID,
+    NOTMUCH_VALUE_FROM,
+    NOTMUCH_VALUE_SUBJECT
 } notmuch_value_t;
 
 /* Xapian (with flint backend) complains if we provide a term longer
@@ -287,12 +271,16 @@ void
 _notmuch_message_ensure_thread_id (notmuch_message_t *message);
 
 void
-_notmuch_message_set_date (notmuch_message_t *message,
-			   const char *date);
-
+_notmuch_message_set_header_values (notmuch_message_t *message,
+				    const char *date,
+				    const char *from,
+				    const char *subject);
 void
 _notmuch_message_sync (notmuch_message_t *message);
 
+notmuch_status_t
+_notmuch_message_delete (notmuch_message_t *message);
+
 void
 _notmuch_message_close (notmuch_message_t *message);
 
diff --git a/lib/notmuch.h b/lib/notmuch.h
index e508309e..9f23a106 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -81,6 +81,9 @@ typedef int notmuch_bool_t;
  * NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: The notmuch_message_thaw
  *	function has been called more times than notmuch_message_freeze.
  *
+ * NOTMUCH_STATUS_UNBALANCED_ATOMIC: notmuch_database_end_atomic has
+ *	been called more times than notmuch_database_begin_atomic.
+ *
  * And finally:
  *
  * NOTMUCH_STATUS_LAST_STATUS: Not an actual status value. Just a way
@@ -97,13 +100,14 @@ typedef enum _notmuch_status {
     NOTMUCH_STATUS_NULL_POINTER,
     NOTMUCH_STATUS_TAG_TOO_LONG,
     NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW,
+    NOTMUCH_STATUS_UNBALANCED_ATOMIC,
 
     NOTMUCH_STATUS_LAST_STATUS
 } notmuch_status_t;
 
 /* Get a string representation of a notmuch_status_t value.
  *
- * The result is readonly.
+ * The result is read-only.
  */
 const char *
 notmuch_status_to_string (notmuch_status_t status);
@@ -214,6 +218,42 @@ notmuch_database_upgrade (notmuch_database_t *database,
 						   double progress),
 			  void *closure);
 
+/* Begin an atomic database operation.
+ *
+ * Any modifications performed between a successful begin and a
+ * notmuch_database_end_atomic will be applied to the database
+ * atomically.  Note that, unlike a typical database transaction, this
+ * only ensures atomicity, not durability; neither begin nor end
+ * necessarily flush modifications to disk.
+ *
+ * Atomic sections may be nested.  begin_atomic and end_atomic must
+ * always be called in pairs.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully entered atomic section.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	atomic section not entered.
+ */
+notmuch_status_t
+notmuch_database_begin_atomic (notmuch_database_t *notmuch);
+
+/* Indicate the end of an atomic database operation.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successfully completed atomic section.
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred;
+ *	atomic section not ended.
+ *
+ * NOTMUCH_STATUS_UNBALANCED_ATOMIC: The database is not currently in
+ *	an atomic section.
+ */
+notmuch_status_t
+notmuch_database_end_atomic (notmuch_database_t *notmuch);
+
 /* Retrieve a directory object from the database for 'path'.
  *
  * Here, 'path' should be a path relative to the path of 'database'
@@ -226,9 +266,10 @@ notmuch_directory_t *
 notmuch_database_get_directory (notmuch_database_t *database,
 				const char *path);
 
-/* Add a new message to the given notmuch database.
+/* Add a new message to the given notmuch database or associate an
+ * additional filename with an existing message.
  *
- * Here,'filename' should be a path relative to the path of
+ * Here, 'filename' should be a path relative to the path of
  * 'database' (see notmuch_database_get_path), or else should be an
  * absolute filename with initial components that match the path of
  * 'database'.
@@ -238,6 +279,10 @@ notmuch_database_get_directory (notmuch_database_t *database,
  * notmuch database will reference the filename, and will not copy the
  * entire contents of the file.
  *
+ * If another message with the same message ID already exists in the
+ * database, rather than creating a new message, this adds 'filename'
+ * to the list of the filenames for the existing message.
+ *
  * If 'message' is not NULL, then, on successful return
  * (NOTMUCH_STATUS_SUCCESS or NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) '*message'
  * will be initialized to a message object that can be used for things
@@ -255,7 +300,7 @@ notmuch_database_get_directory (notmuch_database_t *database,
  * NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: Message has the same message
  *	ID as another message already in the database. The new
  *	filename was successfully added to the message in the database
- *	(if not already present).
+ *	(if not already present) and the existing message is returned.
  *
  * NOTMUCH_STATUS_FILE_ERROR: an error occurred trying to open the
  *	file, (such as permission denied, or file not found,
@@ -272,14 +317,14 @@ notmuch_database_add_message (notmuch_database_t *database,
 			      const char *filename,
 			      notmuch_message_t **message);
 
-/* Remove a message from the given notmuch database.
+/* Remove a message filename from the given notmuch database. If the
+ * message has no more filenames, remove the message.
  *
- * Note that only this particular filename association is removed from
- * the database. If the same message (as determined by the message ID)
- * is still available via other filenames, then the message will
- * persist in the database for those filenames. When the last filename
- * is removed for a particular message, the database content for that
- * message will be entirely removed.
+ * If the same message (as determined by the message ID) is still
+ * available via other filenames, then the message will persist in the
+ * database for those filenames. When the last filename is removed for
+ * a particular message, the database content for that message will be
+ * entirely removed.
  *
  * Return value:
  *
@@ -302,19 +347,57 @@ notmuch_database_remove_message (notmuch_database_t *database,
 
 /* Find a message with the given message_id.
  *
- * If the database contains a message with the given message_id, then
- * a new notmuch_message_t object is returned. The caller should call
- * notmuch_message_destroy when done with the message.
+ * If a message with the given message_id is found then, on successful return
+ * (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to a message
+ * object.  The caller should call notmuch_message_destroy when done with the
+ * message.
+ *
+ * On any failure or when the message is not found, this function initializes
+ * '*message' to NULL. This means, when NOTMUCH_STATUS_SUCCESS is returned, the
+ * caller is supposed to check '*message' for NULL to find out whether the
+ * message with the given message_id was found.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successful return, check '*message'.
  *
- * This function returns NULL in the following situations:
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'message' argument is NULL
  *
- *	* No message is found with the given message_id
- *	* An out-of-memory situation occurs
- *	* A Xapian exception occurs
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory, creating message object
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
  */
-notmuch_message_t *
+notmuch_status_t
 notmuch_database_find_message (notmuch_database_t *database,
-			       const char *message_id);
+			       const char *message_id,
+			       notmuch_message_t **message);
+
+/* Find a message with the given filename.
+ *
+ * If the database contains a message with the given filename then, on
+ * successful return (NOTMUCH_STATUS_SUCCESS) '*message' will be initialized to
+ * a message object. The caller should call notmuch_message_destroy when done
+ * with the message.
+ *
+ * On any failure or when the message is not found, this function initializes
+ * '*message' to NULL. This means, when NOTMUCH_STATUS_SUCCESS is returned, the
+ * caller is supposed to check '*message' for NULL to find out whether the
+ * message with the given filename is found.
+ *
+ * Return value:
+ *
+ * NOTMUCH_STATUS_SUCCESS: Successful return, check '*message'
+ *
+ * NOTMUCH_STATUS_NULL_POINTER: The given 'message' argument is NULL
+ *
+ * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory, creating the message object
+ *
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION: A Xapian exception occurred
+ */
+notmuch_status_t
+notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
+					   const char *filename,
+					   notmuch_message_t **message);
 
 /* Return a list of all tags found in the database.
  *
@@ -510,7 +593,7 @@ notmuch_threads_move_to_next (notmuch_threads_t *threads);
  *
  * It's not strictly necessary to call this function. All memory from
  * the notmuch_threads_t object will be reclaimed when the
- * containg query object is destroyed.
+ * containing query object is destroyed.
  */
 void
 notmuch_threads_destroy (notmuch_threads_t *threads);
@@ -526,6 +609,20 @@ notmuch_threads_destroy (notmuch_threads_t *threads);
 unsigned
 notmuch_query_count_messages (notmuch_query_t *query);
  
+/* Return the number of threads matching a search.
+ *
+ * This function performs a search and returns the number of unique thread IDs
+ * in the matching messages. This is the same as number of threads matching a
+ * search.
+ *
+ * Note that this is a significantly heavier operation than
+ * notmuch_query_count_messages().
+ *
+ * If an error occurs, this function may return 0.
+ */
+unsigned
+notmuch_query_count_threads (notmuch_query_t *query);
+
 /* Get the thread ID of 'thread'.
  *
  * The returned string belongs to 'thread' and as such, should not be
diff --git a/lib/query.cc b/lib/query.cc
index 6f02b04b..b6c0f12d 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -457,3 +457,47 @@ notmuch_query_count_messages (notmuch_query_t *query)
 
     return count;
 }
+
+unsigned
+notmuch_query_count_threads (notmuch_query_t *query)
+{
+    notmuch_messages_t *messages;
+    GHashTable *hash;
+    unsigned int count;
+    notmuch_sort_t sort;
+
+    sort = query->sort;
+    query->sort = NOTMUCH_SORT_UNSORTED;
+    messages = notmuch_query_search_messages (query);
+    query->sort = sort;
+    if (messages == NULL)
+	return 0;
+
+    hash = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
+    if (hash == NULL) {
+	talloc_free (messages);
+	return 0;
+    }
+
+    while (notmuch_messages_valid (messages)) {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	const char *thread_id = notmuch_message_get_thread_id (message);
+	char *thread_id_copy = talloc_strdup (messages, thread_id);
+	if (unlikely (thread_id_copy == NULL)) {
+	    notmuch_message_destroy (message);
+	    count = 0;
+	    goto DONE;
+	}
+	g_hash_table_insert (hash, thread_id_copy, NULL);
+	notmuch_message_destroy (message);
+	notmuch_messages_move_to_next (messages);
+    }
+
+    count = g_hash_table_size (hash);
+
+  DONE:
+    g_hash_table_unref (hash);
+    talloc_free (messages);
+
+    return count;
+}
diff --git a/lib/xutil.c b/lib/xutil.c
deleted file mode 100644
index 268225b8..00000000
--- a/lib/xutil.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/* xutil.c - Various wrapper functions to abort on error.
- *
- * Copyright © 2009 Carl Worth
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
- *
- * Author: Carl Worth <cworth@cworth.org>
- */
-
-#include "notmuch-private.h"
-
-#include <stdio.h>
-
-void *
-xcalloc (size_t nmemb, size_t size)
-{
-    void *ret;
-
-    ret = calloc (nmemb, size);
-    if (ret == NULL) {
-	fprintf (stderr, "Out of memory.\n");
-	exit (1);
-    }
-
-    return ret;
-}
-
-void *
-xmalloc (size_t size)
-{
-    void *ret;
-
-    ret = malloc (size);
-    if (ret == NULL) {
-	fprintf (stderr, "Out of memory.\n");
-	exit (1);
-    }
-
-    return ret;
-}
-
-void *
-xrealloc (void *ptr, size_t size)
-{
-    void *ret;
-
-    ret = realloc (ptr, size);
-    if (ret == NULL) {
-	fprintf (stderr, "Out of memory.\n");
-	exit (1);
-    }
-
-    return ret;
-}
-
-char *
-xstrdup (const char *s)
-{
-    char *ret;
-
-    ret = strdup (s);
-    if (ret == NULL) {
-	fprintf (stderr, "Out of memory.\n");
-	exit (1);
-    }
-
-    return ret;
-}
-
-char *
-xstrndup (const char *s, size_t n)
-{
-    char *ret;
-
-    if (strlen (s) <= n)
-	n = strlen (s);
-
-    ret = malloc (n + 1);
-    if (ret == NULL) {
-	fprintf (stderr, "Out of memory.\n");
-	exit (1);
-    }
-    memcpy (ret, s, n);
-    ret[n] = '\0';
-
-    return ret;
-}
-
-void
-xregcomp (regex_t *preg, const char *regex, int cflags)
-{
-    int rerr;
-
-    rerr = regcomp (preg, regex, cflags);
-    if (rerr) {
-	size_t error_size = regerror (rerr, preg, NULL, 0);
-	char *error = xmalloc (error_size);
-
-	regerror (rerr, preg, error, error_size);
-	INTERNAL_ERROR ("compiling regex %s: %s\n",
-			regex, error);
-    }
-}
-
-int
-xregexec (const regex_t *preg, const char *string,
-	  size_t nmatch, regmatch_t pmatch[], int eflags)
-{
-    unsigned int i;
-    int rerr;
-
-    rerr = regexec (preg, string, nmatch, pmatch, eflags);
-    if (rerr)
-	return rerr;
-
-    for (i = 0; i < nmatch; i++) {
-	if (pmatch[i].rm_so == -1)
-	    INTERNAL_ERROR ("matching regex against %s: Sub-match %d not found\n",
-			    string, i);
-    }
-
-    return 0;
-}
diff --git a/lib/xutil.h b/lib/xutil.h
deleted file mode 100644
index fd77f733..00000000
--- a/lib/xutil.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/* xutil.h - Various wrapper functions to abort on error.
- *
- * Copyright © 2009 Carl Worth
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
- *
- * Author: Carl Worth <cworth@cworth.org>
- */
-
-#ifndef NOTMUCH_XUTIL_H
-#define NOTMUCH_XUTIL_H
-
-#include <stdlib.h>
-#include <sys/types.h>
-#include <regex.h>
-
-#pragma GCC visibility push(hidden)
-
-/* xutil.c */
-void *
-xcalloc (size_t nmemb, size_t size);
-
-void *
-xmalloc (size_t size);
-
-void *
-xrealloc (void *ptrr, size_t size);
-
-char *
-xstrdup (const char *s);
-
-char *
-xstrndup (const char *s, size_t n);
-
-void
-xregcomp (regex_t *preg, const char *regex, int cflags);
-
-int
-xregexec (const regex_t *preg, const char *string,
-	  size_t nmatch, regmatch_t pmatch[], int eflags);
-
-#pragma GCC visibility pop
-
-#endif