Merge tag 'debian/0.23.1-1' into jessie-backports

notmuch Debian 0.23.1-1 upload (same as 0.23.1)

26 files changed, 1238 insertions, 80 deletions

diff --git a/lib/Makefile.local b/lib/Makefile.local
index 3a070907..3d1030a5 100644
--- a/lib/Makefile.local
+++ b/lib/Makefile.local
@@ -39,6 +39,8 @@ libnotmuch_c_srcs =		\
 	$(dir)/message-file.c	\
 	$(dir)/messages.c	\
 	$(dir)/sha1.c		\
+	$(dir)/built-with.c	\
+	$(dir)/string-map.c	\
 	$(dir)/tags.c
 
 libnotmuch_cxx_srcs =		\
@@ -47,7 +49,10 @@ libnotmuch_cxx_srcs =		\
 	$(dir)/directory.cc	\
 	$(dir)/index.cc		\
 	$(dir)/message.cc	\
+	$(dir)/message-property.cc \
 	$(dir)/query.cc		\
+	$(dir)/query-fp.cc      \
+	$(dir)/config.cc	\
 	$(dir)/thread.cc
 
 libnotmuch_modules := $(libnotmuch_c_srcs:.c=.o) $(libnotmuch_cxx_srcs:.cc=.o)
diff --git a/lib/built-with.c b/lib/built-with.c
new file mode 100644
index 00000000..2f1f0b5c
--- /dev/null
+++ b/lib/built-with.c
@@ -0,0 +1,36 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+
+notmuch_bool_t
+notmuch_built_with (const char *name)
+{
+    if (STRNCMP_LITERAL (name, "compact") == 0) {
+	return HAVE_XAPIAN_COMPACT;
+    } else if (STRNCMP_LITERAL (name, "field_processor") == 0) {
+	return HAVE_XAPIAN_FIELD_PROCESSOR;
+    } else if (STRNCMP_LITERAL (name, "retry_lock") == 0) {
+	return HAVE_XAPIAN_DB_RETRY_LOCK;
+    } else {
+	return FALSE;
+    }
+}
diff --git a/lib/config.cc b/lib/config.cc
new file mode 100644
index 00000000..0703b9bb
--- /dev/null
+++ b/lib/config.cc
@@ -0,0 +1,193 @@
+/* config.cc - API for database metadata
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch.h"
+#include "notmuch-private.h"
+#include "database-private.h"
+
+static const std::string CONFIG_PREFIX = "C";
+
+struct _notmuch_config_list {
+    notmuch_database_t *notmuch;
+    Xapian::TermIterator iterator;
+    char *current_key;
+    char *current_val;
+};
+
+static int
+_notmuch_config_list_destroy (notmuch_config_list_t *list)
+{
+    /* invoke destructor w/o deallocating memory */
+    list->iterator.~TermIterator();
+    return 0;
+}
+
+notmuch_status_t
+notmuch_database_set_config (notmuch_database_t *notmuch,
+			     const char *key,
+			     const char *value)
+{
+    notmuch_status_t status;
+    Xapian::WritableDatabase *db;
+
+    status = _notmuch_database_ensure_writable (notmuch);
+    if (status)
+	return status;
+
+    try {
+	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
+	db->set_metadata (CONFIG_PREFIX + key, value);
+    } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	notmuch->exception_reported = TRUE;
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred setting metadata: %s\n",
+			       error.get_msg().c_str());
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_metadata_value (notmuch_database_t *notmuch,
+		 const char *key,
+		 std::string &value)
+{
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    try {
+	value = notmuch->xapian_db->get_metadata (CONFIG_PREFIX + key);
+    } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+	notmuch->exception_reported = TRUE;
+	_notmuch_database_log (notmuch, "Error: A Xapian exception occurred getting metadata: %s\n",
+			       error.get_msg().c_str());
+    }
+    return status;
+}
+
+notmuch_status_t
+notmuch_database_get_config (notmuch_database_t *notmuch,
+			     const char *key,
+			     char **value)
+{
+    std::string strval;
+    notmuch_status_t status;
+
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    status = _metadata_value (notmuch, key, strval);
+    if (status)
+	return status;
+
+    *value = strdup (strval.c_str ());
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_database_get_config_list (notmuch_database_t *notmuch,
+				  const char *prefix,
+				  notmuch_config_list_t **out)
+{
+    notmuch_config_list_t *list = NULL;
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
+    list = talloc (notmuch, notmuch_config_list_t);
+    if (! list) {
+	status = NOTMUCH_STATUS_OUT_OF_MEMORY;
+	goto DONE;
+    }
+
+    talloc_set_destructor (list, _notmuch_config_list_destroy);
+    list->notmuch = notmuch;
+    list->current_key = NULL;
+    list->current_val = NULL;
+
+    try {
+
+	new(&(list->iterator)) Xapian::TermIterator (notmuch->xapian_db->metadata_keys_begin
+						     (CONFIG_PREFIX + (prefix ? prefix : "")));
+
+    } catch (const Xapian::Error &error) {
+	_notmuch_database_log (notmuch, "A Xapian exception occurred getting metadata iterator: %s.\n",
+			       error.get_msg().c_str());
+	notmuch->exception_reported = TRUE;
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    *out = list;
+
+  DONE:
+    if (status && list)
+	talloc_free (list);
+
+    return status;
+}
+
+notmuch_bool_t
+notmuch_config_list_valid (notmuch_config_list_t *metadata)
+{
+    if (metadata->iterator == metadata->notmuch->xapian_db->metadata_keys_end ())
+	return FALSE;
+
+    return TRUE;
+}
+
+const char *
+notmuch_config_list_key (notmuch_config_list_t *list)
+{
+    if (list->current_key)
+	talloc_free (list->current_key);
+
+    list->current_key = talloc_strdup (list, (*list->iterator).c_str () + CONFIG_PREFIX.length ());
+
+    return list->current_key;
+}
+
+const char *
+notmuch_config_list_value (notmuch_config_list_t *list)
+{
+    std::string strval;
+    notmuch_status_t status;
+    const char *key = notmuch_config_list_key (list);
+
+    /* TODO: better error reporting?? */
+    status = _metadata_value (list->notmuch, key, strval);
+    if (status)
+	return NULL;
+
+    if (list->current_val)
+	talloc_free (list->current_val);
+
+    list->current_val = talloc_strdup (list, strval.c_str ());
+    return list->current_val;
+}
+
+void
+notmuch_config_list_move_to_next (notmuch_config_list_t *list)
+{
+    list->iterator++;
+}
+
+void
+notmuch_config_list_destroy (notmuch_config_list_t *list)
+{
+    talloc_free (list);
+}
diff --git a/lib/database-private.h b/lib/database-private.h
index 3fb10f7a..ca71a92f 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -144,6 +144,13 @@ operator&=(_notmuch_features &a, _notmuch_features b)
     return a;
 }
 
+#define NOTMUCH_QUERY_PARSER_FLAGS (Xapian::QueryParser::FLAG_BOOLEAN | \
+				    Xapian::QueryParser::FLAG_PHRASE | \
+				    Xapian::QueryParser::FLAG_LOVEHATE | \
+				    Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE | \
+				    Xapian::QueryParser::FLAG_WILDCARD | \
+				    Xapian::QueryParser::FLAG_PURE_NOT)
+
 struct _notmuch_database {
     notmuch_bool_t exception_reported;
 
@@ -176,6 +183,10 @@ struct _notmuch_database {
     Xapian::TermGenerator *term_gen;
     Xapian::ValueRangeProcessor *value_range_processor;
     Xapian::ValueRangeProcessor *date_range_processor;
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+    Xapian::FieldProcessor *date_field_processor;
+    Xapian::FieldProcessor *query_field_processor;
+#endif
     Xapian::ValueRangeProcessor *last_mod_range_processor;
 };
 
diff --git a/lib/database.cc b/lib/database.cc
index c8c5e261..f0bfe566 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -13,13 +13,14 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
 
 #include "database-private.h"
 #include "parse-time-vrp.h"
+#include "query-fp.h"
 #include "string-util.h"
 
 #include <iostream>
@@ -48,6 +49,12 @@ typedef struct {
 #define STRINGIFY(s) _SUB_STRINGIFY(s)
 #define _SUB_STRINGIFY(s) #s
 
+#if HAVE_XAPIAN_DB_RETRY_LOCK
+#define DB_ACTION (Xapian::DB_CREATE_OR_OPEN | Xapian::DB_RETRY_LOCK)
+#else
+#define DB_ACTION Xapian::DB_CREATE_OR_OPEN
+#endif
+
 /* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
  *
  * We currently have three different types of documents (mail, ghost,
@@ -90,6 +97,9 @@ typedef struct {
  *		        STRING is the name of a file within that
  *		        directory for this mail message.
  *
+ *      property:       Has a property with key=value
+ *                 FIXME: if no = is present, should match on any value
+ *
  *    A mail document also has four values:
  *
  *	TIMESTAMP:	The time_t value corresponding to the message's
@@ -184,6 +194,14 @@ typedef struct {
  *			generated is 1 and the value will be
  *			incremented for each thread ID.
  *
+ *	C*		metadata keys starting with C indicate
+ *			configuration data. It can be managed with the
+ *			n_database_*config* API.  There is a convention
+ *			of hierarchical keys separated by '.' (e.g.
+ *			query.notmuch stores the value for the named
+ *			query 'notmuch'), but it is not enforced by the
+ *			API.
+ *
  * Obsolete metadata
  * -----------------
  *
@@ -216,7 +234,7 @@ typedef struct {
 
 /* With these prefix values we follow the conventions published here:
  *
- * http://xapian.org/docs/omega/termprefixes.html
+ * https://xapian.org/docs/omega/termprefixes.html
  *
  * as much as makes sense. Note that I took some liberty in matching
  * the reserved prefix values to notmuch concepts, (for example, 'G'
@@ -244,11 +262,12 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "is",			"K" },
     { "id",			"Q" },
     { "path",			"P" },
+    { "property",		"XPROPERTY" },
     /*
-     * Without the ":", since this is a multi-letter prefix, Xapian
-     * will add a colon itself if the first letter of the path is
-     * upper-case ASCII. Including the ":" forces there to always be a
-     * colon, which keeps our own logic simpler.
+     * Unconditionally add ':' to reduce potential ambiguity with
+     * overlapping prefixes and/or terms that start with capital
+     * letters. See Xapian document termprefixes.html for related
+     * discussion.
      */
     { "folder",			"XFOLDER:" },
 };
@@ -368,6 +387,22 @@ _notmuch_database_log (notmuch_database_t *notmuch,
 	talloc_free (notmuch->status_string);
 
     notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
+    va_end (va_args);
+}
+
+void
+_notmuch_database_log_append (notmuch_database_t *notmuch,
+		      const char *format,
+		      ...)
+{
+    va_list va_args;
+
+    va_start (va_args, format);
+
+    if (notmuch->status_string)
+	notmuch->status_string = talloc_vasprintf_append (notmuch->status_string, format, va_args);
+    else
+	notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
 
     va_end (va_args);
 }
@@ -930,7 +965,7 @@ notmuch_database_open_verbose (const char *path,
 
 	if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
 	    notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
-							       Xapian::DB_CREATE_OR_OPEN);
+							       DB_ACTION);
 	} else {
 	    notmuch->xapian_db = new Xapian::Database (xapian_path);
 	}
@@ -1000,6 +1035,14 @@ notmuch_database_open_verbose (const char *path,
 	notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
 	notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
 	notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+	/* This currently relies on the query parser to pass anything
+	 * with a .. to the range processor */
+	notmuch->date_field_processor = new DateFieldProcessor();
+	notmuch->query_parser->add_boolean_prefix("date", notmuch->date_field_processor);
+	notmuch->query_field_processor = new QueryFieldProcessor (*notmuch->query_parser, notmuch);
+	notmuch->query_parser->add_boolean_prefix("query", notmuch->query_field_processor);
+#endif
 	notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
 
 	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
@@ -1090,10 +1133,16 @@ notmuch_database_close (notmuch_database_t *notmuch)
     delete notmuch->last_mod_range_processor;
     notmuch->last_mod_range_processor = NULL;
 
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+    delete notmuch->date_field_processor;
+    notmuch->date_field_processor = NULL;
+    delete notmuch->query_field_processor;
+    notmuch->query_field_processor = NULL;
+#endif
+
     return status;
 }
 
-#if HAVE_XAPIAN_COMPACT
 static int
 unlink_cb (const char *path,
 	   unused (const struct stat *sb),
@@ -1277,17 +1326,6 @@ notmuch_database_compact (const char *path,
 
     return ret;
 }
-#else
-notmuch_status_t
-notmuch_database_compact (unused (const char *path),
-			  unused (const char *backup_path),
-			  unused (notmuch_compact_status_cb_t status_cb),
-			  unused (void *closure))
-{
-    _notmuch_database_log (notmuch, "notmuch was compiled against a xapian version lacking compaction support.\n");
-    return NOTMUCH_STATUS_UNSUPPORTED_OPERATION;
-}
-#endif
 
 notmuch_status_t
 notmuch_database_destroy (notmuch_database_t *notmuch)
@@ -1635,8 +1673,8 @@ notmuch_database_begin_atomic (notmuch_database_t *notmuch)
 	notmuch->atomic_nesting > 0)
 	goto DONE;
 
-	if (notmuch_database_needs_upgrade(notmuch))
-		return NOTMUCH_STATUS_UPGRADE_REQUIRED;
+    if (notmuch_database_needs_upgrade (notmuch))
+	return NOTMUCH_STATUS_UPGRADE_REQUIRED;
 
     try {
 	(static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))->begin_transaction (false);
@@ -2168,8 +2206,8 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
      * References header, if available.  If not, fall back to the
      * first message ID in the In-Reply-To header. */
     if (last_ref_message_id) {
-        _notmuch_message_add_term (message, "replyto",
-                                   last_ref_message_id);
+	_notmuch_message_add_term (message, "replyto",
+				   last_ref_message_id);
     } else if (in_reply_to_message_id) {
 	_notmuch_message_add_term (message, "replyto",
 			     in_reply_to_message_id);
@@ -2278,15 +2316,15 @@ _consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch,
     if (stored_id.empty ()) {
 	return NULL;
     } else {
-        Xapian::WritableDatabase *db;
+	Xapian::WritableDatabase *db;
 
 	db = static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db);
 
 	/* Clear the metadata for this message ID. We don't need it
 	 * anymore. */
-        db->set_metadata (metadata_key, "");
+	db->set_metadata (metadata_key, "");
 
-        return talloc_strdup (ctx, stored_id.c_str ());
+	return talloc_strdup (ctx, stored_id.c_str ());
     }
 }
 
diff --git a/lib/directory.cc b/lib/directory.cc
index 78637b3a..5de3319c 100644
--- a/lib/directory.cc
+++ b/lib/directory.cc
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -227,6 +227,9 @@ notmuch_directory_set_mtime (notmuch_directory_t *directory,
 				   Xapian::sortable_serialise (mtime));
 
 	db->replace_document (directory->document_id, directory->doc);
+
+	directory->mtime = mtime;
+
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch,
 		 "A Xapian exception occurred setting directory mtime: %s.\n",
diff --git a/lib/filenames.c b/lib/filenames.c
index 4f7c0d85..63e737dd 100644
--- a/lib/filenames.c
+++ b/lib/filenames.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
diff --git a/lib/gen-version-script.sh b/lib/gen-version-script.sh
index 84770011..5621f2a9 100644
--- a/lib/gen-version-script.sh
+++ b/lib/gen-version-script.sh
@@ -2,7 +2,7 @@ set -eu
 
 # we go through a bit of work to get the unmangled names of the
 # typeinfo symbols because of
-# http://sourceware.org/bugzilla/show_bug.cgi?id=10326
+# https://sourceware.org/bugzilla/show_bug.cgi?id=10326
 
 if [ $# -lt 2 ]; then
     echo Usage: $0 header obj1 obj2 obj3
@@ -17,7 +17,7 @@ nm  $* | awk '$1 ~ "^[0-9a-fA-F][0-9a-fA-F]*$" && $3 ~ "Xapian.*Error" {print $3
 while read sym; do
     demangled=$(c++filt $sym)
     case $demangled in
-	typeinfo*) 
+	typeinfo*)
 	    printf "\t$sym;\n"
 	    ;;
 	*)
diff --git a/lib/index.cc b/lib/index.cc
index f166aefd..8c145540 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -12,7 +12,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -144,7 +144,7 @@ filter_filter (GMimeFilter *gmime_filter, char *inbuf, size_t inlen, size_t pres
 	{9,  ' ',  ' ',  10, 0},
 	{10, '\n', '\n', 11, 10},
 	{11, 'M',  'M',  12, 0},
-	{12, ' ',  '`',  12, 11}  
+	{12, ' ',  '`',  12, 11}
     };
     int next;
 
diff --git a/lib/message-file.c b/lib/message-file.c
index ee305202..db18b163 100644
--- a/lib/message-file.c
+++ b/lib/message-file.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
diff --git a/lib/message-private.h b/lib/message-private.h
new file mode 100644
index 00000000..74199256
--- /dev/null
+++ b/lib/message-private.h
@@ -0,0 +1,16 @@
+#ifndef MESSAGE_PRIVATE_H
+#define MESSAGE_PRIVATE_H
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message);
+
+notmuch_bool_t
+_notmuch_message_frozen (notmuch_message_t *message);
+
+void
+_notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix);
+
+void
+_notmuch_message_invalidate_metadata (notmuch_message_t *message,  const char *prefix_name);
+
+#endif
diff --git a/lib/message-property.cc b/lib/message-property.cc
new file mode 100644
index 00000000..0b13cac3
--- /dev/null
+++ b/lib/message-property.cc
@@ -0,0 +1,146 @@
+/* message-property.cc - Properties are like tags, but (key,value) pairs.
+ * keys are allowed to repeat.
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+#include "database-private.h"
+#include "message-private.h"
+
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value)
+{
+    if (! value)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    *value = _notmuch_string_map_get (_notmuch_message_property_map (message), key);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+static notmuch_status_t
+_notmuch_message_modify_property (notmuch_message_t *message, const char *key, const char *value,
+				  notmuch_bool_t delete_it)
+{
+    notmuch_private_status_t private_status;
+    notmuch_status_t status;
+    char *term = NULL;
+
+    status = _notmuch_database_ensure_writable (_notmuch_message_database (message));
+    if (status)
+	return status;
+
+    if (key == NULL || value == NULL)
+	return NOTMUCH_STATUS_NULL_POINTER;
+
+    if (index (key, '='))
+	return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+
+    term = talloc_asprintf (message, "%s=%s", key, value);
+
+    if (delete_it)
+	private_status = _notmuch_message_remove_term (message, "property", term);
+    else
+	private_status = _notmuch_message_add_term (message, "property", term);
+
+    if (private_status)
+	return COERCE_STATUS (private_status,
+			      "Unhandled error modifying message property");
+    if (! _notmuch_message_frozen (message))
+	_notmuch_message_sync (message);
+
+    if (term)
+	talloc_free (term);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, FALSE);
+}
+
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value)
+{
+    return _notmuch_message_modify_property (message, key, value, TRUE);
+}
+
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key)
+{
+    notmuch_status_t status;
+    const char * term_prefix;
+
+    status = _notmuch_database_ensure_writable (_notmuch_message_database (message));
+    if (status)
+	return status;
+
+    _notmuch_message_invalidate_metadata (message, "property");
+    if (key)
+	term_prefix = talloc_asprintf (message, "%s%s=", _find_prefix ("property"), key);
+    else
+	term_prefix = _find_prefix ("property");
+
+    /* XXX better error reporting ? */
+    _notmuch_message_remove_terms (message, term_prefix);
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact)
+{
+    notmuch_string_map_t *map;
+    map = _notmuch_message_property_map (message);
+    return _notmuch_string_map_iterator_create (map, key, exact);
+}
+
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_valid (properties);
+}
+
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_move_to_next (properties);
+}
+
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_key (properties);
+}
+
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties)
+{
+    return _notmuch_string_map_iterator_value (properties);
+}
+
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties)
+{
+    _notmuch_string_map_iterator_destroy (properties);
+}
diff --git a/lib/message.cc b/lib/message.cc
index 68393055..9d3e8071 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -13,13 +13,14 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
 
 #include "notmuch-private.h"
 #include "database-private.h"
+#include "message-private.h"
 
 #include <stdint.h>
 
@@ -37,6 +38,8 @@ struct visible _notmuch_message {
     notmuch_string_list_t *filename_list;
     char *author;
     notmuch_message_file_t *message_file;
+    notmuch_string_list_t *property_term_list;
+    notmuch_string_map_t *property_map;
     notmuch_message_list_t *replies;
     unsigned long flags;
     /* For flags that are initialized on-demand, lazy_flags indicates
@@ -116,6 +119,8 @@ _notmuch_message_create_for_document (const void *talloc_owner,
     message->filename_list = NULL;
     message->message_file = NULL;
     message->author = NULL;
+    message->property_term_list = NULL;
+    message->property_map = NULL;
 
     message->replies = _notmuch_message_list_create (message);
     if (unlikely (message->replies == NULL)) {
@@ -314,6 +319,7 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
 	*id_prefix = _find_prefix ("id"),
 	*type_prefix = _find_prefix ("type"),
 	*filename_prefix = _find_prefix ("file-direntry"),
+	*property_prefix = _find_prefix ("property"),
 	*replyto_prefix = _find_prefix ("replyto");
 
     /* We do this all in a single pass because Xapian decompresses the
@@ -369,18 +375,28 @@ _notmuch_message_ensure_metadata (notmuch_message_t *message)
 	    _notmuch_database_get_terms_with_prefix (message, i, end,
 						     filename_prefix);
 
+
+    /* Get property terms. Mimic the setup with filenames above */
+    assert (strcmp (filename_prefix, property_prefix) < 0);
+    if (!message->property_map && !message->property_term_list)
+	message->property_term_list =
+	    _notmuch_database_get_terms_with_prefix (message, i, end,
+						     property_prefix);
+
     /* Get reply to */
-    assert (strcmp (filename_prefix, replyto_prefix) < 0);
+    assert (strcmp (property_prefix, replyto_prefix) < 0);
     if (!message->in_reply_to)
 	message->in_reply_to =
 	    _notmuch_message_get_term (message, i, end, replyto_prefix);
+
+
     /* It's perfectly valid for a message to have no In-Reply-To
      * header. For these cases, we return an empty string. */
     if (!message->in_reply_to)
 	message->in_reply_to = talloc_strdup (message, "");
 }
 
-static void
+void
 _notmuch_message_invalidate_metadata (notmuch_message_t *message,
 				      const char *prefix_name)
 {
@@ -405,6 +421,18 @@ _notmuch_message_invalidate_metadata (notmuch_message_t *message,
 	message->filename_term_list = message->filename_list = NULL;
     }
 
+    if (strcmp ("property", prefix_name) == 0) {
+
+	if (message->property_term_list)
+	    talloc_free (message->property_term_list);
+	message->property_term_list = NULL;
+
+	if (message->property_map)
+	    talloc_unlink (message, message->property_map);
+
+	message->property_map = NULL;
+    }
+
     if (strcmp ("replyto", prefix_name) == 0) {
 	talloc_free (message->in_reply_to);
 	message->in_reply_to = NULL;
@@ -525,7 +553,7 @@ notmuch_message_get_replies (notmuch_message_t *message)
     return _notmuch_messages_create (message->replies);
 }
 
-static void
+void
 _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 {
     Xapian::TermIterator i;
@@ -1444,7 +1472,7 @@ notmuch_message_maildir_flags_to_tags (notmuch_message_t *message)
 
     for (i = 0; i < ARRAY_SIZE(flag2tag); i++) {
 	if ((strchr (combined_flags, flag2tag[i].flag) != NULL)
-	    ^ 
+	    ^
 	    flag2tag[i].inverse)
 	{
 	    status = notmuch_message_add_tag (message, flag2tag[i].tag);
@@ -1772,3 +1800,50 @@ _notmuch_message_database (notmuch_message_t *message)
 {
     return message->notmuch;
 }
+
+void
+_notmuch_message_ensure_property_map (notmuch_message_t *message)
+{
+    notmuch_string_node_t *node;
+
+    if (message->property_map)
+	return;
+
+    if (!message->property_term_list)
+	_notmuch_message_ensure_metadata (message);
+
+    message->property_map = _notmuch_string_map_create (message);
+
+    for (node = message->property_term_list->head; node; node = node->next) {
+	const char *key;
+	char *value;
+
+	value = index(node->string, '=');
+	if (!value)
+	    INTERNAL_ERROR ("malformed property term");
+
+	*value = '\0';
+	value++;
+	key = node->string;
+
+	_notmuch_string_map_append (message->property_map, key, value);
+
+    }
+
+    talloc_free (message->property_term_list);
+    message->property_term_list = NULL;
+}
+
+notmuch_string_map_t *
+_notmuch_message_property_map (notmuch_message_t *message)
+{
+    _notmuch_message_ensure_property_map (message);
+
+    return message->property_map;
+}
+
+notmuch_bool_t
+_notmuch_message_frozen (notmuch_message_t *message)
+{
+    return message->frozen;
+}
diff --git a/lib/messages.c b/lib/messages.c
index 0eee5690..b5363bb8 100644
--- a/lib/messages.c
+++ b/lib/messages.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 92807975..7b35fc5b 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -158,8 +158,8 @@ typedef enum _notmuch_private_status {
     ((private_status >= (notmuch_private_status_t) NOTMUCH_STATUS_LAST_STATUS)\
      ?									\
      _internal_error (format " (%s).\n",				\
-                      ##__VA_ARGS__,					\
-                      __location__),					\
+		      ##__VA_ARGS__,					\
+		      __location__),					\
      (notmuch_status_t) NOTMUCH_PRIVATE_STATUS_SUCCESS			\
      :									\
      (notmuch_status_t) private_status)
@@ -196,6 +196,10 @@ void
 _notmuch_database_log (notmuch_database_t *notmuch,
 		       const char *format, ...);
 
+void
+_notmuch_database_log_append (notmuch_database_t *notmuch,
+			      const char *format, ...);
+
 unsigned long
 _notmuch_database_new_revision (notmuch_database_t *notmuch);
 
@@ -477,11 +481,11 @@ _notmuch_mset_messages_move_to_next (notmuch_messages_t *messages);
 
 notmuch_bool_t
 _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
-                              unsigned int doc_id);
+			      unsigned int doc_id);
 
 void
 _notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
-                            unsigned int doc_id);
+			    unsigned int doc_id);
 
 /* querying xapian documents by type (e.g. "mail" or "ghost"): */
 notmuch_status_t
@@ -537,6 +541,39 @@ _notmuch_string_list_append (notmuch_string_list_t *list,
 void
 _notmuch_string_list_sort (notmuch_string_list_t *list);
 
+/* string-map.c */
+typedef struct _notmuch_string_map  notmuch_string_map_t;
+typedef struct _notmuch_string_map_iterator notmuch_string_map_iterator_t;
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx);
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value);
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key);
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     notmuch_bool_t exact);
+
+notmuch_bool_t
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iter);
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iter);
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator);
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator);
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator);
+
 /* tags.c */
 
 notmuch_tags_t *
diff --git a/lib/notmuch.h b/lib/notmuch.h
index cb46fc05..18678c0a 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -56,7 +56,7 @@ NOTMUCH_BEGIN_DECLS
  * version in Makefile.local.
  */
 #define LIBNOTMUCH_MAJOR_VERSION	4
-#define LIBNOTMUCH_MINOR_VERSION	3
+#define LIBNOTMUCH_MINOR_VERSION	4
 #define LIBNOTMUCH_MICRO_VERSION	0
 
 
@@ -93,7 +93,7 @@ NOTMUCH_BEGIN_DECLS
  * #endif
  * @endcode
  */
-#define LIBNOTMUCH_CHECK_VERSION (major, minor, micro)			\
+#define LIBNOTMUCH_CHECK_VERSION(major, minor, micro)			\
     (LIBNOTMUCH_MAJOR_VERSION > (major) ||					\
      (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION > (minor)) || \
      (LIBNOTMUCH_MAJOR_VERSION == (major) && LIBNOTMUCH_MINOR_VERSION == (minor) && \
@@ -180,6 +180,11 @@ typedef enum _notmuch_status {
      */
     NOTMUCH_STATUS_PATH_ERROR,
     /**
+     * One of the arguments violates the preconditions for the
+     * function, in a way not covered by a more specific argument.
+     */
+    NOTMUCH_STATUS_ILLEGAL_ARGUMENT,
+    /**
      * Not an actual status value. Just a way to find out how many
      * valid status values there are.
      */
@@ -206,6 +211,7 @@ typedef struct _notmuch_message notmuch_message_t;
 typedef struct _notmuch_tags notmuch_tags_t;
 typedef struct _notmuch_directory notmuch_directory_t;
 typedef struct _notmuch_filenames notmuch_filenames_t;
+typedef struct _notmuch_config_list notmuch_config_list_t;
 #endif /* __DOXYGEN__ */
 
 /**
@@ -331,7 +337,9 @@ notmuch_database_status_string (const notmuch_database_t *notmuch);
  * functions on objects derived from this database may either behave
  * as if the database had not been closed (e.g., if the required data
  * has been cached) or may fail with a
- * NOTMUCH_STATUS_XAPIAN_EXCEPTION.
+ * NOTMUCH_STATUS_XAPIAN_EXCEPTION. The only further operation
+ * permitted on the database itself is to call
+ * notmuch_database_destroy.
  *
  * notmuch_database_close can be called multiple times.  Later calls
  * have no effect.
@@ -692,7 +700,7 @@ notmuch_database_get_all_tags (notmuch_database_t *db);
  * completely in the future, but it's likely to be a specialized
  * version of the general Xapian query syntax:
  *
- * http://xapian.org/docs/queryparser.html
+ * https://xapian.org/docs/queryparser.html
  *
  * As a special case, passing either a length-zero string, (that is ""),
  * or a string consisting of a single asterisk (that is "*"), will
@@ -1651,6 +1659,183 @@ void
 notmuch_message_destroy (notmuch_message_t *message);
 
 /**
+ * @name Message Properties
+ *
+ * This interface provides the ability to attach arbitrary (key,value)
+ * string pairs to a message, to remove such pairs, and to iterate
+ * over them.  The caller should take some care as to what keys they
+ * add or delete values for, as other subsystems or extensions may
+ * depend on these properties.
+ *
+ */
+/**@{*/
+/**
+ * Retrieve the value for a single property key
+ *
+ * *value* is set to a string owned by the message or NULL if there is
+ * no such key. In the case of multiple values for the given key, the
+ * first one is retrieved.
+ *
+ * @returns
+ * - NOTMUCH_STATUS_NULL_POINTER: *value* may not be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value);
+
+/**
+ * Add a (key,value) pair to a message
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_add_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove a (key,value) pair from a message.
+ *
+ * It is not an error to remove a non-existant (key,value) pair
+ *
+ * @returns
+ * - NOTMUCH_STATUS_ILLEGAL_ARGUMENT: *key* may not contain an '=' character.
+ * - NOTMUCH_STATUS_NULL_POINTER: Neither *key* nor *value* may be NULL.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_remove_property (notmuch_message_t *message, const char *key, const char *value);
+
+/**
+ * Remove all (key,value) pairs from the given message.
+ *
+ * @param[in,out] message  message to operate on.
+ * @param[in]     key      key to delete properties for. If NULL, delete
+ *			   properties for all keys
+ * @returns
+ * - NOTMUCH_STATUS_READ_ONLY_DATABASE: Database was opened in
+ *   read-only mode so message cannot be modified.
+ * - NOTMUCH_STATUS_SUCCESS: No error occured.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_message_remove_all_properties (notmuch_message_t *message, const char *key);
+
+/**
+ * Opaque message property iterator
+ */
+typedef struct _notmuch_string_map_iterator notmuch_message_properties_t;
+
+/**
+ * Get the properties for *message*, returning a
+ * notmuch_message_properties_t object which can be used to iterate
+ * over all properties.
+ *
+ * The notmuch_message_properties_t object is owned by the message and
+ * as such, will only be valid for as long as the message is valid,
+ * (which is until the query from which it derived is destroyed).
+ *
+ * @param[in] message  The message to examine
+ * @param[in] key      key or key prefix
+ * @param[in] exact    if TRUE, require exact match with key. Otherwise
+ *		       treat as prefix.
+ *
+ * Typical usage might be:
+ *
+ *     notmuch_message_properties_t *list;
+ *
+ *     for (list = notmuch_message_get_properties (message, "testkey1", TRUE);
+ *          notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+ *        printf("%s\n", notmuch_message_properties_value(list));
+ *     }
+ *
+ *     notmuch_message_properties_destroy (list);
+ *
+ * Note that there's no explicit destructor needed for the
+ * notmuch_message_properties_t object. (For consistency, we do
+ * provide a notmuch_message_properities_destroy function, but there's
+ * no good reason to call it if the message is about to be destroyed).
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_message_properties_t *
+notmuch_message_get_properties (notmuch_message_t *message, const char *key, notmuch_bool_t exact);
+
+/**
+ * Is the given *properties* iterator pointing at a valid (key,value)
+ * pair.
+ *
+ * When this function returns TRUE,
+ * notmuch_message_properties_{key,value} will return a valid string,
+ * and notmuch_message_properties_move_to_next will do what it
+ * says. Whereas when this function returns FALSE, calling any of
+ * these functions results in undefined behaviour.
+ *
+ * See the documentation of notmuch_message_properties_get for example
+ * code showing how to iterate over a notmuch_message_properties_t
+ * object.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_message_properties_valid (notmuch_message_properties_t *properties);
+
+/**
+ * Move the *properties* iterator to the next (key,value) pair
+ *
+ * If *properties* is already pointing at the last pair then the iterator
+ * will be moved to a point just beyond that last pair, (where
+ * notmuch_message_properties_valid will return FALSE).
+ *
+ * See the documentation of notmuch_message_get_properties for example
+ * code showing how to iterate over a notmuch_message_properties_t object.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_message_properties_move_to_next (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * this could be useful if iterating for a prefix
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_message_properties_key (notmuch_message_properties_t *properties);
+
+/**
+ * Return the key from the current (key,value) pair.
+ *
+ * This could be useful if iterating for a prefix.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_message_properties_value (notmuch_message_properties_t *properties);
+
+
+/**
+ * Destroy a notmuch_message_properties_t object.
+ *
+ * It's not strictly necessary to call this function. All memory from
+ * the notmuch_message_properties_t object will be reclaimed when the
+ * containing message object is destroyed.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_message_properties_destroy (notmuch_message_properties_t *properties);
+
+/**@}*/
+
+/**
  * Is the given 'tags' iterator pointing at a valid tag.
  *
  * When this function returns TRUE, notmuch_tags_get will return a
@@ -1838,6 +2023,91 @@ notmuch_filenames_move_to_next (notmuch_filenames_t *filenames);
 void
 notmuch_filenames_destroy (notmuch_filenames_t *filenames);
 
+
+/**
+ * set config 'key' to 'value'
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_set_config (notmuch_database_t *db, const char *key, const char *value);
+
+/**
+ * retrieve config item 'key', assign to  'value'
+ *
+ * keys which have not been previously set with n_d_set_config will
+ * return an empty string.
+ *
+ * return value is allocated by malloc and should be freed by the
+ * caller.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_get_config (notmuch_database_t *db, const char *key, char **value);
+
+/**
+ * Create an iterator for all config items with keys matching a given prefix
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_status_t
+notmuch_database_get_config_list (notmuch_database_t *db, const char *prefix, notmuch_config_list_t **out);
+
+/**
+ * Is 'config_list' iterator valid (i.e. _key, _value, _move_to_next can be called).
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_config_list_valid (notmuch_config_list_t *config_list);
+
+/**
+ * return key for current config pair
+ *
+ * return value is owned by the iterator, and will be destroyed by the
+ * next call to notmuch_config_list_key or notmuch_config_list_destroy.
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_config_list_key (notmuch_config_list_t *config_list);
+
+/**
+ * return 'value' for current config pair
+ *
+ * return value is owned by the iterator, and will be destroyed by the
+ * next call to notmuch_config_list_value or notmuch config_list_destroy
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+const char *
+notmuch_config_list_value (notmuch_config_list_t *config_list);
+
+
+/**
+ * move 'config_list' iterator to the next pair
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_config_list_move_to_next (notmuch_config_list_t *config_list);
+
+/**
+ * free any resources held by 'config_list'
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+void
+notmuch_config_list_destroy (notmuch_config_list_t *config_list);
+
+/**
+ * interrogate the library for compile time features
+ *
+ * @since libnotmuch 4.4 (notmuch 0.23)
+ */
+notmuch_bool_t
+notmuch_built_with (const char *name);
 /* @} */
 
 NOTMUCH_END_DECLS
diff --git a/lib/parse-time-vrp.cc b/lib/parse-time-vrp.cc
index 03804cf5..dd691494 100644
--- a/lib/parse-time-vrp.cc
+++ b/lib/parse-time-vrp.cc
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Jani Nikula <jani@nikula.org>
  */
@@ -64,3 +64,24 @@ ParseTimeValueRangeProcessor::operator() (std::string &begin, std::string &end)
 
     return valno;
 }
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+/* XXX TODO: is throwing an exception the right thing to do here? */
+Xapian::Query DateFieldProcessor::operator()(const std::string & str) {
+    time_t from, to, now;
+
+    /* Use the same 'now' for begin and end. */
+    if (time (&now) == (time_t) -1)
+	throw Xapian::QueryParserError("Unable to get current time");
+
+    if (parse_time_string (str.c_str (), &from, &now, PARSE_TIME_ROUND_DOWN))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+
+    if (parse_time_string (str.c_str (), &to, &now, PARSE_TIME_ROUND_UP_INCLUSIVE))
+	throw Xapian::QueryParserError ("Didn't understand date specification '" + str + "'");
+
+    return Xapian::Query(Xapian::Query::OP_AND,
+			 Xapian::Query(Xapian::Query::OP_VALUE_GE, 0, Xapian::sortable_serialise ((double) from)),
+			 Xapian::Query(Xapian::Query::OP_VALUE_LE, 0, Xapian::sortable_serialise ((double) to)));
+}
+#endif
diff --git a/lib/parse-time-vrp.h b/lib/parse-time-vrp.h
index 094c4f87..c024dba2 100644
--- a/lib/parse-time-vrp.h
+++ b/lib/parse-time-vrp.h
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Jani Nikula <jani@nikula.org>
  */
@@ -37,4 +37,9 @@ public:
     Xapian::valueno operator() (std::string &begin, std::string &end);
 };
 
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+class DateFieldProcessor : public Xapian::FieldProcessor {
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
 #endif /* NOTMUCH_PARSE_TIME_VRP_H */
diff --git a/lib/query-fp.cc b/lib/query-fp.cc
new file mode 100644
index 00000000..c39f5915
--- /dev/null
+++ b/lib/query-fp.cc
@@ -0,0 +1,43 @@
+/* query-fp.cc - "query:" field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "database-private.h"
+#include "query-fp.h"
+#include <iostream>
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+
+Xapian::Query
+QueryFieldProcessor::operator() (const std::string & name)
+{
+    std::string key = "query." + name;
+    char *expansion;
+    notmuch_status_t status;
+
+    status = notmuch_database_get_config (notmuch, key.c_str (), &expansion);
+    if (status) {
+	throw Xapian::QueryParserError ("error looking up key" + name);
+    }
+
+    return parser.parse_query (expansion, NOTMUCH_QUERY_PARSER_FLAGS);
+}
+#endif
diff --git a/lib/query-fp.h b/lib/query-fp.h
new file mode 100644
index 00000000..d6e4b313
--- /dev/null
+++ b/lib/query-fp.h
@@ -0,0 +1,42 @@
+/* query-fp.h - query field processor glue
+ *
+ * This file is part of notmuch.
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#ifndef NOTMUCH_QUERY_FP_H
+#define NOTMUCH_QUERY_FP_H
+
+#include <xapian.h>
+#include "notmuch.h"
+
+#if HAVE_XAPIAN_FIELD_PROCESSOR
+class QueryFieldProcessor : public Xapian::FieldProcessor {
+ protected:
+    Xapian::QueryParser &parser;
+    notmuch_database_t *notmuch;
+
+ public:
+    QueryFieldProcessor (Xapian::QueryParser &parser_, notmuch_database_t *notmuch_)
+	: parser(parser_), notmuch(notmuch_) { };
+
+    Xapian::Query operator()(const std::string & str);
+};
+#endif
+#endif /* NOTMUCH_QUERY_FP_H */
diff --git a/lib/query.cc b/lib/query.cc
index 77a7926b..53efd4e1 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -220,12 +220,6 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	Xapian::Query string_query, final_query, exclude_query;
 	Xapian::MSet mset;
 	Xapian::MSetIterator iterator;
-	unsigned int flags = (Xapian::QueryParser::FLAG_BOOLEAN |
-			      Xapian::QueryParser::FLAG_PHRASE |
-			      Xapian::QueryParser::FLAG_LOVEHATE |
-			      Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE |
-			      Xapian::QueryParser::FLAG_WILDCARD |
-			      Xapian::QueryParser::FLAG_PURE_NOT);
 
 	if (strcmp (query_string, "") == 0 ||
 	    strcmp (query_string, "*") == 0)
@@ -233,7 +227,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	    final_query = mail_query;
 	} else {
 	    string_query = notmuch->query_parser->
-		parse_query (query_string, flags);
+		parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
 	    final_query = Xapian::Query (Xapian::Query::OP_AND,
 					 mail_query, string_query);
 	}
@@ -282,7 +276,7 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 	case NOTMUCH_SORT_MESSAGE_ID:
 	    enquire.set_sort_by_value (NOTMUCH_VALUE_MESSAGE_ID, FALSE);
 	    break;
-        case NOTMUCH_SORT_UNSORTED:
+	case NOTMUCH_SORT_UNSORTED:
 	    break;
 	}
 
@@ -305,9 +299,10 @@ _notmuch_query_search_documents (notmuch_query_t *query,
 
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch,
-			       "A Xapian exception occurred performing query: %s\n"
+			       "A Xapian exception occurred performing query: %s\n",
+			       error.get_msg().c_str());
+	_notmuch_database_log_append (notmuch,
 			       "Query string was: %s\n",
-			       error.get_msg().c_str(),
 			       query->query_string);
 
 	notmuch->exception_reported = TRUE;
@@ -418,7 +413,7 @@ _notmuch_doc_id_set_contains (notmuch_doc_id_set_t *doc_ids,
 
 void
 _notmuch_doc_id_set_remove (notmuch_doc_id_set_t *doc_ids,
-                            unsigned int doc_id)
+			    unsigned int doc_id)
 {
     if (doc_id < doc_ids->bound)
 	doc_ids->bitmap[DOCIDSET_WORD(doc_id)] &= ~(1 << DOCIDSET_BIT(doc_id));
@@ -579,12 +574,6 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
 						   type));
 	Xapian::Query string_query, final_query, exclude_query;
 	Xapian::MSet mset;
-	unsigned int flags = (Xapian::QueryParser::FLAG_BOOLEAN |
-			      Xapian::QueryParser::FLAG_PHRASE |
-			      Xapian::QueryParser::FLAG_LOVEHATE |
-			      Xapian::QueryParser::FLAG_BOOLEAN_ANY_CASE |
-			      Xapian::QueryParser::FLAG_WILDCARD |
-			      Xapian::QueryParser::FLAG_PURE_NOT);
 
 	if (strcmp (query_string, "") == 0 ||
 	    strcmp (query_string, "*") == 0)
@@ -592,7 +581,7 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
 	    final_query = mail_query;
 	} else {
 	    string_query = notmuch->query_parser->
-		parse_query (query_string, flags);
+		parse_query (query_string, NOTMUCH_QUERY_PARSER_FLAGS);
 	    final_query = Xapian::Query (Xapian::Query::OP_AND,
 					 mail_query, string_query);
 	}
@@ -625,10 +614,11 @@ _notmuch_query_count_documents (notmuch_query_t *query, const char *type, unsign
 
     } catch (const Xapian::Error &error) {
 	_notmuch_database_log (notmuch,
-			       "A Xapian exception occurred performing query: %s\n"
-			       "Query string was: %s\n",
-			       error.get_msg().c_str(),
-			       query->query_string);
+			       "A Xapian exception occurred performing query: %s\n",
+			       error.get_msg().c_str());
+	_notmuch_database_log_append (notmuch,
+				      "Query string was: %s\n",
+				      query->query_string);
 	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
     }
 
diff --git a/lib/sha1.c b/lib/sha1.c
index 94060d57..b7dea1c2 100644
--- a/lib/sha1.c
+++ b/lib/sha1.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
@@ -112,4 +112,3 @@ _notmuch_sha1_of_file (const char *filename)
 
     return result;
 }
-
diff --git a/lib/string-list.c b/lib/string-list.c
index da72746d..43ebe499 100644
--- a/lib/string-list.c
+++ b/lib/string-list.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  *         Austin Clements <aclements@csail.mit.edu>
diff --git a/lib/string-map.c b/lib/string-map.c
new file mode 100644
index 00000000..0bb77e93
--- /dev/null
+++ b/lib/string-map.c
@@ -0,0 +1,228 @@
+/* string-map.c - associative arrays of strings
+ *
+ *
+ * Copyright © 2016 David Bremner
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Author: David Bremner <david@tethera.net>
+ */
+
+#include "notmuch-private.h"
+
+/* Create a new notmuch_string_map_t object, with 'ctx' as its
+ * talloc owner.
+ *
+ * This function can return NULL in case of out-of-memory.
+ */
+
+typedef struct _notmuch_string_pair_t {
+    char *key;
+    char *value;
+} notmuch_string_pair_t;
+
+struct _notmuch_string_map {
+    notmuch_bool_t sorted;
+    size_t length;
+    notmuch_string_pair_t *pairs;
+};
+
+struct _notmuch_string_map_iterator {
+    notmuch_string_pair_t *current;
+    notmuch_bool_t exact;
+    const char *key;
+};
+
+notmuch_string_map_t *
+_notmuch_string_map_create (const void *ctx)
+{
+    notmuch_string_map_t *map;
+
+    map = talloc (ctx, notmuch_string_map_t);
+    if (unlikely (map == NULL))
+	return NULL;
+
+    map->length = 0;
+    map->pairs = NULL;
+    map->sorted = TRUE;
+
+    return map;
+}
+
+void
+_notmuch_string_map_append (notmuch_string_map_t *map,
+			    const char *key,
+			    const char *value)
+{
+
+    map->length++;
+    map->sorted = FALSE;
+
+    if (map->pairs)
+	map->pairs = talloc_realloc (map, map->pairs, notmuch_string_pair_t, map->length + 1);
+    else
+	map->pairs = talloc_array (map, notmuch_string_pair_t, map->length + 1);
+
+    map->pairs[map->length - 1].key = talloc_strdup (map, key);
+    map->pairs[map->length - 1].value = talloc_strdup (map, value);
+
+    /* Add sentinel */
+    map->pairs[map->length].key = NULL;
+    map->pairs[map->length].value = NULL;
+
+}
+
+static int
+cmppair (const void *pa, const void *pb)
+{
+    notmuch_string_pair_t *a = (notmuch_string_pair_t *) pa;
+    notmuch_string_pair_t *b = (notmuch_string_pair_t *) pb;
+
+    return strcmp (a->key, b->key);
+}
+
+static void
+_notmuch_string_map_sort (notmuch_string_map_t *map)
+{
+    if (map->length == 0)
+	return;
+
+    if (map->sorted)
+	return;
+
+    qsort (map->pairs, map->length, sizeof (notmuch_string_pair_t), cmppair);
+
+    map->sorted = TRUE;
+}
+
+static notmuch_bool_t
+string_cmp (const char *a, const char *b, notmuch_bool_t exact)
+{
+    if (exact)
+	return (strcmp (a, b));
+    else
+	return (strncmp (a, b, strlen (a)));
+}
+
+static notmuch_string_pair_t *
+bsearch_first (notmuch_string_pair_t *array, size_t len, const char *key, notmuch_bool_t exact)
+{
+    size_t first = 0;
+    size_t last = len - 1;
+    size_t mid;
+
+    if (len <= 0)
+	return NULL;
+
+    while (last > first) {
+	mid = (first + last) / 2;
+	int sign = string_cmp (key, array[mid].key, exact);
+
+	if (sign <= 0)
+	    last = mid;
+	else
+	    first = mid + 1;
+    }
+
+
+    if (string_cmp (key, array[first].key, exact) == 0)
+	return array + first;
+    else
+	return NULL;
+
+}
+
+const char *
+_notmuch_string_map_get (notmuch_string_map_t *map, const char *key)
+{
+    notmuch_string_pair_t *pair;
+
+    /* this means that calling append invalidates iterators */
+    _notmuch_string_map_sort (map);
+
+    pair = bsearch_first (map->pairs, map->length, key, TRUE);
+    if (! pair)
+	return NULL;
+
+    return pair->value;
+}
+
+notmuch_string_map_iterator_t *
+_notmuch_string_map_iterator_create (notmuch_string_map_t *map, const char *key,
+				     notmuch_bool_t exact)
+{
+    notmuch_string_map_iterator_t *iter;
+
+    _notmuch_string_map_sort (map);
+
+    iter = talloc (map, notmuch_string_map_iterator_t);
+    if (unlikely (iter == NULL))
+	return NULL;
+
+    if (unlikely (talloc_reference (iter, map) == NULL))
+	return NULL;
+
+    iter->key = talloc_strdup (iter, key);
+    iter->exact = exact;
+    iter->current = bsearch_first (map->pairs, map->length, key, exact);
+    return iter;
+}
+
+notmuch_bool_t
+_notmuch_string_map_iterator_valid (notmuch_string_map_iterator_t *iterator)
+{
+    if (iterator->current == NULL)
+	return FALSE;
+
+    /* sentinel */
+    if (iterator->current->key == NULL)
+	return FALSE;
+
+    return (0 == string_cmp (iterator->key, iterator->current->key, iterator->exact));
+
+}
+
+void
+_notmuch_string_map_iterator_move_to_next (notmuch_string_map_iterator_t *iterator)
+{
+
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return;
+
+    (iterator->current)++;
+}
+
+const char *
+_notmuch_string_map_iterator_key (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->key;
+}
+
+const char *
+_notmuch_string_map_iterator_value (notmuch_string_map_iterator_t *iterator)
+{
+    if (! _notmuch_string_map_iterator_valid (iterator))
+	return NULL;
+
+    return iterator->current->value;
+}
+
+void
+_notmuch_string_map_iterator_destroy (notmuch_string_map_iterator_t *iterator)
+{
+    talloc_free (iterator);
+}
diff --git a/lib/tags.c b/lib/tags.c
index b7e5602c..c7d3f66f 100644
--- a/lib/tags.c
+++ b/lib/tags.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */
diff --git a/lib/thread.cc b/lib/thread.cc
index 0c937d76..84ee5298 100644
--- a/lib/thread.cc
+++ b/lib/thread.cc
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ * along with this program.  If not, see https://www.gnu.org/licenses/ .
  *
  * Author: Carl Worth <cworth@cworth.org>
  */