Merge remote-tracking branch 'origin/master' into nmwebnmweb

11 files changed, 174 insertions, 31 deletions

diff --git a/lib/config.cc b/lib/config.cc
index 503a0c8b..2323860d 100644
--- a/lib/config.cc
+++ b/lib/config.cc
@@ -599,6 +599,8 @@ _notmuch_config_key_to_string (notmuch_config_key_t key)
 	return "database.autocommit";
     case NOTMUCH_CONFIG_EXTRA_HEADERS:
 	return "show.extra_headers";
+    case NOTMUCH_CONFIG_INDEX_AS_TEXT:
+	return "index.as_text";
     default:
 	return NULL;
     }
@@ -642,6 +644,7 @@ _notmuch_config_default (notmuch_database_t *notmuch, notmuch_config_key_t key)
 	else
 	    email = _get_email_from_passwd_file (notmuch);
 	return email;
+    case NOTMUCH_CONFIG_INDEX_AS_TEXT:
     case NOTMUCH_CONFIG_NEW_IGNORE:
 	return "";
     case NOTMUCH_CONFIG_AUTOCOMMIT:
diff --git a/lib/database-private.h b/lib/database-private.h
index b9be4e22..61232f1a 100644
--- a/lib/database-private.h
+++ b/lib/database-private.h
@@ -291,6 +291,10 @@ struct _notmuch_database {
 
     /* Track what parameters were specified when opening */
     notmuch_open_param_t params;
+
+    /* list of regular expressions to check for text indexing */
+    regex_t *index_as_text;
+    size_t index_as_text_length;
 };
 
 /* Prior to database version 3, features were implied by the database
diff --git a/lib/database.cc b/lib/database.cc
index c05d70d3..737a3f30 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -1454,9 +1454,11 @@ notmuch_database_remove_message (notmuch_database_t *notmuch,
 							&message);
 
     if (status == NOTMUCH_STATUS_SUCCESS && message) {
-	status = _notmuch_message_remove_filename (message, filename);
+	if (notmuch_message_count_files (message) > 1) {
+	    status = _notmuch_message_remove_filename (message, filename);
+	}
 	if (status == NOTMUCH_STATUS_SUCCESS)
-	    _notmuch_message_delete (message);
+	    status = _notmuch_message_delete (message);
 	else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
 	    _notmuch_message_sync (message);
 
@@ -1573,3 +1575,15 @@ notmuch_database_status_string (const notmuch_database_t *notmuch)
 {
     return notmuch->status_string;
 }
+
+bool
+_notmuch_database_indexable_as_text (notmuch_database_t *notmuch, const char *mime_string)
+{
+    for (size_t i = 0; i < notmuch->index_as_text_length; i++) {
+	if (regexec (&notmuch->index_as_text[i], mime_string, 0, NULL, 0) == 0) {
+	    return true;
+	}
+    }
+
+    return false;
+}
diff --git a/lib/index.cc b/lib/index.cc
index 728bfb22..629dcb22 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -380,6 +380,23 @@ _index_pkcs7_part (notmuch_message_t *message,
 		   GMimeObject *part,
 		   _notmuch_message_crypto_t *msg_crypto);
 
+static bool
+_indexable_as_text (notmuch_message_t *message, GMimeObject *part)
+{
+    GMimeContentType *content_type = g_mime_object_get_content_type (part);
+    notmuch_database_t *notmuch = notmuch_message_get_database (message);
+
+    if (content_type) {
+	char *mime_string = g_mime_content_type_get_mime_type (content_type);
+	if (mime_string) {
+	    bool ret = _notmuch_database_indexable_as_text (notmuch, mime_string);
+	    g_free (mime_string);
+	    return ret;
+	}
+    }
+    return false;
+}
+
 /* Callback to generate terms for each mime part of a message. */
 static void
 _index_mime_part (notmuch_message_t *message,
@@ -497,9 +514,11 @@ _index_mime_part (notmuch_message_t *message,
 	_notmuch_message_add_term (message, "tag", "attachment");
 	_notmuch_message_gen_terms (message, "attachment", filename);
 
-	/* XXX: Would be nice to call out to something here to parse
-	 * the attachment into text and then index that. */
-	goto DONE;
+	if (! _indexable_as_text (message, part)) {
+	    /* XXX: Would be nice to call out to something here to parse
+	     * the attachment into text and then index that. */
+	    goto DONE;
+	}
     }
 
     byte_array = g_byte_array_new ();
diff --git a/lib/message-property.cc b/lib/message-property.cc
index d5afa30c..7f520340 100644
--- a/lib/message-property.cc
+++ b/lib/message-property.cc
@@ -25,6 +25,20 @@
 #include "database-private.h"
 #include "message-private.h"
 
+#define LOG_XAPIAN_EXCEPTION(message, error) _log_xapian_exception (__location__, message, error)
+
+static void
+_log_xapian_exception (const char *where, notmuch_message_t *message,  const Xapian::Error error)
+{
+    notmuch_database_t *notmuch = notmuch_message_get_database (message);
+
+    _notmuch_database_log (notmuch,
+			   "A Xapian exception occurred at %s: %s\n",
+			   where,
+			   error.get_msg ().c_str ());
+    notmuch->exception_reported = true;
+}
+
 notmuch_status_t
 notmuch_message_get_property (notmuch_message_t *message, const char *key, const char **value)
 {
@@ -83,10 +97,15 @@ _notmuch_message_modify_property (notmuch_message_t *message, const char *key, c
 
     term = talloc_asprintf (message, "%s=%s", key, value);
 
-    if (delete_it)
-	private_status = _notmuch_message_remove_term (message, "property", term);
-    else
-	private_status = _notmuch_message_add_term (message, "property", term);
+    try {
+	if (delete_it)
+	    private_status = _notmuch_message_remove_term (message, "property", term);
+	else
+	    private_status = _notmuch_message_add_term (message, "property", term);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
 
     if (private_status)
 	return COERCE_STATUS (private_status,
@@ -123,15 +142,22 @@ _notmuch_message_remove_all_properties (notmuch_message_t *message, const char *
     if (status)
 	return status;
 
-    _notmuch_message_invalidate_metadata (message, "property");
     if (key)
 	term_prefix = talloc_asprintf (message, "%s%s%s", _find_prefix ("property"), key,
 				       prefix ? "" : "=");
     else
 	term_prefix = _find_prefix ("property");
 
-    /* XXX better error reporting ? */
-    _notmuch_message_remove_terms (message, term_prefix);
+    try {
+	/* XXX better error reporting ? */
+	_notmuch_message_remove_terms (message, term_prefix);
+    } catch (Xapian::Error &error) {
+	LOG_XAPIAN_EXCEPTION (message, error);
+	return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+    }
+
+    if (! _notmuch_message_frozen (message))
+	_notmuch_message_sync (message);
 
     return NOTMUCH_STATUS_SUCCESS;
 }
diff --git a/lib/message.cc b/lib/message.cc
index 1c87f8c0..46638f80 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -719,6 +719,8 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 	    /* Ignore failure to remove non-existent term. */
 	}
     }
+
+    _notmuch_message_invalidate_metadata (message, "property");
 }
 
 
@@ -941,6 +943,7 @@ _notmuch_message_add_filename (notmuch_message_t *message,
 {
     const char *relative, *directory;
     notmuch_status_t status;
+    notmuch_private_status_t private_status;
     void *local = talloc_new (message);
     char *direntry;
 
@@ -964,10 +967,17 @@ _notmuch_message_add_filename (notmuch_message_t *message,
 
     /* New file-direntry allows navigating to this message with
      * notmuch_directory_get_child_files() . */
-    status = COERCE_STATUS (_notmuch_message_add_term (message, "file-direntry", direntry),
-			    "adding file-direntry term");
-    if (status)
-	return status;
+    private_status = _notmuch_message_add_term (message, "file-direntry", direntry);
+    switch (private_status) {
+    case NOTMUCH_PRIVATE_STATUS_SUCCESS:
+	break;
+    case NOTMUCH_PRIVATE_STATUS_TERM_TOO_LONG:
+	_notmuch_database_log (message->notmuch, "filename too long for file-direntry term: %s\n",
+			       filename);
+	return NOTMUCH_STATUS_PATH_ERROR;
+    default:
+	return COERCE_STATUS (private_status, "adding file-direntry term");
+    }
 
     status = _notmuch_message_add_folder_terms (message, directory);
     if (status)
@@ -1383,21 +1393,22 @@ _notmuch_message_delete (notmuch_message_t *message)
     if (status)
 	return status;
 
-    message->notmuch->writable_xapian_db->delete_document (message->doc_id);
-
-    /* if this was a ghost to begin with, we are done */
-    private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost);
-    if (private_status)
-	return COERCE_STATUS (private_status,
-			      "Error trying to determine whether message was a ghost");
-    if (is_ghost)
-	return NOTMUCH_STATUS_SUCCESS;
-
-    /* look for a non-ghost message in the same thread */
     try {
 	Xapian::PostingIterator thread_doc, thread_doc_end;
 	Xapian::PostingIterator mail_doc, mail_doc_end;
 
+	/* look for a non-ghost message in the same thread */
+	/* if this was a ghost to begin with, we are done */
+	private_status = _notmuch_message_has_term (message, "type", "ghost", &is_ghost);
+	if (private_status)
+	    return COERCE_STATUS (private_status,
+				  "Error trying to determine whether message was a ghost");
+
+	message->notmuch->writable_xapian_db->delete_document (message->doc_id);
+
+	if (is_ghost)
+	    return NOTMUCH_STATUS_SUCCESS;
+
 	_notmuch_database_find_doc_ids (message->notmuch, "thread", tid, &thread_doc,
 					&thread_doc_end);
 	_notmuch_database_find_doc_ids (message->notmuch, "type", "mail", &mail_doc, &mail_doc_end);
diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h
index 1d3d2b0c..c19ee8e2 100644
--- a/lib/notmuch-private.h
+++ b/lib/notmuch-private.h
@@ -259,6 +259,10 @@ _notmuch_database_filename_to_direntry (void *ctx,
 					notmuch_find_flags_t flags,
 					char **direntry);
 
+bool
+_notmuch_database_indexable_as_text (notmuch_database_t *notmuch,
+				     const char *mime_string);
+
 /* directory.cc */
 
 notmuch_directory_t *
diff --git a/lib/notmuch.h b/lib/notmuch.h
index 0b0540b1..4e2b0fa4 100644
--- a/lib/notmuch.h
+++ b/lib/notmuch.h
@@ -293,6 +293,8 @@ typedef struct _notmuch_indexopts notmuch_indexopts_t;
  *
  * NOTMUCH_STATUS_OUT_OF_MEMORY: Out of memory.
  *
+ * NOTMUCH_STATUS_PATH_ERROR: filename is too long
+ *
  * NOTMUCH_STATUS_FILE_ERROR: An error occurred trying to create the
  *	database file (such as permission denied, or file not found,
  *	etc.), or the database already exists.
@@ -720,7 +722,8 @@ notmuch_database_end_atomic (notmuch_database_t *notmuch);
  *
  * The UUID is a NUL-terminated opaque string that uniquely identifies
  * this database.  Two revision numbers are only comparable if they
- * have the same database UUID.
+ * have the same database UUID. The string 'uuid' is owned by notmuch
+ * and should not be freed or modified by the user.
  */
 unsigned long
 notmuch_database_get_revision (notmuch_database_t *notmuch,
@@ -1170,7 +1173,10 @@ notmuch_query_search_threads_st (notmuch_query_t *query, notmuch_threads_t **out
  *
  *     query = notmuch_query_create (database, query_string);
  *
- *     for (messages = notmuch_query_search_messages (query);
+ *     if (notmuch_query_search_messages (query, &messages) != NOTMUCH_STATUS_SUCCESS)
+ *         return EXIT_FAILURE;
+ *
+ *     for (;
  *          notmuch_messages_valid (messages);
  *          notmuch_messages_move_to_next (messages))
  *     {
@@ -2558,6 +2564,7 @@ typedef enum {
     NOTMUCH_CONFIG_USER_NAME,
     NOTMUCH_CONFIG_AUTOCOMMIT,
     NOTMUCH_CONFIG_EXTRA_HEADERS,
+    NOTMUCH_CONFIG_INDEX_AS_TEXT,
     NOTMUCH_CONFIG_LAST
 } notmuch_config_key_t;
 
diff --git a/lib/open.cc b/lib/open.cc
index 67ff868c..54d1faf3 100644
--- a/lib/open.cc
+++ b/lib/open.cc
@@ -320,6 +320,8 @@ _alloc_notmuch (const char *database_path, const char *config_path, const char *
     notmuch->transaction_count = 0;
     notmuch->transaction_threshold = 0;
     notmuch->view = 1;
+    notmuch->index_as_text = NULL;
+    notmuch->index_as_text_length = 0;
 
     notmuch->params = NOTMUCH_PARAM_NONE;
     if (database_path)
@@ -427,6 +429,53 @@ _load_database_state (notmuch_database_t *notmuch)
 	notmuch, notmuch->xapian_db->get_uuid ().c_str ());
 }
 
+/* XXX This should really be done lazily, but the error reporting path in the indexing code
+ * would need to be redone to report any errors.
+ */
+notmuch_status_t
+_ensure_index_as_text (notmuch_database_t *notmuch, char **message)
+{
+    int nregex = 0;
+    regex_t *regexv = NULL;
+
+    if (notmuch->index_as_text)
+	return NOTMUCH_STATUS_SUCCESS;
+
+    for (notmuch_config_values_t *list = notmuch_config_get_values (notmuch,
+								    NOTMUCH_CONFIG_INDEX_AS_TEXT);
+	 notmuch_config_values_valid (list);
+	 notmuch_config_values_move_to_next (list)) {
+	regex_t *new_regex;
+	int rerr;
+	const char *str = notmuch_config_values_get (list);
+	size_t len = strlen (str);
+
+	/* str must be non-empty, because n_c_get_values skips empty
+	 * strings */
+	assert (len > 0);
+
+	regexv = talloc_realloc (notmuch, regexv, regex_t, nregex + 1);
+	new_regex = &regexv[nregex];
+
+	rerr = regcomp (new_regex, str, REG_EXTENDED | REG_NOSUB);
+	if (rerr) {
+	    size_t error_size = regerror (rerr, new_regex, NULL, 0);
+	    char *error = (char *) talloc_size (str, error_size);
+
+	    regerror (rerr, new_regex, error, error_size);
+	    IGNORE_RESULT (asprintf (message, "Error in index.as_text: %s: %s\n", error, str));
+
+	    return NOTMUCH_STATUS_ILLEGAL_ARGUMENT;
+	}
+	nregex++;
+    }
+
+    notmuch->index_as_text = regexv;
+    notmuch->index_as_text_length = nregex;
+
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 static notmuch_status_t
 _finish_open (notmuch_database_t *notmuch,
 	      const char *profile,
@@ -531,6 +580,10 @@ _finish_open (notmuch_database_t *notmuch,
 	if (status)
 	    goto DONE;
 
+	status = _ensure_index_as_text (notmuch, &message);
+	if (status)
+	    goto DONE;
+
 	autocommit_str = notmuch_config_get (notmuch, NOTMUCH_CONFIG_AUTOCOMMIT);
 	if (unlikely (! autocommit_str)) {
 	    INTERNAL_ERROR ("missing configuration for autocommit");
diff --git a/lib/query.cc b/lib/query.cc
index 707f6222..1c60c122 100644
--- a/lib/query.cc
+++ b/lib/query.cc
@@ -20,6 +20,7 @@
 
 #include "notmuch-private.h"
 #include "database-private.h"
+#include "xapian-extra.h"
 
 #include <glib.h> /* GHashTable, GPtrArray */
 
@@ -186,7 +187,7 @@ _notmuch_query_string_to_xapian_query (notmuch_database_t *notmuch,
 {
     try {
 	if (query_string == "" || query_string == "*") {
-	    output = Xapian::Query::MatchAll;
+	    output = xapian_query_match_all ();
 	} else {
 	    output =
 		notmuch->query_parser->
diff --git a/lib/regexp-fields.cc b/lib/regexp-fields.cc
index 539915d8..3a775261 100644
--- a/lib/regexp-fields.cc
+++ b/lib/regexp-fields.cc
@@ -25,6 +25,7 @@
 #include "regexp-fields.h"
 #include "notmuch-private.h"
 #include "database-private.h"
+#include "xapian-extra.h"
 
 notmuch_status_t
 compile_regex (regex_t &regexp, const char *str, std::string &msg)
@@ -200,7 +201,7 @@ RegexpFieldProcessor::operator() (const std::string & str)
     if (str.empty ()) {
 	if (options & NOTMUCH_FIELD_PROBABILISTIC) {
 	    return Xapian::Query (Xapian::Query::OP_AND_NOT,
-				  Xapian::Query::MatchAll,
+				  xapian_query_match_all (),
 				  Xapian::Query (Xapian::Query::OP_WILDCARD, term_prefix));
 	} else {
 	    return Xapian::Query (term_prefix);