From: David Bremner <david@tethera.net>
Date: Tue, 16 Apr 2019 01:46:16 +0000 (-0300)
Subject: n_m_remove_indexed_terms: reduce number of Xapian API calls.
X-Git-Tag: archive/debian/0.29_rc0-1~42
X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=commitdiff_plain;h=97939170b3e8b31f88520932aa20bf1b98e469c3

n_m_remove_indexed_terms: reduce number of Xapian API calls.

Previously this functioned scanned every term attached to a given
Xapian document. It turns out we know how to read only the terms we
need to preserve (and we might have already done so). This commit
replaces many calls to Xapian::Document::remove_term with one call to
::clear_terms, and a (typically much smaller) number of calls to
::add_term. Roughly speaking this is based on the assumption that most
messages have more text than they have tags.

According to the performance test suite, this yields a roughly 40%
speedup on "notmuch reindex '*'"
---

diff --git a/lib/message.cc b/lib/message.cc
index 38a48933..4d1b08d6 100644
--- a/lib/message.cc
+++ b/lib/message.cc
@@ -716,6 +716,8 @@ _notmuch_message_remove_terms (notmuch_message_t *message, const char *prefix)
 
 /* Remove all terms generated by indexing, i.e. not tags or
  * properties, along with any automatic tags*/
+/* According to Xapian API docs, none of these calls throw
+ * exceptions */
 notmuch_private_status_t
 _notmuch_message_remove_indexed_terms (notmuch_message_t *message)
 {
@@ -727,45 +729,53 @@ _notmuch_message_remove_indexed_terms (notmuch_message_t *message)
 	tag_prefix = _find_prefix ("tag"),
 	type_prefix = _find_prefix ("type");
 
-    for (i = message->doc.termlist_begin ();
-	 i != message->doc.termlist_end (); i++) {
+    /* Make sure we have the data to restore to Xapian*/
+    _notmuch_message_ensure_metadata (message,NULL);
 
-	const std::string term = *i;
+    /* Empirically, it turns out to be faster to remove all the terms,
+     * and add back the ones we want. */
+    message->doc.clear_terms ();
+    message->modified = true;
 
-	if (term.compare (0, type_prefix.size (), type_prefix) == 0)
-	    continue;
+    /* still a mail message */
+    message->doc.add_term (type_prefix + "mail");
 
-	if (term.compare (0, id_prefix.size (), id_prefix) == 0)
-	    continue;
+    /* Put back message-id */
+    message->doc.add_term (id_prefix + message->message_id);
 
-	if (term.compare (0, property_prefix.size (), property_prefix) == 0)
-	    continue;
+    /* Put back non-automatic tags */
+    for (notmuch_tags_t *tags = notmuch_message_get_tags (message);
+	 notmuch_tags_valid (tags);
+	 notmuch_tags_move_to_next (tags)) {
 
-	if (term.compare (0, tag_prefix.size (), tag_prefix) == 0 &&
-	    term.compare (1, strlen("encrypted"), "encrypted") != 0 &&
-	    term.compare (1, strlen("signed"), "signed") != 0 &&
-	    term.compare (1, strlen("attachment"), "attachment") != 0)
-	    continue;
+	const char *tag = notmuch_tags_get (tags);
 
-	try {
-	    message->doc.remove_term ((*i));
-	    message->modified = true;
-	} catch (const Xapian::InvalidArgumentError) {
-	    /* Ignore failure to remove non-existent term. */
-	} catch (const Xapian::Error &error) {
-	    notmuch_database_t *notmuch = message->notmuch;
-
-	    if (!notmuch->exception_reported) {
-		_notmuch_database_log(notmuch_message_get_database (message), "A Xapian exception occurred creating message: %s\n",
-				      error.get_msg().c_str());
-		notmuch->exception_reported = true;
-	    }
-	    return NOTMUCH_PRIVATE_STATUS_XAPIAN_EXCEPTION;
+	if (STRNCMP_LITERAL (tag, "encrypted") != 0 &&
+	    STRNCMP_LITERAL (tag, "signed") != 0 &&
+	    STRNCMP_LITERAL (tag, "attachment") != 0) {
+	    std::string term = tag_prefix + tag;
+	    message->doc.add_term(term);
 	}
     }
+
+    /* Put back properties */
+    notmuch_message_properties_t *list;
+
+    for (list = notmuch_message_get_properties (message, "", false);
+	 notmuch_message_properties_valid (list); notmuch_message_properties_move_to_next (list)) {
+	std::string term = property_prefix +
+	    notmuch_message_properties_key(list) + "=" +
+	    notmuch_message_properties_value(list);
+
+	message->doc.add_term(term);
+    }
+
+    notmuch_message_properties_destroy (list);
+
     return NOTMUCH_PRIVATE_STATUS_SUCCESS;
 }
 
+
 /* Return true if p points at "new" or "cur". */
 static bool is_maildir (const char *p)
 {