notmuch-index-message: Start generating correct thread_id values.

[notmuch] / notmuch-index-message.cc

diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc
index 51c9f049d99190c7c1240ed450f9f92f91091bf1..1ed3b4c92d182e4ad9eed30752d2148cb5117a5c 100644 (file)
--- a/notmuch-index-message.cc
+++ b/notmuch-index-message.cc
@@ -33,6 +33,9 @@ using namespace std;
 
 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
 
+/* Xapian complains if we provide a term longer than this. */
+#define NOTMUCH_MAX_TERM 245
+
 /* These prefix values are specifically chosen to be compatible
  * with sup, (http://sup.rubyforge.org), written by
  * William Morgan <wmorgan-sup@masanjin.net>, and released
@@ -109,7 +112,8 @@ add_term (Xapian::Document doc,
 
     term = g_strdup_printf ("%s%s", prefix, value);
 
-    doc.add_term (term);
+    if (strlen (term) <= NOTMUCH_MAX_TERM)
+       doc.add_term (term);
 
     g_free (term);
 }
@@ -204,16 +208,68 @@ skip_re_in_subject (const char *subject)
     return s;
 }
 
+Xapian::Document
+find_message (Xapian::Database db, const char *message_id)
+{
+    Xapian::PostingIterator i;
+    char *term;
+
+    term = g_strdup_printf ("%s%s", find_prefix ("msgid"), message_id);
+    i = db.postlist_begin (term);
+    if (i != db.postlist_end (term))
+       return db.get_document (*i);
+    else
+       return Xapian::Document ();
+}
+
+static char *
+find_thread_id (Xapian::Database db, GPtrArray *parents)
+{
+    Xapian::Document doc;
+    GHashTable *thread_ids;
+    GList *keys, *l;
+    GString *result = NULL;
+    unsigned int i;
+    string value_string;
+    const char *value;
+
+    thread_ids = g_hash_table_new (g_str_hash, g_str_equal);
+
+    for (i = 0; i < parents->len; i++) {
+       doc = find_message (db, (char *) g_ptr_array_index (parents, i));
+       value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
+       value = value_string.c_str();
+       if (strlen (value))
+           g_hash_table_insert (thread_ids, strdup (value), NULL);
+    }
+
+    keys = g_hash_table_get_keys (thread_ids);
+    for (l = keys; l; l = l->next) {
+       char *id = (char *) l->data;
+       if (result == NULL) {
+           result = g_string_new (id);
+       } else {
+           g_string_append_printf (result, ",%s", id);
+       }
+       free (id);
+    }
+
+    if (result)
+       return g_string_free (result, FALSE);
+    else
+       return NULL;
+}
+
 /* Add a term for each message-id in the References header of the
  * message. */
 static void
-add_terms_references (Xapian::Document doc,
-                     GMimeMessage *message)
+parse_references (GPtrArray *array,
+                 const char *refs)
 {
-    const char *refs, *end, *next;
-    char *term;
+    const char *end, *next;
 
-    refs = g_mime_object_get_header (GMIME_OBJECT (message), "references");
+    if (refs == NULL)
+       return;
 
     while (*refs) {
        while (*refs && isspace (*refs))
@@ -228,9 +284,7 @@ add_terms_references (Xapian::Document doc,
        if (end > refs && *end == '>')
            end--;
        if (end > refs) {
-           term = g_strndup (refs, end - refs + 1);
-           add_term (doc, "ref", term);
-           g_free (term);
+           g_ptr_array_add (array, g_strndup (refs, end - refs + 1));
        }
        refs = next;
     }
@@ -307,7 +361,9 @@ gen_terms_body (Xapian::TermGenerator term_gen,
 
        /* Skip signatures */
        /* XXX: Should only do this if "near" the end of the message. */
-       if (strncmp (body_line, "-- ", 3) == 0)
+       if (strncmp (body_line, "-- ", 3) == 0 ||
+           strncmp (body_line, "----------", 10) == 0 ||
+           strncmp (body_line, "__________", 10) == 0)
            break;
 
        gen_terms (term_gen, "body", body_line);
@@ -330,14 +386,16 @@ index_file (Xapian::WritableDatabase db,
     GMimeParser *parser;
     GMimeMessage *message;
     InternetAddressList *addresses;
+    GPtrArray *parents;
 
     FILE *file;
 
-    const char *value, *from;
+    const char *value, *from, *thread_id;
 
     time_t time;
     struct tm gm_time_tm;
     char date_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */
+    unsigned int i;
 
     file = fopen (filename, "r");
     if (! file) {
@@ -375,7 +433,22 @@ index_file (Xapian::WritableDatabase db,
     gen_terms_body (term_gen, filename,
                    g_mime_parser_get_headers_end (parser));
 
-    add_terms_references (doc, message);
+    parents = g_ptr_array_new ();
+
+    value = g_mime_object_get_header (GMIME_OBJECT (message), "references");
+    parse_references (parents, value);
+
+    value = g_mime_object_get_header (GMIME_OBJECT (message), "in-reply-to");
+    parse_references (parents, value);
+
+    for (i = 0; i < parents->len; i++)
+       add_term (doc, "ref", (char *) g_ptr_array_index (parents, i));
+
+    thread_id = find_thread_id (db, parents);
+
+    for (i = 0; i < parents->len; i++)
+       g_free (g_ptr_array_index (parents, i));
+    g_ptr_array_free (parents, TRUE);
 
     from = g_mime_message_get_sender (message);
     addresses = internet_address_list_parse_string (from);
@@ -405,10 +478,17 @@ index_file (Xapian::WritableDatabase db,
 
     value = g_mime_message_get_message_id (message);
     add_term (doc, "msgid", value);
-    add_term (doc, "thread", value);
-
     doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, value);
-    doc.add_value (NOTMUCH_VALUE_THREAD, value);
+
+    if (thread_id) {
+       add_term (doc, "thread", thread_id);
+       doc.add_value (NOTMUCH_VALUE_THREAD, thread_id);
+       free ((void *) thread_id);
+    } else {
+       /* If not referenced thread, use the message ID */
+       add_term (doc, "thread", value);
+       doc.add_value (NOTMUCH_VALUE_THREAD, value);
+    }
 
     doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time));