X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=notmuch-index-message.cc;h=88eb2c52f5754a00c569a0ad9a9d33dc448a5964;hp=79b3bf3845c14ca1a1de487db68312daaad2f43e;hb=ed320cb45ba38c5fbf98ca2ed34abb34f5a5b13b;hpb=3253954233c378a0176eb9695fa7908c1ca21c2e diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc index 79b3bf38..88eb2c52 100644 --- a/notmuch-index-message.cc +++ b/notmuch-index-message.cc @@ -33,6 +33,9 @@ using namespace std; #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) +/* Xapian complains if we provide a term longer than this. */ +#define NOTMUCH_MAX_TERM 245 + /* These prefix values are specifically chosen to be compatible * with sup, (http://sup.rubyforge.org), written by * William Morgan , and released @@ -109,7 +112,8 @@ add_term (Xapian::Document doc, term = g_strdup_printf ("%s%s", prefix, value); - doc.add_term (term); + if (strlen (term) <= NOTMUCH_MAX_TERM) + doc.add_term (term); g_free (term); } @@ -204,16 +208,68 @@ skip_re_in_subject (const char *subject) return s; } +Xapian::Document +find_message (Xapian::Database db, const char *message_id) +{ + Xapian::PostingIterator i; + char *term; + + term = g_strdup_printf ("%s%s", find_prefix ("msgid"), message_id); + i = db.postlist_begin (term); + if (i != db.postlist_end (term)) + return db.get_document (*i); + else + return Xapian::Document (); +} + +static char * +find_thread_id (Xapian::Database db, GPtrArray *parents) +{ + Xapian::Document doc; + GHashTable *thread_ids; + GList *keys, *l; + GString *result = NULL; + unsigned int i; + string value_string; + const char *value; + + thread_ids = g_hash_table_new (g_str_hash, g_str_equal); + + for (i = 0; i < parents->len; i++) { + doc = find_message (db, (char *) g_ptr_array_index (parents, i)); + value_string = doc.get_value (NOTMUCH_VALUE_THREAD); + value = value_string.c_str(); + if (strlen (value)) + g_hash_table_insert (thread_ids, strdup (value), NULL); + } + + keys = g_hash_table_get_keys (thread_ids); + for (l = keys; l; l = l->next) { + char *id = (char *) l->data; + if (result == NULL) { + result = g_string_new (id); + } else { + g_string_append_printf (result, ",%s", id); + } + free (id); + } + + if (result) + return g_string_free (result, FALSE); + else + return NULL; +} + /* Add a term for each message-id in the References header of the * message. */ static void -add_terms_references (Xapian::Document doc, - GMimeMessage *message) +parse_references (GPtrArray *array, + const char *refs) { - const char *refs, *end, *next; - char *term; + const char *end, *next; - refs = g_mime_object_get_header (GMIME_OBJECT (message), "references"); + if (refs == NULL) + return; while (*refs) { while (*refs && isspace (*refs)) @@ -228,9 +284,7 @@ add_terms_references (Xapian::Document doc, if (end > refs && *end == '>') end--; if (end > refs) { - term = g_strndup (refs, end - refs + 1); - add_term (doc, "ref", term); - g_free (term); + g_ptr_array_add (array, g_strndup (refs, end - refs + 1)); } refs = next; } @@ -307,7 +361,9 @@ gen_terms_body (Xapian::TermGenerator term_gen, /* Skip signatures */ /* XXX: Should only do this if "near" the end of the message. */ - if (strncmp (body_line, "-- ", 3) == 0) + if (strncmp (body_line, "-- ", 3) == 0 || + strncmp (body_line, "----------", 10) == 0 || + strncmp (body_line, "__________", 10) == 0) break; gen_terms (term_gen, "body", body_line); @@ -330,14 +386,17 @@ index_file (Xapian::WritableDatabase db, GMimeParser *parser; GMimeMessage *message; InternetAddressList *addresses; + GPtrArray *parents; FILE *file; - const char *value, *from; + const char *subject, *refs, *in_reply_to, *from; + const char *message_id, *thread_id; time_t time; struct tm gm_time_tm; char date_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ + unsigned int i; file = fopen (filename, "r"); if (! file) { @@ -367,15 +426,31 @@ index_file (Xapian::WritableDatabase db, addresses = g_mime_message_get_all_recipients (message); gen_terms_address_names (term_gen, addresses, "to_name"); - value = g_mime_message_get_subject (message); - value = skip_re_in_subject (value); - gen_terms (term_gen, "subject", value); - gen_terms (term_gen, "body", value); + subject = g_mime_message_get_subject (message); + subject = skip_re_in_subject (subject); + gen_terms (term_gen, "subject", subject); + gen_terms (term_gen, "body", subject); gen_terms_body (term_gen, filename, g_mime_parser_get_headers_end (parser)); - add_terms_references (doc, message); + parents = g_ptr_array_new (); + + refs = g_mime_object_get_header (GMIME_OBJECT (message), "references"); + parse_references (parents, refs); + + in_reply_to = g_mime_object_get_header (GMIME_OBJECT (message), + "in-reply-to"); + parse_references (parents, in_reply_to); + + for (i = 0; i < parents->len; i++) + add_term (doc, "ref", (char *) g_ptr_array_index (parents, i)); + + thread_id = find_thread_id (db, parents); + + for (i = 0; i < parents->len; i++) + g_free (g_ptr_array_index (parents, i)); + g_ptr_array_free (parents, TRUE); from = g_mime_message_get_sender (message); addresses = internet_address_list_parse_string (from); @@ -403,12 +478,19 @@ index_file (Xapian::WritableDatabase db, add_term (doc, "type", "mail"); add_term (doc, "source_id", "1"); - value = g_mime_message_get_message_id (message); - add_term (doc, "msgid", value); - add_term (doc, "thread", value); - - doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, value); - doc.add_value (NOTMUCH_VALUE_THREAD, value); + message_id = g_mime_message_get_message_id (message); + add_term (doc, "msgid", message_id); + doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id); + + if (thread_id) { + add_term (doc, "thread", thread_id); + doc.add_value (NOTMUCH_VALUE_THREAD, thread_id); + free ((void *) thread_id); + } else { + /* If not referenced thread, use the message ID */ + add_term (doc, "thread", message_id); + doc.add_value (NOTMUCH_VALUE_THREAD, message_id); + } doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time)); @@ -419,19 +501,30 @@ index_file (Xapian::WritableDatabase db, g_object_unref (stream); } +static void +usage (const char *argv0) +{ + fprintf (stderr, "Usage: %s \n", argv0); + fprintf (stderr, "\n"); + fprintf (stderr, "Messages to be indexed are read from stdnin as absolute filenames\n"); + fprintf (stderr, "one file per line."); +} + int main (int argc, char **argv) { - const char *database_path, *filename; + const char *database_path; + char *filename; + GIOChannel *channel; + GIOStatus gio_status; + GError *error = NULL; - if (argc < 3) { - fprintf (stderr, "Usage: %s \n", - argv[0]); + if (argc < 2) { + usage (argv[0]); exit (1); } database_path = argv[1]; - filename = argv[2]; g_mime_init (0); @@ -444,7 +537,24 @@ main (int argc, char **argv) term_gen = Xapian::TermGenerator (); - index_file (db, term_gen, filename); + channel = g_io_channel_unix_new (fileno (stdin)); + + while (1) { + gio_status = g_io_channel_read_line (channel, &filename, + NULL, NULL, &error); + if (gio_status == G_IO_STATUS_EOF) + break; + if (gio_status != G_IO_STATUS_NORMAL) { + fprintf (stderr, "An error occurred reading from stdin: %s\n", + error->message); + exit (1); + } + + g_strchomp (filename); + index_file (db, term_gen, filename); + + g_free (filename); + } } catch (const Xapian::Error &error) { cerr << "A Xapian exception occurred: " << error.get_msg () << endl;