X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=notmuch-index-message.cc;h=dd3fbb44687ba1859be696bf86ae9ee12a86b8dd;hp=44b1b03a1e180bc8625b13dcd02f8ddf73f1a453;hb=dcebf35ec9d177720a120fd8601f966d169d2edf;hpb=9bf3cda34c40089085c1ce612fd5027ae9a2f314 diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc index 44b1b03a..dd3fbb44 100644 --- a/notmuch-index-message.cc +++ b/notmuch-index-message.cc @@ -17,11 +17,44 @@ * Author: Carl Worth */ +/* This indexer creates a Xapian mail index that is remarkably similar + * to that created by sup. The big difference, (and the thing that + * will keep a notmuch index from being used by sup directly), is that + * sup expects a serialized ruby data structure in the document's data + * field, but notmuch just puts the mail's filename there (trusting + * that the email client can get the data in needs from the filename). + * + * Note: One bug here is that sup actually merges together fields such + * as To, CC, Bcc etc. when finding multiple emails with the same + * message ID. To support something similar, notmuch should list + * multiple files in the data field. + * + * Other differences between sup and notmuch-index identified so far: + * + * o sup supports encrypted mime parts by prompting for a passphrase + * to decrypt the message. So far, notmuch doesn't support this, + * both because I'm lazy to code it, and I also think doing so + * would present a security leak. + * + * o sup and notmuch have different heuristics for identifying (and + * thus ignoring) signatures. For example, sup considers a line + * consisting of two hypens as a signature separator, while + * notmuch expects those two hyphens to be followed by a space + * character. + * + * o sup as been seen to split some numbers before indexing + * them. For example, the number 1754 in an email message was + * indexed by sup as separate terms 17 and 54. I couldn't find any + * explanation for this behavior and did not try to replicate it + * in notmuch. + */ + #include #include #include #include #include +#include #include @@ -33,6 +66,9 @@ using namespace std; #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) +/* Xapian complains if we provide a term longer than this. */ +#define NOTMUCH_MAX_TERM 245 + /* These prefix values are specifically chosen to be compatible * with sup, (http://sup.rubyforge.org), written by * William Morgan , and released @@ -92,7 +128,40 @@ find_prefix (const char *name) return ""; } -int TERM_COMBINED = 0; +/* "128 bits of thread-id ought to be enough for anybody" */ +#define NOTMUCH_THREAD_ID_BITS 128 +#define NOTMUCH_THREAD_ID_DIGITS (NOTMUCH_THREAD_ID_BITS / 4) +typedef struct _thread_id { + char str[NOTMUCH_THREAD_ID_DIGITS + 1]; +} thread_id_t; + +static void +thread_id_generate (thread_id_t *thread_id) +{ + FILE *urandom; + uint32_t value; + char *s; + int i; + + urandom = fopen ("/dev/urandom", "r"); + if (urandom == NULL) { + fprintf (stderr, "Error opening /dev/urandom: %s\n", + strerror (errno)); + fprintf (stderr, "Perhaps notmuch needs some portability fixes for your platform?\n"); + exit (1); + } + + s = thread_id->str; + for (i = 0; i < NOTMUCH_THREAD_ID_DIGITS; i += 8) { + fread ((void *) &value, sizeof (value), 1, urandom); + sprintf (s, "%08x", value); + s += 8; + } + + fclose (urandom); + + printf ("Generated thread id: %s\n", thread_id->str); +} static void add_term (Xapian::Document doc, @@ -109,7 +178,8 @@ add_term (Xapian::Document doc, term = g_strdup_printf ("%s%s", prefix, value); - doc.add_term (term); + if (strlen (term) <= NOTMUCH_MAX_TERM) + doc.add_term (term); g_free (term); } @@ -134,12 +204,44 @@ gen_terms_address_name (Xapian::TermGenerator term_gen, InternetAddress *address, const char *prefix_name) { - const char *name; - - name = internet_address_get_name (address); + if (INTERNET_ADDRESS_IS_MAILBOX(address)) { + const char *name; + int own_name = 0; + + name = internet_address_get_name (address); + + /* In the absence of a name, we'll strip the part before the @ + * from the address. */ + if (! name) { + InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); + const char *addr = internet_address_mailbox_get_addr (mailbox); + const char *at; + + at = strchr (addr, '@'); + if (at) { + name = strndup (addr, at - addr); + own_name = 1; + } + } - if (name) - gen_terms (term_gen, prefix_name, name); + if (name) + gen_terms (term_gen, prefix_name, name); + + if (own_name) + free ((void *) name); + } else if (INTERNET_ADDRESS_IS_GROUP (address)) { + InternetAddressGroup *group = INTERNET_ADDRESS_GROUP (address); + InternetAddressList *list = internet_address_group_get_members(group); + if (list) { + int length = internet_address_list_length(list); + int i; + + for (i = 0; i < length; i++) + gen_terms_address_name(term_gen, + internet_address_list_get_address(list, i), + prefix_name); + } + } } static void @@ -150,6 +252,9 @@ gen_terms_address_names (Xapian::TermGenerator term_gen, int i; InternetAddress *address; + if (addresses == NULL) + return; + for (i = 0; i < internet_address_list_length (addresses); i++) { address = internet_address_list_get_address (addresses, i); gen_terms_address_name (term_gen, address, address_type); @@ -163,13 +268,27 @@ add_term_address_addr (Xapian::Document doc, InternetAddress *address, const char *prefix_name) { - InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); - const char *addr; - - addr = internet_address_mailbox_get_addr (mailbox); - - if (addr) - add_term (doc, prefix_name, addr); + if (INTERNET_ADDRESS_IS_MAILBOX(address)) { + InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); + const char *addr; + + addr = internet_address_mailbox_get_addr (mailbox); + + if (addr) + add_term (doc, prefix_name, addr); + } else if (INTERNET_ADDRESS_IS_GROUP (address)) { + InternetAddressGroup *group = INTERNET_ADDRESS_GROUP (address); + InternetAddressList *list = internet_address_group_get_members(group); + if (list) { + int length = internet_address_list_length(list); + int i; + + for (i = 0; i < length; i++) + add_term_address_addr(doc, + internet_address_list_get_address(list, i), + prefix_name); + } + } } static void @@ -180,6 +299,9 @@ add_terms_address_addrs (Xapian::Document doc, int i; InternetAddress *address; + if (addresses == NULL) + return; + for (i = 0; i < internet_address_list_length (addresses); i++) { address = internet_address_list_get_address (addresses, i); add_term_address_addr (doc, address, address_type); @@ -187,115 +309,314 @@ add_terms_address_addrs (Xapian::Document doc, } } -/* Generate terms for the body of a message, given the filename of the - * message and the offset at which the headers of the message end, - * (and hence the body begins). */ +static const char * +skip_re_in_subject (const char *subject) +{ + const char *s = subject; + + if (subject == NULL) + return NULL; + + while (*s) { + while (*s && isspace (*s)) + s++; + if (strncasecmp (s, "re:", 3) == 0) + s += 3; + else + break; + } + + return s; +} + static void -gen_terms_body (Xapian::TermGenerator term_gen, - const char * filename, - gint64 body_offset) +find_messages_by_term (Xapian::Database db, + const char *prefix_name, + const char *value, + Xapian::PostingIterator *begin, + Xapian::PostingIterator *end) { - GIOChannel *channel; - GIOStatus gio_status; - GError *error = NULL; - char *p, *body_line = NULL, *prev_line = NULL; + Xapian::PostingIterator i; + char *term; - channel = g_io_channel_new_file (filename, "r", &error); - if (channel == NULL) { - fprintf (stderr, "Error: %s\n", error->message); - exit (1); + term = g_strdup_printf ("%s%s", find_prefix (prefix_name), value); + + *begin = db.postlist_begin (term); + + if (end) + *end = db.postlist_end (term); + + free (term); +} + +Xapian::Document +find_message_by_docid (Xapian::Database db, Xapian::docid docid) +{ + return db.get_document (docid); +} + +Xapian::Document +find_message_by_message_id (Xapian::Database db, const char *message_id) +{ + Xapian::PostingIterator i, end; + + find_messages_by_term (db, "msgid", message_id, &i, &end); + + if (i != end) + return find_message_by_docid (db, *i); + else + return Xapian::Document (); +} + +static void +insert_thread_id (GHashTable *thread_ids, Xapian::Document doc) +{ + string value_string; + const char *value, *id, *comma; + + value_string = doc.get_value (NOTMUCH_VALUE_THREAD); + value = value_string.c_str(); + if (strlen (value)) { + id = value; + while (*id) { + comma = strchr (id, ','); + if (comma == NULL) + comma = id + strlen (id); + g_hash_table_insert (thread_ids, + strndup (id, comma - id), NULL); + id = comma; + if (*id) + id++; + } } +} - gio_status = g_io_channel_seek_position (channel, body_offset, - G_SEEK_SET, &error); - if (gio_status != G_IO_STATUS_NORMAL) { - fprintf (stderr, "Error: %s\n", error->message); - exit (1); +/* Return one or more thread_ids, (as a GPtrArray of strings), for the + * given message based on looking into the database for any messages + * referenced in parents, and also for any messages in the database + * referencing message_id. + * + * Caller should free all strings in the array and the array itself, + * (g_ptr_array_free) when done. */ +static GPtrArray * +find_thread_ids (Xapian::Database db, + GPtrArray *parents, + const char *message_id) +{ + Xapian::PostingIterator child, children_end; + Xapian::Document doc; + GHashTable *thread_ids; + GList *keys, *l; + unsigned int i; + const char *parent_message_id; + GPtrArray *result; + + thread_ids = g_hash_table_new (g_str_hash, g_str_equal); + + find_messages_by_term (db, "ref", message_id, &child, &children_end); + for ( ; child != children_end; child++) { + doc = find_message_by_docid (db, *child); + insert_thread_id (thread_ids, doc); } - while (1) { - if (body_line) - g_free (body_line); + for (i = 0; i < parents->len; i++) { + parent_message_id = (char *) g_ptr_array_index (parents, i); + doc = find_message_by_message_id (db, parent_message_id); + insert_thread_id (thread_ids, doc); + } + + result = g_ptr_array_new (); + + keys = g_hash_table_get_keys (thread_ids); + for (l = keys; l; l = l->next) { + char *id = (char *) l->data; + g_ptr_array_add (result, id); + } + + return result; +} + +/* Add a term for each message-id in the References header of the + * message. */ +static void +parse_references (GPtrArray *array, + const char *refs_str) +{ + GMimeReferences *refs, *r; + const char *message_id; - gio_status = g_io_channel_read_line (channel, &body_line, - NULL, NULL, &error); - if (gio_status == G_IO_STATUS_EOF) + if (refs_str == NULL) + return; + + refs = g_mime_references_decode (refs_str); + + for (r = refs; r; r = r->next) { + message_id = g_mime_references_get_message_id (r); + g_ptr_array_add (array, g_strdup (message_id)); + } + + g_mime_references_free (refs); +} + +/* Given a string representing the body of a message, generate terms + * for it, (skipping quoted portions and signatures). */ +static void +gen_terms_body_str (Xapian::TermGenerator term_gen, + char *body) +{ + char *line, *line_end, *next_line; + + if (body == NULL) + return; + + next_line = body; + + while (1) { + line = next_line; + if (*line == '\0') break; - if (gio_status != G_IO_STATUS_NORMAL) { - fprintf (stderr, "Error: %s\n", error->message); - exit (1); + + next_line = strchr (line, '\n'); + if (next_line == NULL) { + next_line = line + strlen (line); } + line_end = next_line - 1; - if (strlen (body_line) == 0) - continue; + /* Get to the next non-blank line. */ + while (*next_line == '\n') + next_line++; - /* If the line looks like it might be introducing a quote, - * save it until we see if the next line begins a quote. */ - p = body_line + strlen (body_line) - 1; - while (p > body_line and isspace (*p)) - p--; - if (*p == ':') { - prev_line = body_line; - body_line = NULL; + /* Skip blank lines. */ + if (line_end < line) continue; - } - /* Skip quoted lines, (and previous lines that introduced them) */ - if (body_line[0] == '>') { - if (prev_line) { - g_free (prev_line); - prev_line = NULL; - } + /* Skip lines that are quotes. */ + if (*line == '>') continue; - } - /* Now that we're not looking at a quote we can add the prev_line */ - if (prev_line) { - gen_terms (term_gen, "body", prev_line); - g_free (prev_line); - prev_line = NULL; - } + /* Also skip lines introducing a quote on the next line. */ + if (*line_end == ':' && *next_line == '>') + continue; - /* Skip signatures */ + /* Finally, bail as soon as we see a signature. */ /* XXX: Should only do this if "near" the end of the message. */ - if (strncmp (body_line, "-- ", 3) == 0) + if (strncmp (line, "-- ", 3) == 0 || + strncmp (line, "----------", 10) == 0 || + strncmp (line, "__________", 10) == 0) break; - gen_terms (term_gen, "body", body_line); + *(line_end + 1) = '\0'; + gen_terms (term_gen, "body", line); } +} - if (body_line) - g_free (body_line); - g_io_channel_close (channel); -} +/* Callback to generate terms for each mime part of a message. */ +static void +gen_terms_part (Xapian::TermGenerator term_gen, + GMimeObject *part) +{ + GMimeStream *stream; + GMimeDataWrapper *wrapper; + GByteArray *byte_array; + GMimeContentDisposition *disposition; + char *body; + + if (GMIME_IS_MULTIPART (part)) { + GMimeMultipart *multipart = GMIME_MULTIPART (part); + int i; + + for (i = 0; i < g_mime_multipart_get_count (multipart); i++) { + if (GMIME_IS_MULTIPART_SIGNED (multipart)) { + /* Don't index the signature. */ + if (i == 1) + continue; + if (i > 1) + fprintf (stderr, "Warning: Unexpected extra parts of mutlipart/signed. Indexing anyway.\n"); + } + gen_terms_part (term_gen, + g_mime_multipart_get_part (multipart, i)); + } + return; + } + if (GMIME_IS_MESSAGE_PART (part)) { + GMimeMessage *message; -int -main (int argc, char **argv) + message = g_mime_message_part_get_message (GMIME_MESSAGE_PART (part)); + + gen_terms_part (term_gen, g_mime_message_get_mime_part (message)); + + return; + } + + if (! (GMIME_IS_PART (part))) { + fprintf (stderr, "Warning: Not indexing unknown mime part: %s.\n", + g_type_name (G_OBJECT_TYPE (part))); + return; + } + + disposition = g_mime_object_get_content_disposition (part); + if (disposition && + strcmp (disposition->disposition, GMIME_DISPOSITION_ATTACHMENT) == 0) + { + const char *filename = g_mime_part_get_filename (GMIME_PART (part)); + const char *extension; + + add_term (term_gen.get_document (), "label", "attachment"); + gen_terms (term_gen, "attachment", filename); + + if (filename) { + extension = strchr (filename, '.'); + if (extension) { + add_term (term_gen.get_document (), "attachment_extension", + extension + 1); + } + } + + return; + } + + byte_array = g_byte_array_new (); + + stream = g_mime_stream_mem_new_with_byte_array (byte_array); + g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE); + wrapper = g_mime_part_get_content_object (GMIME_PART (part)); + if (wrapper) + g_mime_data_wrapper_write_to_stream (wrapper, stream); + + g_object_unref (stream); + + g_byte_array_append (byte_array, (guint8 *) "\0", 1); + body = (char *) g_byte_array_free (byte_array, FALSE); + + gen_terms_body_str (term_gen, body); + + free (body); +} + +static void +index_file (Xapian::WritableDatabase db, + Xapian::TermGenerator term_gen, + const char *filename) { + Xapian::Document doc; + GMimeStream *stream; GMimeParser *parser; GMimeMessage *message; InternetAddressList *addresses; + GPtrArray *parents, *thread_ids; - const char *database_path, *filename; FILE *file; - const char *value, *from; + const char *subject, *refs, *in_reply_to, *from; + const char *message_id; time_t time; struct tm gm_time_tm; char date_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ - - if (argc < 3) { - fprintf (stderr, "Usage: %s \n", - argv[0]); - exit (1); - } - - database_path = argv[1]; - filename = argv[2]; + unsigned int i; file = fopen (filename, "r"); if (! file) { @@ -303,93 +624,209 @@ main (int argc, char **argv) exit (1); } - g_mime_init (0); - stream = g_mime_stream_file_new (file); parser = g_mime_parser_new_with_stream (stream); message = g_mime_parser_construct_message (parser); - try { - Xapian::WritableDatabase db; - Xapian::TermGenerator term_gen; - Xapian::Document doc; + doc = Xapian::Document (); - doc = Xapian::Document (); + doc.set_data (filename); - doc.set_data (filename); + term_gen.set_stemmer (Xapian::Stem ("english")); - db = Xapian::WritableDatabase (database_path, - Xapian::DB_CREATE_OR_OPEN); + term_gen.set_document (doc); - term_gen = Xapian::TermGenerator (); - term_gen.set_stemmer (Xapian::Stem ("english")); + from = g_mime_message_get_sender (message); + addresses = internet_address_list_parse_string (from); - term_gen.set_document (doc); + gen_terms_address_names (term_gen, addresses, "from_name"); - from = g_mime_message_get_sender (message); - addresses = internet_address_list_parse_string (from); + addresses = g_mime_message_get_all_recipients (message); + gen_terms_address_names (term_gen, addresses, "to_name"); - gen_terms_address_names (term_gen, addresses, "from_name"); + subject = g_mime_message_get_subject (message); + subject = skip_re_in_subject (subject); + gen_terms (term_gen, "subject", subject); + gen_terms (term_gen, "body", subject); - addresses = g_mime_message_get_all_recipients (message); - gen_terms_address_names (term_gen, addresses, "to_name"); + gen_terms_part (term_gen, g_mime_message_get_mime_part (message)); - value = g_mime_message_get_subject (message); - gen_terms (term_gen, "subject", value); - gen_terms (term_gen, "body", value); + parents = g_ptr_array_new (); - gen_terms_body (term_gen, filename, - g_mime_parser_get_headers_end (parser)); + refs = g_mime_object_get_header (GMIME_OBJECT (message), "references"); + parse_references (parents, refs); - from = g_mime_message_get_sender (message); - addresses = internet_address_list_parse_string (from); + in_reply_to = g_mime_object_get_header (GMIME_OBJECT (message), + "in-reply-to"); + parse_references (parents, in_reply_to); - add_terms_address_addrs (doc, addresses, "from_email"); + for (i = 0; i < parents->len; i++) + add_term (doc, "ref", (char *) g_ptr_array_index (parents, i)); - add_terms_address_addrs (doc, - g_mime_message_get_all_recipients (message), - "to_email"); + message_id = g_mime_message_get_message_id (message); - g_mime_message_get_date (message, &time, NULL); + thread_ids = find_thread_ids (db, parents, message_id); - gmtime_r (&time, &gm_time_tm); + for (i = 0; i < parents->len; i++) + g_free (g_ptr_array_index (parents, i)); + g_ptr_array_free (parents, TRUE); - if (strftime (date_str, sizeof (date_str), - "%Y%m%d%H%M%S", &gm_time_tm) == 0) { - fprintf (stderr, "Internal error formatting time\n"); - exit (1); - } + from = g_mime_message_get_sender (message); + addresses = internet_address_list_parse_string (from); - add_term (doc, "date", date_str); + add_terms_address_addrs (doc, addresses, "from_email"); - add_term (doc, "label", "inbox"); - add_term (doc, "label", "unread"); - add_term (doc, "type", "mail"); + add_terms_address_addrs (doc, + g_mime_message_get_all_recipients (message), + "to_email"); - value = g_mime_message_get_message_id (message); - add_term (doc, "msgid", value); + g_mime_message_get_date (message, &time, NULL); - add_term (doc, "source_id", "1"); + gmtime_r (&time, &gm_time_tm); - add_term (doc, "thread", value); + if (strftime (date_str, sizeof (date_str), + "%Y%m%d%H%M%S", &gm_time_tm) == 0) { + fprintf (stderr, "Internal error formatting time\n"); + exit (1); + } - doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, value); - doc.add_value (NOTMUCH_VALUE_THREAD, value); + add_term (doc, "date", date_str); - doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time)); + add_term (doc, "label", "inbox"); + add_term (doc, "label", "unread"); + add_term (doc, "type", "mail"); + add_term (doc, "source_id", "1"); - db.add_document (doc); + if (message_id) { + add_term (doc, "msgid", message_id); + doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id); + } - } catch (const Xapian::Error &error) { - cerr << "A Xapian exception occurred: " << error.get_msg () << endl; - exit (1); + if (thread_ids->len) { + unsigned int i; + GString *thread_id; + char *id; + + for (i = 0; i < thread_ids->len; i++) { + id = (char *) thread_ids->pdata[i]; + + add_term (doc, "thread", id); + + if (i == 0) + thread_id = g_string_new (id); + else + g_string_append_printf (thread_id, ",%s", id); + + free (id); + } + g_ptr_array_free (thread_ids, TRUE); + + doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str); + + g_string_free (thread_id, TRUE); + } else if (message_id) { + /* If not part of any existing thread, generate a new thread_id. */ + thread_id_t thread_id; + + thread_id_generate (&thread_id); + + add_term (doc, "thread", thread_id.str); + doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str); } + doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time)); + + db.add_document (doc); + g_object_unref (message); g_object_unref (parser); g_object_unref (stream); +} + +static void +usage (const char *argv0) +{ + fprintf (stderr, "Usage: %s \n", argv0); + fprintf (stderr, "\n"); + fprintf (stderr, "Messages to be indexed are read from stdnin as absolute filenames\n"); + fprintf (stderr, "one file per line."); +} + +int +main (int argc, char **argv) +{ + const char *database_path; + char *filename; + GIOChannel *channel; + GIOStatus gio_status; + GError *error = NULL; + int count; + struct timeval tv_start, tv_last, tv_now; + double elapsed; + + if (argc < 2) { + usage (argv[0]); + exit (1); + } + + database_path = argv[1]; + + g_mime_init (0); + + try { + Xapian::WritableDatabase db; + Xapian::TermGenerator term_gen; + + db = Xapian::WritableDatabase (database_path, + Xapian::DB_CREATE_OR_OPEN); + + term_gen = Xapian::TermGenerator (); + + channel = g_io_channel_unix_new (fileno (stdin)); + + count = 0; + + gettimeofday (&tv_start, NULL); + tv_last = tv_start; + + while (1) { + gio_status = g_io_channel_read_line (channel, &filename, + NULL, NULL, &error); + if (gio_status == G_IO_STATUS_EOF) + break; + if (gio_status != G_IO_STATUS_NORMAL) { + fprintf (stderr, "An error occurred reading from stdin: %s\n", + error->message); + exit (1); + } + + g_strchomp (filename); + index_file (db, term_gen, filename); + + g_free (filename); + + count++; + if (count % 1000 == 0) { + gettimeofday (&tv_now, NULL); + printf ("Indexed %d messages (%g messages/second)\n", + count, 1000 / ((tv_now.tv_sec - tv_last.tv_sec) + + (tv_now.tv_usec - tv_last.tv_usec) / 1e6)); + tv_last = tv_now; + } + } + + gettimeofday (&tv_now, NULL); + elapsed = (tv_now.tv_sec - tv_start.tv_sec + + (tv_now.tv_usec - tv_start.tv_usec) / 1e6); + printf ("Completed indexing of %d messages in %g seconds (%g messages/second)\n", + count, elapsed, count / elapsed); + + } catch (const Xapian::Error &error) { + cerr << "A Xapian exception occurred: " << error.get_msg () << endl; + exit (1); + } return 0; }