X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=notmuch-index-message.cc;h=1922dd373856a32efb8c0f6f0ac4c857554da4ca;hp=cf99e6e43d7762e25fa812f307ab3cf65d452827;hb=be72bf30703c5df814d4f572c3d9132d39700b36;hpb=c55c34f4a08829c2734168a912c0ad371b934d30 diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc index cf99e6e4..1922dd37 100644 --- a/notmuch-index-message.cc +++ b/notmuch-index-message.cc @@ -23,10 +23,19 @@ #include #include +#include + #include +#include + +using namespace std; + #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0])) +/* Xapian complains if we provide a term longer than this. */ +#define NOTMUCH_MAX_TERM 245 + /* These prefix values are specifically chosen to be compatible * with sup, (http://sup.rubyforge.org), written by * William Morgan , and released @@ -61,6 +70,15 @@ prefix_t BOOLEAN_PREFIX[] = { { "ref", "R" } }; +/* Similarly, these value numbers are also chosen to be sup + * compatible. */ + +typedef enum { + NOTMUCH_VALUE_MESSAGE_ID = 0, + NOTMUCH_VALUE_THREAD = 1, + NOTMUCH_VALUE_DATE = 2 +} notmuch_value_t; + static const char * find_prefix (const char *name) { @@ -80,48 +98,74 @@ find_prefix (const char *name) int TERM_COMBINED = 0; static void -print_term (const char *prefix_name, const char *value) +add_term (Xapian::Document doc, + const char *prefix_name, + const char *value) { const char *prefix; + char *term; if (value == NULL) return; prefix = find_prefix (prefix_name); - if (TERM_COMBINED) - printf ("\"%s%s\", ", prefix, value); - else - printf ("[\"%s\", \"%s\"], ", value, prefix); + term = g_strdup_printf ("%s%s", prefix, value); + + if (strlen (term) <= NOTMUCH_MAX_TERM) + doc.add_term (term); + + g_free (term); +} + +static void +gen_terms (Xapian::TermGenerator term_gen, + const char *prefix_name, + const char *text) +{ + const char *prefix; + + if (text == NULL) + return; + + prefix = find_prefix (prefix_name); + + term_gen.index_text (text, 1, prefix); } static void -add_address_name (InternetAddress *address, const char *prefix_name) +gen_terms_address_name (Xapian::TermGenerator term_gen, + InternetAddress *address, + const char *prefix_name) { const char *name; name = internet_address_get_name (address); if (name) - print_term (prefix_name, name); + gen_terms (term_gen, prefix_name, name); } static void -add_address_names (InternetAddressList *addresses, const char *address_type) +gen_terms_address_names (Xapian::TermGenerator term_gen, + InternetAddressList *addresses, + const char *address_type) { int i; InternetAddress *address; for (i = 0; i < internet_address_list_length (addresses); i++) { address = internet_address_list_get_address (addresses, i); - add_address_name (address, address_type); - add_address_name (address, "name"); - add_address_name (address, "body"); + gen_terms_address_name (term_gen, address, address_type); + gen_terms_address_name (term_gen, address, "name"); + gen_terms_address_name (term_gen, address, "body"); } } static void -add_address_addr (InternetAddress *address, const char *prefix_name) +add_term_address_addr (Xapian::Document doc, + InternetAddress *address, + const char *prefix_name) { InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address); const char *addr; @@ -129,46 +173,266 @@ add_address_addr (InternetAddress *address, const char *prefix_name) addr = internet_address_mailbox_get_addr (mailbox); if (addr) - print_term (prefix_name, addr); + add_term (doc, prefix_name, addr); } static void -add_address_addrs (InternetAddressList *addresses, const char *address_type) +add_terms_address_addrs (Xapian::Document doc, + InternetAddressList *addresses, + const char *address_type) { int i; InternetAddress *address; for (i = 0; i < internet_address_list_length (addresses); i++) { address = internet_address_list_get_address (addresses, i); - add_address_addr (address, address_type); - add_address_addr (address, "email"); + add_term_address_addr (doc, address, address_type); + add_term_address_addr (doc, address, "email"); } } -int -main (int argc, char **argv) +static const char * +skip_re_in_subject (const char *subject) +{ + const char *s = subject; + + while (*s) { + while (*s && isspace (*s)) + s++; + if (strncasecmp (s, "re:", 3) == 0) + s += 3; + else + break; + } + + return s; +} + +static void +find_messages_by_term (Xapian::Database db, + const char *prefix_name, + const char *value, + Xapian::PostingIterator *begin, + Xapian::PostingIterator *end) +{ + Xapian::PostingIterator i; + char *term; + + term = g_strdup_printf ("%s%s", find_prefix (prefix_name), value); + + *begin = db.postlist_begin (term); + + if (end) + *end = db.postlist_end (term); + + free (term); +} + +Xapian::Document +find_message_by_docid (Xapian::Database db, Xapian::docid docid) +{ + return db.get_document (docid); +} + +Xapian::Document +find_message_by_message_id (Xapian::Database db, const char *message_id) { + Xapian::PostingIterator i, end; + + find_messages_by_term (db, "msgid", message_id, &i, &end); + + if (i != end) + return find_message_by_docid (db, *i); + else + return Xapian::Document (); +} + +static void +insert_thread_id (GHashTable *thread_ids, Xapian::Document doc) +{ + string value_string; + const char *value; + + value_string = doc.get_value (NOTMUCH_VALUE_THREAD); + value = value_string.c_str(); + if (strlen (value)) + g_hash_table_insert (thread_ids, strdup (value), NULL); +} + +static char * +find_thread_id (Xapian::Database db, + GPtrArray *parents, + const char *message_id) +{ + Xapian::PostingIterator child, children_end; + Xapian::Document doc; + GHashTable *thread_ids; + GList *keys, *l; + GString *result = NULL; + unsigned int i; + const char *parent_message_id; + + thread_ids = g_hash_table_new (g_str_hash, g_str_equal); + + find_messages_by_term (db, "ref", message_id, &child, &children_end); + for ( ; child != children_end; child++) { + doc = find_message_by_docid (db, *child); + insert_thread_id (thread_ids, doc); + } + + for (i = 0; i < parents->len; i++) { + parent_message_id = (char *) g_ptr_array_index (parents, i); + doc = find_message_by_message_id (db, parent_message_id); + insert_thread_id (thread_ids, doc); + } + + keys = g_hash_table_get_keys (thread_ids); + for (l = keys; l; l = l->next) { + char *id = (char *) l->data; + if (result == NULL) { + result = g_string_new (id); + } else { + g_string_append_printf (result, ",%s", id); + } + free (id); + } + + if (result) + return g_string_free (result, FALSE); + else + return NULL; +} + +/* Add a term for each message-id in the References header of the + * message. */ +static void +parse_references (GPtrArray *array, + const char *refs_str) +{ + GMimeReferences *refs, *r; + const char *message_id; + + if (refs_str == NULL) + return; + + refs = g_mime_references_decode (refs_str); + + for (r = refs; r; r = r->next) { + message_id = g_mime_references_get_message_id (r); + g_ptr_array_add (array, g_strdup (message_id)); + } + + g_mime_references_free (refs); +} + +/* Given a string representing the body of a message, generate terms + * for it, (skipping quoted portions and signatures). */ +static void +gen_terms_body_str (Xapian::TermGenerator term_gen, + char *body) +{ + char *line, *line_end, *next_line; + + if (body == NULL) + return; + + next_line = body; + + while (1) { + line = next_line; + if (*line == '\0') + break; + + next_line = strchr (line, '\n'); + if (next_line == NULL) { + next_line = line + strlen (line); + } + line_end = next_line - 1; + + /* Trim whitespace. */ + while (*next_line && isspace (*next_line)) + next_line++; + + /* Skip lines that are quotes. */ + if (*line == '>') + continue; + + /* Also skip lines introducing a quote on the next line. */ + while (line_end > line && isspace (*line_end)) + line_end--; + + if (*line_end == ':' && *next_line == '>') + continue; + + /* Finally, bail as soon as we see a signature. */ + /* XXX: Should only do this if "near" the end of the message. */ + if (strncmp (line, "-- ", 3) == 0 || + strncmp (line, "----------", 10) == 0 || + strncmp (line, "__________", 10) == 0) + break; + + *(line_end + 1) = '\0'; + gen_terms (term_gen, "body", line); + } +} + + +/* Callback to generate terms for each mime part of a message. */ +static void +gen_terms_part (GMimeObject *parent, + GMimeObject *part, + gpointer user_data) +{ + Xapian::TermGenerator *term_gen = (Xapian::TermGenerator *) user_data; + GMimeStream *stream; + GMimeDataWrapper *wrapper; + GByteArray *byte_array; + char *body; + + if (! GMIME_IS_PART (part)) { + fprintf (stderr, "Warning: Not indexing unknown mime part: %s.\n", + g_type_name (G_OBJECT_TYPE (part))); + return; + } + + byte_array = g_byte_array_new (); + + stream = g_mime_stream_mem_new_with_byte_array (byte_array); + g_mime_stream_mem_set_owner (GMIME_STREAM_MEM (stream), FALSE); + wrapper = g_mime_part_get_content_object (GMIME_PART (part)); + g_mime_data_wrapper_write_to_stream (wrapper, stream); + + g_object_unref (stream); + + body = (char *) g_byte_array_free (byte_array, FALSE); + + gen_terms_body_str (*term_gen, body); + + free (body); +} + +static void +index_file (Xapian::WritableDatabase db, + Xapian::TermGenerator term_gen, + const char *filename) +{ + Xapian::Document doc; + GMimeStream *stream; GMimeParser *parser; GMimeMessage *message; InternetAddressList *addresses; + GPtrArray *parents; - const char *filename; FILE *file; - const char *value, *from; + const char *subject, *refs, *in_reply_to, *from; + const char *message_id, *thread_id; time_t time; struct tm gm_time_tm; - char time_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ - - if (argc < 2) { - fprintf (stderr, "Usage: %s \n", - argv[0]); - exit (1); - } - - filename = argv[1]; + char date_str[16]; /* YYYYMMDDHHMMSS + 1 for Y100k compatibility ;-) */ + unsigned int i; file = fopen (filename, "r"); if (! file) { @@ -176,68 +440,162 @@ main (int argc, char **argv) exit (1); } - g_mime_init (0); - stream = g_mime_stream_file_new (file); parser = g_mime_parser_new_with_stream (stream); message = g_mime_parser_construct_message (parser); - printf ("text is:\n["); + doc = Xapian::Document (); + + doc.set_data (filename); + + term_gen.set_stemmer (Xapian::Stem ("english")); + + term_gen.set_document (doc); + from = g_mime_message_get_sender (message); addresses = internet_address_list_parse_string (from); - add_address_names (addresses, "from_name"); + gen_terms_address_names (term_gen, addresses, "from_name"); + + addresses = g_mime_message_get_all_recipients (message); + gen_terms_address_names (term_gen, addresses, "to_name"); + + subject = g_mime_message_get_subject (message); + subject = skip_re_in_subject (subject); + gen_terms (term_gen, "subject", subject); + gen_terms (term_gen, "body", subject); + + g_mime_message_foreach (message, gen_terms_part, &term_gen); + + parents = g_ptr_array_new (); + + refs = g_mime_object_get_header (GMIME_OBJECT (message), "references"); + parse_references (parents, refs); + + in_reply_to = g_mime_object_get_header (GMIME_OBJECT (message), + "in-reply-to"); + parse_references (parents, in_reply_to); - add_address_names (g_mime_message_get_all_recipients (message), - "to_name"); + for (i = 0; i < parents->len; i++) + add_term (doc, "ref", (char *) g_ptr_array_index (parents, i)); - value = g_mime_message_get_subject (message); - print_term ("subject", value); - print_term ("body", value); + message_id = g_mime_message_get_message_id (message); - printf ("]\nterms is:\n["); + thread_id = find_thread_id (db, parents, message_id); - TERM_COMBINED = 1; + for (i = 0; i < parents->len; i++) + g_free (g_ptr_array_index (parents, i)); + g_ptr_array_free (parents, TRUE); from = g_mime_message_get_sender (message); addresses = internet_address_list_parse_string (from); - add_address_addrs (addresses, "from_email"); + add_terms_address_addrs (doc, addresses, "from_email"); - add_address_addrs (g_mime_message_get_all_recipients (message), - "to_email"); + add_terms_address_addrs (doc, + g_mime_message_get_all_recipients (message), + "to_email"); g_mime_message_get_date (message, &time, NULL); gmtime_r (&time, &gm_time_tm); - if (strftime (time_str, sizeof (time_str), + if (strftime (date_str, sizeof (date_str), "%Y%m%d%H%M%S", &gm_time_tm) == 0) { fprintf (stderr, "Internal error formatting time\n"); exit (1); } - print_term ("date", time_str); + add_term (doc, "date", date_str); - print_term ("label", "inbox"); - print_term ("label", "unread"); - print_term ("type", "mail"); + add_term (doc, "label", "inbox"); + add_term (doc, "label", "unread"); + add_term (doc, "type", "mail"); + add_term (doc, "source_id", "1"); - value = g_mime_message_get_message_id (message); - print_term ("msgid", value); + add_term (doc, "msgid", message_id); + doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id); - print_term ("source_id", "1"); + if (thread_id) { + add_term (doc, "thread", thread_id); + doc.add_value (NOTMUCH_VALUE_THREAD, thread_id); + free ((void *) thread_id); + } else { + /* If not referenced thread, use the message ID */ + add_term (doc, "thread", message_id); + doc.add_value (NOTMUCH_VALUE_THREAD, message_id); + } - value = g_mime_message_get_message_id (message); - print_term ("thread", value); + doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time)); - printf ("]\n"); + db.add_document (doc); g_object_unref (message); g_object_unref (parser); g_object_unref (stream); +} + +static void +usage (const char *argv0) +{ + fprintf (stderr, "Usage: %s \n", argv0); + fprintf (stderr, "\n"); + fprintf (stderr, "Messages to be indexed are read from stdnin as absolute filenames\n"); + fprintf (stderr, "one file per line."); +} + +int +main (int argc, char **argv) +{ + const char *database_path; + char *filename; + GIOChannel *channel; + GIOStatus gio_status; + GError *error = NULL; + + if (argc < 2) { + usage (argv[0]); + exit (1); + } + + database_path = argv[1]; + + g_mime_init (0); + + try { + Xapian::WritableDatabase db; + Xapian::TermGenerator term_gen; + + db = Xapian::WritableDatabase (database_path, + Xapian::DB_CREATE_OR_OPEN); + + term_gen = Xapian::TermGenerator (); + + channel = g_io_channel_unix_new (fileno (stdin)); + + while (1) { + gio_status = g_io_channel_read_line (channel, &filename, + NULL, NULL, &error); + if (gio_status == G_IO_STATUS_EOF) + break; + if (gio_status != G_IO_STATUS_NORMAL) { + fprintf (stderr, "An error occurred reading from stdin: %s\n", + error->message); + exit (1); + } + + g_strchomp (filename); + index_file (db, term_gen, filename); + + g_free (filename); + } + + } catch (const Xapian::Error &error) { + cerr << "A Xapian exception occurred: " << error.get_msg () << endl; + exit (1); + } return 0; }