xapian-dump: Rewrite to generate C code as output.

[notmuch] / notmuch-index-message.cc

diff --git a/notmuch-index-message.cc b/notmuch-index-message.cc
index 02f250b419f01dd5d643f4f8d98d61fa13531567..e3de80f1ec63e9d7877e11485425f3d368ccfc83 100644 (file)
--- a/notmuch-index-message.cc
+++ b/notmuch-index-message.cc
@@ -128,7 +128,40 @@ find_prefix (const char *name)
     return "";
 }
 
-int TERM_COMBINED = 0;
+/* "128 bits of thread-id ought to be enough for anybody" */
+#define NOTMUCH_THREAD_ID_BITS  128
+#define NOTMUCH_THREAD_ID_DIGITS (NOTMUCH_THREAD_ID_BITS / 4)
+typedef struct _thread_id {
+    char str[NOTMUCH_THREAD_ID_DIGITS + 1];
+} thread_id_t;
+
+static void
+thread_id_generate (thread_id_t *thread_id)
+{
+    FILE *urandom;
+    uint32_t value;
+    char *s;
+    int i;
+
+    urandom = fopen ("/dev/urandom", "r");
+    if (urandom == NULL) {
+       fprintf (stderr, "Error opening /dev/urandom: %s\n",
+                strerror (errno));
+       fprintf (stderr, "Perhaps notmuch needs some portability fixes for your platform?\n");
+       exit (1);
+    }
+
+    s = thread_id->str;
+    for (i = 0; i < NOTMUCH_THREAD_ID_DIGITS; i += 8) {
+       fread ((void *) &value, sizeof (value), 1, urandom);
+       sprintf (s, "%08x", value);
+       s += 8;
+    }
+
+    fclose (urandom);
+
+    printf ("Generated thread id: %s\n", thread_id->str);
+}
 
 static void
 add_term (Xapian::Document doc,
@@ -378,7 +411,8 @@ find_thread_ids (Xapian::Database db,
     const char *parent_message_id;
     GPtrArray *result;
 
-    thread_ids = g_hash_table_new (g_str_hash, g_str_equal);
+    thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                       free, NULL);
 
     find_messages_by_term (db, "ref", message_id, &child, &children_end);
     for ( ; child != children_end; child++) {
@@ -399,6 +433,13 @@ find_thread_ids (Xapian::Database db,
        char *id = (char *) l->data;
        g_ptr_array_add (result, id);
     }
+    g_list_free (keys);
+
+    /* We're done with the hash table, but we've taken the pointers to
+     * the allocated strings and put them into our result array, so
+     * tell the hash not to free them on its way out. */
+    g_hash_table_steal_all (thread_ids);
+    g_hash_table_unref (thread_ids);
 
     return result;
 }
@@ -453,6 +494,10 @@ gen_terms_body_str (Xapian::TermGenerator term_gen,
        while (*next_line == '\n')
            next_line++;
 
+       /* Skip blank lines. */
+       if (line_end < line)
+           continue;
+
        /* Skip lines that are quotes. */
        if (*line == '>')
            continue;
@@ -662,8 +707,10 @@ index_file (Xapian::WritableDatabase db,
     add_term (doc, "type", "mail");
     add_term (doc, "source_id", "1");
 
-    add_term (doc, "msgid", message_id);
-    doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
+    if (message_id) {
+       add_term (doc, "msgid", message_id);
+       doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
+    }
 
     if (thread_ids->len) {
        unsigned int i;
@@ -687,10 +734,14 @@ index_file (Xapian::WritableDatabase db,
        doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str);
 
        g_string_free (thread_id, TRUE);
-    } else {
-       /* If not referenced thread, use the message ID */
-       add_term (doc, "thread", message_id);
-       doc.add_value (NOTMUCH_VALUE_THREAD, message_id);
+    } else if (message_id) {
+       /* If not part of any existing thread, generate a new thread_id. */
+       thread_id_t thread_id;
+
+       thread_id_generate (&thread_id);
+
+       add_term (doc, "thread", thread_id.str);
+       doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str);
     }
 
     doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time));
@@ -720,7 +771,8 @@ main (int argc, char **argv)
     GIOStatus gio_status;
     GError *error = NULL;
     int count;
-    struct timeval tv_start, tv_now;
+    struct timeval tv_start, tv_last, tv_now;
+    double elapsed;
 
     if (argc < 2) {
        usage (argv[0]);
@@ -745,6 +797,7 @@ main (int argc, char **argv)
        count = 0;
 
        gettimeofday (&tv_start, NULL);
+       tv_last = tv_start;
 
        while (1) {
            gio_status = g_io_channel_read_line (channel, &filename,
@@ -766,11 +819,20 @@ main (int argc, char **argv)
            if (count % 1000 == 0) {
                gettimeofday (&tv_now, NULL);
                printf ("Indexed %d messages (%g messages/second)\n",
-                       count, count / ((tv_now.tv_sec - tv_start.tv_sec) +
-                                       (tv_now.tv_usec - tv_start.tv_usec) / 1e6));
+                       count, 1000 / ((tv_now.tv_sec - tv_last.tv_sec) +
+                                      (tv_now.tv_usec - tv_last.tv_usec) / 1e6));
+               tv_last = tv_now;
            }
        }
 
+       g_io_channel_unref (channel);
+
+       gettimeofday (&tv_now, NULL);
+       elapsed = (tv_now.tv_sec - tv_start.tv_sec +
+                  (tv_now.tv_usec - tv_start.tv_usec) / 1e6);
+       printf ("Completed indexing of %d messages in %g seconds (%g messages/second)\n",
+               count, elapsed, count / elapsed);
+
     } catch (const Xapian::Error &error) {
        cerr << "A Xapian exception occurred: " << error.get_msg () << endl;
        exit (1);