database: Add private find_unique_doc_id and find_unique_document functions

[notmuch] / database.cc

diff --git a/database.cc b/database.cc
index 6d62109e73c11e758987c5203839e2ef91aa8291..442b850d2264874442837fe2ec8e279d03fdc352 100644 (file)
--- a/database.cc
+++ b/database.cc
@@ -36,6 +36,8 @@ notmuch_status_to_string (notmuch_status_t status)
        return "No error occurred";
     case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
        return "A Xapian exception occurred";
        return "No error occurred";
     case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
        return "A Xapian exception occurred";

+    case NOTMUCH_STATUS_FILE_ERROR:
+       return "Something went wrong trying to read or write a file";

     case NOTMUCH_STATUS_FILE_NOT_EMAIL:
        return "File is not an email";
     case NOTMUCH_STATUS_NULL_POINTER:
     case NOTMUCH_STATUS_FILE_NOT_EMAIL:
        return "File is not an email";
     case NOTMUCH_STATUS_NULL_POINTER:


@@ -48,42 +50,6 @@ notmuch_status_to_string (notmuch_status_t status)
     }
 }
 
     }
 }
 

-/* "128 bits of thread-id ought to be enough for anybody" */
-#define NOTMUCH_THREAD_ID_BITS  128
-#define NOTMUCH_THREAD_ID_DIGITS (NOTMUCH_THREAD_ID_BITS / 4)
-typedef struct _thread_id {
-    char str[NOTMUCH_THREAD_ID_DIGITS + 1];
-} thread_id_t;
-
-static void
-thread_id_generate (thread_id_t *thread_id)
-{
-    static int seeded = 0;
-    FILE *dev_random;
-    uint32_t value;
-    char *s;
-    int i;
-
-    if (! seeded) {
-       dev_random = fopen ("/dev/random", "r");
-       if (dev_random == NULL) {
-           srand (time (NULL));
-       } else {
-           fread ((void *) &value, sizeof (value), 1, dev_random);
-           srand (value);
-           fclose (dev_random);
-       }
-       seeded = 1;
-    }
-
-    s = thread_id->str;
-    for (i = 0; i < NOTMUCH_THREAD_ID_DIGITS; i += 8) {
-       value = rand ();
-       sprintf (s, "%08x", value);
-       s += 8;
-    }
-}
-

 /* XXX: We should drop this function and convert all callers to call
  * _notmuch_message_add_term instead. */
 static void
 /* XXX: We should drop this function and convert all callers to call
  * _notmuch_message_add_term instead. */
 static void


@@ -108,29 +74,67 @@ add_term (Xapian::Document doc,
 }
 
 static void
 }
 
 static void

-find_messages_by_term (Xapian::Database *db,
-                      const char *prefix_name,
-                      const char *value,
-                      Xapian::PostingIterator *begin,
-                      Xapian::PostingIterator *end)
+find_doc_ids (notmuch_database_t *notmuch,
+             const char *prefix_name,
+             const char *value,
+             Xapian::PostingIterator *begin,
+             Xapian::PostingIterator *end)

 {
     Xapian::PostingIterator i;
     char *term;
 
     term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
 
 {
     Xapian::PostingIterator i;
     char *term;
 
     term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
 

-    *begin = db->postlist_begin (term);
+    *begin = notmuch->xapian_db->postlist_begin (term);

 
 

-    if (end)
-       *end = db->postlist_end (term);
+    *end = notmuch->xapian_db->postlist_end (term);

 
     free (term);
 }
 
 
     free (term);
 }
 

-Xapian::Document
-find_message_by_docid (Xapian::Database *db, Xapian::docid docid)
+static notmuch_private_status_t
+find_unique_doc_id (notmuch_database_t *notmuch,
+                   const char *prefix_name,
+                   const char *value,
+                   unsigned int *doc_id)
+{
+    Xapian::PostingIterator i, end;
+
+    find_doc_ids (notmuch, prefix_name, value, &i, &end);
+
+    if (i == end) {
+       *doc_id = 0;
+       return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+    } else {
+       *doc_id = *i;
+       return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+    }
+}
+
+static Xapian::Document
+find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
+{
+    return notmuch->xapian_db->get_document (doc_id);
+}
+
+static notmuch_private_status_t
+find_unique_document (notmuch_database_t *notmuch,
+                     const char *prefix_name,
+                     const char *value,
+                     Xapian::Document *document,
+                     unsigned int *doc_id)

 {
 {

-    return db->get_document (docid);
+    notmuch_private_status_t status;
+
+    status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
+
+    if (status) {
+       *document = Xapian::Document ();
+       return status;
+    }
+
+    *document = find_document_for_doc_id (notmuch, *doc_id);
+    return NOTMUCH_PRIVATE_STATUS_SUCCESS;

 }
 
 static void
 }
 
 static void


@@ -160,15 +164,15 @@ notmuch_message_t *
 notmuch_database_find_message (notmuch_database_t *notmuch,
                               const char *message_id)
 {
 notmuch_database_find_message (notmuch_database_t *notmuch,
                               const char *message_id)
 {

-    Xapian::PostingIterator i, end;
+    notmuch_private_status_t status;
+    unsigned int doc_id;

 
 

-    find_messages_by_term (notmuch->xapian_db,
-                          "msgid", message_id, &i, &end);
+    status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);

 
 

-    if (i == end)
+    if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)

        return NULL;
 
        return NULL;
 

-    return _notmuch_message_create (notmuch, notmuch, *i);
+    return _notmuch_message_create (notmuch, notmuch, doc_id);

 }
 
 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
 }
 
 /* Return one or more thread_ids, (as a GPtrArray of strings), for the


@@ -183,7 +187,6 @@ find_thread_ids (notmuch_database_t *notmuch,
                 GPtrArray *parents,
                 const char *message_id)
 {
                 GPtrArray *parents,
                 const char *message_id)
 {

-    Xapian::WritableDatabase *db = notmuch->xapian_db;

     Xapian::PostingIterator child, children_end;
     Xapian::Document doc;
     GHashTable *thread_ids;
     Xapian::PostingIterator child, children_end;
     Xapian::Document doc;
     GHashTable *thread_ids;


@@ -195,9 +198,9 @@ find_thread_ids (notmuch_database_t *notmuch,
     thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
                                        free, NULL);
 
     thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
                                        free, NULL);
 

-    find_messages_by_term (db, "ref", message_id, &child, &children_end);
+    find_doc_ids (notmuch, "ref", message_id, &child, &children_end);

     for ( ; child != children_end; child++) {
     for ( ; child != children_end; child++) {

-       doc = find_message_by_docid (db, *child);
+       doc = find_document_for_doc_id (notmuch, *child);

        insert_thread_id (thread_ids, doc);
     }
 
        insert_thread_id (thread_ids, doc);
     }
 


@@ -485,47 +488,38 @@ notmuch_status_t
 notmuch_database_add_message (notmuch_database_t *notmuch,
                              const char *filename)
 {
 notmuch_database_add_message (notmuch_database_t *notmuch,
                              const char *filename)
 {

-    Xapian::WritableDatabase *db = notmuch->xapian_db;
-    Xapian::Document doc;
-    notmuch_message_file_t *message;
+    notmuch_message_file_t *message_file;
+    notmuch_message_t *message;
+    notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;

 
     GPtrArray *parents, *thread_ids;
 
     const char *refs, *in_reply_to, *date, *header;
 
     GPtrArray *parents, *thread_ids;
 
     const char *refs, *in_reply_to, *date, *header;

-    const char *from, *to, *subject;
+    const char *from, *to, *subject, *old_filename;

     char *message_id;
 
     char *message_id;
 

-    time_t time_value;

     unsigned int i;
 
     unsigned int i;
 

-    message = notmuch_message_file_open (filename);
+    message_file = notmuch_message_file_open (filename);
+    if (message_file == NULL) {
+       ret = NOTMUCH_STATUS_FILE_ERROR;
+       goto DONE;
+    }

 
 

-    notmuch_message_file_restrict_headers (message,
+    notmuch_message_file_restrict_headers (message_file,

                                           "date",
                                           "from",
                                           "in-reply-to",
                                           "message-id",
                                           "references",
                                           "subject",
                                           "date",
                                           "from",
                                           "in-reply-to",
                                           "message-id",
                                           "references",
                                           "subject",

+                                          "to",

                                           (char *) NULL);
 
     try {
                                           (char *) NULL);
 
     try {

-       doc.set_data (filename);
-
-       add_term (doc, "type", "mail");
-
-       parents = g_ptr_array_new ();
+       /* The first order of business is to find/create a message ID. */

 
 

-       refs = notmuch_message_file_get_header (message, "references");
-       parse_references (parents, refs);
-
-       in_reply_to = notmuch_message_file_get_header (message, "in-reply-to");
-       parse_references (parents, in_reply_to);
-
-       for (i = 0; i < parents->len; i++)
-           add_term (doc, "ref", (char *) g_ptr_array_index (parents, i));
-
-       header = notmuch_message_file_get_header (message, "message-id");
+       header = notmuch_message_file_get_header (message_file, "message-id");

        if (header) {
            message_id = parse_message_id (header, NULL);
            /* So the header value isn't RFC-compliant, but it's
        if (header) {
            message_id = parse_message_id (header, NULL);
            /* So the header value isn't RFC-compliant, but it's


@@ -533,20 +527,82 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
            if (message_id == NULL)
                message_id = xstrdup (header);
        } else {
            if (message_id == NULL)
                message_id = xstrdup (header);
        } else {

-           /* XXX: Should generate a message_id here, (such as a SHA1
-            * sum of the message itself) */
-           message_id = NULL;
+           /* No message-id at all, let's generate one by taking a
+            * hash over the file's contents. */
+           char *sha1 = notmuch_sha1_of_file (filename);
+
+           /* If that failed too, something is really wrong. Give up. */
+           if (sha1 == NULL) {
+               ret = NOTMUCH_STATUS_FILE_ERROR;
+               goto DONE;
+           }
+
+           message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
+           free (sha1);
+       }
+
+       /* Now that we have a message ID, we get a message object,
+        * (which may or may not reference an existing document in the
+        * database). */
+
+       /* Use NULL for owner since we want to free this locally. */
+
+       /* XXX: This call can fail by either out-of-memory or an
+        * "impossible" Xapian exception. We should rewrite it to
+        * allow us to propagate the error status. */
+       message = _notmuch_message_create_for_message_id (NULL, notmuch,
+                                                         message_id);
+       if (message == NULL) {
+           fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
+           fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
+           exit (1);
+       }
+
+       /* Has a message previously been added with the same ID? */
+       old_filename = notmuch_message_get_filename (message);
+       if (old_filename && strlen (old_filename)) {
+           /* XXX: This is too noisy to actually print, and what do we
+            * really expect the user to do? Go manually delete a
+            * redundant message or merge two similar messages?
+            * Instead we should handle this transparently.
+            *
+            * What we likely want to move to is adding both filenames
+            * to the database so that subsequent indexing will pick up
+            * terms from both files.
+            */
+#if 0
+           fprintf (stderr,
+                    "Note: Attempting to add a message with a duplicate message ID:\n"
+                    "Old: %s\n"   "New: %s\n",
+                    old_filename, filename);
+           fprintf (stderr, "The old filename will be used, but any new terms\n"
+                    "from the new message will added to the database.\n");
+#endif
+       } else {
+           _notmuch_message_set_filename (message, filename);
+           _notmuch_message_add_term (message, "type", "mail");

        }
 
        }
 

+       /* Next, find the thread(s) to which this message belongs. */
+       parents = g_ptr_array_new ();
+
+       refs = notmuch_message_file_get_header (message_file, "references");
+       parse_references (parents, refs);
+
+       in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
+       parse_references (parents, in_reply_to);
+
+       for (i = 0; i < parents->len; i++)
+           _notmuch_message_add_term (message, "ref",
+                                      (char *) g_ptr_array_index (parents, i));
+

        thread_ids = find_thread_ids (notmuch, parents, message_id);
 
        thread_ids = find_thread_ids (notmuch, parents, message_id);
 

+       free (message_id);
+

        for (i = 0; i < parents->len; i++)
            g_free (g_ptr_array_index (parents, i));
        g_ptr_array_free (parents, TRUE);
        for (i = 0; i < parents->len; i++)
            g_free (g_ptr_array_index (parents, i));
        g_ptr_array_free (parents, TRUE);

-       if (message_id) {
-           add_term (doc, "msgid", message_id);
-           doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
-       }

 
        if (thread_ids->len) {
            unsigned int i;
 
        if (thread_ids->len) {
            unsigned int i;


@@ -555,7 +611,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
            for (i = 0; i < thread_ids->len; i++) {
                id = (char *) thread_ids->pdata[i];
 
            for (i = 0; i < thread_ids->len; i++) {
                id = (char *) thread_ids->pdata[i];

-               add_term (doc, "thread", id);
+               _notmuch_message_add_thread_id (message, id);

                if (i == 0)
                    thread_id = g_string_new (id);
                else
                if (i == 0)
                    thread_id = g_string_new (id);
                else


@@ -563,47 +619,41 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 
                free (id);
            }
 
                free (id);
            }

-           doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str);

            g_string_free (thread_id, TRUE);
            g_string_free (thread_id, TRUE);

-       } else if (message_id) {
-           /* If not part of any existing thread, generate a new thread_id. */
-           thread_id_t thread_id;
-
-           thread_id_generate (&thread_id);
-           add_term (doc, "thread", thread_id.str);
-           doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str);
+       } else {
+           _notmuch_message_ensure_thread_id (message);

        }
 
        g_ptr_array_free (thread_ids, TRUE);
 
        }
 
        g_ptr_array_free (thread_ids, TRUE);
 

-       free (message_id);
-
-       date = notmuch_message_file_get_header (message, "date");
-       time_value = notmuch_parse_date (date, NULL);
+       date = notmuch_message_file_get_header (message_file, "date");
+       _notmuch_message_set_date (message, date);

 
 

-       doc.add_value (NOTMUCH_VALUE_DATE,
-                      Xapian::sortable_serialise (time_value));
-
-       from = notmuch_message_file_get_header (message, "from");
-       subject = notmuch_message_file_get_header (message, "subject");
-       to = notmuch_message_file_get_header (message, "to");
+       from = notmuch_message_file_get_header (message_file, "from");
+       subject = notmuch_message_file_get_header (message_file, "subject");
+       to = notmuch_message_file_get_header (message_file, "to");

 
        if (from == NULL &&
            subject == NULL &&
            to == NULL)
        {
 
        if (from == NULL &&
            subject == NULL &&
            to == NULL)
        {

-           notmuch_message_file_close (message);
-           return NOTMUCH_STATUS_FILE_NOT_EMAIL;
+           ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+           goto DONE;

        } else {
        } else {

-           db->add_document (doc);
+           _notmuch_message_sync (message);

        }
     } catch (const Xapian::Error &error) {
        fprintf (stderr, "A Xapian exception occurred: %s.\n",
                 error.get_msg().c_str());
        }
     } catch (const Xapian::Error &error) {
        fprintf (stderr, "A Xapian exception occurred: %s.\n",
                 error.get_msg().c_str());

-       return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+       ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+       goto DONE;

     }
 
     }
 

-    notmuch_message_file_close (message);
+  DONE:
+    if (message)
+       notmuch_message_destroy (message);
+    if (message_file)
+       notmuch_message_file_close (message_file);

 
 

-    return NOTMUCH_STATUS_SUCCESS;
+    return ret;

 }
 }