X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=lib%2Fdatabase.cc;h=3974e2ed3a35c1278443095d452dbd1243b7fef2;hp=4e687060c09d52cb5dce8024958f61eb79a99678;hb=08757767de5c148cea8e7b97274789f851fbd854;hpb=54ec8a0fd87ae7a32c8208be67a345c0eeea7f69 diff --git a/lib/database.cc b/lib/database.cc index 4e687060..3974e2ed 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -50,8 +50,8 @@ typedef struct { /* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION): * - * We currently have two different types of documents (mail and - * directory) and also some metadata. + * We currently have three different types of documents (mail, ghost, + * and directory) and also some metadata. * * Mail document * ------------- @@ -109,6 +109,15 @@ typedef struct { * * The data portion of a mail document is empty. * + * Ghost mail document [if NOTMUCH_FEATURE_GHOSTS] + * ----------------------------------------------- + * A ghost mail document is like a mail document, but where we don't + * have the message content. These are used to track thread reference + * information for messages we haven't received. + * + * A ghost mail document has type: ghost; id and thread fields that + * are identical to the mail document fields; and a MESSAGE_ID value. + * * Directory document * ------------------ * A directory document is used by a client of the notmuch library to @@ -172,6 +181,13 @@ typedef struct { * generated is 1 and the value will be * incremented for each thread ID. * + * Obsolete metadata + * ----------------- + * + * If ! NOTMUCH_FEATURE_GHOSTS, there are no ghost mail documents. + * Instead, the database has the following additional database + * metadata: + * * thread_id_* A pre-allocated thread ID for a particular * message. This is actually an arbitrarily large * family of metadata name. Any particular name is @@ -238,6 +254,7 @@ static prefix_t PROBABILISTIC_PREFIX[]= { { "from", "XFROM" }, { "to", "XTO" }, { "attachment", "XATTACHMENT" }, + { "mimetype", "XMIMETYPE"}, { "subject", "XSUBJECT"}, }; @@ -286,6 +303,13 @@ static const struct { "from/subject/message-ID in database", "w" }, { NOTMUCH_FEATURE_BOOL_FOLDER, "exact folder:/path: search", "rw" }, + { NOTMUCH_FEATURE_GHOSTS, + "mail documents for missing messages", "w"}, + /* Knowledge of the index mime-types are not required for reading + * a database because a reader will just be unable to query + * them. */ + { NOTMUCH_FEATURE_INDEXED_MIMETYPES, + "indexed MIME types", "w"}, }; const char * @@ -628,9 +652,10 @@ notmuch_database_create (const char *path, notmuch_database_t **database) if (status) goto DONE; - /* Upgrade doesn't add this feature to existing databases, but new - * databases have it. */ + /* Upgrade doesn't add these feature to existing databases, but + * new databases have them. */ notmuch->features |= NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES; + notmuch->features |= NOTMUCH_FEATURE_INDEXED_MIMETYPES; status = notmuch_database_upgrade (notmuch, NULL, NULL); if (status) { @@ -1213,6 +1238,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, notmuch_bool_t timer_is_active = FALSE; enum _notmuch_features target_features, new_features; notmuch_status_t status; + notmuch_private_status_t private_status; unsigned int count = 0, total = 0; status = _notmuch_database_ensure_writable (notmuch); @@ -1257,6 +1283,13 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++) ++total; } + if (new_features & NOTMUCH_FEATURE_GHOSTS) { + /* The ghost message upgrade converts all thread_id_* + * metadata values into ghost message documents. */ + t_end = db->metadata_keys_end ("thread_id_"); + for (t = db->metadata_keys_begin ("thread_id_"); t != t_end; ++t) + ++total; + } /* Perform the upgrade in a transaction. */ db->begin_transaction (true); @@ -1360,10 +1393,64 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, } } + /* Perform metadata upgrades. */ + + /* Prior to NOTMUCH_FEATURE_GHOSTS, thread IDs for missing + * messages were stored as database metadata. Change these to + * ghost messages. + */ + if (new_features & NOTMUCH_FEATURE_GHOSTS) { + notmuch_message_t *message; + std::string message_id, thread_id; + + t_end = db->metadata_keys_end (NOTMUCH_METADATA_THREAD_ID_PREFIX); + for (t = db->metadata_keys_begin (NOTMUCH_METADATA_THREAD_ID_PREFIX); + t != t_end; ++t) { + if (do_progress_notify) { + progress_notify (closure, (double) count / total); + do_progress_notify = 0; + } + + message_id = (*t).substr ( + strlen (NOTMUCH_METADATA_THREAD_ID_PREFIX)); + thread_id = db->get_metadata (*t); + + /* Create ghost message */ + message = _notmuch_message_create_for_message_id ( + notmuch, message_id.c_str (), &private_status); + if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) { + /* Document already exists; ignore the stored thread ID */ + } else if (private_status == + NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { + private_status = _notmuch_message_initialize_ghost ( + message, thread_id.c_str ()); + if (! private_status) + _notmuch_message_sync (message); + } + + if (private_status) { + fprintf (stderr, + "Upgrade failed while creating ghost messages.\n"); + status = COERCE_STATUS (private_status, "Unexpected status from _notmuch_message_initialize_ghost"); + goto DONE; + } + + /* Clear saved metadata thread ID */ + db->set_metadata (*t, ""); + + ++count; + } + } + + status = NOTMUCH_STATUS_SUCCESS; db->set_metadata ("features", _print_features (local, notmuch->features)); db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); - db->commit_transaction (); + DONE: + if (status == NOTMUCH_STATUS_SUCCESS) + db->commit_transaction (); + else + db->cancel_transaction (); if (timer_is_active) { /* Now stop the timer. */ @@ -1379,7 +1466,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, } talloc_free (local); - return NOTMUCH_STATUS_SUCCESS; + return status; } notmuch_status_t @@ -1734,6 +1821,12 @@ _get_metadata_thread_id_key (void *ctx, const char *message_id) message_id); } +static notmuch_status_t +_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch, + void *ctx, + const char *message_id, + const char **thread_id_ret); + /* Find the thread ID to which the message with 'message_id' belongs. * * Note: 'thread_id_ret' must not be NULL! @@ -1742,15 +1835,58 @@ _get_metadata_thread_id_key (void *ctx, const char *message_id) * * Note: If there is no message in the database with the given * 'message_id' then a new thread_id will be allocated for this - * message and stored in the database metadata, (where this same + * message ID and stored in the database metadata so that the * thread ID can be looked up if the message is added to the database - * later). + * later. */ static notmuch_status_t _resolve_message_id_to_thread_id (notmuch_database_t *notmuch, void *ctx, const char *message_id, const char **thread_id_ret) +{ + notmuch_private_status_t status; + notmuch_message_t *message; + + if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) + return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id, + thread_id_ret); + + /* Look for this message (regular or ghost) */ + message = _notmuch_message_create_for_message_id ( + notmuch, message_id, &status); + if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) { + /* Message exists */ + *thread_id_ret = talloc_steal ( + ctx, notmuch_message_get_thread_id (message)); + } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { + /* Message did not exist. Give it a fresh thread ID and + * populate this message as a ghost message. */ + *thread_id_ret = talloc_strdup ( + ctx, _notmuch_database_generate_thread_id (notmuch)); + if (! *thread_id_ret) { + status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY; + } else { + status = _notmuch_message_initialize_ghost (message, *thread_id_ret); + if (status == 0) + /* Commit the new ghost message */ + _notmuch_message_sync (message); + } + } else { + /* Create failed. Fall through. */ + } + + notmuch_message_destroy (message); + + return COERCE_STATUS (status, "Error creating ghost message"); +} + +/* Pre-ghost messages _resolve_message_id_to_thread_id */ +static notmuch_status_t +_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch, + void *ctx, + const char *message_id, + const char **thread_id_ret) { notmuch_status_t status; notmuch_message_t *message; @@ -1958,13 +2094,47 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch, return ret; } -/* Given a (mostly empty) 'message' and its corresponding +/* Fetch and clear the stored thread_id for message, or NULL if none. */ +static char * +_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch, + notmuch_message_t *message) +{ + const char *message_id; + string stored_id; + char *metadata_key; + + message_id = notmuch_message_get_message_id (message); + metadata_key = _get_metadata_thread_id_key (ctx, message_id); + + /* Check if we have already seen related messages to this one. + * If we have then use the thread_id that we stored at that time. + */ + stored_id = notmuch->xapian_db->get_metadata (metadata_key); + if (stored_id.empty ()) { + return NULL; + } else { + Xapian::WritableDatabase *db; + + db = static_cast (notmuch->xapian_db); + + /* Clear the metadata for this message ID. We don't need it + * anymore. */ + db->set_metadata (metadata_key, ""); + + return talloc_strdup (ctx, stored_id.c_str ()); + } +} + +/* Given a blank or ghost 'message' and its corresponding * 'message_file' link it to existing threads in the database. * - * The first check is in the metadata of the database to see if we - * have pre-allocated a thread_id in advance for this message, (which - * would have happened if a message was previously added that - * referenced this one). + * First, if is_ghost, this retrieves the thread ID already stored in + * the message (which will be the case if a message was previously + * added that referenced this one). If the message is blank + * (!is_ghost), it doesn't have a thread ID yet (we'll generate one + * later in this function). If the database does not support ghost + * messages, this checks for a thread ID stored in database metadata + * for this message ID. * * Second, we look at 'message_file' and its link-relevant headers * (References and In-Reply-To) for message IDs. @@ -1972,12 +2142,12 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch, * Finally, we look in the database for existing message that * reference 'message'. * - * In all cases, we assign to the current message the first thread_id - * found (through either parent or child). We will also merge any - * existing, distinct threads where this message belongs to both, - * (which is not uncommon when messages are processed out of order). + * In all cases, we assign to the current message the first thread ID + * found. We will also merge any existing, distinct threads where this + * message belongs to both, (which is not uncommon when messages are + * processed out of order). * - * Finally, if no thread ID has been found through parent or child, we + * Finally, if no thread ID has been found through referenced messages, we * call _notmuch_message_generate_thread_id to generate a new thread * ID. This should only happen for new, top-level messages, (no * References or In-Reply-To header in this message, and no previously @@ -1986,44 +2156,46 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch, static notmuch_status_t _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_message_t *message, - notmuch_message_file_t *message_file) + notmuch_message_file_t *message_file, + notmuch_bool_t is_ghost) { + void *local = talloc_new (NULL); notmuch_status_t status; - const char *message_id, *thread_id = NULL; - char *metadata_key; - string stored_id; - - message_id = notmuch_message_get_message_id (message); - metadata_key = _get_metadata_thread_id_key (message, message_id); - - /* Check if we have already seen related messages to this one. - * If we have then use the thread_id that we stored at that time. - */ - stored_id = notmuch->xapian_db->get_metadata (metadata_key); - if (! stored_id.empty()) { - Xapian::WritableDatabase *db; - - db = static_cast (notmuch->xapian_db); - - /* Clear the metadata for this message ID. We don't need it - * anymore. */ - db->set_metadata (metadata_key, ""); - thread_id = stored_id.c_str(); + const char *thread_id = NULL; - _notmuch_message_add_term (message, "thread", thread_id); + /* Check if the message already had a thread ID */ + if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { + if (is_ghost) + thread_id = notmuch_message_get_thread_id (message); + } else { + thread_id = _consume_metadata_thread_id (local, notmuch, message); + if (thread_id) + _notmuch_message_add_term (message, "thread", thread_id); } - talloc_free (metadata_key); status = _notmuch_database_link_message_to_parents (notmuch, message, message_file, &thread_id); if (status) - return status; + goto DONE; - status = _notmuch_database_link_message_to_children (notmuch, message, - &thread_id); - if (status) - return status; + if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) { + /* In general, it shouldn't be necessary to link children, + * since the earlier indexing of those children will have + * stored a thread ID for the missing parent. However, prior + * to ghost messages, these stored thread IDs were NOT + * rewritten during thread merging (and there was no + * performant way to do so), so if indexed children were + * pulled into a different thread ID by a merge, it was + * necessary to pull them *back* into the stored thread ID of + * the parent. With ghost messages, we just rewrite the + * stored thread IDs during merging, so this workaround isn't + * necessary. */ + status = _notmuch_database_link_message_to_children (notmuch, message, + &thread_id); + if (status) + goto DONE; + } /* If not part of any existing thread, generate a new thread ID. */ if (thread_id == NULL) { @@ -2032,7 +2204,10 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, _notmuch_message_add_term (message, "thread", thread_id); } - return NOTMUCH_STATUS_SUCCESS; + DONE: + talloc_free (local); + + return status; } notmuch_status_t @@ -2044,6 +2219,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, notmuch_message_t *message = NULL; notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; notmuch_private_status_t private_status; + notmuch_bool_t is_ghost = false; const char *date, *header; const char *from, *to, *subject; @@ -2136,12 +2312,20 @@ notmuch_database_add_message (notmuch_database_t *notmuch, _notmuch_message_add_filename (message, filename); - /* Is this a newly created message object? */ - if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { + /* Is this a newly created message object or a ghost + * message? We have to be slightly careful: if this is a + * blank message, it's not safe to call + * notmuch_message_get_flag yet. */ + if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND || + (is_ghost = notmuch_message_get_flag ( + message, NOTMUCH_MESSAGE_FLAG_GHOST))) { _notmuch_message_add_term (message, "type", "mail"); + if (is_ghost) + /* Convert ghost message to a regular message */ + _notmuch_message_remove_term (message, "type", "ghost"); ret = _notmuch_database_link_message (notmuch, message, - message_file); + message_file, is_ghost); if (ret) goto DONE;