X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=lib%2Fdatabase.cc;h=45c42607802d79db683f7cb77683d68aaa831a4a;hp=91d43298f3795a30aa833196c2f196e49382d671;hb=df8885f62c13f77b2b16cbb211e3a727945870b0;hpb=610f0e09929a5f351f7c1c3850ac7e0d83ffe388 diff --git a/lib/database.cc b/lib/database.cc index 91d43298..45c42607 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -24,7 +24,9 @@ #include #include +#include #include +#include #include /* g_free, GPtrArray, GHashTable */ #include /* g_type_init */ @@ -40,7 +42,7 @@ typedef struct { const char *prefix; } prefix_t; -#define NOTMUCH_DATABASE_VERSION 1 +#define NOTMUCH_DATABASE_VERSION 2 #define STRINGIFY(s) _SUB_STRINGIFY(s) #define _SUB_STRINGIFY(s) #s @@ -98,8 +100,8 @@ typedef struct { * In addition, terms from the content of the message are added with * "from", "to", "attachment", and "subject" prefixes for use by the * user in searching. Similarly, terms from the path of the mail - * message are added with a "folder" prefix. But the database doesn't - * really care itself about any of these. + * message are added with "folder" and "path" prefixes. But the + * database doesn't really care itself about any of these. * * The data portion of a mail document is empty. * @@ -206,7 +208,15 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = { { "thread", "G" }, { "tag", "K" }, { "is", "K" }, - { "id", "Q" } + { "id", "Q" }, + { "path", "P" }, + /* + * Without the ":", since this is a multi-letter prefix, Xapian + * will add a colon itself if the first letter of the path is + * upper-case ASCII. Including the ":" forces there to always be a + * colon, which keeps our own logic simpler. + */ + { "folder", "XFOLDER:" }, }; static prefix_t PROBABILISTIC_PREFIX[]= { @@ -214,7 +224,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= { { "to", "XTO" }, { "attachment", "XATTACHMENT" }, { "subject", "XSUBJECT"}, - { "folder", "XFOLDER"} }; const char * @@ -268,6 +277,8 @@ notmuch_status_to_string (notmuch_status_t status) return "Unbalanced number of calls to notmuch_message_freeze/thaw"; case NOTMUCH_STATUS_UNBALANCED_ATOMIC: return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic"; + case NOTMUCH_STATUS_UNSUPPORTED_OPERATION: + return "Unsupported operation"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; @@ -345,7 +356,7 @@ _message_id_compressed (void *ctx, const char *message_id) { char *sha1, *compressed; - sha1 = notmuch_sha1_of_string (message_id); + sha1 = _notmuch_sha1_of_string (message_id); compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1); free (sha1); @@ -501,8 +512,10 @@ _parse_message_id (void *ctx, const char *message_id, const char **next) * 'message_id' in the result (to avoid mass confusion when a single * message references itself cyclically---and yes, mail messages are * not infrequent in the wild that do this---don't ask me why). -*/ -static void + * + * Return the last reference parsed, if it is not equal to message_id. + */ +static char * parse_references (void *ctx, const char *message_id, GHashTable *hash, @@ -511,7 +524,7 @@ parse_references (void *ctx, char *ref; if (refs == NULL || *refs == '\0') - return; + return NULL; while (*refs) { ref = _parse_message_id (ctx, refs, &refs); @@ -519,6 +532,17 @@ parse_references (void *ctx, if (ref && strcmp (ref, message_id)) g_hash_table_insert (hash, ref, NULL); } + + /* The return value of this function is used to add a parent + * reference to the database. We should avoid making a message + * its own parent, thus the following check. + */ + + if (ref && strcmp(ref, message_id)) { + return ref; + } else { + return NULL; + } } notmuch_status_t @@ -636,11 +660,13 @@ notmuch_database_open (const char *path, } /* Initialize the GLib type system and threads */ +#if !GLIB_CHECK_VERSION(2, 35, 1) g_type_init (); +#endif /* Initialize gmime */ if (! initialized) { - g_mime_init (0); + g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS); initialized = 1; } @@ -748,14 +774,17 @@ notmuch_database_open (const char *path, return status; } -void +notmuch_status_t notmuch_database_close (notmuch_database_t *notmuch) { + notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; + try { if (notmuch->xapian_db != NULL && notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE) (static_cast (notmuch->xapian_db))->flush (); } catch (const Xapian::Error &error) { + status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; if (! notmuch->exception_reported) { fprintf (stderr, "Error: A Xapian exception occurred flushing database: %s\n", error.get_msg().c_str()); @@ -769,7 +798,9 @@ notmuch_database_close (notmuch_database_t *notmuch) try { notmuch->xapian_db->close(); } catch (const Xapian::Error &error) { - /* do nothing */ + /* don't clobber previous error status */ + if (status == NOTMUCH_STATUS_SUCCESS) + status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; } } @@ -783,13 +814,206 @@ notmuch_database_close (notmuch_database_t *notmuch) notmuch->value_range_processor = NULL; delete notmuch->date_range_processor; notmuch->date_range_processor = NULL; + + return status; +} + +#if HAVE_XAPIAN_COMPACT +static int +unlink_cb (const char *path, + unused (const struct stat *sb), + unused (int type), + unused (struct FTW *ftw)) +{ + return remove (path); +} + +static int +rmtree (const char *path) +{ + return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS); +} + +class NotmuchCompactor : public Xapian::Compactor +{ + notmuch_compact_status_cb_t status_cb; + void *status_closure; + +public: + NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) : + status_cb (cb), status_closure (closure) { } + + virtual void + set_status (const std::string &table, const std::string &status) + { + char *msg; + + if (status_cb == NULL) + return; + + if (status.length () == 0) + msg = talloc_asprintf (NULL, "compacting table %s", table.c_str()); + else + msg = talloc_asprintf (NULL, " %s", status.c_str()); + + if (msg == NULL) { + return; + } + + status_cb (msg, status_closure); + talloc_free (msg); + } +}; + +/* Compacts the given database, optionally saving the original database + * in backup_path. Additionally, a callback function can be provided to + * give the user feedback on the progress of the (likely long-lived) + * compaction process. + * + * The backup path must point to a directory on the same volume as the + * original database. Passing a NULL backup_path will result in the + * uncompacted database being deleted after compaction has finished. + * Note that the database write lock will be held during the + * compaction process to protect data integrity. + */ +notmuch_status_t +notmuch_database_compact (const char *path, + const char *backup_path, + notmuch_compact_status_cb_t status_cb, + void *closure) +{ + void *local; + char *notmuch_path, *xapian_path, *compact_xapian_path; + notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; + notmuch_database_t *notmuch = NULL; + struct stat statbuf; + notmuch_bool_t keep_backup; + + local = talloc_new (NULL); + if (! local) + return NOTMUCH_STATUS_OUT_OF_MEMORY; + + ret = notmuch_database_open (path, NOTMUCH_DATABASE_MODE_READ_WRITE, ¬much); + if (ret) { + goto DONE; + } + + if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + + if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + + if (! (compact_xapian_path = talloc_asprintf (local, "%s.compact", xapian_path))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + + if (backup_path == NULL) { + if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + keep_backup = FALSE; + } + else { + keep_backup = TRUE; + } + + if (stat (backup_path, &statbuf) != -1) { + fprintf (stderr, "Path already exists: %s\n", backup_path); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + if (errno != ENOENT) { + fprintf (stderr, "Unknown error while stat()ing path: %s\n", + strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + /* Unconditionally attempt to remove old work-in-progress database (if + * any). This is "protected" by database lock. If this fails due to write + * errors (etc), the following code will fail and provide error message. + */ + (void) rmtree (compact_xapian_path); + + try { + NotmuchCompactor compactor (status_cb, closure); + + compactor.set_renumber (false); + compactor.add_source (xapian_path); + compactor.set_destdir (compact_xapian_path); + compactor.compact (); + } catch (const Xapian::Error &error) { + fprintf (stderr, "Error while compacting: %s\n", error.get_msg().c_str()); + ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION; + goto DONE; + } + + if (rename (xapian_path, backup_path)) { + fprintf (stderr, "Error moving %s to %s: %s\n", + xapian_path, backup_path, strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + if (rename (compact_xapian_path, xapian_path)) { + fprintf (stderr, "Error moving %s to %s: %s\n", + compact_xapian_path, xapian_path, strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + if (! keep_backup) { + if (rmtree (backup_path)) { + fprintf (stderr, "Error removing old database %s: %s\n", + backup_path, strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + } + + DONE: + if (notmuch) { + notmuch_status_t ret2; + + ret2 = notmuch_database_destroy (notmuch); + + /* don't clobber previous error status */ + if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS) + ret = ret2; + } + + talloc_free (local); + + return ret; } +#else +notmuch_status_t +notmuch_database_compact (unused (const char *path), + unused (const char *backup_path), + unused (notmuch_compact_status_cb_t status_cb), + unused (void *closure)) +{ + fprintf (stderr, "notmuch was compiled against a xapian version lacking compaction support.\n"); + return NOTMUCH_STATUS_UNSUPPORTED_OPERATION; +} +#endif -void +notmuch_status_t notmuch_database_destroy (notmuch_database_t *notmuch) { - notmuch_database_close (notmuch); + notmuch_status_t status; + + status = notmuch_database_close (notmuch); talloc_free (notmuch); + + return status; } const char * @@ -968,6 +1192,40 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, } } + /* + * Prior to version 2, the "folder:" prefix was probabilistic and + * stemmed. Change it to the current boolean prefix. Add "path:" + * prefixes while at it. + */ + if (version < 2) { + notmuch_query_t *query = notmuch_query_create (notmuch, ""); + notmuch_messages_t *messages; + notmuch_message_t *message; + + count = 0; + total = notmuch_query_count_messages (query); + + for (messages = notmuch_query_search_messages (query); + notmuch_messages_valid (messages); + notmuch_messages_move_to_next (messages)) { + if (do_progress_notify) { + progress_notify (closure, (double) count / total); + do_progress_notify = 0; + } + + message = notmuch_messages_get (messages); + + _notmuch_message_upgrade_folder (message); + _notmuch_message_sync (message); + + notmuch_message_destroy (message); + + count++; + } + + notmuch_query_destroy (query); + } + db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); db->flush (); @@ -1116,7 +1374,7 @@ _notmuch_database_get_directory_db_path (const char *path) int term_len = strlen (_find_prefix ("directory")) + strlen (path); if (term_len > NOTMUCH_TERM_MAX) - return notmuch_sha1_of_string (path); + return _notmuch_sha1_of_string (path); else return path; } @@ -1365,7 +1623,7 @@ _notmuch_database_generate_doc_id (notmuch_database_t *notmuch) notmuch->last_doc_id++; if (notmuch->last_doc_id == 0) - INTERNAL_ERROR ("Xapian document IDs are exhausted.\n"); + INTERNAL_ERROR ("Xapian document IDs are exhausted.\n"); return notmuch->last_doc_id; } @@ -1510,28 +1768,33 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, { GHashTable *parents = NULL; const char *refs, *in_reply_to, *in_reply_to_message_id; + const char *last_ref_message_id, *this_message_id; GList *l, *keys = NULL; notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; parents = g_hash_table_new_full (g_str_hash, g_str_equal, _my_talloc_free_for_g_hash, NULL); - - refs = notmuch_message_file_get_header (message_file, "references"); - parse_references (message, notmuch_message_get_message_id (message), - parents, refs); - - in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to"); - parse_references (message, notmuch_message_get_message_id (message), - parents, in_reply_to); - - /* Carefully avoid adding any self-referential in-reply-to term. */ - in_reply_to_message_id = _parse_message_id (message, in_reply_to, NULL); - if (in_reply_to_message_id && - strcmp (in_reply_to_message_id, - notmuch_message_get_message_id (message))) - { + this_message_id = notmuch_message_get_message_id (message); + + refs = _notmuch_message_file_get_header (message_file, "references"); + last_ref_message_id = parse_references (message, + this_message_id, + parents, refs); + + in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to"); + in_reply_to_message_id = parse_references (message, + this_message_id, + parents, in_reply_to); + + /* For the parent of this message, use the last message ID of the + * References header, if available. If not, fall back to the + * first message ID in the In-Reply-To header. */ + if (last_ref_message_id) { + _notmuch_message_add_term (message, "replyto", + last_ref_message_id); + } else if (in_reply_to_message_id) { _notmuch_message_add_term (message, "replyto", - _parse_message_id (message, in_reply_to, NULL)); + in_reply_to_message_id); } keys = g_hash_table_get_keys (parents); @@ -1716,7 +1979,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (ret) return ret; - message_file = notmuch_message_file_open (filename); + message_file = _notmuch_message_file_open (filename); if (message_file == NULL) return NOTMUCH_STATUS_FILE_ERROR; @@ -1726,15 +1989,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (ret) goto DONE; - notmuch_message_file_restrict_headers (message_file, - "date", - "from", - "in-reply-to", - "message-id", - "references", - "subject", - "to", - (char *) NULL); + /* Parse message up front to get better error status. */ + ret = _notmuch_message_file_parse (message_file); + if (ret) + goto DONE; try { /* Before we do any real work, (especially before doing a @@ -1742,9 +2000,9 @@ notmuch_database_add_message (notmuch_database_t *notmuch, * let's make sure that what we're looking at looks like an * actual email message. */ - from = notmuch_message_file_get_header (message_file, "from"); - subject = notmuch_message_file_get_header (message_file, "subject"); - to = notmuch_message_file_get_header (message_file, "to"); + from = _notmuch_message_file_get_header (message_file, "from"); + subject = _notmuch_message_file_get_header (message_file, "subject"); + to = _notmuch_message_file_get_header (message_file, "to"); if ((from == NULL || *from == '\0') && (subject == NULL || *subject == '\0') && @@ -1757,7 +2015,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, /* Now that we're sure it's mail, the first order of business * is to find a message ID (or else create one ourselves). */ - header = notmuch_message_file_get_header (message_file, "message-id"); + header = _notmuch_message_file_get_header (message_file, "message-id"); if (header && *header != '\0') { message_id = _parse_message_id (message_file, header, NULL); @@ -1778,7 +2036,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (message_id == NULL ) { /* No message-id at all, let's generate one by taking a * hash over the file's contents. */ - char *sha1 = notmuch_sha1_of_file (filename); + char *sha1 = _notmuch_sha1_of_file (filename); /* If that failed too, something is really wrong. Give up. */ if (sha1 == NULL) { @@ -1818,10 +2076,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (ret) goto DONE; - date = notmuch_message_file_get_header (message_file, "date"); + date = _notmuch_message_file_get_header (message_file, "date"); _notmuch_message_set_header_values (message, date, from, subject); - ret = _notmuch_message_index_file (message, filename); + ret = _notmuch_message_index_file (message, message_file); if (ret) goto DONE; } else { @@ -1847,7 +2105,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, } if (message_file) - notmuch_message_file_close (message_file); + _notmuch_message_file_close (message_file); ret2 = notmuch_database_end_atomic (notmuch); if ((ret == NOTMUCH_STATUS_SUCCESS ||