X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=lib%2Fdatabase.cc;h=511618893d50ac7ba5c5af692e98151138a84824;hp=5cc076587a88785a5801eafcc56df2abef997d20;hb=ec573cd54fb3ea98f37a3c3612b00fb16e34578b;hpb=8d6aa603ef3a9680c7032cabf59e3673f773dea3 diff --git a/lib/database.cc b/lib/database.cc index 5cc07658..51161889 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -20,11 +20,14 @@ #include "database-private.h" #include "parse-time-vrp.h" +#include "string-util.h" #include #include +#include #include +#include #include /* g_free, GPtrArray, GHashTable */ #include /* g_type_init */ @@ -40,7 +43,7 @@ typedef struct { const char *prefix; } prefix_t; -#define NOTMUCH_DATABASE_VERSION 1 +#define NOTMUCH_DATABASE_VERSION 3 #define STRINGIFY(s) _SUB_STRINGIFY(s) #define _SUB_STRINGIFY(s) #s @@ -52,9 +55,12 @@ typedef struct { * * Mail document * ------------- - * A mail document is associated with a particular email message file - * on disk. It is indexed with the following prefixed terms which the - * database uses to construct threads, etc.: + * A mail document is associated with a particular email message. It + * is stored in one or more files on disk (though only one has its + * content indexed) and is uniquely identified by its "id" field + * (which is generally the message ID). It is indexed with the + * following prefixed terms which the database uses to construct + * threads, etc.: * * Single terms of given prefix: * @@ -98,8 +104,8 @@ typedef struct { * In addition, terms from the content of the message are added with * "from", "to", "attachment", and "subject" prefixes for use by the * user in searching. Similarly, terms from the path of the mail - * message are added with a "folder" prefix. But the database doesn't - * really care itself about any of these. + * message are added with "folder" and "path" prefixes. But the + * database doesn't really care itself about any of these. * * The data portion of a mail document is empty. * @@ -149,6 +155,17 @@ typedef struct { * changes are made to the database (such as by * indexing new fields). * + * features The set of features supported by this + * database. This consists of a set of + * '\n'-separated lines, where each is a feature + * name, a '\t', and compatibility flags. If the + * compatibility flags contain 'w', then the + * opener must support this feature to safely + * write this database. If the compatibility + * flags contain 'r', then the opener must + * support this feature to read this database. + * Introduced in database version 3. + * * last_thread_id The last thread ID generated. This is stored * as a 16-byte hexadecimal ASCII representation * of a 64-bit unsigned integer. The first ID @@ -206,7 +223,15 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = { { "thread", "G" }, { "tag", "K" }, { "is", "K" }, - { "id", "Q" } + { "id", "Q" }, + { "path", "P" }, + /* + * Without the ":", since this is a multi-letter prefix, Xapian + * will add a colon itself if the first letter of the path is + * upper-case ASCII. Including the ":" forces there to always be a + * colon, which keeps our own logic simpler. + */ + { "folder", "XFOLDER:" }, }; static prefix_t PROBABILISTIC_PREFIX[]= { @@ -214,7 +239,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= { { "to", "XTO" }, { "attachment", "XATTACHMENT" }, { "subject", "XSUBJECT"}, - { "folder", "XFOLDER"} }; const char * @@ -242,6 +266,28 @@ _find_prefix (const char *name) return ""; } +static const struct { + /* NOTMUCH_FEATURE_* value. */ + _notmuch_features value; + /* Feature name as it appears in the database. This name should + * be appropriate for displaying to the user if an older version + * of notmuch doesn't support this feature. */ + const char *name; + /* Compatibility flags when this feature is declared. */ + const char *flags; +} feature_names[] = { + { NOTMUCH_FEATURE_FILE_TERMS, + "multiple paths per message", "rw" }, + { NOTMUCH_FEATURE_DIRECTORY_DOCS, + "relative directory paths", "rw" }, + /* Header values are not required for reading a database because a + * reader can just refer to the message file. */ + { NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES, + "from/subject/message-ID in database", "w" }, + { NOTMUCH_FEATURE_BOOL_FOLDER, + "exact folder:/path: search", "rw" }, +}; + const char * notmuch_status_to_string (notmuch_status_t status) { @@ -268,6 +314,10 @@ notmuch_status_to_string (notmuch_status_t status) return "Unbalanced number of calls to notmuch_message_freeze/thaw"; case NOTMUCH_STATUS_UNBALANCED_ATOMIC: return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic"; + case NOTMUCH_STATUS_UNSUPPORTED_OPERATION: + return "Unsupported operation"; + case NOTMUCH_STATUS_UPGRADE_REQUIRED: + return "Operation requires a database upgrade"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; @@ -345,7 +395,7 @@ _message_id_compressed (void *ctx, const char *message_id) { char *sha1, *compressed; - sha1 = notmuch_sha1_of_string (message_id); + sha1 = _notmuch_sha1_of_string (message_id); compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1); free (sha1); @@ -510,7 +560,7 @@ parse_references (void *ctx, GHashTable *hash, const char *refs) { - char *ref; + char *ref, *last_ref = NULL; if (refs == NULL || *refs == '\0') return NULL; @@ -518,20 +568,17 @@ parse_references (void *ctx, while (*refs) { ref = _parse_message_id (ctx, refs, &refs); - if (ref && strcmp (ref, message_id)) + if (ref && strcmp (ref, message_id)) { g_hash_table_insert (hash, ref, NULL); + last_ref = ref; + } } /* The return value of this function is used to add a parent * reference to the database. We should avoid making a message - * its own parent, thus the following check. + * its own parent, thus the above check. */ - - if (ref && strcmp(ref, message_id)) { - return ref; - } else { - return NULL; - } + return last_ref; } notmuch_status_t @@ -580,6 +627,11 @@ notmuch_database_create (const char *path, notmuch_database_t **database) ¬much); if (status) goto DONE; + + /* Upgrade doesn't add this feature to existing databases, but new + * databases have it. */ + notmuch->features |= NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES; + status = notmuch_database_upgrade (notmuch, NULL, NULL); if (status) { notmuch_database_close(notmuch); @@ -608,6 +660,83 @@ _notmuch_database_ensure_writable (notmuch_database_t *notmuch) return NOTMUCH_STATUS_SUCCESS; } +/* Parse a database features string from the given database version. + * Returns the feature bit set. + * + * For version < 3, this ignores the features string and returns a + * hard-coded set of features. + * + * If there are unrecognized features that are required to open the + * database in mode (which should be 'r' or 'w'), return a + * comma-separated list of unrecognized but required features in + * *incompat_out suitable for presenting to the user. *incompat_out + * will be allocated from ctx. + */ +static _notmuch_features +_parse_features (const void *ctx, const char *features, unsigned int version, + char mode, char **incompat_out) +{ + _notmuch_features res = static_cast<_notmuch_features>(0); + unsigned int namelen, i; + size_t llen = 0; + const char *flags; + + /* Prior to database version 3, features were implied by the + * version number. */ + if (version == 0) + return NOTMUCH_FEATURES_V0; + else if (version == 1) + return NOTMUCH_FEATURES_V1; + else if (version == 2) + return NOTMUCH_FEATURES_V2; + + /* Parse the features string */ + while ((features = strtok_len_c (features + llen, "\n", &llen)) != NULL) { + flags = strchr (features, '\t'); + if (! flags || flags > features + llen) + continue; + namelen = flags - features; + + for (i = 0; i < ARRAY_SIZE (feature_names); ++i) { + if (strlen (feature_names[i].name) == namelen && + strncmp (feature_names[i].name, features, namelen) == 0) { + res |= feature_names[i].value; + break; + } + } + + if (i == ARRAY_SIZE (feature_names) && incompat_out) { + /* Unrecognized feature */ + const char *have = strchr (flags, mode); + if (have && have < features + llen) { + /* This feature is required to access this database in + * 'mode', but we don't understand it. */ + if (! *incompat_out) + *incompat_out = talloc_strdup (ctx, ""); + *incompat_out = talloc_asprintf_append_buffer ( + *incompat_out, "%s%.*s", **incompat_out ? ", " : "", + namelen, features); + } + } + } + + return res; +} + +static char * +_print_features (const void *ctx, unsigned int features) +{ + unsigned int i; + char *res = talloc_strdup (ctx, ""); + + for (i = 0; i < ARRAY_SIZE (feature_names); ++i) + if (features & feature_names[i].value) + res = talloc_asprintf_append_buffer ( + res, "%s\t%s\n", feature_names[i].name, feature_names[i].flags); + + return res; +} + notmuch_status_t notmuch_database_open (const char *path, notmuch_database_mode_t mode, @@ -616,7 +745,7 @@ notmuch_database_open (const char *path, notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; void *local = talloc_new (NULL); notmuch_database_t *notmuch = NULL; - char *notmuch_path, *xapian_path; + char *notmuch_path, *xapian_path, *incompat_features; struct stat st; int err; unsigned int i, version; @@ -655,7 +784,7 @@ notmuch_database_open (const char *path, /* Initialize gmime */ if (! initialized) { - g_mime_init (0); + g_mime_init (GMIME_ENABLE_RFC2047_WORKAROUNDS); initialized = 1; } @@ -666,7 +795,6 @@ notmuch_database_open (const char *path, if (notmuch->path[strlen (notmuch->path) - 1] == '/') notmuch->path[strlen (notmuch->path) - 1] = '\0'; - notmuch->needs_upgrade = FALSE; notmuch->mode = mode; notmuch->atomic_nesting = 0; try { @@ -675,37 +803,44 @@ notmuch_database_open (const char *path, if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) { notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path, Xapian::DB_CREATE_OR_OPEN); - version = notmuch_database_get_version (notmuch); - - if (version > NOTMUCH_DATABASE_VERSION) { - fprintf (stderr, - "Error: Notmuch database at %s\n" - " has a newer database format version (%u) than supported by this\n" - " version of notmuch (%u). Refusing to open this database in\n" - " read-write mode.\n", - notmuch_path, version, NOTMUCH_DATABASE_VERSION); - notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY; - notmuch_database_destroy (notmuch); - notmuch = NULL; - status = NOTMUCH_STATUS_FILE_ERROR; - goto DONE; - } - - if (version < NOTMUCH_DATABASE_VERSION) - notmuch->needs_upgrade = TRUE; } else { notmuch->xapian_db = new Xapian::Database (xapian_path); - version = notmuch_database_get_version (notmuch); - if (version > NOTMUCH_DATABASE_VERSION) - { - fprintf (stderr, - "Warning: Notmuch database at %s\n" - " has a newer database format version (%u) than supported by this\n" - " version of notmuch (%u). Some operations may behave incorrectly,\n" - " (but the database will not be harmed since it is being opened\n" - " in read-only mode).\n", - notmuch_path, version, NOTMUCH_DATABASE_VERSION); - } + } + + /* Check version. As of database version 3, we represent + * changes in terms of features, so assume a version bump + * means a dramatically incompatible change. */ + version = notmuch_database_get_version (notmuch); + if (version > NOTMUCH_DATABASE_VERSION) { + fprintf (stderr, + "Error: Notmuch database at %s\n" + " has a newer database format version (%u) than supported by this\n" + " version of notmuch (%u).\n", + notmuch_path, version, NOTMUCH_DATABASE_VERSION); + notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY; + notmuch_database_destroy (notmuch); + notmuch = NULL; + status = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + /* Check features. */ + incompat_features = NULL; + notmuch->features = _parse_features ( + local, notmuch->xapian_db->get_metadata ("features").c_str (), + version, mode == NOTMUCH_DATABASE_MODE_READ_WRITE ? 'w' : 'r', + &incompat_features); + if (incompat_features) { + fprintf (stderr, + "Error: Notmuch database at %s\n" + " requires features (%s)\n" + " not supported by this version of notmuch.\n", + notmuch_path, incompat_features); + notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY; + notmuch_database_destroy (notmuch); + notmuch = NULL; + status = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; } notmuch->last_doc_id = notmuch->xapian_db->get_lastdocid (); @@ -763,14 +898,17 @@ notmuch_database_open (const char *path, return status; } -void +notmuch_status_t notmuch_database_close (notmuch_database_t *notmuch) { + notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; + try { if (notmuch->xapian_db != NULL && notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE) (static_cast (notmuch->xapian_db))->flush (); } catch (const Xapian::Error &error) { + status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; if (! notmuch->exception_reported) { fprintf (stderr, "Error: A Xapian exception occurred flushing database: %s\n", error.get_msg().c_str()); @@ -784,7 +922,9 @@ notmuch_database_close (notmuch_database_t *notmuch) try { notmuch->xapian_db->close(); } catch (const Xapian::Error &error) { - /* do nothing */ + /* don't clobber previous error status */ + if (status == NOTMUCH_STATUS_SUCCESS) + status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; } } @@ -798,13 +938,206 @@ notmuch_database_close (notmuch_database_t *notmuch) notmuch->value_range_processor = NULL; delete notmuch->date_range_processor; notmuch->date_range_processor = NULL; + + return status; +} + +#if HAVE_XAPIAN_COMPACT +static int +unlink_cb (const char *path, + unused (const struct stat *sb), + unused (int type), + unused (struct FTW *ftw)) +{ + return remove (path); +} + +static int +rmtree (const char *path) +{ + return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS); +} + +class NotmuchCompactor : public Xapian::Compactor +{ + notmuch_compact_status_cb_t status_cb; + void *status_closure; + +public: + NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) : + status_cb (cb), status_closure (closure) { } + + virtual void + set_status (const std::string &table, const std::string &status) + { + char *msg; + + if (status_cb == NULL) + return; + + if (status.length () == 0) + msg = talloc_asprintf (NULL, "compacting table %s", table.c_str()); + else + msg = talloc_asprintf (NULL, " %s", status.c_str()); + + if (msg == NULL) { + return; + } + + status_cb (msg, status_closure); + talloc_free (msg); + } +}; + +/* Compacts the given database, optionally saving the original database + * in backup_path. Additionally, a callback function can be provided to + * give the user feedback on the progress of the (likely long-lived) + * compaction process. + * + * The backup path must point to a directory on the same volume as the + * original database. Passing a NULL backup_path will result in the + * uncompacted database being deleted after compaction has finished. + * Note that the database write lock will be held during the + * compaction process to protect data integrity. + */ +notmuch_status_t +notmuch_database_compact (const char *path, + const char *backup_path, + notmuch_compact_status_cb_t status_cb, + void *closure) +{ + void *local; + char *notmuch_path, *xapian_path, *compact_xapian_path; + notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS; + notmuch_database_t *notmuch = NULL; + struct stat statbuf; + notmuch_bool_t keep_backup; + + local = talloc_new (NULL); + if (! local) + return NOTMUCH_STATUS_OUT_OF_MEMORY; + + ret = notmuch_database_open (path, NOTMUCH_DATABASE_MODE_READ_WRITE, ¬much); + if (ret) { + goto DONE; + } + + if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + + if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + + if (! (compact_xapian_path = talloc_asprintf (local, "%s.compact", xapian_path))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + + if (backup_path == NULL) { + if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) { + ret = NOTMUCH_STATUS_OUT_OF_MEMORY; + goto DONE; + } + keep_backup = FALSE; + } + else { + keep_backup = TRUE; + } + + if (stat (backup_path, &statbuf) != -1) { + fprintf (stderr, "Path already exists: %s\n", backup_path); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + if (errno != ENOENT) { + fprintf (stderr, "Unknown error while stat()ing path: %s\n", + strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + /* Unconditionally attempt to remove old work-in-progress database (if + * any). This is "protected" by database lock. If this fails due to write + * errors (etc), the following code will fail and provide error message. + */ + (void) rmtree (compact_xapian_path); + + try { + NotmuchCompactor compactor (status_cb, closure); + + compactor.set_renumber (false); + compactor.add_source (xapian_path); + compactor.set_destdir (compact_xapian_path); + compactor.compact (); + } catch (const Xapian::Error &error) { + fprintf (stderr, "Error while compacting: %s\n", error.get_msg().c_str()); + ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION; + goto DONE; + } + + if (rename (xapian_path, backup_path)) { + fprintf (stderr, "Error moving %s to %s: %s\n", + xapian_path, backup_path, strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + if (rename (compact_xapian_path, xapian_path)) { + fprintf (stderr, "Error moving %s to %s: %s\n", + compact_xapian_path, xapian_path, strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + + if (! keep_backup) { + if (rmtree (backup_path)) { + fprintf (stderr, "Error removing old database %s: %s\n", + backup_path, strerror (errno)); + ret = NOTMUCH_STATUS_FILE_ERROR; + goto DONE; + } + } + + DONE: + if (notmuch) { + notmuch_status_t ret2; + + ret2 = notmuch_database_destroy (notmuch); + + /* don't clobber previous error status */ + if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS) + ret = ret2; + } + + talloc_free (local); + + return ret; } +#else +notmuch_status_t +notmuch_database_compact (unused (const char *path), + unused (const char *backup_path), + unused (notmuch_compact_status_cb_t status_cb), + unused (void *closure)) +{ + fprintf (stderr, "notmuch was compiled against a xapian version lacking compaction support.\n"); + return NOTMUCH_STATUS_UNSUPPORTED_OPERATION; +} +#endif -void +notmuch_status_t notmuch_database_destroy (notmuch_database_t *notmuch) { - notmuch_database_close (notmuch); + notmuch_status_t status; + + status = notmuch_database_close (notmuch); talloc_free (notmuch); + + return status; } const char * @@ -839,7 +1172,9 @@ notmuch_database_get_version (notmuch_database_t *notmuch) notmuch_bool_t notmuch_database_needs_upgrade (notmuch_database_t *notmuch) { - return notmuch->needs_upgrade; + return notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE && + ((NOTMUCH_FEATURES_CURRENT & ~notmuch->features) || + (notmuch_database_get_version (notmuch) < NOTMUCH_DATABASE_VERSION)); } static volatile sig_atomic_t do_progress_notify = 0; @@ -868,11 +1203,13 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, double progress), void *closure) { + void *local = talloc_new (NULL); + Xapian::TermIterator t, t_end; Xapian::WritableDatabase *db; struct sigaction action; struct itimerval timerval; notmuch_bool_t timer_is_active = FALSE; - unsigned int version; + enum _notmuch_features target_features, new_features; notmuch_status_t status; unsigned int count = 0, total = 0; @@ -882,9 +1219,10 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, db = static_cast (notmuch->xapian_db); - version = notmuch_database_get_version (notmuch); + target_features = notmuch->features | NOTMUCH_FEATURES_CURRENT; + new_features = NOTMUCH_FEATURES_CURRENT & ~notmuch->features; - if (version >= NOTMUCH_DATABASE_VERSION) + if (! new_features) return NOTMUCH_STATUS_SUCCESS; if (progress_notify) { @@ -905,18 +1243,33 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, timer_is_active = TRUE; } - /* Before version 1, each message document had its filename in the - * data field. Copy that into the new format by calling - * notmuch_message_add_filename. - */ - if (version < 1) { + /* Figure out how much total work we need to do. */ + if (new_features & + (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER)) { + notmuch_query_t *query = notmuch_query_create (notmuch, ""); + total += notmuch_query_count_messages (query); + notmuch_query_destroy (query); + } + if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) { + t_end = db->allterms_end ("XTIMESTAMP"); + for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++) + ++total; + } + + /* Perform the upgrade in a transaction. */ + db->begin_transaction (true); + + /* Set the target features so we write out changes in the desired + * format. */ + notmuch->features = target_features; + + /* Perform per-message upgrades. */ + if (new_features & + (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER)) { notmuch_query_t *query = notmuch_query_create (notmuch, ""); notmuch_messages_t *messages; notmuch_message_t *message; char *filename; - Xapian::TermIterator t, t_end; - - total = notmuch_query_count_messages (query); for (messages = notmuch_query_search_messages (query); notmuch_messages_valid (messages); @@ -929,12 +1282,27 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, message = notmuch_messages_get (messages); - filename = _notmuch_message_talloc_copy_data (message); - if (filename && *filename != '\0') { - _notmuch_message_add_filename (message, filename); - _notmuch_message_sync (message); + /* Before version 1, each message document had its + * filename in the data field. Copy that into the new + * format by calling notmuch_message_add_filename. + */ + if (new_features & NOTMUCH_FEATURE_FILE_TERMS) { + filename = _notmuch_message_talloc_copy_data (message); + if (filename && *filename != '\0') { + _notmuch_message_add_filename (message, filename); + _notmuch_message_clear_data (message); + } + talloc_free (filename); } - talloc_free (filename); + + /* Prior to version 2, the "folder:" prefix was + * probabilistic and stemmed. Change it to the current + * boolean prefix. Add "path:" prefixes while at it. + */ + if (new_features & NOTMUCH_FEATURE_BOOL_FOLDER) + _notmuch_message_upgrade_folder (message); + + _notmuch_message_sync (message); notmuch_message_destroy (message); @@ -942,11 +1310,14 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, } notmuch_query_destroy (query); + } - /* Also, before version 1 we stored directory timestamps in - * XTIMESTAMP documents instead of the current XDIRECTORY - * documents. So copy those as well. */ + /* Perform per-directory upgrades. */ + /* Before version 1 we stored directory timestamps in + * XTIMESTAMP documents instead of the current XDIRECTORY + * documents. So copy those as well. */ + if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) { t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP"); for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP"); @@ -979,72 +1350,18 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, NOTMUCH_FIND_CREATE, &status); notmuch_directory_set_mtime (directory, mtime); notmuch_directory_destroy (directory); - } - } - } - - db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); - db->flush (); - - /* Now that the upgrade is complete we can remove the old data - * and documents that are no longer needed. */ - if (version < 1) { - notmuch_query_t *query = notmuch_query_create (notmuch, ""); - notmuch_messages_t *messages; - notmuch_message_t *message; - char *filename; - for (messages = notmuch_query_search_messages (query); - notmuch_messages_valid (messages); - notmuch_messages_move_to_next (messages)) - { - if (do_progress_notify) { - progress_notify (closure, (double) count / total); - do_progress_notify = 0; - } - - message = notmuch_messages_get (messages); - - filename = _notmuch_message_talloc_copy_data (message); - if (filename && *filename != '\0') { - _notmuch_message_clear_data (message); - _notmuch_message_sync (message); + db->delete_document (*p); } - talloc_free (filename); - notmuch_message_destroy (message); + ++count; } - - notmuch_query_destroy (query); } - if (version < 1) { - Xapian::TermIterator t, t_end; - - t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP"); - - for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP"); - t != t_end; - t++) - { - Xapian::PostingIterator p, p_end; - std::string term = *t; - - p_end = notmuch->xapian_db->postlist_end (term); - - for (p = notmuch->xapian_db->postlist_begin (term); - p != p_end; - p++) - { - if (do_progress_notify) { - progress_notify (closure, (double) count / total); - do_progress_notify = 0; - } + db->set_metadata ("features", _print_features (local, notmuch->features)); + db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); - db->delete_document (*p); - } - } - } + db->commit_transaction (); if (timer_is_active) { /* Now stop the timer. */ @@ -1059,6 +1376,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, sigaction (SIGALRM, &action, NULL); } + talloc_free (local); return NOTMUCH_STATUS_SUCCESS; } @@ -1131,7 +1449,7 @@ _notmuch_database_get_directory_db_path (const char *path) int term_len = strlen (_find_prefix ("directory")) + strlen (path); if (term_len > NOTMUCH_TERM_MAX) - return notmuch_sha1_of_string (path); + return _notmuch_sha1_of_string (path); else return path; } @@ -1380,7 +1698,7 @@ _notmuch_database_generate_doc_id (notmuch_database_t *notmuch) notmuch->last_doc_id++; if (notmuch->last_doc_id == 0) - INTERNAL_ERROR ("Xapian document IDs are exhausted.\n"); + INTERNAL_ERROR ("Xapian document IDs are exhausted.\n"); return notmuch->last_doc_id; } @@ -1533,12 +1851,12 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, _my_talloc_free_for_g_hash, NULL); this_message_id = notmuch_message_get_message_id (message); - refs = notmuch_message_file_get_header (message_file, "references"); + refs = _notmuch_message_file_get_header (message_file, "references"); last_ref_message_id = parse_references (message, this_message_id, parents, refs); - in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to"); + in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to"); in_reply_to_message_id = parse_references (message, this_message_id, parents, in_reply_to); @@ -1736,7 +2054,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (ret) return ret; - message_file = notmuch_message_file_open (filename); + message_file = _notmuch_message_file_open (filename); if (message_file == NULL) return NOTMUCH_STATUS_FILE_ERROR; @@ -1746,15 +2064,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (ret) goto DONE; - notmuch_message_file_restrict_headers (message_file, - "date", - "from", - "in-reply-to", - "message-id", - "references", - "subject", - "to", - (char *) NULL); + /* Parse message up front to get better error status. */ + ret = _notmuch_message_file_parse (message_file); + if (ret) + goto DONE; try { /* Before we do any real work, (especially before doing a @@ -1762,9 +2075,9 @@ notmuch_database_add_message (notmuch_database_t *notmuch, * let's make sure that what we're looking at looks like an * actual email message. */ - from = notmuch_message_file_get_header (message_file, "from"); - subject = notmuch_message_file_get_header (message_file, "subject"); - to = notmuch_message_file_get_header (message_file, "to"); + from = _notmuch_message_file_get_header (message_file, "from"); + subject = _notmuch_message_file_get_header (message_file, "subject"); + to = _notmuch_message_file_get_header (message_file, "to"); if ((from == NULL || *from == '\0') && (subject == NULL || *subject == '\0') && @@ -1777,7 +2090,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, /* Now that we're sure it's mail, the first order of business * is to find a message ID (or else create one ourselves). */ - header = notmuch_message_file_get_header (message_file, "message-id"); + header = _notmuch_message_file_get_header (message_file, "message-id"); if (header && *header != '\0') { message_id = _parse_message_id (message_file, header, NULL); @@ -1798,7 +2111,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (message_id == NULL ) { /* No message-id at all, let's generate one by taking a * hash over the file's contents. */ - char *sha1 = notmuch_sha1_of_file (filename); + char *sha1 = _notmuch_sha1_of_file (filename); /* If that failed too, something is really wrong. Give up. */ if (sha1 == NULL) { @@ -1838,10 +2151,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch, if (ret) goto DONE; - date = notmuch_message_file_get_header (message_file, "date"); + date = _notmuch_message_file_get_header (message_file, "date"); _notmuch_message_set_header_values (message, date, from, subject); - ret = _notmuch_message_index_file (message, filename); + ret = _notmuch_message_index_file (message, message_file); if (ret) goto DONE; } else { @@ -1867,7 +2180,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, } if (message_file) - notmuch_message_file_close (message_file); + _notmuch_message_file_close (message_file); ret2 = notmuch_database_end_atomic (notmuch); if ((ret == NOTMUCH_STATUS_SUCCESS || @@ -1915,6 +2228,9 @@ notmuch_database_find_message_by_filename (notmuch_database_t *notmuch, if (message_ret == NULL) return NOTMUCH_STATUS_NULL_POINTER; + if (! (notmuch->features & NOTMUCH_FEATURE_FILE_TERMS)) + return NOTMUCH_STATUS_UPGRADE_REQUIRED; + /* return NULL on any failure */ *message_ret = NULL;