From: Carl Worth Date: Fri, 8 Jan 2010 02:26:31 +0000 (-0800) Subject: lib: Implement versioning in the database and provide upgrade function. X-Git-Tag: 0.1~151 X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=commitdiff_plain;h=909f52bd8c4bdfa11cd3e75e3d0959e0293689bd;ds=sidebyside lib: Implement versioning in the database and provide upgrade function. The recent support for renames in the database is our first time (since notmuch has had more than a single user) that we have a database format change. To support smooth upgrades we now encode a database format version number in the Xapian metadata. Going forward notmuch will emit a warning if used to read from a database with a newer version than it natively supports, and will refuse to write to a database with a newer version. The library also provides functions to query the database format version: notmuch_database_get_version to ask if notmuch wants a newer version than that: notmuch_database_needs_upgrade and a function to actually perform that upgrade: notmuch_database_upgrade --- diff --git a/lib/database-private.h b/lib/database-private.h index 643b0507..5891584e 100644 --- a/lib/database-private.h +++ b/lib/database-private.h @@ -33,6 +33,8 @@ struct _notmuch_database { Xapian::QueryParser *query_parser; Xapian::TermGenerator *term_gen; Xapian::ValueRangeProcessor *value_range_processor; + + notmuch_bool_t needs_upgrade; }; /* Convert tags from Xapian internal format to notmuch format. diff --git a/lib/database.cc b/lib/database.cc index 3de7f295..6b8c9989 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -22,6 +22,8 @@ #include +#include +#include #include #include /* g_free, GPtrArray, GHashTable */ @@ -35,7 +37,12 @@ typedef struct { const char *prefix; } prefix_t; -/* Here's the current schema for our database: +#define NOTMUCH_DATABASE_VERSION 1 + +#define STRINGIFY(s) _SUB_STRINGIFY(s) +#define _SUB_STRINGIFY(s) #s + +/* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION): * * We currently have two different types of documents: mail and directory. * @@ -467,6 +474,7 @@ notmuch_database_create (const char *path) notmuch = notmuch_database_open (path, NOTMUCH_DATABASE_MODE_READ_WRITE); + notmuch_database_upgrade (notmuch, NULL, NULL); DONE: if (notmuch_path) @@ -494,7 +502,7 @@ notmuch_database_open (const char *path, char *notmuch_path = NULL, *xapian_path = NULL; struct stat st; int err; - unsigned int i; + unsigned int i, version; if (asprintf (¬much_path, "%s/%s", path, ".notmuch") == -1) { notmuch_path = NULL; @@ -522,13 +530,42 @@ notmuch_database_open (const char *path, if (notmuch->path[strlen (notmuch->path) - 1] == '/') notmuch->path[strlen (notmuch->path) - 1] = '\0'; + notmuch->needs_upgrade = FALSE; notmuch->mode = mode; try { if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) { notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path, Xapian::DB_CREATE_OR_OPEN); + version = notmuch_database_get_version (notmuch); + + if (version > NOTMUCH_DATABASE_VERSION) { + fprintf (stderr, + "Error: Notmuch database at %s\n" + " has a newer database format version (%u) than supported by this\n" + " version of notmuch (%u). Refusing to open this database in\n" + " read-write mode.\n", + notmuch_path, version, NOTMUCH_DATABASE_VERSION); + notmuch->mode = NOTMUCH_DATABASE_MODE_READ_ONLY; + notmuch_database_close (notmuch); + notmuch = NULL; + goto DONE; + } + + if (version < NOTMUCH_DATABASE_VERSION) + notmuch->needs_upgrade = TRUE; } else { notmuch->xapian_db = new Xapian::Database (xapian_path); + version = notmuch_database_get_version (notmuch); + if (version > NOTMUCH_DATABASE_VERSION) + { + fprintf (stderr, + "Warning: Notmuch database at %s\n" + " has a newer database format version (%u) than supported by this\n" + " version of notmuch (%u). Some operations may behave incorrectly,\n" + " (but the database will not be harmed since it is being opened\n" + " in read-only mode).\n", + notmuch_path, version, NOTMUCH_DATABASE_VERSION); + } } notmuch->query_parser = new Xapian::QueryParser; notmuch->term_gen = new Xapian::TermGenerator; @@ -592,6 +629,181 @@ notmuch_database_get_path (notmuch_database_t *notmuch) return notmuch->path; } +unsigned int +notmuch_database_get_version (notmuch_database_t *notmuch) +{ + unsigned int version; + string version_string; + const char *str; + char *end; + + version_string = notmuch->xapian_db->get_metadata ("version"); + if (version_string.empty ()) + return 0; + + str = version_string.c_str (); + if (str == NULL || *str == '\0') + return 0; + + version = strtoul (str, &end, 10); + if (*end != '\0') + INTERNAL_ERROR ("Malformed database version: %s", str); + + return version; +} + +notmuch_bool_t +notmuch_database_needs_upgrade (notmuch_database_t *notmuch) +{ + return notmuch->needs_upgrade; +} + +static volatile sig_atomic_t do_progress_notify = 0; + +static void +handle_sigalrm (unused (int signal)) +{ + do_progress_notify = 1; +} + +/* Upgrade the current database. + * + * After opening a database in read-write mode, the client should + * check if an upgrade is needed (notmuch_database_needs_upgrade) and + * if so, upgrade with this function before making any modifications. + * + * The optional progress_notify callback can be used by the caller to + * provide progress indication to the user. If non-NULL it will be + * called periodically with 'count' as the number of messages upgraded + * so far and 'total' the overall number of messages that will be + * converted. + */ +notmuch_status_t +notmuch_database_upgrade (notmuch_database_t *notmuch, + void (*progress_notify) (void *closure, + unsigned int count, + unsigned int total), + void *closure) +{ + Xapian::WritableDatabase *db; + struct sigaction action; + struct itimerval timerval; + notmuch_bool_t timer_is_active = FALSE; + unsigned int version; + notmuch_status_t status; + + status = _notmuch_database_ensure_writable (notmuch); + if (status) + return status; + + db = static_cast (notmuch->xapian_db); + + version = notmuch_database_get_version (notmuch); + + if (version >= NOTMUCH_DATABASE_VERSION) + return NOTMUCH_STATUS_SUCCESS; + + if (progress_notify) { + /* Setup our handler for SIGALRM */ + memset (&action, 0, sizeof (struct sigaction)); + action.sa_handler = handle_sigalrm; + sigemptyset (&action.sa_mask); + action.sa_flags = SA_RESTART; + sigaction (SIGALRM, &action, NULL); + + /* Then start a timer to send SIGALRM once per second. */ + timerval.it_interval.tv_sec = 1; + timerval.it_interval.tv_usec = 0; + timerval.it_value.tv_sec = 1; + timerval.it_value.tv_usec = 0; + setitimer (ITIMER_REAL, &timerval, NULL); + + timer_is_active = TRUE; + } + + /* Before version 1, each message document had its filename in the + * data field. Move that into the new format by calling + * notmuch_message_add_filename. + */ + if (version < 1) { + unsigned int count = 0, total; + notmuch_query_t *query = notmuch_query_create (notmuch, ""); + notmuch_messages_t *messages; + notmuch_message_t *message; + + total = notmuch_query_count_messages (query); + + for (messages = notmuch_query_search_messages (query); + notmuch_messages_has_more (messages); + notmuch_messages_advance (messages)) + { + if (do_progress_notify) + progress_notify (closure, count, total); + + message = notmuch_messages_get (messages); + + _notmuch_message_upgrade_filename_storage (message); + + count++; + } + } + + /* Also, before version 1 we stored directory timestamps in + * XTIMESTAMP documents instead of the current XDIRECTORY + * documents. So convert those as well. */ + if (version < 1) { + Xapian::TermIterator t, t_end; + + t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP"); + + for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP"); + t != t_end; + t++) + { + Xapian::PostingIterator p, p_end; + std::string term = *t; + + p_end = notmuch->xapian_db->postlist_end (term); + + for (p = notmuch->xapian_db->postlist_begin (term); + p != p_end; + p++) + { + Xapian::Document document; + time_t mtime; + notmuch_directory_t *directory; + + document = find_document_for_doc_id (notmuch, *p); + mtime = Xapian::sortable_unserialise ( + document.get_value (NOTMUCH_VALUE_TIMESTAMP)); + + directory = notmuch_database_get_directory (notmuch, + term.c_str() + 10); + notmuch_directory_set_mtime (directory, mtime); + notmuch_directory_destroy (directory); + } + } + } + + db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION)); + db->flush (); + + if (timer_is_active) { + /* Now stop the timer. */ + timerval.it_interval.tv_sec = 0; + timerval.it_interval.tv_usec = 0; + timerval.it_value.tv_sec = 0; + timerval.it_value.tv_usec = 0; + setitimer (ITIMER_REAL, &timerval, NULL); + + /* And disable the signal handler. */ + action.sa_handler = SIG_IGN; + sigaction (SIGALRM, &action, NULL); + } + + return NOTMUCH_STATUS_SUCCESS; +} + /* We allow the user to use arbitrarily long paths for directories. But * we have a term-length limit. So if we exceed that, we'll use the * SHA-1 of the path for the database term. diff --git a/lib/message.cc b/lib/message.cc index ea9c8bd9..baeaa469 100644 --- a/lib/message.cc +++ b/lib/message.cc @@ -416,6 +416,24 @@ _notmuch_message_add_filename (notmuch_message_t *message, return NOTMUCH_STATUS_SUCCESS; } +/* Move the filename from the data field (as it was in database format + * version 0) to a file-direntry term instead (as in database format + * version 1). + */ +void +_notmuch_message_upgrade_filename_storage (notmuch_message_t *message) +{ + char *filename; + + filename = talloc_strdup (message, message->doc.get_data ().c_str ()); + if (filename && *filename != '\0') { + _notmuch_message_add_filename (message, filename); + message->doc.set_data (""); + _notmuch_message_sync (message); + } + talloc_free (filename); +} + const char * notmuch_message_get_filename (notmuch_message_t *message) { @@ -441,7 +459,10 @@ notmuch_message_get_filename (notmuch_message_t *message) { /* A message document created by an old version of notmuch * (prior to rename support) will have the filename in the - * data of the document rather than as a file-direntry term. */ + * data of the document rather than as a file-direntry term. + * + * It would be nice to do the upgrade of the document directly + * here, but the database is likely open in read-only mode. */ const char *data; data = message->doc.get_data ().c_str (); diff --git a/lib/notmuch-private.h b/lib/notmuch-private.h index 27b1317d..f9adea72 100644 --- a/lib/notmuch-private.h +++ b/lib/notmuch-private.h @@ -238,6 +238,9 @@ _notmuch_message_gen_terms (notmuch_message_t *message, const char *prefix_name, const char *text); +void +_notmuch_message_upgrade_filename_storage (notmuch_message_t *message); + notmuch_status_t _notmuch_message_add_filename (notmuch_message_t *message, const char *filename); diff --git a/lib/notmuch.h b/lib/notmuch.h index eae0fb50..d8508dfd 100644 --- a/lib/notmuch.h +++ b/lib/notmuch.h @@ -183,15 +183,43 @@ notmuch_database_close (notmuch_database_t *database); const char * notmuch_database_get_path (notmuch_database_t *database); +/* Return the database format version of the given database. */ +unsigned int +notmuch_database_get_version (notmuch_database_t *database); + +/* Does this database need to be upgraded before writing to it? + * + * If this function returns TRUE then no functions that modify the + * database (notmuch_database_add_message, notmuch_message_add_tag, + * notmuch_directory_set_mtime, etc.) will work unless the function + * notmuch_database_upgrade is called successfully first. */ +notmuch_bool_t +notmuch_database_needs_upgrade (notmuch_database_t *database); + +/* Upgrade the current database. + * + * After opening a database in read-write mode, the client should + * check if an upgrade is needed (notmuch_database_needs_upgrade) and + * if so, upgrade with this function before making any modifications. + * + * The optional progress_notify callback can be used by the caller to + * provide progress indication to the user. If non-NULL it will be + * called periodically with 'count' as the number of messages upgraded + * so far and 'total' the overall number of messages that will be + * converted. + */ +notmuch_status_t +notmuch_database_upgrade (notmuch_database_t *database, + void (*progress_notify) (void *closure, + unsigned int count, + unsigned int total), + void *closure); + /* Retrieve a directory object from the database for 'path'. * * Here, 'path' should be a path relative to the path of 'database' * (see notmuch_database_get_path), or else should be an absolute path * with initial components that match the path of 'database'. - * - * Note: The resulting notmuch_directory_t object will represent the - * state as it currently exists in the database, (and will not reflect - * subsequent changes). */ notmuch_directory_t * notmuch_database_get_directory (notmuch_database_t *database, @@ -990,7 +1018,7 @@ notmuch_directory_set_mtime (notmuch_directory_t *directory, /* Get the mtime of a directory, (as previously stored with * notmuch_directory_set_mtime). * - * Returns 0 if not mtime has previously been stored for this + * Returns 0 if no mtime has previously been stored for this * directory.*/ time_t notmuch_directory_get_mtime (notmuch_directory_t *directory);