X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=lib%2Fdatabase.cc;h=386dcd17adc4d0a91ab5eb10479907b254d32187;hp=6d0e5a63fd81c6b4163860e84b1e6af8258394bb;hb=e0e8586fc72763571f9eafeb831c7bc79a15273d;hpb=33c8777a967ece2dd4bbda7e83a4e07c195abf51 diff --git a/lib/database.cc b/lib/database.cc index 6d0e5a63..386dcd17 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -13,13 +13,14 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see http://www.gnu.org/licenses/ . + * along with this program. If not, see https://www.gnu.org/licenses/ . * * Author: Carl Worth */ #include "database-private.h" #include "parse-time-vrp.h" +#include "query-fp.h" #include "string-util.h" #include @@ -41,6 +42,7 @@ using namespace std; typedef struct { const char *name; const char *prefix; + notmuch_field_flag_t flags; } prefix_t; #define NOTMUCH_DATABASE_VERSION 3 @@ -48,6 +50,12 @@ typedef struct { #define STRINGIFY(s) _SUB_STRINGIFY(s) #define _SUB_STRINGIFY(s) #s +#if HAVE_XAPIAN_DB_RETRY_LOCK +#define DB_ACTION (Xapian::DB_CREATE_OR_OPEN | Xapian::DB_RETRY_LOCK) +#else +#define DB_ACTION Xapian::DB_CREATE_OR_OPEN +#endif + /* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION): * * We currently have three different types of documents (mail, ghost, @@ -90,6 +98,9 @@ typedef struct { * STRING is the name of a file within that * directory for this mail message. * + * property: Has a property with key=value + * FIXME: if no = is present, should match on any value + * * A mail document also has four values: * * TIMESTAMP: The time_t value corresponding to the message's @@ -184,6 +195,14 @@ typedef struct { * generated is 1 and the value will be * incremented for each thread ID. * + * C* metadata keys starting with C indicate + * configuration data. It can be managed with the + * n_database_*config* API. There is a convention + * of hierarchical keys separated by '.' (e.g. + * query.notmuch stores the value for the named + * query 'notmuch'), but it is not enforced by the + * API. + * * Obsolete metadata * ----------------- * @@ -216,7 +235,7 @@ typedef struct { /* With these prefix values we follow the conventions published here: * - * http://xapian.org/docs/omega/termprefixes.html + * https://xapian.org/docs/omega/termprefixes.html * * as much as makes sense. Note that I took some liberty in matching * the reserved prefix values to notmuch concepts, (for example, 'G' @@ -229,36 +248,38 @@ typedef struct { * nearly universal to all mail messages). */ -static prefix_t BOOLEAN_PREFIX_INTERNAL[] = { - { "type", "T" }, - { "reference", "XREFERENCE" }, - { "replyto", "XREPLYTO" }, - { "directory", "XDIRECTORY" }, - { "file-direntry", "XFDIRENTRY" }, - { "directory-direntry", "XDDIRENTRY" }, -}; - -static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = { - { "thread", "G" }, - { "tag", "K" }, - { "is", "K" }, - { "id", "Q" }, - { "path", "P" }, +static const +prefix_t prefix_table[] = { + /* name term prefix flags */ + { "type", "T", NOTMUCH_FIELD_NO_FLAGS }, + { "reference", "XREFERENCE", NOTMUCH_FIELD_NO_FLAGS }, + { "replyto", "XREPLYTO", NOTMUCH_FIELD_NO_FLAGS }, + { "directory", "XDIRECTORY", NOTMUCH_FIELD_NO_FLAGS }, + { "file-direntry", "XFDIRENTRY", NOTMUCH_FIELD_NO_FLAGS }, + { "directory-direntry", "XDDIRENTRY", NOTMUCH_FIELD_NO_FLAGS }, + { "thread", "G", NOTMUCH_FIELD_EXTERNAL }, + { "tag", "K", NOTMUCH_FIELD_EXTERNAL }, + { "is", "K", NOTMUCH_FIELD_EXTERNAL }, + { "id", "Q", NOTMUCH_FIELD_EXTERNAL }, + { "path", "P", NOTMUCH_FIELD_EXTERNAL }, + { "property", "XPROPERTY", NOTMUCH_FIELD_EXTERNAL }, /* - * Without the ":", since this is a multi-letter prefix, Xapian - * will add a colon itself if the first letter of the path is - * upper-case ASCII. Including the ":" forces there to always be a - * colon, which keeps our own logic simpler. + * Unconditionally add ':' to reduce potential ambiguity with + * overlapping prefixes and/or terms that start with capital + * letters. See Xapian document termprefixes.html for related + * discussion. */ - { "folder", "XFOLDER:" }, -}; - -static prefix_t PROBABILISTIC_PREFIX[]= { - { "from", "XFROM" }, - { "to", "XTO" }, - { "attachment", "XATTACHMENT" }, - { "mimetype", "XMIMETYPE"}, - { "subject", "XSUBJECT"}, + { "folder", "XFOLDER:", NOTMUCH_FIELD_EXTERNAL }, + { "from", "XFROM", NOTMUCH_FIELD_EXTERNAL | + NOTMUCH_FIELD_PROBABILISTIC }, + { "to", "XTO", NOTMUCH_FIELD_EXTERNAL | + NOTMUCH_FIELD_PROBABILISTIC }, + { "attachment", "XATTACHMENT", NOTMUCH_FIELD_EXTERNAL | + NOTMUCH_FIELD_PROBABILISTIC }, + { "mimetype", "XMIMETYPE", NOTMUCH_FIELD_EXTERNAL | + NOTMUCH_FIELD_PROBABILISTIC }, + { "subject", "XSUBJECT", NOTMUCH_FIELD_EXTERNAL | + NOTMUCH_FIELD_PROBABILISTIC }, }; const char * @@ -266,19 +287,9 @@ _find_prefix (const char *name) { unsigned int i; - for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_INTERNAL); i++) { - if (strcmp (name, BOOLEAN_PREFIX_INTERNAL[i].name) == 0) - return BOOLEAN_PREFIX_INTERNAL[i].prefix; - } - - for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_EXTERNAL); i++) { - if (strcmp (name, BOOLEAN_PREFIX_EXTERNAL[i].name) == 0) - return BOOLEAN_PREFIX_EXTERNAL[i].prefix; - } - - for (i = 0; i < ARRAY_SIZE (PROBABILISTIC_PREFIX); i++) { - if (strcmp (name, PROBABILISTIC_PREFIX[i].name) == 0) - return PROBABILISTIC_PREFIX[i].prefix; + for (i = 0; i < ARRAY_SIZE (prefix_table); i++) { + if (strcmp (name, prefix_table[i].name) == 0) + return prefix_table[i].prefix; } INTERNAL_ERROR ("No prefix exists for '%s'\n", name); @@ -368,6 +379,22 @@ _notmuch_database_log (notmuch_database_t *notmuch, talloc_free (notmuch->status_string); notmuch->status_string = talloc_vasprintf (notmuch, format, va_args); + va_end (va_args); +} + +void +_notmuch_database_log_append (notmuch_database_t *notmuch, + const char *format, + ...) +{ + va_list va_args; + + va_start (va_args, format); + + if (notmuch->status_string) + notmuch->status_string = talloc_vasprintf_append (notmuch->status_string, format, va_args); + else + notmuch->status_string = talloc_vasprintf (notmuch, format, va_args); va_end (va_args); } @@ -617,7 +644,7 @@ parse_references (void *ctx, ref = _parse_message_id (ctx, refs, &refs); if (ref && strcmp (ref, message_id)) { - g_hash_table_insert (hash, ref, NULL); + g_hash_table_add (hash, ref); last_ref = ref; } } @@ -626,7 +653,7 @@ parse_references (void *ctx, * reference to the database. We should avoid making a message * its own parent, thus the above check. */ - return last_ref; + return talloc_strdup(ctx, last_ref); } notmuch_status_t @@ -930,7 +957,7 @@ notmuch_database_open_verbose (const char *path, if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) { notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path, - Xapian::DB_CREATE_OR_OPEN); + DB_ACTION); } else { notmuch->xapian_db = new Xapian::Database (xapian_path); } @@ -1000,6 +1027,18 @@ notmuch_database_open_verbose (const char *path, notmuch->term_gen->set_stemmer (Xapian::Stem ("english")); notmuch->value_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP); notmuch->date_range_processor = new ParseTimeValueRangeProcessor (NOTMUCH_VALUE_TIMESTAMP); +#if HAVE_XAPIAN_FIELD_PROCESSOR + /* This currently relies on the query parser to pass anything + * with a .. to the range processor */ + { + Xapian::FieldProcessor * date_fp = new DateFieldProcessor(); + Xapian::FieldProcessor * query_fp = + new QueryFieldProcessor (*notmuch->query_parser, notmuch); + + notmuch->query_parser->add_boolean_prefix("date", date_fp->release ()); + notmuch->query_parser->add_boolean_prefix("query", query_fp->release ()); + } +#endif notmuch->last_mod_range_processor = new Xapian::NumberValueRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:"); notmuch->query_parser->set_default_op (Xapian::Query::OP_AND); @@ -1010,15 +1049,16 @@ notmuch_database_open_verbose (const char *path, notmuch->query_parser->add_valuerangeprocessor (notmuch->date_range_processor); notmuch->query_parser->add_valuerangeprocessor (notmuch->last_mod_range_processor); - for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_EXTERNAL); i++) { - prefix_t *prefix = &BOOLEAN_PREFIX_EXTERNAL[i]; - notmuch->query_parser->add_boolean_prefix (prefix->name, - prefix->prefix); - } - - for (i = 0; i < ARRAY_SIZE (PROBABILISTIC_PREFIX); i++) { - prefix_t *prefix = &PROBABILISTIC_PREFIX[i]; - notmuch->query_parser->add_prefix (prefix->name, prefix->prefix); + for (i = 0; i < ARRAY_SIZE (prefix_table); i++) { + const prefix_t *prefix = &prefix_table[i]; + if (prefix->flags & NOTMUCH_FIELD_EXTERNAL) { + if (prefix->flags & NOTMUCH_FIELD_PROBABILISTIC) { + notmuch->query_parser->add_prefix (prefix->name, prefix->prefix); + } else { + notmuch->query_parser->add_boolean_prefix (prefix->name, + prefix->prefix); + } + } } } catch (const Xapian::Error &error) { IGNORE_RESULT (asprintf (&message, "A Xapian exception occurred opening database: %s\n", @@ -1093,7 +1133,6 @@ notmuch_database_close (notmuch_database_t *notmuch) return status; } -#if HAVE_XAPIAN_COMPACT static int unlink_cb (const char *path, unused (const struct stat *sb), @@ -1277,17 +1316,6 @@ notmuch_database_compact (const char *path, return ret; } -#else -notmuch_status_t -notmuch_database_compact (unused (const char *path), - unused (const char *backup_path), - unused (notmuch_compact_status_cb_t status_cb), - unused (void *closure)) -{ - _notmuch_database_log (notmuch, "notmuch was compiled against a xapian version lacking compaction support.\n"); - return NOTMUCH_STATUS_UNSUPPORTED_OPERATION; -} -#endif notmuch_status_t notmuch_database_destroy (notmuch_database_t *notmuch) @@ -1372,6 +1400,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, enum _notmuch_features target_features, new_features; notmuch_status_t status; notmuch_private_status_t private_status; + notmuch_query_t *query = NULL; unsigned int count = 0, total = 0; status = _notmuch_database_ensure_writable (notmuch); @@ -1408,9 +1437,16 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, if (new_features & (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER | NOTMUCH_FEATURE_LAST_MOD)) { - notmuch_query_t *query = notmuch_query_create (notmuch, ""); - total += notmuch_query_count_messages (query); + query = notmuch_query_create (notmuch, ""); + unsigned msg_count; + + status = notmuch_query_count_messages_st (query, &msg_count); + if (status) + goto DONE; + + total += msg_count; notmuch_query_destroy (query); + query = NULL; } if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) { t_end = db->allterms_end ("XTIMESTAMP"); @@ -1436,14 +1472,16 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, if (new_features & (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER | NOTMUCH_FEATURE_LAST_MOD)) { - notmuch_query_t *query = notmuch_query_create (notmuch, ""); notmuch_messages_t *messages; notmuch_message_t *message; char *filename; - /* XXX: this should use the _st version, but needs an error - path */ - for (messages = notmuch_query_search_messages (query); + query = notmuch_query_create (notmuch, ""); + + status = notmuch_query_search_messages_st (query, &messages); + if (status) + goto DONE; + for (; notmuch_messages_valid (messages); notmuch_messages_move_to_next (messages)) { @@ -1490,6 +1528,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, } notmuch_query_destroy (query); + query = NULL; } /* Perform per-directory upgrades. */ @@ -1610,6 +1649,9 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, sigaction (SIGALRM, &action, NULL); } + if (query) + notmuch_query_destroy (query); + talloc_free (local); return status; } @@ -1621,6 +1663,9 @@ notmuch_database_begin_atomic (notmuch_database_t *notmuch) notmuch->atomic_nesting > 0) goto DONE; + if (notmuch_database_needs_upgrade (notmuch)) + return NOTMUCH_STATUS_UPGRADE_REQUIRED; + try { (static_cast (notmuch->xapian_db))->begin_transaction (false); } catch (const Xapian::Error &error) { @@ -1656,7 +1701,7 @@ notmuch_database_end_atomic (notmuch_database_t *notmuch) * However, we rely on flushing to test atomicity. */ const char *thresh = getenv ("XAPIAN_FLUSH_THRESHOLD"); if (thresh && atoi (thresh) == 1) - db->flush (); + db->commit (); } catch (const Xapian::Error &error) { _notmuch_database_log (notmuch, "A Xapian exception occurred committing transaction: %s.\n", error.get_msg().c_str()); @@ -1744,18 +1789,11 @@ _notmuch_database_split_path (void *ctx, slash = path + strlen (path) - 1; /* First, skip trailing slashes. */ - while (slash != path) { - if (*slash != '/') - break; - + while (slash != path && *slash == '/') --slash; - } /* Then, find a slash. */ - while (slash != path) { - if (*slash == '/') - break; - + while (slash != path && *slash != '/') { if (basename) *basename = slash; @@ -1763,12 +1801,8 @@ _notmuch_database_split_path (void *ctx, } /* Finally, skip multiple slashes. */ - while (slash != path) { - if (*slash != '/') - break; - + while (slash != path && *(slash - 1) == '/') --slash; - } if (slash == path) { if (directory) @@ -1777,7 +1811,7 @@ _notmuch_database_split_path (void *ctx, *basename = path; } else { if (directory) - *directory = talloc_strndup (ctx, path, slash - path + 1); + *directory = talloc_strndup (ctx, path, slash - path); } return NOTMUCH_STATUS_SUCCESS; @@ -2162,8 +2196,8 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch, * References header, if available. If not, fall back to the * first message ID in the In-Reply-To header. */ if (last_ref_message_id) { - _notmuch_message_add_term (message, "replyto", - last_ref_message_id); + _notmuch_message_add_term (message, "replyto", + last_ref_message_id); } else if (in_reply_to_message_id) { _notmuch_message_add_term (message, "replyto", in_reply_to_message_id); @@ -2272,15 +2306,15 @@ _consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch, if (stored_id.empty ()) { return NULL; } else { - Xapian::WritableDatabase *db; + Xapian::WritableDatabase *db; db = static_cast (notmuch->xapian_db); /* Clear the metadata for this message ID. We don't need it * anymore. */ - db->set_metadata (metadata_key, ""); + db->set_metadata (metadata_key, ""); - return talloc_strdup (ctx, stored_id.c_str ()); + return talloc_strdup (ctx, stored_id.c_str ()); } }