1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see https://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
22 #include "parse-time-vrp.h"
24 #include "thread-fp.h"
25 #include "regexp-fields.h"
26 #include "string-util.h"
35 #include <glib.h> /* g_free, GPtrArray, GHashTable */
36 #include <glib-object.h> /* g_type_init */
38 #include <gmime/gmime.h> /* g_mime_init */
45 notmuch_field_flag_t flags;
48 #define NOTMUCH_DATABASE_VERSION 3
50 #define STRINGIFY(s) _SUB_STRINGIFY (s)
51 #define _SUB_STRINGIFY(s) #s
53 #if HAVE_XAPIAN_DB_RETRY_LOCK
54 #define DB_ACTION (Xapian::DB_CREATE_OR_OPEN | Xapian::DB_RETRY_LOCK)
56 #define DB_ACTION Xapian::DB_CREATE_OR_OPEN
59 #define LOG_XAPIAN_EXCEPTION(message, error) _log_xapian_exception (__location__, message, error)
62 _log_xapian_exception (const char *where, notmuch_database_t *notmuch, const Xapian::Error error) {
63 _notmuch_database_log (notmuch,
64 "A Xapian exception occurred at %s: %s\n",
66 error.get_msg ().c_str ());
67 notmuch->exception_reported = true;
70 notmuch_database_mode_t
71 _notmuch_database_mode (notmuch_database_t *notmuch)
73 if (notmuch->writable_xapian_db)
74 return NOTMUCH_DATABASE_MODE_READ_WRITE;
76 return NOTMUCH_DATABASE_MODE_READ_ONLY;
79 /* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION):
81 * We currently have three different types of documents (mail, ghost,
82 * and directory) and also some metadata.
84 * There are two kinds of prefixes used in notmuch. There are the
85 * human friendly 'prefix names' like "thread:", which are also used
86 * in the query parser, and the actual prefix terms in the database
87 * (e.g. "G"). The correspondence is maintained in the file scope data
88 * structure 'prefix_table'.
92 * A mail document is associated with a particular email message. It
93 * is stored in one or more files on disk and is uniquely identified
94 * by its "id" field (which is generally the message ID). It is
95 * indexed with the following prefixed terms which the database uses
96 * to construct threads, etc.:
98 * Single terms of given prefix:
102 * id: Unique ID of mail. This is from the Message-ID header
103 * if present and not too long (see NOTMUCH_MESSAGE_ID_MAX).
104 * If it's present and too long, then we use
105 * "notmuch-sha1-<sha1_sum_of_message_id>".
106 * If this header is not present, we use
107 * "notmuch-sha1-<sha1_sum_of_entire_file>".
109 * thread: The ID of the thread to which the mail belongs
111 * replyto: The ID from the In-Reply-To header of the mail (if any).
113 * Multiple terms of given prefix:
115 * reference: All message IDs from In-Reply-To and References
116 * headers in the message.
118 * tag: Any tags associated with this message by the user.
120 * file-direntry: A colon-separated pair of values
121 * (INTEGER:STRING), where INTEGER is the
122 * document ID of a directory document, and
123 * STRING is the name of a file within that
124 * directory for this mail message.
126 * property: Has a property with key=value
127 * FIXME: if no = is present, should match on any value
129 * A mail document also has four values:
131 * TIMESTAMP: The time_t value corresponding to the message's
134 * MESSAGE_ID: The unique ID of the mail mess (see "id" above)
136 * FROM: The value of the "From" header
138 * SUBJECT: The value of the "Subject" header
140 * LAST_MOD: The revision number as of the last tag or
143 * The prefixed terms described above are also searchable without an
144 * explicit field name, but as of notmuch 0.29 this is due to
145 * query-parser setup, not extra terms in the database. In addition,
146 * terms from the content of the message are added without a prefix
147 * for use by the user in searching. Note that the prefix name "body"
148 * is used to refer to the empty prefix string in the database.
150 * The path of the containing folder is added with the "folder" prefix
151 * (see _notmuch_message_add_folder_terms). Sub-paths of the the path
152 * of the mail message are added with the "path" prefix.
154 * The data portion of a mail document is empty.
156 * Ghost mail document [if NOTMUCH_FEATURE_GHOSTS]
157 * -----------------------------------------------
158 * A ghost mail document is like a mail document, but where we don't
159 * have the message content. These are used to track thread reference
160 * information for messages we haven't received.
162 * A ghost mail document has type: ghost; id and thread fields that
163 * are identical to the mail document fields; and a MESSAGE_ID value.
167 * A directory document is used by a client of the notmuch library to
168 * maintain data necessary to allow for efficient polling of mail
171 * All directory documents contain one term:
173 * directory: The directory path (relative to the database path)
174 * Or the SHA1 sum of the directory path (if the
175 * path itself is too long to fit in a Xapian
178 * And all directory documents for directories other than top-level
179 * directories also contain the following term:
181 * directory-direntry: A colon-separated pair of values
182 * (INTEGER:STRING), where INTEGER is the
183 * document ID of the parent directory
184 * document, and STRING is the name of this
185 * directory within that parent.
187 * All directory documents have a single value:
189 * TIMESTAMP: The mtime of the directory (at last scan)
191 * The data portion of a directory document contains the path of the
192 * directory (relative to the database path).
196 * Xapian allows us to store arbitrary name-value pairs as
197 * "metadata". We currently use the following metadata names with the
200 * version The database schema version, (which is distinct
201 * from both the notmuch package version (see
202 * notmuch --version) and the libnotmuch library
203 * version. The version is stored as an base-10
204 * ASCII integer. The initial database version
205 * was 1, (though a schema existed before that
206 * were no "version" database value existed at
207 * all). Successive versions are allocated as
208 * changes are made to the database (such as by
209 * indexing new fields).
211 * features The set of features supported by this
212 * database. This consists of a set of
213 * '\n'-separated lines, where each is a feature
214 * name, a '\t', and compatibility flags. If the
215 * compatibility flags contain 'w', then the
216 * opener must support this feature to safely
217 * write this database. If the compatibility
218 * flags contain 'r', then the opener must
219 * support this feature to read this database.
220 * Introduced in database version 3.
222 * last_thread_id The last thread ID generated. This is stored
223 * as a 16-byte hexadecimal ASCII representation
224 * of a 64-bit unsigned integer. The first ID
225 * generated is 1 and the value will be
226 * incremented for each thread ID.
228 * C* metadata keys starting with C indicate
229 * configuration data. It can be managed with the
230 * n_database_*config* API. There is a convention
231 * of hierarchical keys separated by '.' (e.g.
232 * query.notmuch stores the value for the named
233 * query 'notmuch'), but it is not enforced by the
239 * If ! NOTMUCH_FEATURE_GHOSTS, there are no ghost mail documents.
240 * Instead, the database has the following additional database
243 * thread_id_* A pre-allocated thread ID for a particular
244 * message. This is actually an arbitrarily large
245 * family of metadata name. Any particular name is
246 * formed by concatenating "thread_id_" with a message
247 * ID (or the SHA1 sum of a message ID if it is very
248 * long---see description of 'id' in the mail
249 * document). The value stored is a thread ID.
251 * These thread ID metadata values are stored
252 * whenever a message references a parent message
253 * that does not yet exist in the database. A
254 * thread ID will be allocated and stored, and if
255 * the message is later added, the stored thread
256 * ID will be used (and the metadata value will
259 * Even before a message is added, it's
260 * pre-allocated thread ID is useful so that all
261 * descendant messages that reference this common
262 * parent can be recognized as belonging to the
266 /* With these prefix values we follow the conventions published here:
268 * https://xapian.org/docs/omega/termprefixes.html
270 * as much as makes sense. Note that I took some liberty in matching
271 * the reserved prefix values to notmuch concepts, (for example, 'G'
272 * is documented as "newsGroup (or similar entity - e.g. a web forum
273 * name)", for which I think the thread is the closest analogue in
274 * notmuch. This in spite of the fact that we will eventually be
275 * storing mailing-list messages where 'G' for "mailing list name"
276 * might be even a closer analogue. I'm treating the single-character
277 * prefixes preferentially for core notmuch concepts (which will be
278 * nearly universal to all mail messages).
282 prefix_t prefix_table[] = {
283 /* name term prefix flags */
284 { "type", "T", NOTMUCH_FIELD_NO_FLAGS },
285 { "reference", "XREFERENCE", NOTMUCH_FIELD_NO_FLAGS },
286 { "replyto", "XREPLYTO", NOTMUCH_FIELD_NO_FLAGS },
287 { "directory", "XDIRECTORY", NOTMUCH_FIELD_NO_FLAGS },
288 { "file-direntry", "XFDIRENTRY", NOTMUCH_FIELD_NO_FLAGS },
289 { "directory-direntry", "XDDIRENTRY", NOTMUCH_FIELD_NO_FLAGS },
290 { "body", "", NOTMUCH_FIELD_EXTERNAL |
291 NOTMUCH_FIELD_PROBABILISTIC },
292 { "thread", "G", NOTMUCH_FIELD_EXTERNAL |
293 NOTMUCH_FIELD_PROCESSOR },
294 { "tag", "K", NOTMUCH_FIELD_EXTERNAL |
295 NOTMUCH_FIELD_PROCESSOR },
296 { "is", "K", NOTMUCH_FIELD_EXTERNAL |
297 NOTMUCH_FIELD_PROCESSOR },
298 { "id", "Q", NOTMUCH_FIELD_EXTERNAL },
299 { "mid", "Q", NOTMUCH_FIELD_EXTERNAL |
300 NOTMUCH_FIELD_PROCESSOR },
301 { "path", "P", NOTMUCH_FIELD_EXTERNAL |
302 NOTMUCH_FIELD_PROCESSOR },
303 { "property", "XPROPERTY", NOTMUCH_FIELD_EXTERNAL },
305 * Unconditionally add ':' to reduce potential ambiguity with
306 * overlapping prefixes and/or terms that start with capital
307 * letters. See Xapian document termprefixes.html for related
310 { "folder", "XFOLDER:", NOTMUCH_FIELD_EXTERNAL |
311 NOTMUCH_FIELD_PROCESSOR },
312 { "date", NULL, NOTMUCH_FIELD_EXTERNAL |
313 NOTMUCH_FIELD_PROCESSOR },
314 { "query", NULL, NOTMUCH_FIELD_EXTERNAL |
315 NOTMUCH_FIELD_PROCESSOR },
316 { "from", "XFROM", NOTMUCH_FIELD_EXTERNAL |
317 NOTMUCH_FIELD_PROBABILISTIC |
318 NOTMUCH_FIELD_PROCESSOR },
319 { "to", "XTO", NOTMUCH_FIELD_EXTERNAL |
320 NOTMUCH_FIELD_PROBABILISTIC },
321 { "attachment", "XATTACHMENT", NOTMUCH_FIELD_EXTERNAL |
322 NOTMUCH_FIELD_PROBABILISTIC },
323 { "mimetype", "XMIMETYPE", NOTMUCH_FIELD_EXTERNAL |
324 NOTMUCH_FIELD_PROBABILISTIC },
325 { "subject", "XSUBJECT", NOTMUCH_FIELD_EXTERNAL |
326 NOTMUCH_FIELD_PROBABILISTIC |
327 NOTMUCH_FIELD_PROCESSOR },
331 _setup_query_field_default (const prefix_t *prefix, notmuch_database_t *notmuch)
334 notmuch->query_parser->add_prefix ("", prefix->prefix);
335 if (prefix->flags & NOTMUCH_FIELD_PROBABILISTIC)
336 notmuch->query_parser->add_prefix (prefix->name, prefix->prefix);
338 notmuch->query_parser->add_boolean_prefix (prefix->name, prefix->prefix);
341 notmuch_string_map_iterator_t *
342 _notmuch_database_user_headers (notmuch_database_t *notmuch)
344 return _notmuch_string_map_iterator_create (notmuch->user_header, "", false);
348 _user_prefix (void *ctx, const char *name)
350 return talloc_asprintf (ctx, "XU%s:", name);
353 static notmuch_status_t
354 _setup_user_query_fields (notmuch_database_t *notmuch)
356 notmuch_config_list_t *list;
357 notmuch_status_t status;
359 notmuch->user_prefix = _notmuch_string_map_create (notmuch);
360 if (notmuch->user_prefix == NULL)
361 return NOTMUCH_STATUS_OUT_OF_MEMORY;
363 notmuch->user_header = _notmuch_string_map_create (notmuch);
364 if (notmuch->user_header == NULL)
365 return NOTMUCH_STATUS_OUT_OF_MEMORY;
367 status = notmuch_database_get_config_list (notmuch, CONFIG_HEADER_PREFIX, &list);
371 for (; notmuch_config_list_valid (list); notmuch_config_list_move_to_next (list)) {
373 prefix_t query_field;
375 const char *key = notmuch_config_list_key (list)
376 + sizeof (CONFIG_HEADER_PREFIX) - 1;
378 _notmuch_string_map_append (notmuch->user_prefix,
380 _user_prefix (notmuch, key));
382 _notmuch_string_map_append (notmuch->user_header,
384 notmuch_config_list_value (list));
386 query_field.name = talloc_strdup (notmuch, key);
387 query_field.prefix = _user_prefix (notmuch, key);
388 query_field.flags = NOTMUCH_FIELD_PROBABILISTIC
389 | NOTMUCH_FIELD_EXTERNAL;
391 _setup_query_field_default (&query_field, notmuch);
394 notmuch_config_list_destroy (list);
396 return NOTMUCH_STATUS_SUCCESS;
400 _setup_query_field (const prefix_t *prefix, notmuch_database_t *notmuch)
402 if (prefix->flags & NOTMUCH_FIELD_PROCESSOR) {
403 Xapian::FieldProcessor *fp;
405 if (STRNCMP_LITERAL (prefix->name, "date") == 0)
406 fp = (new DateFieldProcessor(NOTMUCH_VALUE_TIMESTAMP))->release ();
407 else if (STRNCMP_LITERAL(prefix->name, "query") == 0)
408 fp = (new QueryFieldProcessor (*notmuch->query_parser, notmuch))->release ();
409 else if (STRNCMP_LITERAL (prefix->name, "thread") == 0)
410 fp = (new ThreadFieldProcessor (*notmuch->query_parser, notmuch))->release ();
412 fp = (new RegexpFieldProcessor (prefix->name, prefix->flags,
413 *notmuch->query_parser, notmuch))->release ();
415 /* we treat all field-processor fields as boolean in order to get the raw input */
417 notmuch->query_parser->add_prefix ("", prefix->prefix);
418 notmuch->query_parser->add_boolean_prefix (prefix->name, fp);
420 _setup_query_field_default (prefix, notmuch);
425 _find_prefix (const char *name)
429 for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
430 if (strcmp (name, prefix_table[i].name) == 0)
431 return prefix_table[i].prefix;
434 INTERNAL_ERROR ("No prefix exists for '%s'\n", name);
439 /* Like find prefix, but include the possibility of user defined
440 * prefixes specific to this database */
443 _notmuch_database_prefix (notmuch_database_t *notmuch, const char *name)
447 /*XXX TODO: reduce code duplication */
448 for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
449 if (strcmp (name, prefix_table[i].name) == 0)
450 return prefix_table[i].prefix;
453 if (notmuch->user_prefix)
454 return _notmuch_string_map_get (notmuch->user_prefix, name);
460 notmuch_status_to_string (notmuch_status_t status)
463 case NOTMUCH_STATUS_SUCCESS:
464 return "No error occurred";
465 case NOTMUCH_STATUS_OUT_OF_MEMORY:
466 return "Out of memory";
467 case NOTMUCH_STATUS_READ_ONLY_DATABASE:
468 return "Attempt to write to a read-only database";
469 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
470 return "A Xapian exception occurred";
471 case NOTMUCH_STATUS_FILE_ERROR:
472 return "Something went wrong trying to read or write a file";
473 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
474 return "File is not an email";
475 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
476 return "Message ID is identical to a message in database";
477 case NOTMUCH_STATUS_NULL_POINTER:
478 return "Erroneous NULL pointer";
479 case NOTMUCH_STATUS_TAG_TOO_LONG:
480 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
481 case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW:
482 return "Unbalanced number of calls to notmuch_message_freeze/thaw";
483 case NOTMUCH_STATUS_UNBALANCED_ATOMIC:
484 return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic";
485 case NOTMUCH_STATUS_UNSUPPORTED_OPERATION:
486 return "Unsupported operation";
487 case NOTMUCH_STATUS_UPGRADE_REQUIRED:
488 return "Operation requires a database upgrade";
489 case NOTMUCH_STATUS_PATH_ERROR:
490 return "Path supplied is illegal for this function";
491 case NOTMUCH_STATUS_MALFORMED_CRYPTO_PROTOCOL:
492 return "Crypto protocol missing, malformed, or unintelligible";
493 case NOTMUCH_STATUS_FAILED_CRYPTO_CONTEXT_CREATION:
494 return "Crypto engine initialization failure";
495 case NOTMUCH_STATUS_UNKNOWN_CRYPTO_PROTOCOL:
496 return "Unknown crypto protocol";
498 case NOTMUCH_STATUS_LAST_STATUS:
499 return "Unknown error status value";
504 _notmuch_database_log (notmuch_database_t *notmuch,
510 va_start (va_args, format);
512 if (notmuch->status_string)
513 talloc_free (notmuch->status_string);
515 notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
520 _notmuch_database_log_append (notmuch_database_t *notmuch,
526 va_start (va_args, format);
528 if (notmuch->status_string)
529 notmuch->status_string = talloc_vasprintf_append (notmuch->status_string, format, va_args);
531 notmuch->status_string = talloc_vasprintf (notmuch, format, va_args);
537 find_doc_ids_for_term (notmuch_database_t *notmuch,
539 Xapian::PostingIterator *begin,
540 Xapian::PostingIterator *end)
542 *begin = notmuch->xapian_db->postlist_begin (term);
544 *end = notmuch->xapian_db->postlist_end (term);
548 _notmuch_database_find_doc_ids (notmuch_database_t *notmuch,
549 const char *prefix_name,
551 Xapian::PostingIterator *begin,
552 Xapian::PostingIterator *end)
556 term = talloc_asprintf (notmuch, "%s%s",
557 _find_prefix (prefix_name), value);
559 find_doc_ids_for_term (notmuch, term, begin, end);
564 notmuch_private_status_t
565 _notmuch_database_find_unique_doc_id (notmuch_database_t *notmuch,
566 const char *prefix_name,
568 unsigned int *doc_id)
570 Xapian::PostingIterator i, end;
572 _notmuch_database_find_doc_ids (notmuch, prefix_name, value, &i, &end);
576 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
581 #if DEBUG_DATABASE_SANITY
585 INTERNAL_ERROR ("Term %s:%s is not unique as expected.\n",
589 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
592 static Xapian::Document
593 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
595 return notmuch->xapian_db->get_document (doc_id);
598 /* Generate a compressed version of 'message_id' of the form:
600 * notmuch-sha1-<sha1_sum_of_message_id>
603 _notmuch_message_id_compressed (void *ctx, const char *message_id)
605 char *sha1, *compressed;
607 sha1 = _notmuch_sha1_of_string (message_id);
609 compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1);
616 notmuch_database_find_message (notmuch_database_t *notmuch,
617 const char *message_id,
618 notmuch_message_t **message_ret)
620 notmuch_private_status_t status;
623 if (message_ret == NULL)
624 return NOTMUCH_STATUS_NULL_POINTER;
626 if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX)
627 message_id = _notmuch_message_id_compressed (notmuch, message_id);
630 status = _notmuch_database_find_unique_doc_id (notmuch, "id",
631 message_id, &doc_id);
633 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
636 *message_ret = _notmuch_message_create (notmuch, notmuch, doc_id,
638 if (*message_ret == NULL)
639 return NOTMUCH_STATUS_OUT_OF_MEMORY;
642 return NOTMUCH_STATUS_SUCCESS;
643 } catch (const Xapian::Error &error) {
644 _notmuch_database_log (notmuch, "A Xapian exception occurred finding message: %s.\n",
645 error.get_msg ().c_str ());
646 notmuch->exception_reported = true;
648 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
653 notmuch_database_create (const char *path, notmuch_database_t **database)
655 char *status_string = NULL;
656 notmuch_status_t status;
658 status = notmuch_database_create_verbose (path, database,
662 fputs (status_string, stderr);
663 free (status_string);
670 notmuch_database_create_verbose (const char *path,
671 notmuch_database_t **database,
672 char **status_string)
674 notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
675 notmuch_database_t *notmuch = NULL;
676 char *notmuch_path = NULL;
677 char *message = NULL;
682 message = strdup ("Error: Cannot create a database for a NULL path.\n");
683 status = NOTMUCH_STATUS_NULL_POINTER;
687 if (path[0] != '/') {
688 message = strdup ("Error: Database path must be absolute.\n");
689 status = NOTMUCH_STATUS_PATH_ERROR;
693 err = stat (path, &st);
695 IGNORE_RESULT (asprintf (&message, "Error: Cannot create database at %s: %s.\n",
696 path, strerror (errno)));
697 status = NOTMUCH_STATUS_FILE_ERROR;
701 if (! S_ISDIR (st.st_mode)) {
702 IGNORE_RESULT (asprintf (&message, "Error: Cannot create database at %s: "
703 "Not a directory.\n",
705 status = NOTMUCH_STATUS_FILE_ERROR;
709 notmuch_path = talloc_asprintf (NULL, "%s/%s", path, ".notmuch");
711 err = mkdir (notmuch_path, 0755);
714 IGNORE_RESULT (asprintf (&message, "Error: Cannot create directory %s: %s.\n",
715 notmuch_path, strerror (errno)));
716 status = NOTMUCH_STATUS_FILE_ERROR;
720 status = notmuch_database_open_verbose (path,
721 NOTMUCH_DATABASE_MODE_READ_WRITE,
726 /* Upgrade doesn't add these feature to existing databases, but
727 * new databases have them. */
728 notmuch->features |= NOTMUCH_FEATURE_FROM_SUBJECT_ID_VALUES;
729 notmuch->features |= NOTMUCH_FEATURE_INDEXED_MIMETYPES;
730 notmuch->features |= NOTMUCH_FEATURE_UNPREFIX_BODY_ONLY;
732 status = notmuch_database_upgrade (notmuch, NULL, NULL);
734 notmuch_database_close (notmuch);
740 talloc_free (notmuch_path);
744 *status_string = message;
751 talloc_free (notmuch);
756 _notmuch_database_ensure_writable (notmuch_database_t *notmuch)
758 if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY) {
759 _notmuch_database_log (notmuch, "Cannot write to a read-only database.\n");
760 return NOTMUCH_STATUS_READ_ONLY_DATABASE;
763 return NOTMUCH_STATUS_SUCCESS;
766 /* Allocate a revision number for the next change. */
768 _notmuch_database_new_revision (notmuch_database_t *notmuch)
770 unsigned long new_revision = notmuch->revision + 1;
772 /* If we're in an atomic section, hold off on updating the
773 * committed revision number until we commit the atomic section.
775 if (notmuch->atomic_nesting)
776 notmuch->atomic_dirty = true;
778 notmuch->revision = new_revision;
784 notmuch_database_open (const char *path,
785 notmuch_database_mode_t mode,
786 notmuch_database_t **database)
788 char *status_string = NULL;
789 notmuch_status_t status;
791 status = notmuch_database_open_verbose (path, mode, database,
795 fputs (status_string, stderr);
796 free (status_string);
803 notmuch_database_open_verbose (const char *path,
804 notmuch_database_mode_t mode,
805 notmuch_database_t **database,
806 char **status_string)
808 notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
809 void *local = talloc_new (NULL);
810 notmuch_database_t *notmuch = NULL;
811 char *notmuch_path, *xapian_path, *incompat_features;
812 char *message = NULL;
815 unsigned int i, version;
816 static int initialized = 0;
819 message = strdup ("Error: Cannot open a database for a NULL path.\n");
820 status = NOTMUCH_STATUS_NULL_POINTER;
824 if (path[0] != '/') {
825 message = strdup ("Error: Database path must be absolute.\n");
826 status = NOTMUCH_STATUS_PATH_ERROR;
830 if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
831 message = strdup ("Out of memory\n");
832 status = NOTMUCH_STATUS_OUT_OF_MEMORY;
836 err = stat (notmuch_path, &st);
838 IGNORE_RESULT (asprintf (&message, "Error opening database at %s: %s\n",
839 notmuch_path, strerror (errno)));
840 status = NOTMUCH_STATUS_FILE_ERROR;
844 if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) {
845 message = strdup ("Out of memory\n");
846 status = NOTMUCH_STATUS_OUT_OF_MEMORY;
850 /* Initialize the GLib type system and threads */
851 #if ! GLIB_CHECK_VERSION (2, 35, 1)
855 /* Initialize gmime */
861 notmuch = talloc_zero (NULL, notmuch_database_t);
862 notmuch->exception_reported = false;
863 notmuch->status_string = NULL;
864 notmuch->path = talloc_strdup (notmuch, path);
866 strip_trailing (notmuch->path, '/');
868 notmuch->writable_xapian_db = NULL;
869 notmuch->atomic_nesting = 0;
872 string last_thread_id;
875 if (mode == NOTMUCH_DATABASE_MODE_READ_WRITE) {
876 notmuch->writable_xapian_db = new Xapian::WritableDatabase (xapian_path,
878 notmuch->xapian_db = notmuch->writable_xapian_db;
880 notmuch->xapian_db = new Xapian::Database (xapian_path);
883 /* Check version. As of database version 3, we represent
884 * changes in terms of features, so assume a version bump
885 * means a dramatically incompatible change. */
886 version = notmuch_database_get_version (notmuch);
887 if (version > NOTMUCH_DATABASE_VERSION) {
888 IGNORE_RESULT (asprintf (&message,
889 "Error: Notmuch database at %s\n"
890 " has a newer database format version (%u) than supported by this\n"
891 " version of notmuch (%u).\n",
892 notmuch_path, version, NOTMUCH_DATABASE_VERSION));
893 notmuch_database_destroy (notmuch);
895 status = NOTMUCH_STATUS_FILE_ERROR;
899 /* Check features. */
900 incompat_features = NULL;
901 notmuch->features = _notmuch_database_parse_features (
902 local, notmuch->xapian_db->get_metadata ("features").c_str (),
903 version, mode == NOTMUCH_DATABASE_MODE_READ_WRITE ? 'w' : 'r',
905 if (incompat_features) {
906 IGNORE_RESULT (asprintf (&message,
907 "Error: Notmuch database at %s\n"
908 " requires features (%s)\n"
909 " not supported by this version of notmuch.\n",
910 notmuch_path, incompat_features));
911 notmuch_database_destroy (notmuch);
913 status = NOTMUCH_STATUS_FILE_ERROR;
917 notmuch->last_doc_id = notmuch->xapian_db->get_lastdocid ();
918 last_thread_id = notmuch->xapian_db->get_metadata ("last_thread_id");
919 if (last_thread_id.empty ()) {
920 notmuch->last_thread_id = 0;
925 str = last_thread_id.c_str ();
926 notmuch->last_thread_id = strtoull (str, &end, 16);
928 INTERNAL_ERROR ("Malformed database last_thread_id: %s", str);
931 /* Get current highest revision number. */
932 last_mod = notmuch->xapian_db->get_value_upper_bound (
933 NOTMUCH_VALUE_LAST_MOD);
934 if (last_mod.empty ())
935 notmuch->revision = 0;
937 notmuch->revision = Xapian::sortable_unserialise (last_mod);
938 notmuch->uuid = talloc_strdup (
939 notmuch, notmuch->xapian_db->get_uuid ().c_str ());
941 notmuch->query_parser = new Xapian::QueryParser;
942 notmuch->term_gen = new Xapian::TermGenerator;
943 notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
944 notmuch->value_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_TIMESTAMP);
945 notmuch->date_range_processor = new ParseTimeRangeProcessor (NOTMUCH_VALUE_TIMESTAMP, "date:");
946 notmuch->last_mod_range_processor = new Xapian::NumberRangeProcessor (NOTMUCH_VALUE_LAST_MOD, "lastmod:");
947 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
948 notmuch->query_parser->set_database (*notmuch->xapian_db);
949 notmuch->query_parser->set_stemmer (Xapian::Stem ("english"));
950 notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);
951 notmuch->query_parser->add_rangeprocessor (notmuch->value_range_processor);
952 notmuch->query_parser->add_rangeprocessor (notmuch->date_range_processor);
953 notmuch->query_parser->add_rangeprocessor (notmuch->last_mod_range_processor);
955 for (i = 0; i < ARRAY_SIZE (prefix_table); i++) {
956 const prefix_t *prefix = &prefix_table[i];
957 if (prefix->flags & NOTMUCH_FIELD_EXTERNAL) {
958 _setup_query_field (prefix, notmuch);
961 status = _setup_user_query_fields (notmuch);
962 } catch (const Xapian::Error &error) {
963 IGNORE_RESULT (asprintf (&message, "A Xapian exception occurred opening database: %s\n",
964 error.get_msg ().c_str ()));
965 notmuch_database_destroy (notmuch);
967 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
975 *status_string = message;
983 talloc_free (notmuch);
986 notmuch->open = true;
992 notmuch_database_close (notmuch_database_t *notmuch)
994 notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
996 /* Many Xapian objects (and thus notmuch objects) hold references to
997 * the database, so merely deleting the database may not suffice to
998 * close it. Thus, we explicitly close it here. */
1001 /* If there's an outstanding transaction, it's unclear if
1002 * closing the Xapian database commits everything up to
1003 * that transaction, or may discard committed (but
1004 * unflushed) transactions. To be certain, explicitly
1005 * cancel any outstanding transaction before closing. */
1006 if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_WRITE &&
1007 notmuch->atomic_nesting)
1008 notmuch->writable_xapian_db->cancel_transaction ();
1010 /* Close the database. This implicitly flushes
1011 * outstanding changes. */
1012 notmuch->xapian_db->close ();
1013 } catch (const Xapian::Error &error) {
1014 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1015 if (! notmuch->exception_reported) {
1016 _notmuch_database_log (notmuch, "Error: A Xapian exception occurred closing database: %s\n",
1017 error.get_msg ().c_str ());
1021 notmuch->open = false;
1026 _notmuch_database_reopen (notmuch_database_t *notmuch)
1028 if (_notmuch_database_mode (notmuch) != NOTMUCH_DATABASE_MODE_READ_ONLY)
1029 return NOTMUCH_STATUS_UNSUPPORTED_OPERATION;
1032 notmuch->xapian_db->reopen ();
1033 } catch (const Xapian::Error &error) {
1034 if (! notmuch->exception_reported) {
1035 _notmuch_database_log (notmuch, "Error: A Xapian exception reopening database: %s\n",
1036 error.get_msg ().c_str ());
1037 notmuch->exception_reported = true;
1039 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1044 return NOTMUCH_STATUS_SUCCESS;
1048 unlink_cb (const char *path,
1049 unused (const struct stat *sb),
1051 unused (struct FTW *ftw))
1053 return remove (path);
1057 rmtree (const char *path)
1059 return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
1062 class NotmuchCompactor : public Xapian::Compactor
1064 notmuch_compact_status_cb_t status_cb;
1065 void *status_closure;
1068 NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) :
1069 status_cb (cb), status_closure (closure)
1074 set_status (const std::string &table, const std::string &status)
1078 if (status_cb == NULL)
1081 if (status.length () == 0)
1082 msg = talloc_asprintf (NULL, "compacting table %s", table.c_str ());
1084 msg = talloc_asprintf (NULL, " %s", status.c_str ());
1090 status_cb (msg, status_closure);
1095 /* Compacts the given database, optionally saving the original database
1096 * in backup_path. Additionally, a callback function can be provided to
1097 * give the user feedback on the progress of the (likely long-lived)
1098 * compaction process.
1100 * The backup path must point to a directory on the same volume as the
1101 * original database. Passing a NULL backup_path will result in the
1102 * uncompacted database being deleted after compaction has finished.
1103 * Note that the database write lock will be held during the
1104 * compaction process to protect data integrity.
1107 notmuch_database_compact (const char *path,
1108 const char *backup_path,
1109 notmuch_compact_status_cb_t status_cb,
1113 char *notmuch_path, *xapian_path, *compact_xapian_path;
1114 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
1115 notmuch_database_t *notmuch = NULL;
1116 struct stat statbuf;
1118 char *message = NULL;
1120 local = talloc_new (NULL);
1122 return NOTMUCH_STATUS_OUT_OF_MEMORY;
1124 ret = notmuch_database_open_verbose (path,
1125 NOTMUCH_DATABASE_MODE_READ_WRITE,
1129 if (status_cb) status_cb (message, closure);
1133 if (! (notmuch_path = talloc_asprintf (local, "%s/%s", path, ".notmuch"))) {
1134 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1138 if (! (xapian_path = talloc_asprintf (local, "%s/%s", notmuch_path, "xapian"))) {
1139 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1143 if (! (compact_xapian_path = talloc_asprintf (local, "%s.compact", xapian_path))) {
1144 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1148 if (backup_path == NULL) {
1149 if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) {
1150 ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
1153 keep_backup = false;
1158 if (stat (backup_path, &statbuf) != -1) {
1159 _notmuch_database_log (notmuch, "Path already exists: %s\n", backup_path);
1160 ret = NOTMUCH_STATUS_FILE_ERROR;
1163 if (errno != ENOENT) {
1164 _notmuch_database_log (notmuch, "Unknown error while stat()ing path: %s\n",
1166 ret = NOTMUCH_STATUS_FILE_ERROR;
1170 /* Unconditionally attempt to remove old work-in-progress database (if
1171 * any). This is "protected" by database lock. If this fails due to write
1172 * errors (etc), the following code will fail and provide error message.
1174 (void) rmtree (compact_xapian_path);
1177 NotmuchCompactor compactor (status_cb, closure);
1178 notmuch->xapian_db->compact (compact_xapian_path, Xapian::DBCOMPACT_NO_RENUMBER, 0, compactor);
1179 } catch (const Xapian::Error &error) {
1180 _notmuch_database_log (notmuch, "Error while compacting: %s\n", error.get_msg ().c_str ());
1181 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1185 if (rename (xapian_path, backup_path)) {
1186 _notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
1187 xapian_path, backup_path, strerror (errno));
1188 ret = NOTMUCH_STATUS_FILE_ERROR;
1192 if (rename (compact_xapian_path, xapian_path)) {
1193 _notmuch_database_log (notmuch, "Error moving %s to %s: %s\n",
1194 compact_xapian_path, xapian_path, strerror (errno));
1195 ret = NOTMUCH_STATUS_FILE_ERROR;
1199 if (! keep_backup) {
1200 if (rmtree (backup_path)) {
1201 _notmuch_database_log (notmuch, "Error removing old database %s: %s\n",
1202 backup_path, strerror (errno));
1203 ret = NOTMUCH_STATUS_FILE_ERROR;
1210 notmuch_status_t ret2;
1212 const char *str = notmuch_database_status_string (notmuch);
1213 if (status_cb && str)
1214 status_cb (str, closure);
1216 ret2 = notmuch_database_destroy (notmuch);
1218 /* don't clobber previous error status */
1219 if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS)
1223 talloc_free (local);
1229 notmuch_database_destroy (notmuch_database_t *notmuch)
1231 notmuch_status_t status;
1233 status = notmuch_database_close (notmuch);
1235 delete notmuch->term_gen;
1236 notmuch->term_gen = NULL;
1237 delete notmuch->query_parser;
1238 notmuch->query_parser = NULL;
1239 delete notmuch->xapian_db;
1240 notmuch->xapian_db = NULL;
1241 delete notmuch->value_range_processor;
1242 notmuch->value_range_processor = NULL;
1243 delete notmuch->date_range_processor;
1244 notmuch->date_range_processor = NULL;
1245 delete notmuch->last_mod_range_processor;
1246 notmuch->last_mod_range_processor = NULL;
1248 talloc_free (notmuch);
1254 notmuch_database_get_path (notmuch_database_t *notmuch)
1256 return notmuch->path;
1260 notmuch_database_get_version (notmuch_database_t *notmuch)
1262 unsigned int version;
1263 string version_string;
1268 version_string = notmuch->xapian_db->get_metadata ("version");
1269 } catch (const Xapian::Error &error) {
1270 LOG_XAPIAN_EXCEPTION (notmuch, error);
1274 if (version_string.empty ())
1277 str = version_string.c_str ();
1278 if (str == NULL || *str == '\0')
1281 version = strtoul (str, &end, 10);
1283 INTERNAL_ERROR ("Malformed database version: %s", str);
1289 notmuch_database_needs_upgrade (notmuch_database_t *notmuch)
1291 unsigned int version;
1293 if (_notmuch_database_mode (notmuch) != NOTMUCH_DATABASE_MODE_READ_WRITE)
1296 if (NOTMUCH_FEATURES_CURRENT & ~notmuch->features)
1299 version = notmuch_database_get_version (notmuch);
1301 return (version > 0 && version < NOTMUCH_DATABASE_VERSION);
1304 static volatile sig_atomic_t do_progress_notify = 0;
1307 handle_sigalrm (unused (int signal))
1309 do_progress_notify = 1;
1312 /* Upgrade the current database.
1314 * After opening a database in read-write mode, the client should
1315 * check if an upgrade is needed (notmuch_database_needs_upgrade) and
1316 * if so, upgrade with this function before making any modifications.
1318 * The optional progress_notify callback can be used by the caller to
1319 * provide progress indication to the user. If non-NULL it will be
1320 * called periodically with 'count' as the number of messages upgraded
1321 * so far and 'total' the overall number of messages that will be
1325 notmuch_database_upgrade (notmuch_database_t *notmuch,
1326 void (*progress_notify) (void *closure,
1330 void *local = talloc_new (NULL);
1331 Xapian::TermIterator t, t_end;
1332 Xapian::WritableDatabase *db;
1333 struct sigaction action;
1334 struct itimerval timerval;
1335 bool timer_is_active = false;
1336 enum _notmuch_features target_features, new_features;
1337 notmuch_status_t status;
1338 notmuch_private_status_t private_status;
1339 notmuch_query_t *query = NULL;
1340 unsigned int count = 0, total = 0;
1342 status = _notmuch_database_ensure_writable (notmuch);
1346 db = notmuch->writable_xapian_db;
1348 target_features = notmuch->features | NOTMUCH_FEATURES_CURRENT;
1349 new_features = NOTMUCH_FEATURES_CURRENT & ~notmuch->features;
1351 if (! notmuch_database_needs_upgrade (notmuch))
1352 return NOTMUCH_STATUS_SUCCESS;
1354 if (progress_notify) {
1355 /* Set up our handler for SIGALRM */
1356 memset (&action, 0, sizeof (struct sigaction));
1357 action.sa_handler = handle_sigalrm;
1358 sigemptyset (&action.sa_mask);
1359 action.sa_flags = SA_RESTART;
1360 sigaction (SIGALRM, &action, NULL);
1362 /* Then start a timer to send SIGALRM once per second. */
1363 timerval.it_interval.tv_sec = 1;
1364 timerval.it_interval.tv_usec = 0;
1365 timerval.it_value.tv_sec = 1;
1366 timerval.it_value.tv_usec = 0;
1367 setitimer (ITIMER_REAL, &timerval, NULL);
1369 timer_is_active = true;
1372 /* Figure out how much total work we need to do. */
1374 (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
1375 NOTMUCH_FEATURE_LAST_MOD)) {
1376 query = notmuch_query_create (notmuch, "");
1379 status = notmuch_query_count_messages (query, &msg_count);
1384 notmuch_query_destroy (query);
1387 if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
1388 t_end = db->allterms_end ("XTIMESTAMP");
1389 for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++)
1392 if (new_features & NOTMUCH_FEATURE_GHOSTS) {
1393 /* The ghost message upgrade converts all thread_id_*
1394 * metadata values into ghost message documents. */
1395 t_end = db->metadata_keys_end ("thread_id_");
1396 for (t = db->metadata_keys_begin ("thread_id_"); t != t_end; ++t)
1400 /* Perform the upgrade in a transaction. */
1401 db->begin_transaction (true);
1403 /* Set the target features so we write out changes in the desired
1405 notmuch->features = target_features;
1407 /* Perform per-message upgrades. */
1409 (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER |
1410 NOTMUCH_FEATURE_LAST_MOD)) {
1411 notmuch_messages_t *messages;
1412 notmuch_message_t *message;
1415 query = notmuch_query_create (notmuch, "");
1417 status = notmuch_query_search_messages (query, &messages);
1421 notmuch_messages_valid (messages);
1422 notmuch_messages_move_to_next (messages)) {
1423 if (do_progress_notify) {
1424 progress_notify (closure, (double) count / total);
1425 do_progress_notify = 0;
1428 message = notmuch_messages_get (messages);
1430 /* Before version 1, each message document had its
1431 * filename in the data field. Copy that into the new
1432 * format by calling notmuch_message_add_filename.
1434 if (new_features & NOTMUCH_FEATURE_FILE_TERMS) {
1435 filename = _notmuch_message_talloc_copy_data (message);
1436 if (filename && *filename != '\0') {
1437 _notmuch_message_add_filename (message, filename);
1438 _notmuch_message_clear_data (message);
1440 talloc_free (filename);
1443 /* Prior to version 2, the "folder:" prefix was
1444 * probabilistic and stemmed. Change it to the current
1445 * boolean prefix. Add "path:" prefixes while at it.
1447 if (new_features & NOTMUCH_FEATURE_BOOL_FOLDER)
1448 _notmuch_message_upgrade_folder (message);
1450 /* Prior to NOTMUCH_FEATURE_LAST_MOD, messages did not
1451 * track modification revisions. Give all messages the
1452 * next available revision; since we just started tracking
1453 * revisions for this database, that will be 1.
1455 if (new_features & NOTMUCH_FEATURE_LAST_MOD)
1456 _notmuch_message_upgrade_last_mod (message);
1458 _notmuch_message_sync (message);
1460 notmuch_message_destroy (message);
1465 notmuch_query_destroy (query);
1469 /* Perform per-directory upgrades. */
1471 /* Before version 1 we stored directory timestamps in
1472 * XTIMESTAMP documents instead of the current XDIRECTORY
1473 * documents. So copy those as well. */
1474 if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) {
1475 t_end = notmuch->xapian_db->allterms_end ("XTIMESTAMP");
1477 for (t = notmuch->xapian_db->allterms_begin ("XTIMESTAMP");
1480 Xapian::PostingIterator p, p_end;
1481 std::string term = *t;
1483 p_end = notmuch->xapian_db->postlist_end (term);
1485 for (p = notmuch->xapian_db->postlist_begin (term);
1488 Xapian::Document document;
1490 notmuch_directory_t *directory;
1492 if (do_progress_notify) {
1493 progress_notify (closure, (double) count / total);
1494 do_progress_notify = 0;
1497 document = find_document_for_doc_id (notmuch, *p);
1498 mtime = Xapian::sortable_unserialise (
1499 document.get_value (NOTMUCH_VALUE_TIMESTAMP));
1501 directory = _notmuch_directory_find_or_create (notmuch, term.c_str () + 10,
1502 NOTMUCH_FIND_CREATE, &status);
1503 notmuch_directory_set_mtime (directory, mtime);
1504 notmuch_directory_destroy (directory);
1506 db->delete_document (*p);
1513 /* Perform metadata upgrades. */
1515 /* Prior to NOTMUCH_FEATURE_GHOSTS, thread IDs for missing
1516 * messages were stored as database metadata. Change these to
1519 if (new_features & NOTMUCH_FEATURE_GHOSTS) {
1520 notmuch_message_t *message;
1521 std::string message_id, thread_id;
1523 t_end = db->metadata_keys_end (NOTMUCH_METADATA_THREAD_ID_PREFIX);
1524 for (t = db->metadata_keys_begin (NOTMUCH_METADATA_THREAD_ID_PREFIX);
1526 if (do_progress_notify) {
1527 progress_notify (closure, (double) count / total);
1528 do_progress_notify = 0;
1531 message_id = (*t).substr (
1532 strlen (NOTMUCH_METADATA_THREAD_ID_PREFIX));
1533 thread_id = db->get_metadata (*t);
1535 /* Create ghost message */
1536 message = _notmuch_message_create_for_message_id (
1537 notmuch, message_id.c_str (), &private_status);
1538 if (private_status == NOTMUCH_PRIVATE_STATUS_SUCCESS) {
1539 /* Document already exists; ignore the stored thread ID */
1540 } else if (private_status ==
1541 NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
1542 private_status = _notmuch_message_initialize_ghost (
1543 message, thread_id.c_str ());
1544 if (! private_status)
1545 _notmuch_message_sync (message);
1548 if (private_status) {
1549 _notmuch_database_log (notmuch,
1550 "Upgrade failed while creating ghost messages.\n");
1551 status = COERCE_STATUS (private_status, "Unexpected status from _notmuch_message_initialize_ghost");
1555 /* Clear saved metadata thread ID */
1556 db->set_metadata (*t, "");
1562 status = NOTMUCH_STATUS_SUCCESS;
1563 db->set_metadata ("features", _notmuch_database_print_features (local, notmuch->features));
1564 db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
1567 if (status == NOTMUCH_STATUS_SUCCESS)
1568 db->commit_transaction ();
1570 db->cancel_transaction ();
1572 if (timer_is_active) {
1573 /* Now stop the timer. */
1574 timerval.it_interval.tv_sec = 0;
1575 timerval.it_interval.tv_usec = 0;
1576 timerval.it_value.tv_sec = 0;
1577 timerval.it_value.tv_usec = 0;
1578 setitimer (ITIMER_REAL, &timerval, NULL);
1580 /* And disable the signal handler. */
1581 action.sa_handler = SIG_IGN;
1582 sigaction (SIGALRM, &action, NULL);
1586 notmuch_query_destroy (query);
1588 talloc_free (local);
1593 notmuch_database_begin_atomic (notmuch_database_t *notmuch)
1595 if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY ||
1596 notmuch->atomic_nesting > 0)
1599 if (notmuch_database_needs_upgrade (notmuch))
1600 return NOTMUCH_STATUS_UPGRADE_REQUIRED;
1603 notmuch->writable_xapian_db->begin_transaction (false);
1604 } catch (const Xapian::Error &error) {
1605 _notmuch_database_log (notmuch, "A Xapian exception occurred beginning transaction: %s.\n",
1606 error.get_msg ().c_str ());
1607 notmuch->exception_reported = true;
1608 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1612 notmuch->atomic_nesting++;
1613 return NOTMUCH_STATUS_SUCCESS;
1617 notmuch_database_end_atomic (notmuch_database_t *notmuch)
1619 Xapian::WritableDatabase *db;
1621 if (notmuch->atomic_nesting == 0)
1622 return NOTMUCH_STATUS_UNBALANCED_ATOMIC;
1624 if (_notmuch_database_mode (notmuch) == NOTMUCH_DATABASE_MODE_READ_ONLY ||
1625 notmuch->atomic_nesting > 1)
1628 db = notmuch->writable_xapian_db;
1630 db->commit_transaction ();
1632 /* This is a hack for testing. Xapian never flushes on a
1633 * non-flushed commit, even if the flush threshold is 1.
1634 * However, we rely on flushing to test atomicity. */
1635 const char *thresh = getenv ("XAPIAN_FLUSH_THRESHOLD");
1636 if (thresh && atoi (thresh) == 1)
1638 } catch (const Xapian::Error &error) {
1639 _notmuch_database_log (notmuch, "A Xapian exception occurred committing transaction: %s.\n",
1640 error.get_msg ().c_str ());
1641 notmuch->exception_reported = true;
1642 return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1645 if (notmuch->atomic_dirty) {
1646 ++notmuch->revision;
1647 notmuch->atomic_dirty = false;
1651 notmuch->atomic_nesting--;
1652 return NOTMUCH_STATUS_SUCCESS;
1656 notmuch_database_get_revision (notmuch_database_t *notmuch,
1660 *uuid = notmuch->uuid;
1661 return notmuch->revision;
1664 /* We allow the user to use arbitrarily long paths for directories. But
1665 * we have a term-length limit. So if we exceed that, we'll use the
1666 * SHA-1 of the path for the database term.
1668 * Note: This function may return the original value of 'path'. If it
1669 * does not, then the caller is responsible to free() the returned
1673 _notmuch_database_get_directory_db_path (const char *path)
1675 int term_len = strlen (_find_prefix ("directory")) + strlen (path);
1677 if (term_len > NOTMUCH_TERM_MAX)
1678 return _notmuch_sha1_of_string (path);
1683 /* Given a path, split it into two parts: the directory part is all
1684 * components except for the last, and the basename is that last
1685 * component. Getting the return-value for either part is optional
1686 * (the caller can pass NULL).
1688 * The original 'path' can represent either a regular file or a
1689 * directory---the splitting will be carried out in the same way in
1690 * either case. Trailing slashes on 'path' will be ignored, and any
1691 * cases of multiple '/' characters appearing in series will be
1692 * treated as a single '/'.
1694 * Allocation (if any) will have 'ctx' as the talloc owner. But
1695 * pointers will be returned within the original path string whenever
1698 * Note: If 'path' is non-empty and contains no non-trailing slash,
1699 * (that is, consists of a filename with no parent directory), then
1700 * the directory returned will be an empty string. However, if 'path'
1701 * is an empty string, then both directory and basename will be
1705 _notmuch_database_split_path (void *ctx,
1707 const char **directory,
1708 const char **basename)
1712 if (path == NULL || *path == '\0') {
1717 return NOTMUCH_STATUS_SUCCESS;
1720 /* Find the last slash (not counting a trailing slash), if any. */
1722 slash = path + strlen (path) - 1;
1724 /* First, skip trailing slashes. */
1725 while (slash != path && *slash == '/')
1728 /* Then, find a slash. */
1729 while (slash != path && *slash != '/') {
1736 /* Finally, skip multiple slashes. */
1737 while (slash != path && *(slash - 1) == '/')
1740 if (slash == path) {
1742 *directory = talloc_strdup (ctx, "");
1747 *directory = talloc_strndup (ctx, path, slash - path);
1750 return NOTMUCH_STATUS_SUCCESS;
1753 /* Find the document ID of the specified directory.
1755 * If (flags & NOTMUCH_FIND_CREATE), a new directory document will be
1756 * created if one does not exist for 'path'. Otherwise, if the
1757 * directory document does not exist, this sets *directory_id to
1758 * ((unsigned int)-1) and returns NOTMUCH_STATUS_SUCCESS.
1761 _notmuch_database_find_directory_id (notmuch_database_t *notmuch,
1763 notmuch_find_flags_t flags,
1764 unsigned int *directory_id)
1766 notmuch_directory_t *directory;
1767 notmuch_status_t status;
1771 return NOTMUCH_STATUS_SUCCESS;
1774 directory = _notmuch_directory_find_or_create (notmuch, path, flags, &status);
1775 if (status || ! directory) {
1780 *directory_id = _notmuch_directory_get_document_id (directory);
1782 notmuch_directory_destroy (directory);
1784 return NOTMUCH_STATUS_SUCCESS;
1788 _notmuch_database_get_directory_path (void *ctx,
1789 notmuch_database_t *notmuch,
1790 unsigned int doc_id)
1792 Xapian::Document document;
1794 document = find_document_for_doc_id (notmuch, doc_id);
1796 return talloc_strdup (ctx, document.get_data ().c_str ());
1799 /* Given a legal 'filename' for the database, (either relative to
1800 * database path or absolute with initial components identical to
1801 * database path), return a new string (with 'ctx' as the talloc
1802 * owner) suitable for use as a direntry term value.
1804 * If (flags & NOTMUCH_FIND_CREATE), the necessary directory documents
1805 * will be created in the database as needed. Otherwise, if the
1806 * necessary directory documents do not exist, this sets
1807 * *direntry to NULL and returns NOTMUCH_STATUS_SUCCESS.
1810 _notmuch_database_filename_to_direntry (void *ctx,
1811 notmuch_database_t *notmuch,
1812 const char *filename,
1813 notmuch_find_flags_t flags,
1816 const char *relative, *directory, *basename;
1817 Xapian::docid directory_id;
1818 notmuch_status_t status;
1820 relative = _notmuch_database_relative_path (notmuch, filename);
1822 status = _notmuch_database_split_path (ctx, relative,
1823 &directory, &basename);
1827 status = _notmuch_database_find_directory_id (notmuch, directory, flags,
1829 if (status || directory_id == (unsigned int) -1) {
1834 *direntry = talloc_asprintf (ctx, "%u:%s", directory_id, basename);
1836 return NOTMUCH_STATUS_SUCCESS;
1839 /* Given a legal 'path' for the database, return the relative path.
1841 * The return value will be a pointer to the original path contents,
1842 * and will be either the original string (if 'path' was relative) or
1843 * a portion of the string (if path was absolute and begins with the
1847 _notmuch_database_relative_path (notmuch_database_t *notmuch,
1850 const char *db_path, *relative;
1851 unsigned int db_path_len;
1853 db_path = notmuch_database_get_path (notmuch);
1854 db_path_len = strlen (db_path);
1858 if (*relative == '/') {
1859 while (*relative == '/' && *(relative + 1) == '/')
1862 if (strncmp (relative, db_path, db_path_len) == 0) {
1863 relative += db_path_len;
1864 while (*relative == '/')
1873 notmuch_database_get_directory (notmuch_database_t *notmuch,
1875 notmuch_directory_t **directory)
1877 notmuch_status_t status;
1879 if (directory == NULL)
1880 return NOTMUCH_STATUS_NULL_POINTER;
1884 *directory = _notmuch_directory_find_or_create (notmuch, path,
1885 NOTMUCH_FIND_LOOKUP, &status);
1886 } catch (const Xapian::Error &error) {
1887 _notmuch_database_log (notmuch, "A Xapian exception occurred getting directory: %s.\n",
1888 error.get_msg ().c_str ());
1889 notmuch->exception_reported = true;
1890 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1895 /* Allocate a document ID that satisfies the following criteria:
1897 * 1. The ID does not exist for any document in the Xapian database
1899 * 2. The ID was not previously returned from this function
1901 * 3. The ID is the smallest integer satisfying (1) and (2)
1903 * This function will trigger an internal error if these constraints
1904 * cannot all be satisfied, (that is, the pool of available document
1905 * IDs has been exhausted).
1908 _notmuch_database_generate_doc_id (notmuch_database_t *notmuch)
1910 assert (notmuch->last_doc_id >= notmuch->xapian_db->get_lastdocid ());
1912 notmuch->last_doc_id++;
1914 if (notmuch->last_doc_id == 0)
1915 INTERNAL_ERROR ("Xapian document IDs are exhausted.\n");
1917 return notmuch->last_doc_id;
1921 notmuch_database_remove_message (notmuch_database_t *notmuch,
1922 const char *filename)
1924 notmuch_status_t status;
1925 notmuch_message_t *message;
1927 status = notmuch_database_find_message_by_filename (notmuch, filename,
1930 if (status == NOTMUCH_STATUS_SUCCESS && message) {
1931 status = _notmuch_message_remove_filename (message, filename);
1932 if (status == NOTMUCH_STATUS_SUCCESS)
1933 _notmuch_message_delete (message);
1934 else if (status == NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID)
1935 _notmuch_message_sync (message);
1937 notmuch_message_destroy (message);
1944 notmuch_database_find_message_by_filename (notmuch_database_t *notmuch,
1945 const char *filename,
1946 notmuch_message_t **message_ret)
1949 const char *prefix = _find_prefix ("file-direntry");
1950 char *direntry, *term;
1951 Xapian::PostingIterator i, end;
1952 notmuch_status_t status;
1954 if (message_ret == NULL)
1955 return NOTMUCH_STATUS_NULL_POINTER;
1957 if (! (notmuch->features & NOTMUCH_FEATURE_FILE_TERMS))
1958 return NOTMUCH_STATUS_UPGRADE_REQUIRED;
1960 /* return NULL on any failure */
1961 *message_ret = NULL;
1963 local = talloc_new (notmuch);
1966 status = _notmuch_database_filename_to_direntry (
1967 local, notmuch, filename, NOTMUCH_FIND_LOOKUP, &direntry);
1968 if (status || ! direntry)
1971 term = talloc_asprintf (local, "%s%s", prefix, direntry);
1973 find_doc_ids_for_term (notmuch, term, &i, &end);
1976 notmuch_private_status_t private_status;
1978 *message_ret = _notmuch_message_create (notmuch, notmuch, *i,
1980 if (*message_ret == NULL)
1981 status = NOTMUCH_STATUS_OUT_OF_MEMORY;
1983 } catch (const Xapian::Error &error) {
1984 _notmuch_database_log (notmuch, "Error: A Xapian exception occurred finding message by filename: %s\n",
1985 error.get_msg ().c_str ());
1986 notmuch->exception_reported = true;
1987 status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
1991 talloc_free (local);
1993 if (status && *message_ret) {
1994 notmuch_message_destroy (*message_ret);
1995 *message_ret = NULL;
2000 notmuch_string_list_t *
2001 _notmuch_database_get_terms_with_prefix (void *ctx, Xapian::TermIterator &i,
2002 Xapian::TermIterator &end,
2005 int prefix_len = strlen (prefix);
2006 notmuch_string_list_t *list;
2008 list = _notmuch_string_list_create (ctx);
2009 if (unlikely (list == NULL))
2012 for (i.skip_to (prefix); i != end; i++) {
2013 /* Terminate loop at first term without desired prefix. */
2014 if (strncmp ((*i).c_str (), prefix, prefix_len))
2017 _notmuch_string_list_append (list, (*i).c_str () + prefix_len);
2024 notmuch_database_get_all_tags (notmuch_database_t *db)
2026 Xapian::TermIterator i, end;
2027 notmuch_string_list_t *tags;
2030 i = db->xapian_db->allterms_begin ();
2031 end = db->xapian_db->allterms_end ();
2032 tags = _notmuch_database_get_terms_with_prefix (db, i, end,
2033 _find_prefix ("tag"));
2034 _notmuch_string_list_sort (tags);
2035 return _notmuch_tags_create (db, tags);
2036 } catch (const Xapian::Error &error) {
2037 _notmuch_database_log (db, "A Xapian exception occurred getting tags: %s.\n",
2038 error.get_msg ().c_str ());
2039 db->exception_reported = true;
2045 notmuch_database_status_string (const notmuch_database_t *notmuch)
2047 return notmuch->status_string;