1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
31 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
33 /* These prefix values are specifically chosen to be compatible
34 * with sup, (http://sup.rubyforge.org), written by
35 * William Morgan <wmorgan-sup@masanjin.net>, and released
36 * under the GNU GPL v2.
44 prefix_t BOOLEAN_PREFIX[] = {
50 { "timestamp", "KTS" },
54 _find_prefix (const char *name)
58 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++)
59 if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0)
60 return BOOLEAN_PREFIX[i].prefix;
62 fprintf (stderr, "Internal error: No prefix exists for '%s'\n", name);
69 notmuch_status_to_string (notmuch_status_t status)
72 case NOTMUCH_STATUS_SUCCESS:
73 return "No error occurred";
74 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
75 return "A Xapian exception occurred";
76 case NOTMUCH_STATUS_FILE_ERROR:
77 return "Something went wrong trying to read or write a file";
78 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
79 return "File is not an email";
80 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
81 return "Message ID is identical to a message in database";
82 case NOTMUCH_STATUS_NULL_POINTER:
83 return "Erroneous NULL pointer";
84 case NOTMUCH_STATUS_TAG_TOO_LONG:
85 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
87 case NOTMUCH_STATUS_LAST_STATUS:
88 return "Unknown error status value";
92 /* XXX: We should drop this function and convert all callers to call
93 * _notmuch_message_add_term instead. */
95 add_term (Xapian::Document doc,
96 const char *prefix_name,
105 prefix = _find_prefix (prefix_name);
107 term = g_strdup_printf ("%s%s", prefix, value);
109 if (strlen (term) <= NOTMUCH_TERM_MAX)
116 find_doc_ids (notmuch_database_t *notmuch,
117 const char *prefix_name,
119 Xapian::PostingIterator *begin,
120 Xapian::PostingIterator *end)
122 Xapian::PostingIterator i;
125 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
127 *begin = notmuch->xapian_db->postlist_begin (term);
129 *end = notmuch->xapian_db->postlist_end (term);
134 static notmuch_private_status_t
135 find_unique_doc_id (notmuch_database_t *notmuch,
136 const char *prefix_name,
138 unsigned int *doc_id)
140 Xapian::PostingIterator i, end;
142 find_doc_ids (notmuch, prefix_name, value, &i, &end);
146 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
149 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
153 static Xapian::Document
154 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
156 return notmuch->xapian_db->get_document (doc_id);
159 static notmuch_private_status_t
160 find_unique_document (notmuch_database_t *notmuch,
161 const char *prefix_name,
163 Xapian::Document *document,
164 unsigned int *doc_id)
166 notmuch_private_status_t status;
168 status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
171 *document = Xapian::Document ();
175 *document = find_document_for_doc_id (notmuch, *doc_id);
176 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
180 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
183 const char *value, *id, *comma;
185 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
186 value = value_string.c_str();
187 if (strlen (value)) {
190 comma = strchr (id, ',');
192 comma = id + strlen (id);
193 g_hash_table_insert (thread_ids,
194 strndup (id, comma - id), NULL);
203 notmuch_database_find_message (notmuch_database_t *notmuch,
204 const char *message_id)
206 notmuch_private_status_t status;
209 status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);
211 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
214 return _notmuch_message_create (notmuch, notmuch, doc_id);
217 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
218 * given message based on looking into the database for any messages
219 * referenced in parents, and also for any messages in the database
220 * referencing message_id.
222 * Caller should free all strings in the array and the array itself,
223 * (g_ptr_array_free) when done. */
225 find_thread_ids (notmuch_database_t *notmuch,
227 const char *message_id)
229 Xapian::PostingIterator child, children_end;
230 Xapian::Document doc;
231 GHashTable *thread_ids;
234 const char *parent_message_id;
237 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
240 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
241 for ( ; child != children_end; child++) {
242 doc = find_document_for_doc_id (notmuch, *child);
243 insert_thread_id (thread_ids, doc);
246 for (i = 0; i < parents->len; i++) {
247 notmuch_message_t *parent;
248 notmuch_thread_ids_t *ids;
250 parent_message_id = (char *) g_ptr_array_index (parents, i);
251 parent = notmuch_database_find_message (notmuch, parent_message_id);
255 for (ids = notmuch_message_get_thread_ids (parent);
256 notmuch_thread_ids_has_more (ids);
257 notmuch_thread_ids_advance (ids))
261 id = notmuch_thread_ids_get (ids);
262 g_hash_table_insert (thread_ids, strdup (id), NULL);
265 notmuch_message_destroy (parent);
268 result = g_ptr_array_new ();
270 keys = g_hash_table_get_keys (thread_ids);
271 for (l = keys; l; l = l->next) {
272 char *id = (char *) l->data;
273 g_ptr_array_add (result, id);
277 /* We're done with the hash table, but we've taken the pointers to
278 * the allocated strings and put them into our result array, so
279 * tell the hash not to free them on its way out. */
280 g_hash_table_steal_all (thread_ids);
281 g_hash_table_unref (thread_ids);
286 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
287 * a (potentially nested) parenthesized sequence with '\' used to
288 * escape any character (including parentheses).
290 * If the sequence to be skipped continues to the end of the string,
291 * then 'str' will be left pointing at the final terminating '\0'
295 skip_space_and_comments (const char **str)
300 while (*s && (isspace (*s) || *s == '(')) {
301 while (*s && isspace (*s))
306 while (*s && nesting) {
322 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
323 * comments, and the '<' and '>' delimeters.
325 * If not NULL, then *next will be made to point to the first character
326 * not parsed, (possibly pointing to the final '\0' terminator.
328 * Returns a newly allocated string which the caller should free()
331 * Returns NULL if there is any error parsing the message-id. */
333 parse_message_id (const char *message_id, const char **next)
338 if (message_id == NULL)
343 skip_space_and_comments (&s);
345 /* Skip any unstructured text as well. */
346 while (*s && *s != '<')
357 skip_space_and_comments (&s);
360 while (*end && *end != '>')
369 if (end > s && *end == '>')
374 result = strndup (s, end - s + 1);
376 /* Finally, collapse any whitespace that is within the message-id
382 for (r = result, len = strlen (r); *r; r++, len--)
383 if (*r == ' ' || *r == '\t')
384 memmove (r, r+1, len);
390 /* Parse a References header value, putting a copy of each referenced
391 * message-id into 'array'. */
393 parse_references (GPtrArray *array,
402 ref = parse_message_id (refs, &refs);
405 g_ptr_array_add (array, ref);
410 notmuch_database_default_path (void)
412 if (getenv ("NOTMUCH_BASE"))
413 return strdup (getenv ("NOTMUCH_BASE"));
415 return g_strdup_printf ("%s/mail", getenv ("HOME"));
419 notmuch_database_create (const char *path)
421 notmuch_database_t *notmuch = NULL;
422 char *notmuch_path = NULL;
425 char *local_path = NULL;
428 path = local_path = notmuch_database_default_path ();
430 err = stat (path, &st);
432 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
433 path, strerror (errno));
437 if (! S_ISDIR (st.st_mode)) {
438 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
443 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
445 err = mkdir (notmuch_path, 0755);
448 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
449 notmuch_path, strerror (errno));
453 notmuch = notmuch_database_open (path);
465 notmuch_database_open (const char *path)
467 notmuch_database_t *notmuch = NULL;
468 char *notmuch_path = NULL, *xapian_path = NULL;
471 char *local_path = NULL;
474 path = local_path = notmuch_database_default_path ();
476 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
478 err = stat (notmuch_path, &st);
480 fprintf (stderr, "Error opening database at %s: %s\n",
481 notmuch_path, strerror (errno));
485 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
487 notmuch = talloc (NULL, notmuch_database_t);
488 notmuch->path = talloc_strdup (notmuch, path);
491 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
492 Xapian::DB_CREATE_OR_OPEN);
493 notmuch->query_parser = new Xapian::QueryParser;
494 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
495 notmuch->query_parser->set_database (*notmuch->xapian_db);
496 } catch (const Xapian::Error &error) {
497 fprintf (stderr, "A Xapian exception occurred: %s\n",
498 error.get_msg().c_str());
513 notmuch_database_close (notmuch_database_t *notmuch)
515 delete notmuch->query_parser;
516 delete notmuch->xapian_db;
517 talloc_free (notmuch);
521 notmuch_database_get_path (notmuch_database_t *notmuch)
523 return notmuch->path;
526 notmuch_private_status_t
527 find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
528 Xapian::Document *doc, unsigned int *doc_id)
530 return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
533 /* We allow the user to use arbitrarily long keys for timestamps,
534 * (they're for filesystem paths after all, which have no limit we
535 * know about). But we have a term-length limit. So if we exceed that,
536 * we'll use the SHA-1 of the user's key as the actual key for
537 * constructing a database term.
539 * Caution: This function returns a newly allocated string which the
540 * caller should free() when finished.
543 timestamp_db_key (const char *key)
545 int term_len = strlen (_find_prefix ("timestamp")) + strlen (key);
547 if (term_len > NOTMUCH_TERM_MAX)
548 return notmuch_sha1_of_string (key);
554 notmuch_database_set_timestamp (notmuch_database_t *notmuch,
555 const char *key, time_t timestamp)
557 Xapian::Document doc;
559 notmuch_private_status_t status;
560 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
563 db_key = timestamp_db_key (key);
566 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
568 doc.add_value (0, Xapian::sortable_serialise (timestamp));
570 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
571 char *term = talloc_asprintf (NULL, "%s%s",
572 _find_prefix ("timestamp"), db_key);
576 notmuch->xapian_db->add_document (doc);
578 notmuch->xapian_db->replace_document (doc_id, doc);
581 } catch (Xapian::Error &error) {
582 fprintf (stderr, "A Xapian exception occurred: %s.\n",
583 error.get_msg().c_str());
584 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
594 notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
596 Xapian::Document doc;
598 notmuch_private_status_t status;
602 db_key = timestamp_db_key (key);
605 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
607 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
610 ret = Xapian::sortable_unserialise (doc.get_value (0));
611 } catch (Xapian::Error &error) {
623 notmuch_database_add_message (notmuch_database_t *notmuch,
624 const char *filename)
626 notmuch_message_file_t *message_file;
627 notmuch_message_t *message;
628 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
630 GPtrArray *parents, *thread_ids;
632 const char *refs, *in_reply_to, *date, *header;
633 const char *from, *to, *subject, *old_filename;
638 message_file = notmuch_message_file_open (filename);
639 if (message_file == NULL) {
640 ret = NOTMUCH_STATUS_FILE_ERROR;
644 notmuch_message_file_restrict_headers (message_file,
655 /* The first order of business is to find/create a message ID. */
657 header = notmuch_message_file_get_header (message_file, "message-id");
659 message_id = parse_message_id (header, NULL);
660 /* So the header value isn't RFC-compliant, but it's
661 * better than no message-id at all. */
662 if (message_id == NULL)
663 message_id = xstrdup (header);
665 /* No message-id at all, let's generate one by taking a
666 * hash over the file's contents. */
667 char *sha1 = notmuch_sha1_of_file (filename);
669 /* If that failed too, something is really wrong. Give up. */
671 ret = NOTMUCH_STATUS_FILE_ERROR;
675 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
679 /* Now that we have a message ID, we get a message object,
680 * (which may or may not reference an existing document in the
683 /* Use NULL for owner since we want to free this locally. */
685 /* XXX: This call can fail by either out-of-memory or an
686 * "impossible" Xapian exception. We should rewrite it to
687 * allow us to propagate the error status. */
688 message = _notmuch_message_create_for_message_id (NULL, notmuch,
690 if (message == NULL) {
691 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
692 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
696 /* Has a message previously been added with the same ID? */
697 old_filename = notmuch_message_get_filename (message);
698 if (old_filename && strlen (old_filename)) {
699 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
702 _notmuch_message_set_filename (message, filename);
703 _notmuch_message_add_term (message, "type", "mail");
706 /* Next, find the thread(s) to which this message belongs. */
707 parents = g_ptr_array_new ();
709 refs = notmuch_message_file_get_header (message_file, "references");
710 parse_references (parents, refs);
712 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
713 parse_references (parents, in_reply_to);
715 for (i = 0; i < parents->len; i++)
716 _notmuch_message_add_term (message, "ref",
717 (char *) g_ptr_array_index (parents, i));
719 thread_ids = find_thread_ids (notmuch, parents, message_id);
723 for (i = 0; i < parents->len; i++)
724 g_free (g_ptr_array_index (parents, i));
725 g_ptr_array_free (parents, TRUE);
727 if (thread_ids->len) {
732 for (i = 0; i < thread_ids->len; i++) {
733 id = (char *) thread_ids->pdata[i];
734 _notmuch_message_add_thread_id (message, id);
736 thread_id = g_string_new (id);
738 g_string_append_printf (thread_id, ",%s", id);
742 g_string_free (thread_id, TRUE);
744 _notmuch_message_ensure_thread_id (message);
747 g_ptr_array_free (thread_ids, TRUE);
749 date = notmuch_message_file_get_header (message_file, "date");
750 _notmuch_message_set_date (message, date);
752 from = notmuch_message_file_get_header (message_file, "from");
753 subject = notmuch_message_file_get_header (message_file, "subject");
754 to = notmuch_message_file_get_header (message_file, "to");
760 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
763 _notmuch_message_sync (message);
765 } catch (const Xapian::Error &error) {
766 fprintf (stderr, "A Xapian exception occurred: %s.\n",
767 error.get_msg().c_str());
768 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
774 notmuch_message_destroy (message);
776 notmuch_message_file_close (message_file);