1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
31 #define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
33 /* These prefix values are specifically chosen to be compatible
34 * with sup, (http://sup.rubyforge.org), written by
35 * William Morgan <wmorgan-sup@masanjin.net>, and released
36 * under the GNU GPL v2.
44 prefix_t NORMAL_PREFIX[] = {
47 { "from_name", "FN" },
53 prefix_t BOOLEAN_PREFIX[] = {
55 { "from_email", "FE" },
62 { "attachment_extension", "O" },
66 { "timestamp", "KTS" },
70 _find_prefix (const char *name)
74 for (i = 0; i < ARRAY_SIZE (NORMAL_PREFIX); i++)
75 if (strcmp (name, NORMAL_PREFIX[i].name) == 0)
76 return NORMAL_PREFIX[i].prefix;
78 for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++)
79 if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0)
80 return BOOLEAN_PREFIX[i].prefix;
82 fprintf (stderr, "Internal error: No prefix exists for '%s'\n", name);
89 notmuch_status_to_string (notmuch_status_t status)
92 case NOTMUCH_STATUS_SUCCESS:
93 return "No error occurred";
94 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
95 return "A Xapian exception occurred";
96 case NOTMUCH_STATUS_FILE_ERROR:
97 return "Something went wrong trying to read or write a file";
98 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
99 return "File is not an email";
100 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
101 return "Message ID is identical to a message in database";
102 case NOTMUCH_STATUS_NULL_POINTER:
103 return "Erroneous NULL pointer";
104 case NOTMUCH_STATUS_TAG_TOO_LONG:
105 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
107 case NOTMUCH_STATUS_LAST_STATUS:
108 return "Unknown error status value";
112 /* XXX: We should drop this function and convert all callers to call
113 * _notmuch_message_add_term instead. */
115 add_term (Xapian::Document doc,
116 const char *prefix_name,
125 prefix = _find_prefix (prefix_name);
127 term = g_strdup_printf ("%s%s", prefix, value);
129 if (strlen (term) <= NOTMUCH_TERM_MAX)
136 find_doc_ids (notmuch_database_t *notmuch,
137 const char *prefix_name,
139 Xapian::PostingIterator *begin,
140 Xapian::PostingIterator *end)
142 Xapian::PostingIterator i;
145 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
147 *begin = notmuch->xapian_db->postlist_begin (term);
149 *end = notmuch->xapian_db->postlist_end (term);
154 static notmuch_private_status_t
155 find_unique_doc_id (notmuch_database_t *notmuch,
156 const char *prefix_name,
158 unsigned int *doc_id)
160 Xapian::PostingIterator i, end;
162 find_doc_ids (notmuch, prefix_name, value, &i, &end);
166 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
169 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
173 static Xapian::Document
174 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
176 return notmuch->xapian_db->get_document (doc_id);
179 static notmuch_private_status_t
180 find_unique_document (notmuch_database_t *notmuch,
181 const char *prefix_name,
183 Xapian::Document *document,
184 unsigned int *doc_id)
186 notmuch_private_status_t status;
188 status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
191 *document = Xapian::Document ();
195 *document = find_document_for_doc_id (notmuch, *doc_id);
196 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
200 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
203 const char *value, *id, *comma;
205 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
206 value = value_string.c_str();
207 if (strlen (value)) {
210 comma = strchr (id, ',');
212 comma = id + strlen (id);
213 g_hash_table_insert (thread_ids,
214 strndup (id, comma - id), NULL);
223 notmuch_database_find_message (notmuch_database_t *notmuch,
224 const char *message_id)
226 notmuch_private_status_t status;
229 status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);
231 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
234 return _notmuch_message_create (notmuch, notmuch, doc_id);
237 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
238 * given message based on looking into the database for any messages
239 * referenced in parents, and also for any messages in the database
240 * referencing message_id.
242 * Caller should free all strings in the array and the array itself,
243 * (g_ptr_array_free) when done. */
245 find_thread_ids (notmuch_database_t *notmuch,
247 const char *message_id)
249 Xapian::PostingIterator child, children_end;
250 Xapian::Document doc;
251 GHashTable *thread_ids;
254 const char *parent_message_id;
257 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
260 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
261 for ( ; child != children_end; child++) {
262 doc = find_document_for_doc_id (notmuch, *child);
263 insert_thread_id (thread_ids, doc);
266 for (i = 0; i < parents->len; i++) {
267 notmuch_message_t *parent;
268 notmuch_thread_ids_t *ids;
270 parent_message_id = (char *) g_ptr_array_index (parents, i);
271 parent = notmuch_database_find_message (notmuch, parent_message_id);
275 for (ids = notmuch_message_get_thread_ids (parent);
276 notmuch_thread_ids_has_more (ids);
277 notmuch_thread_ids_advance (ids))
281 id = notmuch_thread_ids_get (ids);
282 g_hash_table_insert (thread_ids, strdup (id), NULL);
285 notmuch_message_destroy (parent);
288 result = g_ptr_array_new ();
290 keys = g_hash_table_get_keys (thread_ids);
291 for (l = keys; l; l = l->next) {
292 char *id = (char *) l->data;
293 g_ptr_array_add (result, id);
297 /* We're done with the hash table, but we've taken the pointers to
298 * the allocated strings and put them into our result array, so
299 * tell the hash not to free them on its way out. */
300 g_hash_table_steal_all (thread_ids);
301 g_hash_table_unref (thread_ids);
306 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
307 * a (potentially nested) parenthesized sequence with '\' used to
308 * escape any character (including parentheses).
310 * If the sequence to be skipped continues to the end of the string,
311 * then 'str' will be left pointing at the final terminating '\0'
315 skip_space_and_comments (const char **str)
320 while (*s && (isspace (*s) || *s == '(')) {
321 while (*s && isspace (*s))
326 while (*s && nesting) {
342 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
343 * comments, and the '<' and '>' delimeters.
345 * If not NULL, then *next will be made to point to the first character
346 * not parsed, (possibly pointing to the final '\0' terminator.
348 * Returns a newly allocated string which the caller should free()
351 * Returns NULL if there is any error parsing the message-id. */
353 parse_message_id (const char *message_id, const char **next)
358 if (message_id == NULL)
363 skip_space_and_comments (&s);
365 /* Skip any unstructured text as well. */
366 while (*s && *s != '<')
377 skip_space_and_comments (&s);
380 while (*end && *end != '>')
389 if (end > s && *end == '>')
394 result = strndup (s, end - s + 1);
396 /* Finally, collapse any whitespace that is within the message-id
402 for (r = result, len = strlen (r); *r; r++, len--)
403 if (*r == ' ' || *r == '\t')
404 memmove (r, r+1, len);
410 /* Parse a References header value, putting a copy of each referenced
411 * message-id into 'array'. */
413 parse_references (GPtrArray *array,
422 ref = parse_message_id (refs, &refs);
425 g_ptr_array_add (array, ref);
430 notmuch_database_default_path (void)
432 if (getenv ("NOTMUCH_BASE"))
433 return strdup (getenv ("NOTMUCH_BASE"));
435 return g_strdup_printf ("%s/mail", getenv ("HOME"));
439 notmuch_database_create (const char *path)
441 notmuch_database_t *notmuch = NULL;
442 char *notmuch_path = NULL;
445 char *local_path = NULL;
448 path = local_path = notmuch_database_default_path ();
450 err = stat (path, &st);
452 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
453 path, strerror (errno));
457 if (! S_ISDIR (st.st_mode)) {
458 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
463 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
465 err = mkdir (notmuch_path, 0755);
468 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
469 notmuch_path, strerror (errno));
473 notmuch = notmuch_database_open (path);
485 notmuch_database_open (const char *path)
487 notmuch_database_t *notmuch = NULL;
488 char *notmuch_path = NULL, *xapian_path = NULL;
491 char *local_path = NULL;
494 path = local_path = notmuch_database_default_path ();
496 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
498 err = stat (notmuch_path, &st);
500 fprintf (stderr, "Error opening database at %s: %s\n",
501 notmuch_path, strerror (errno));
505 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
507 notmuch = talloc (NULL, notmuch_database_t);
508 notmuch->path = talloc_strdup (notmuch, path);
511 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
512 Xapian::DB_CREATE_OR_OPEN);
513 notmuch->query_parser = new Xapian::QueryParser;
514 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
515 notmuch->query_parser->set_database (*notmuch->xapian_db);
516 } catch (const Xapian::Error &error) {
517 fprintf (stderr, "A Xapian exception occurred: %s\n",
518 error.get_msg().c_str());
533 notmuch_database_close (notmuch_database_t *notmuch)
535 delete notmuch->query_parser;
536 delete notmuch->xapian_db;
537 talloc_free (notmuch);
541 notmuch_database_get_path (notmuch_database_t *notmuch)
543 return notmuch->path;
546 notmuch_private_status_t
547 find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
548 Xapian::Document *doc, unsigned int *doc_id)
550 return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
553 /* We allow the user to use arbitrarily long keys for timestamps,
554 * (they're for filesystem paths after all, which have no limit we
555 * know about). But we have a term-length limit. So if we exceed that,
556 * we'll use the SHA-1 of the user's key as the actual key for
557 * constructing a database term.
559 * Caution: This function returns a newly allocated string which the
560 * caller should free() when finished.
563 timestamp_db_key (const char *key)
565 if (strlen (key) + 1 > NOTMUCH_TERM_MAX) {
566 return notmuch_sha1_of_string (key);
573 notmuch_database_set_timestamp (notmuch_database_t *notmuch,
574 const char *key, time_t timestamp)
576 Xapian::Document doc;
578 notmuch_private_status_t status;
579 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
582 db_key = timestamp_db_key (key);
585 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
587 doc.add_value (0, Xapian::sortable_serialise (timestamp));
589 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
590 char *term = talloc_asprintf (NULL, "%s%s",
591 _find_prefix ("timestamp"), db_key);
595 notmuch->xapian_db->add_document (doc);
597 notmuch->xapian_db->replace_document (doc_id, doc);
600 } catch (Xapian::Error &error) {
601 fprintf (stderr, "A Xapian exception occurred: %s.\n",
602 error.get_msg().c_str());
603 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
613 notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
615 Xapian::Document doc;
617 notmuch_private_status_t status;
621 db_key = timestamp_db_key (key);
624 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
626 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
629 ret = Xapian::sortable_unserialise (doc.get_value (0));
630 } catch (Xapian::Error &error) {
642 notmuch_database_add_message (notmuch_database_t *notmuch,
643 const char *filename)
645 notmuch_message_file_t *message_file;
646 notmuch_message_t *message;
647 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
649 GPtrArray *parents, *thread_ids;
651 const char *refs, *in_reply_to, *date, *header;
652 const char *from, *to, *subject, *old_filename;
657 message_file = notmuch_message_file_open (filename);
658 if (message_file == NULL) {
659 ret = NOTMUCH_STATUS_FILE_ERROR;
663 notmuch_message_file_restrict_headers (message_file,
674 /* The first order of business is to find/create a message ID. */
676 header = notmuch_message_file_get_header (message_file, "message-id");
678 message_id = parse_message_id (header, NULL);
679 /* So the header value isn't RFC-compliant, but it's
680 * better than no message-id at all. */
681 if (message_id == NULL)
682 message_id = xstrdup (header);
684 /* No message-id at all, let's generate one by taking a
685 * hash over the file's contents. */
686 char *sha1 = notmuch_sha1_of_file (filename);
688 /* If that failed too, something is really wrong. Give up. */
690 ret = NOTMUCH_STATUS_FILE_ERROR;
694 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
698 /* Now that we have a message ID, we get a message object,
699 * (which may or may not reference an existing document in the
702 /* Use NULL for owner since we want to free this locally. */
704 /* XXX: This call can fail by either out-of-memory or an
705 * "impossible" Xapian exception. We should rewrite it to
706 * allow us to propagate the error status. */
707 message = _notmuch_message_create_for_message_id (NULL, notmuch,
709 if (message == NULL) {
710 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
711 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
715 /* Has a message previously been added with the same ID? */
716 old_filename = notmuch_message_get_filename (message);
717 if (old_filename && strlen (old_filename)) {
718 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
721 _notmuch_message_set_filename (message, filename);
722 _notmuch_message_add_term (message, "type", "mail");
725 /* Next, find the thread(s) to which this message belongs. */
726 parents = g_ptr_array_new ();
728 refs = notmuch_message_file_get_header (message_file, "references");
729 parse_references (parents, refs);
731 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
732 parse_references (parents, in_reply_to);
734 for (i = 0; i < parents->len; i++)
735 _notmuch_message_add_term (message, "ref",
736 (char *) g_ptr_array_index (parents, i));
738 thread_ids = find_thread_ids (notmuch, parents, message_id);
742 for (i = 0; i < parents->len; i++)
743 g_free (g_ptr_array_index (parents, i));
744 g_ptr_array_free (parents, TRUE);
746 if (thread_ids->len) {
751 for (i = 0; i < thread_ids->len; i++) {
752 id = (char *) thread_ids->pdata[i];
753 _notmuch_message_add_thread_id (message, id);
755 thread_id = g_string_new (id);
757 g_string_append_printf (thread_id, ",%s", id);
761 g_string_free (thread_id, TRUE);
763 _notmuch_message_ensure_thread_id (message);
766 g_ptr_array_free (thread_ids, TRUE);
768 date = notmuch_message_file_get_header (message_file, "date");
769 _notmuch_message_set_date (message, date);
771 from = notmuch_message_file_get_header (message_file, "from");
772 subject = notmuch_message_file_get_header (message_file, "subject");
773 to = notmuch_message_file_get_header (message_file, "to");
779 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
782 _notmuch_message_sync (message);
784 } catch (const Xapian::Error &error) {
785 fprintf (stderr, "A Xapian exception occurred: %s.\n",
786 error.get_msg().c_str());
787 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
793 notmuch_message_destroy (message);
795 notmuch_message_file_close (message_file);