1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
32 notmuch_status_to_string (notmuch_status_t status)
35 case NOTMUCH_STATUS_SUCCESS:
36 return "No error occurred";
37 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
38 return "A Xapian exception occurred";
39 case NOTMUCH_STATUS_FILE_ERROR:
40 return "Something went wrong trying to read or write a file";
41 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
42 return "File is not an email";
43 case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID:
44 return "Message ID is identical to a message in database";
45 case NOTMUCH_STATUS_NULL_POINTER:
46 return "Erroneous NULL pointer";
47 case NOTMUCH_STATUS_TAG_TOO_LONG:
48 return "Tag value is too long (exceeds NOTMUCH_TAG_MAX)";
50 case NOTMUCH_STATUS_LAST_STATUS:
51 return "Unknown error status value";
55 /* XXX: We should drop this function and convert all callers to call
56 * _notmuch_message_add_term instead. */
58 add_term (Xapian::Document doc,
59 const char *prefix_name,
68 prefix = _find_prefix (prefix_name);
70 term = g_strdup_printf ("%s%s", prefix, value);
72 if (strlen (term) <= NOTMUCH_TERM_MAX)
79 find_doc_ids (notmuch_database_t *notmuch,
80 const char *prefix_name,
82 Xapian::PostingIterator *begin,
83 Xapian::PostingIterator *end)
85 Xapian::PostingIterator i;
88 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
90 *begin = notmuch->xapian_db->postlist_begin (term);
92 *end = notmuch->xapian_db->postlist_end (term);
97 static notmuch_private_status_t
98 find_unique_doc_id (notmuch_database_t *notmuch,
99 const char *prefix_name,
101 unsigned int *doc_id)
103 Xapian::PostingIterator i, end;
105 find_doc_ids (notmuch, prefix_name, value, &i, &end);
109 return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
112 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
116 static Xapian::Document
117 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
119 return notmuch->xapian_db->get_document (doc_id);
122 static notmuch_private_status_t
123 find_unique_document (notmuch_database_t *notmuch,
124 const char *prefix_name,
126 Xapian::Document *document,
127 unsigned int *doc_id)
129 notmuch_private_status_t status;
131 status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
134 *document = Xapian::Document ();
138 *document = find_document_for_doc_id (notmuch, *doc_id);
139 return NOTMUCH_PRIVATE_STATUS_SUCCESS;
143 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
146 const char *value, *id, *comma;
148 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
149 value = value_string.c_str();
150 if (strlen (value)) {
153 comma = strchr (id, ',');
155 comma = id + strlen (id);
156 g_hash_table_insert (thread_ids,
157 strndup (id, comma - id), NULL);
166 notmuch_database_find_message (notmuch_database_t *notmuch,
167 const char *message_id)
169 notmuch_private_status_t status;
172 status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);
174 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
177 return _notmuch_message_create (notmuch, notmuch, doc_id);
180 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
181 * given message based on looking into the database for any messages
182 * referenced in parents, and also for any messages in the database
183 * referencing message_id.
185 * Caller should free all strings in the array and the array itself,
186 * (g_ptr_array_free) when done. */
188 find_thread_ids (notmuch_database_t *notmuch,
190 const char *message_id)
192 Xapian::PostingIterator child, children_end;
193 Xapian::Document doc;
194 GHashTable *thread_ids;
197 const char *parent_message_id;
200 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
203 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
204 for ( ; child != children_end; child++) {
205 doc = find_document_for_doc_id (notmuch, *child);
206 insert_thread_id (thread_ids, doc);
209 for (i = 0; i < parents->len; i++) {
210 notmuch_message_t *parent;
211 notmuch_thread_ids_t *ids;
213 parent_message_id = (char *) g_ptr_array_index (parents, i);
214 parent = notmuch_database_find_message (notmuch, parent_message_id);
218 for (ids = notmuch_message_get_thread_ids (parent);
219 notmuch_thread_ids_has_more (ids);
220 notmuch_thread_ids_advance (ids))
224 id = notmuch_thread_ids_get (ids);
225 g_hash_table_insert (thread_ids, strdup (id), NULL);
228 notmuch_message_destroy (parent);
231 result = g_ptr_array_new ();
233 keys = g_hash_table_get_keys (thread_ids);
234 for (l = keys; l; l = l->next) {
235 char *id = (char *) l->data;
236 g_ptr_array_add (result, id);
240 /* We're done with the hash table, but we've taken the pointers to
241 * the allocated strings and put them into our result array, so
242 * tell the hash not to free them on its way out. */
243 g_hash_table_steal_all (thread_ids);
244 g_hash_table_unref (thread_ids);
249 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
250 * a (potentially nested) parenthesized sequence with '\' used to
251 * escape any character (including parentheses).
253 * If the sequence to be skipped continues to the end of the string,
254 * then 'str' will be left pointing at the final terminating '\0'
258 skip_space_and_comments (const char **str)
263 while (*s && (isspace (*s) || *s == '(')) {
264 while (*s && isspace (*s))
269 while (*s && nesting) {
285 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
286 * comments, and the '<' and '>' delimeters.
288 * If not NULL, then *next will be made to point to the first character
289 * not parsed, (possibly pointing to the final '\0' terminator.
291 * Returns a newly allocated string which the caller should free()
294 * Returns NULL if there is any error parsing the message-id. */
296 parse_message_id (const char *message_id, const char **next)
301 if (message_id == NULL)
306 skip_space_and_comments (&s);
308 /* Skip any unstructured text as well. */
309 while (*s && *s != '<')
320 skip_space_and_comments (&s);
323 while (*end && *end != '>')
332 if (end > s && *end == '>')
337 result = strndup (s, end - s + 1);
339 /* Finally, collapse any whitespace that is within the message-id
345 for (r = result, len = strlen (r); *r; r++, len--)
346 if (*r == ' ' || *r == '\t')
347 memmove (r, r+1, len);
353 /* Parse a References header value, putting a copy of each referenced
354 * message-id into 'array'. */
356 parse_references (GPtrArray *array,
365 ref = parse_message_id (refs, &refs);
368 g_ptr_array_add (array, ref);
373 notmuch_database_default_path (void)
375 if (getenv ("NOTMUCH_BASE"))
376 return strdup (getenv ("NOTMUCH_BASE"));
378 return g_strdup_printf ("%s/mail", getenv ("HOME"));
382 notmuch_database_create (const char *path)
384 notmuch_database_t *notmuch = NULL;
385 char *notmuch_path = NULL;
388 char *local_path = NULL;
391 path = local_path = notmuch_database_default_path ();
393 err = stat (path, &st);
395 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
396 path, strerror (errno));
400 if (! S_ISDIR (st.st_mode)) {
401 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
406 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
408 err = mkdir (notmuch_path, 0755);
411 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
412 notmuch_path, strerror (errno));
416 notmuch = notmuch_database_open (path);
428 notmuch_database_open (const char *path)
430 notmuch_database_t *notmuch = NULL;
431 char *notmuch_path = NULL, *xapian_path = NULL;
434 char *local_path = NULL;
437 path = local_path = notmuch_database_default_path ();
439 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
441 err = stat (notmuch_path, &st);
443 fprintf (stderr, "Error opening database at %s: %s\n",
444 notmuch_path, strerror (errno));
448 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
450 notmuch = talloc (NULL, notmuch_database_t);
451 notmuch->path = talloc_strdup (notmuch, path);
454 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
455 Xapian::DB_CREATE_OR_OPEN);
456 notmuch->query_parser = new Xapian::QueryParser;
457 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
458 notmuch->query_parser->set_database (*notmuch->xapian_db);
459 } catch (const Xapian::Error &error) {
460 fprintf (stderr, "A Xapian exception occurred: %s\n",
461 error.get_msg().c_str());
476 notmuch_database_close (notmuch_database_t *notmuch)
478 delete notmuch->query_parser;
479 delete notmuch->xapian_db;
480 talloc_free (notmuch);
484 notmuch_database_get_path (notmuch_database_t *notmuch)
486 return notmuch->path;
489 notmuch_private_status_t
490 find_timestamp_document (notmuch_database_t *notmuch, const char *db_key,
491 Xapian::Document *doc, unsigned int *doc_id)
493 return find_unique_document (notmuch, "timestamp", db_key, doc, doc_id);
496 /* We allow the user to use arbitrarily long keys for timestamps,
497 * (they're for filesystem paths after all, which have no limit we
498 * know about). But we have a term-length limit. So if we exceed that,
499 * we'll use the SHA-1 of the user's key as the actual key for
500 * constructing a database term.
502 * Caution: This function returns a newly allocated string which the
503 * caller should free() when finished.
506 timestamp_db_key (const char *key)
508 if (strlen (key) + 1 > NOTMUCH_TERM_MAX) {
509 return notmuch_sha1_of_string (key);
516 notmuch_database_set_timestamp (notmuch_database_t *notmuch,
517 const char *key, time_t timestamp)
519 Xapian::Document doc;
521 notmuch_private_status_t status;
522 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
525 db_key = timestamp_db_key (key);
528 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
530 doc.add_value (0, Xapian::sortable_serialise (timestamp));
532 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) {
533 char *term = talloc_asprintf (NULL, "%s%s",
534 _find_prefix ("timestamp"), db_key);
538 notmuch->xapian_db->add_document (doc);
540 notmuch->xapian_db->replace_document (doc_id, doc);
543 } catch (Xapian::Error &error) {
544 fprintf (stderr, "A Xapian exception occurred: %s.\n",
545 error.get_msg().c_str());
546 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
556 notmuch_database_get_timestamp (notmuch_database_t *notmuch, const char *key)
558 Xapian::Document doc;
560 notmuch_private_status_t status;
564 db_key = timestamp_db_key (key);
567 status = find_timestamp_document (notmuch, db_key, &doc, &doc_id);
569 if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
572 ret = Xapian::sortable_unserialise (doc.get_value (0));
573 } catch (Xapian::Error &error) {
585 notmuch_database_add_message (notmuch_database_t *notmuch,
586 const char *filename)
588 notmuch_message_file_t *message_file;
589 notmuch_message_t *message;
590 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
592 GPtrArray *parents, *thread_ids;
594 const char *refs, *in_reply_to, *date, *header;
595 const char *from, *to, *subject, *old_filename;
600 message_file = notmuch_message_file_open (filename);
601 if (message_file == NULL) {
602 ret = NOTMUCH_STATUS_FILE_ERROR;
606 notmuch_message_file_restrict_headers (message_file,
617 /* The first order of business is to find/create a message ID. */
619 header = notmuch_message_file_get_header (message_file, "message-id");
621 message_id = parse_message_id (header, NULL);
622 /* So the header value isn't RFC-compliant, but it's
623 * better than no message-id at all. */
624 if (message_id == NULL)
625 message_id = xstrdup (header);
627 /* No message-id at all, let's generate one by taking a
628 * hash over the file's contents. */
629 char *sha1 = notmuch_sha1_of_file (filename);
631 /* If that failed too, something is really wrong. Give up. */
633 ret = NOTMUCH_STATUS_FILE_ERROR;
637 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
641 /* Now that we have a message ID, we get a message object,
642 * (which may or may not reference an existing document in the
645 /* Use NULL for owner since we want to free this locally. */
647 /* XXX: This call can fail by either out-of-memory or an
648 * "impossible" Xapian exception. We should rewrite it to
649 * allow us to propagate the error status. */
650 message = _notmuch_message_create_for_message_id (NULL, notmuch,
652 if (message == NULL) {
653 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
654 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
658 /* Has a message previously been added with the same ID? */
659 old_filename = notmuch_message_get_filename (message);
660 if (old_filename && strlen (old_filename)) {
661 ret = NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID;
664 _notmuch_message_set_filename (message, filename);
665 _notmuch_message_add_term (message, "type", "mail");
668 /* Next, find the thread(s) to which this message belongs. */
669 parents = g_ptr_array_new ();
671 refs = notmuch_message_file_get_header (message_file, "references");
672 parse_references (parents, refs);
674 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
675 parse_references (parents, in_reply_to);
677 for (i = 0; i < parents->len; i++)
678 _notmuch_message_add_term (message, "ref",
679 (char *) g_ptr_array_index (parents, i));
681 thread_ids = find_thread_ids (notmuch, parents, message_id);
685 for (i = 0; i < parents->len; i++)
686 g_free (g_ptr_array_index (parents, i));
687 g_ptr_array_free (parents, TRUE);
689 if (thread_ids->len) {
694 for (i = 0; i < thread_ids->len; i++) {
695 id = (char *) thread_ids->pdata[i];
696 _notmuch_message_add_thread_id (message, id);
698 thread_id = g_string_new (id);
700 g_string_append_printf (thread_id, ",%s", id);
704 g_string_free (thread_id, TRUE);
706 _notmuch_message_ensure_thread_id (message);
709 g_ptr_array_free (thread_ids, TRUE);
711 date = notmuch_message_file_get_header (message_file, "date");
712 _notmuch_message_set_date (message, date);
714 from = notmuch_message_file_get_header (message_file, "from");
715 subject = notmuch_message_file_get_header (message_file, "subject");
716 to = notmuch_message_file_get_header (message_file, "to");
722 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
725 _notmuch_message_sync (message);
727 } catch (const Xapian::Error &error) {
728 fprintf (stderr, "A Xapian exception occurred: %s.\n",
729 error.get_msg().c_str());
730 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
736 notmuch_message_destroy (message);
738 notmuch_message_file_close (message_file);