1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
32 notmuch_status_to_string (notmuch_status_t status)
35 case NOTMUCH_STATUS_SUCCESS:
36 return "No error occurred";
37 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
38 return "A Xapian exception occurred";
39 case NOTMUCH_STATUS_FILE_ERROR:
40 return "Something went wrong trying to read or write a file";
41 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
42 return "File is not an email";
43 case NOTMUCH_STATUS_NULL_POINTER:
44 return "Erroneous NULL pointer";
45 case NOTMUCH_STATUS_TAG_TOO_LONG:
46 return "Tag value is too long";
48 case NOTMUCH_STATUS_LAST_STATUS:
49 return "Unknown error status value";
53 /* XXX: We should drop this function and convert all callers to call
54 * _notmuch_message_add_term instead. */
56 add_term (Xapian::Document doc,
57 const char *prefix_name,
66 prefix = _find_prefix (prefix_name);
68 term = g_strdup_printf ("%s%s", prefix, value);
70 if (strlen (term) <= NOTMUCH_TERM_MAX)
77 find_doc_ids (notmuch_database_t *notmuch,
78 const char *prefix_name,
80 Xapian::PostingIterator *begin,
81 Xapian::PostingIterator *end)
83 Xapian::PostingIterator i;
86 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
88 *begin = notmuch->xapian_db->postlist_begin (term);
90 *end = notmuch->xapian_db->postlist_end (term);
96 find_message_by_docid (Xapian::Database *db, Xapian::docid docid)
98 return db->get_document (docid);
102 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
105 const char *value, *id, *comma;
107 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
108 value = value_string.c_str();
109 if (strlen (value)) {
112 comma = strchr (id, ',');
114 comma = id + strlen (id);
115 g_hash_table_insert (thread_ids,
116 strndup (id, comma - id), NULL);
125 notmuch_database_find_message (notmuch_database_t *notmuch,
126 const char *message_id)
128 Xapian::PostingIterator i, end;
130 find_doc_ids (notmuch, "msgid", message_id, &i, &end);
135 return _notmuch_message_create (notmuch, notmuch, *i);
138 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
139 * given message based on looking into the database for any messages
140 * referenced in parents, and also for any messages in the database
141 * referencing message_id.
143 * Caller should free all strings in the array and the array itself,
144 * (g_ptr_array_free) when done. */
146 find_thread_ids (notmuch_database_t *notmuch,
148 const char *message_id)
150 Xapian::WritableDatabase *db = notmuch->xapian_db;
151 Xapian::PostingIterator child, children_end;
152 Xapian::Document doc;
153 GHashTable *thread_ids;
156 const char *parent_message_id;
159 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
162 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
163 for ( ; child != children_end; child++) {
164 doc = find_message_by_docid (db, *child);
165 insert_thread_id (thread_ids, doc);
168 for (i = 0; i < parents->len; i++) {
169 notmuch_message_t *parent;
170 notmuch_thread_ids_t *ids;
172 parent_message_id = (char *) g_ptr_array_index (parents, i);
173 parent = notmuch_database_find_message (notmuch, parent_message_id);
177 for (ids = notmuch_message_get_thread_ids (parent);
178 notmuch_thread_ids_has_more (ids);
179 notmuch_thread_ids_advance (ids))
183 id = notmuch_thread_ids_get (ids);
184 g_hash_table_insert (thread_ids, strdup (id), NULL);
187 notmuch_message_destroy (parent);
190 result = g_ptr_array_new ();
192 keys = g_hash_table_get_keys (thread_ids);
193 for (l = keys; l; l = l->next) {
194 char *id = (char *) l->data;
195 g_ptr_array_add (result, id);
199 /* We're done with the hash table, but we've taken the pointers to
200 * the allocated strings and put them into our result array, so
201 * tell the hash not to free them on its way out. */
202 g_hash_table_steal_all (thread_ids);
203 g_hash_table_unref (thread_ids);
208 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
209 * a (potentially nested) parenthesized sequence with '\' used to
210 * escape any character (including parentheses).
212 * If the sequence to be skipped continues to the end of the string,
213 * then 'str' will be left pointing at the final terminating '\0'
217 skip_space_and_comments (const char **str)
222 while (*s && (isspace (*s) || *s == '(')) {
223 while (*s && isspace (*s))
228 while (*s && nesting) {
244 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
245 * comments, and the '<' and '>' delimeters.
247 * If not NULL, then *next will be made to point to the first character
248 * not parsed, (possibly pointing to the final '\0' terminator.
250 * Returns a newly allocated string which the caller should free()
253 * Returns NULL if there is any error parsing the message-id. */
255 parse_message_id (const char *message_id, const char **next)
260 if (message_id == NULL)
265 skip_space_and_comments (&s);
267 /* Skip any unstructured text as well. */
268 while (*s && *s != '<')
279 skip_space_and_comments (&s);
282 while (*end && *end != '>')
291 if (end > s && *end == '>')
296 result = strndup (s, end - s + 1);
298 /* Finally, collapse any whitespace that is within the message-id
304 for (r = result, len = strlen (r); *r; r++, len--)
305 if (*r == ' ' || *r == '\t')
306 memmove (r, r+1, len);
312 /* Parse a References header value, putting a copy of each referenced
313 * message-id into 'array'. */
315 parse_references (GPtrArray *array,
324 ref = parse_message_id (refs, &refs);
327 g_ptr_array_add (array, ref);
332 notmuch_database_default_path (void)
334 if (getenv ("NOTMUCH_BASE"))
335 return strdup (getenv ("NOTMUCH_BASE"));
337 return g_strdup_printf ("%s/mail", getenv ("HOME"));
341 notmuch_database_create (const char *path)
343 notmuch_database_t *notmuch = NULL;
344 char *notmuch_path = NULL;
347 char *local_path = NULL;
350 path = local_path = notmuch_database_default_path ();
352 err = stat (path, &st);
354 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
355 path, strerror (errno));
359 if (! S_ISDIR (st.st_mode)) {
360 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
365 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
367 err = mkdir (notmuch_path, 0755);
370 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
371 notmuch_path, strerror (errno));
375 notmuch = notmuch_database_open (path);
387 notmuch_database_open (const char *path)
389 notmuch_database_t *notmuch = NULL;
390 char *notmuch_path = NULL, *xapian_path = NULL;
393 char *local_path = NULL;
396 path = local_path = notmuch_database_default_path ();
398 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
400 err = stat (notmuch_path, &st);
402 fprintf (stderr, "Error opening database at %s: %s\n",
403 notmuch_path, strerror (errno));
407 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
409 notmuch = talloc (NULL, notmuch_database_t);
410 notmuch->path = talloc_strdup (notmuch, path);
413 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
414 Xapian::DB_CREATE_OR_OPEN);
415 notmuch->query_parser = new Xapian::QueryParser;
416 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
417 notmuch->query_parser->set_database (*notmuch->xapian_db);
418 } catch (const Xapian::Error &error) {
419 fprintf (stderr, "A Xapian exception occurred: %s\n",
420 error.get_msg().c_str());
435 notmuch_database_close (notmuch_database_t *notmuch)
437 delete notmuch->query_parser;
438 delete notmuch->xapian_db;
439 talloc_free (notmuch);
443 notmuch_database_get_path (notmuch_database_t *notmuch)
445 return notmuch->path;
449 notmuch_database_add_message (notmuch_database_t *notmuch,
450 const char *filename)
452 notmuch_message_file_t *message_file;
453 notmuch_message_t *message;
454 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
456 GPtrArray *parents, *thread_ids;
458 const char *refs, *in_reply_to, *date, *header;
459 const char *from, *to, *subject, *old_filename;
464 message_file = notmuch_message_file_open (filename);
465 if (message_file == NULL) {
466 ret = NOTMUCH_STATUS_FILE_ERROR;
470 notmuch_message_file_restrict_headers (message_file,
481 /* The first order of business is to find/create a message ID. */
483 header = notmuch_message_file_get_header (message_file, "message-id");
485 message_id = parse_message_id (header, NULL);
486 /* So the header value isn't RFC-compliant, but it's
487 * better than no message-id at all. */
488 if (message_id == NULL)
489 message_id = xstrdup (header);
491 /* No message-id at all, let's generate one by taking a
492 * hash over the file's contents. */
493 char *sha1 = notmuch_sha1_of_file (filename);
495 /* If that failed too, something is really wrong. Give up. */
497 ret = NOTMUCH_STATUS_FILE_ERROR;
501 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
505 /* Now that we have a message ID, we get a message object,
506 * (which may or may not reference an existing document in the
509 /* Use NULL for owner since we want to free this locally. */
511 /* XXX: This call can fail by either out-of-memory or an
512 * "impossible" Xapian exception. We should rewrite it to
513 * allow us to propagate the error status. */
514 message = _notmuch_message_create_for_message_id (NULL, notmuch,
516 if (message == NULL) {
517 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
518 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
522 /* Has a message previously been added with the same ID? */
523 old_filename = notmuch_message_get_filename (message);
524 if (old_filename && strlen (old_filename)) {
525 /* XXX: This is too noisy to actually print, and what do we
526 * really expect the user to do? Go manually delete a
527 * redundant message or merge two similar messages?
528 * Instead we should handle this transparently.
530 * What we likely want to move to is adding both filenames
531 * to the database so that subsequent indexing will pick up
532 * terms from both files.
536 "Note: Attempting to add a message with a duplicate message ID:\n"
537 "Old: %s\n" "New: %s\n",
538 old_filename, filename);
539 fprintf (stderr, "The old filename will be used, but any new terms\n"
540 "from the new message will added to the database.\n");
543 _notmuch_message_set_filename (message, filename);
544 _notmuch_message_add_term (message, "type", "mail");
547 /* Next, find the thread(s) to which this message belongs. */
548 parents = g_ptr_array_new ();
550 refs = notmuch_message_file_get_header (message_file, "references");
551 parse_references (parents, refs);
553 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
554 parse_references (parents, in_reply_to);
556 for (i = 0; i < parents->len; i++)
557 _notmuch_message_add_term (message, "ref",
558 (char *) g_ptr_array_index (parents, i));
560 thread_ids = find_thread_ids (notmuch, parents, message_id);
564 for (i = 0; i < parents->len; i++)
565 g_free (g_ptr_array_index (parents, i));
566 g_ptr_array_free (parents, TRUE);
568 if (thread_ids->len) {
573 for (i = 0; i < thread_ids->len; i++) {
574 id = (char *) thread_ids->pdata[i];
575 _notmuch_message_add_thread_id (message, id);
577 thread_id = g_string_new (id);
579 g_string_append_printf (thread_id, ",%s", id);
583 g_string_free (thread_id, TRUE);
585 _notmuch_message_ensure_thread_id (message);
588 g_ptr_array_free (thread_ids, TRUE);
590 date = notmuch_message_file_get_header (message_file, "date");
591 _notmuch_message_set_date (message, date);
593 from = notmuch_message_file_get_header (message_file, "from");
594 subject = notmuch_message_file_get_header (message_file, "subject");
595 to = notmuch_message_file_get_header (message_file, "to");
601 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
604 _notmuch_message_sync (message);
606 } catch (const Xapian::Error &error) {
607 fprintf (stderr, "A Xapian exception occurred: %s.\n",
608 error.get_msg().c_str());
609 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
615 notmuch_message_destroy (message);
617 notmuch_message_file_close (message_file);