1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
32 notmuch_status_to_string (notmuch_status_t status)
35 case NOTMUCH_STATUS_SUCCESS:
36 return "No error occurred";
37 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
38 return "A Xapian exception occurred";
39 case NOTMUCH_STATUS_FILE_ERROR:
40 return "Something went wrong trying to read or write a file";
41 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
42 return "File is not an email";
43 case NOTMUCH_STATUS_NULL_POINTER:
44 return "Erroneous NULL pointer";
45 case NOTMUCH_STATUS_TAG_TOO_LONG:
46 return "Tag value is too long";
48 case NOTMUCH_STATUS_LAST_STATUS:
49 return "Unknown error status value";
53 /* XXX: We should drop this function and convert all callers to call
54 * _notmuch_message_add_term instead. */
56 add_term (Xapian::Document doc,
57 const char *prefix_name,
66 prefix = _find_prefix (prefix_name);
68 term = g_strdup_printf ("%s%s", prefix, value);
70 if (strlen (term) <= NOTMUCH_TERM_MAX)
77 find_doc_ids (notmuch_database_t *notmuch,
78 const char *prefix_name,
80 Xapian::PostingIterator *begin,
81 Xapian::PostingIterator *end)
83 Xapian::PostingIterator i;
86 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
88 *begin = notmuch->xapian_db->postlist_begin (term);
90 *end = notmuch->xapian_db->postlist_end (term);
95 static Xapian::Document
96 find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
98 return notmuch->xapian_db->get_document (doc_id);
102 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
105 const char *value, *id, *comma;
107 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
108 value = value_string.c_str();
109 if (strlen (value)) {
112 comma = strchr (id, ',');
114 comma = id + strlen (id);
115 g_hash_table_insert (thread_ids,
116 strndup (id, comma - id), NULL);
125 notmuch_database_find_message (notmuch_database_t *notmuch,
126 const char *message_id)
128 Xapian::PostingIterator i, end;
130 find_doc_ids (notmuch, "msgid", message_id, &i, &end);
135 return _notmuch_message_create (notmuch, notmuch, *i);
138 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
139 * given message based on looking into the database for any messages
140 * referenced in parents, and also for any messages in the database
141 * referencing message_id.
143 * Caller should free all strings in the array and the array itself,
144 * (g_ptr_array_free) when done. */
146 find_thread_ids (notmuch_database_t *notmuch,
148 const char *message_id)
150 Xapian::PostingIterator child, children_end;
151 Xapian::Document doc;
152 GHashTable *thread_ids;
155 const char *parent_message_id;
158 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
161 find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
162 for ( ; child != children_end; child++) {
163 doc = find_document_for_doc_id (notmuch, *child);
164 insert_thread_id (thread_ids, doc);
167 for (i = 0; i < parents->len; i++) {
168 notmuch_message_t *parent;
169 notmuch_thread_ids_t *ids;
171 parent_message_id = (char *) g_ptr_array_index (parents, i);
172 parent = notmuch_database_find_message (notmuch, parent_message_id);
176 for (ids = notmuch_message_get_thread_ids (parent);
177 notmuch_thread_ids_has_more (ids);
178 notmuch_thread_ids_advance (ids))
182 id = notmuch_thread_ids_get (ids);
183 g_hash_table_insert (thread_ids, strdup (id), NULL);
186 notmuch_message_destroy (parent);
189 result = g_ptr_array_new ();
191 keys = g_hash_table_get_keys (thread_ids);
192 for (l = keys; l; l = l->next) {
193 char *id = (char *) l->data;
194 g_ptr_array_add (result, id);
198 /* We're done with the hash table, but we've taken the pointers to
199 * the allocated strings and put them into our result array, so
200 * tell the hash not to free them on its way out. */
201 g_hash_table_steal_all (thread_ids);
202 g_hash_table_unref (thread_ids);
207 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
208 * a (potentially nested) parenthesized sequence with '\' used to
209 * escape any character (including parentheses).
211 * If the sequence to be skipped continues to the end of the string,
212 * then 'str' will be left pointing at the final terminating '\0'
216 skip_space_and_comments (const char **str)
221 while (*s && (isspace (*s) || *s == '(')) {
222 while (*s && isspace (*s))
227 while (*s && nesting) {
243 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
244 * comments, and the '<' and '>' delimeters.
246 * If not NULL, then *next will be made to point to the first character
247 * not parsed, (possibly pointing to the final '\0' terminator.
249 * Returns a newly allocated string which the caller should free()
252 * Returns NULL if there is any error parsing the message-id. */
254 parse_message_id (const char *message_id, const char **next)
259 if (message_id == NULL)
264 skip_space_and_comments (&s);
266 /* Skip any unstructured text as well. */
267 while (*s && *s != '<')
278 skip_space_and_comments (&s);
281 while (*end && *end != '>')
290 if (end > s && *end == '>')
295 result = strndup (s, end - s + 1);
297 /* Finally, collapse any whitespace that is within the message-id
303 for (r = result, len = strlen (r); *r; r++, len--)
304 if (*r == ' ' || *r == '\t')
305 memmove (r, r+1, len);
311 /* Parse a References header value, putting a copy of each referenced
312 * message-id into 'array'. */
314 parse_references (GPtrArray *array,
323 ref = parse_message_id (refs, &refs);
326 g_ptr_array_add (array, ref);
331 notmuch_database_default_path (void)
333 if (getenv ("NOTMUCH_BASE"))
334 return strdup (getenv ("NOTMUCH_BASE"));
336 return g_strdup_printf ("%s/mail", getenv ("HOME"));
340 notmuch_database_create (const char *path)
342 notmuch_database_t *notmuch = NULL;
343 char *notmuch_path = NULL;
346 char *local_path = NULL;
349 path = local_path = notmuch_database_default_path ();
351 err = stat (path, &st);
353 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
354 path, strerror (errno));
358 if (! S_ISDIR (st.st_mode)) {
359 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
364 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
366 err = mkdir (notmuch_path, 0755);
369 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
370 notmuch_path, strerror (errno));
374 notmuch = notmuch_database_open (path);
386 notmuch_database_open (const char *path)
388 notmuch_database_t *notmuch = NULL;
389 char *notmuch_path = NULL, *xapian_path = NULL;
392 char *local_path = NULL;
395 path = local_path = notmuch_database_default_path ();
397 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
399 err = stat (notmuch_path, &st);
401 fprintf (stderr, "Error opening database at %s: %s\n",
402 notmuch_path, strerror (errno));
406 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
408 notmuch = talloc (NULL, notmuch_database_t);
409 notmuch->path = talloc_strdup (notmuch, path);
412 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
413 Xapian::DB_CREATE_OR_OPEN);
414 notmuch->query_parser = new Xapian::QueryParser;
415 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
416 notmuch->query_parser->set_database (*notmuch->xapian_db);
417 } catch (const Xapian::Error &error) {
418 fprintf (stderr, "A Xapian exception occurred: %s\n",
419 error.get_msg().c_str());
434 notmuch_database_close (notmuch_database_t *notmuch)
436 delete notmuch->query_parser;
437 delete notmuch->xapian_db;
438 talloc_free (notmuch);
442 notmuch_database_get_path (notmuch_database_t *notmuch)
444 return notmuch->path;
448 notmuch_database_add_message (notmuch_database_t *notmuch,
449 const char *filename)
451 notmuch_message_file_t *message_file;
452 notmuch_message_t *message;
453 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
455 GPtrArray *parents, *thread_ids;
457 const char *refs, *in_reply_to, *date, *header;
458 const char *from, *to, *subject, *old_filename;
463 message_file = notmuch_message_file_open (filename);
464 if (message_file == NULL) {
465 ret = NOTMUCH_STATUS_FILE_ERROR;
469 notmuch_message_file_restrict_headers (message_file,
480 /* The first order of business is to find/create a message ID. */
482 header = notmuch_message_file_get_header (message_file, "message-id");
484 message_id = parse_message_id (header, NULL);
485 /* So the header value isn't RFC-compliant, but it's
486 * better than no message-id at all. */
487 if (message_id == NULL)
488 message_id = xstrdup (header);
490 /* No message-id at all, let's generate one by taking a
491 * hash over the file's contents. */
492 char *sha1 = notmuch_sha1_of_file (filename);
494 /* If that failed too, something is really wrong. Give up. */
496 ret = NOTMUCH_STATUS_FILE_ERROR;
500 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
504 /* Now that we have a message ID, we get a message object,
505 * (which may or may not reference an existing document in the
508 /* Use NULL for owner since we want to free this locally. */
510 /* XXX: This call can fail by either out-of-memory or an
511 * "impossible" Xapian exception. We should rewrite it to
512 * allow us to propagate the error status. */
513 message = _notmuch_message_create_for_message_id (NULL, notmuch,
515 if (message == NULL) {
516 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
517 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
521 /* Has a message previously been added with the same ID? */
522 old_filename = notmuch_message_get_filename (message);
523 if (old_filename && strlen (old_filename)) {
524 /* XXX: This is too noisy to actually print, and what do we
525 * really expect the user to do? Go manually delete a
526 * redundant message or merge two similar messages?
527 * Instead we should handle this transparently.
529 * What we likely want to move to is adding both filenames
530 * to the database so that subsequent indexing will pick up
531 * terms from both files.
535 "Note: Attempting to add a message with a duplicate message ID:\n"
536 "Old: %s\n" "New: %s\n",
537 old_filename, filename);
538 fprintf (stderr, "The old filename will be used, but any new terms\n"
539 "from the new message will added to the database.\n");
542 _notmuch_message_set_filename (message, filename);
543 _notmuch_message_add_term (message, "type", "mail");
546 /* Next, find the thread(s) to which this message belongs. */
547 parents = g_ptr_array_new ();
549 refs = notmuch_message_file_get_header (message_file, "references");
550 parse_references (parents, refs);
552 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
553 parse_references (parents, in_reply_to);
555 for (i = 0; i < parents->len; i++)
556 _notmuch_message_add_term (message, "ref",
557 (char *) g_ptr_array_index (parents, i));
559 thread_ids = find_thread_ids (notmuch, parents, message_id);
563 for (i = 0; i < parents->len; i++)
564 g_free (g_ptr_array_index (parents, i));
565 g_ptr_array_free (parents, TRUE);
567 if (thread_ids->len) {
572 for (i = 0; i < thread_ids->len; i++) {
573 id = (char *) thread_ids->pdata[i];
574 _notmuch_message_add_thread_id (message, id);
576 thread_id = g_string_new (id);
578 g_string_append_printf (thread_id, ",%s", id);
582 g_string_free (thread_id, TRUE);
584 _notmuch_message_ensure_thread_id (message);
587 g_ptr_array_free (thread_ids, TRUE);
589 date = notmuch_message_file_get_header (message_file, "date");
590 _notmuch_message_set_date (message, date);
592 from = notmuch_message_file_get_header (message_file, "from");
593 subject = notmuch_message_file_get_header (message_file, "subject");
594 to = notmuch_message_file_get_header (message_file, "to");
600 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
603 _notmuch_message_sync (message);
605 } catch (const Xapian::Error &error) {
606 fprintf (stderr, "A Xapian exception occurred: %s.\n",
607 error.get_msg().c_str());
608 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
614 notmuch_message_destroy (message);
616 notmuch_message_file_close (message_file);