1 /* database.cc - The database interfaces of the notmuch mail library
3 * Copyright © 2009 Carl Worth
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see http://www.gnu.org/licenses/ .
18 * Author: Carl Worth <cworth@cworth.org>
21 #include "database-private.h"
27 #include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
32 notmuch_status_to_string (notmuch_status_t status)
35 case NOTMUCH_STATUS_SUCCESS:
36 return "No error occurred";
37 case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
38 return "A Xapian exception occurred";
39 case NOTMUCH_STATUS_FILE_ERROR:
40 return "Something went wrong trying to read or write a file";
41 case NOTMUCH_STATUS_FILE_NOT_EMAIL:
42 return "File is not an email";
43 case NOTMUCH_STATUS_NULL_POINTER:
44 return "Erroneous NULL pointer";
45 case NOTMUCH_STATUS_TAG_TOO_LONG:
46 return "Tag value is too long";
48 case NOTMUCH_STATUS_LAST_STATUS:
49 return "Unknown error status value";
53 /* XXX: We should drop this function and convert all callers to call
54 * _notmuch_message_add_term instead. */
56 add_term (Xapian::Document doc,
57 const char *prefix_name,
66 prefix = _find_prefix (prefix_name);
68 term = g_strdup_printf ("%s%s", prefix, value);
70 if (strlen (term) <= NOTMUCH_TERM_MAX)
77 find_messages_by_term (Xapian::Database *db,
78 const char *prefix_name,
80 Xapian::PostingIterator *begin,
81 Xapian::PostingIterator *end)
83 Xapian::PostingIterator i;
86 term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
88 *begin = db->postlist_begin (term);
91 *end = db->postlist_end (term);
97 find_message_by_docid (Xapian::Database *db, Xapian::docid docid)
99 return db->get_document (docid);
103 insert_thread_id (GHashTable *thread_ids, Xapian::Document doc)
106 const char *value, *id, *comma;
108 value_string = doc.get_value (NOTMUCH_VALUE_THREAD);
109 value = value_string.c_str();
110 if (strlen (value)) {
113 comma = strchr (id, ',');
115 comma = id + strlen (id);
116 g_hash_table_insert (thread_ids,
117 strndup (id, comma - id), NULL);
126 notmuch_database_find_message (notmuch_database_t *notmuch,
127 const char *message_id)
129 Xapian::PostingIterator i, end;
131 find_messages_by_term (notmuch->xapian_db,
132 "msgid", message_id, &i, &end);
137 return _notmuch_message_create (notmuch, notmuch, *i);
140 /* Return one or more thread_ids, (as a GPtrArray of strings), for the
141 * given message based on looking into the database for any messages
142 * referenced in parents, and also for any messages in the database
143 * referencing message_id.
145 * Caller should free all strings in the array and the array itself,
146 * (g_ptr_array_free) when done. */
148 find_thread_ids (notmuch_database_t *notmuch,
150 const char *message_id)
152 Xapian::WritableDatabase *db = notmuch->xapian_db;
153 Xapian::PostingIterator child, children_end;
154 Xapian::Document doc;
155 GHashTable *thread_ids;
158 const char *parent_message_id;
161 thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
164 find_messages_by_term (db, "ref", message_id, &child, &children_end);
165 for ( ; child != children_end; child++) {
166 doc = find_message_by_docid (db, *child);
167 insert_thread_id (thread_ids, doc);
170 for (i = 0; i < parents->len; i++) {
171 notmuch_message_t *parent;
172 notmuch_thread_ids_t *ids;
174 parent_message_id = (char *) g_ptr_array_index (parents, i);
175 parent = notmuch_database_find_message (notmuch, parent_message_id);
179 for (ids = notmuch_message_get_thread_ids (parent);
180 notmuch_thread_ids_has_more (ids);
181 notmuch_thread_ids_advance (ids))
185 id = notmuch_thread_ids_get (ids);
186 g_hash_table_insert (thread_ids, strdup (id), NULL);
189 notmuch_message_destroy (parent);
192 result = g_ptr_array_new ();
194 keys = g_hash_table_get_keys (thread_ids);
195 for (l = keys; l; l = l->next) {
196 char *id = (char *) l->data;
197 g_ptr_array_add (result, id);
201 /* We're done with the hash table, but we've taken the pointers to
202 * the allocated strings and put them into our result array, so
203 * tell the hash not to free them on its way out. */
204 g_hash_table_steal_all (thread_ids);
205 g_hash_table_unref (thread_ids);
210 /* Advance 'str' past any whitespace or RFC 822 comments. A comment is
211 * a (potentially nested) parenthesized sequence with '\' used to
212 * escape any character (including parentheses).
214 * If the sequence to be skipped continues to the end of the string,
215 * then 'str' will be left pointing at the final terminating '\0'
219 skip_space_and_comments (const char **str)
224 while (*s && (isspace (*s) || *s == '(')) {
225 while (*s && isspace (*s))
230 while (*s && nesting) {
246 /* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
247 * comments, and the '<' and '>' delimeters.
249 * If not NULL, then *next will be made to point to the first character
250 * not parsed, (possibly pointing to the final '\0' terminator.
252 * Returns a newly allocated string which the caller should free()
255 * Returns NULL if there is any error parsing the message-id. */
257 parse_message_id (const char *message_id, const char **next)
262 if (message_id == NULL)
267 skip_space_and_comments (&s);
269 /* Skip any unstructured text as well. */
270 while (*s && *s != '<')
281 skip_space_and_comments (&s);
284 while (*end && *end != '>')
293 if (end > s && *end == '>')
298 result = strndup (s, end - s + 1);
300 /* Finally, collapse any whitespace that is within the message-id
306 for (r = result, len = strlen (r); *r; r++, len--)
307 if (*r == ' ' || *r == '\t')
308 memmove (r, r+1, len);
314 /* Parse a References header value, putting a copy of each referenced
315 * message-id into 'array'. */
317 parse_references (GPtrArray *array,
326 ref = parse_message_id (refs, &refs);
329 g_ptr_array_add (array, ref);
334 notmuch_database_default_path (void)
336 if (getenv ("NOTMUCH_BASE"))
337 return strdup (getenv ("NOTMUCH_BASE"));
339 return g_strdup_printf ("%s/mail", getenv ("HOME"));
343 notmuch_database_create (const char *path)
345 notmuch_database_t *notmuch = NULL;
346 char *notmuch_path = NULL;
349 char *local_path = NULL;
352 path = local_path = notmuch_database_default_path ();
354 err = stat (path, &st);
356 fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
357 path, strerror (errno));
361 if (! S_ISDIR (st.st_mode)) {
362 fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
367 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
369 err = mkdir (notmuch_path, 0755);
372 fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
373 notmuch_path, strerror (errno));
377 notmuch = notmuch_database_open (path);
389 notmuch_database_open (const char *path)
391 notmuch_database_t *notmuch = NULL;
392 char *notmuch_path = NULL, *xapian_path = NULL;
395 char *local_path = NULL;
398 path = local_path = notmuch_database_default_path ();
400 notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
402 err = stat (notmuch_path, &st);
404 fprintf (stderr, "Error opening database at %s: %s\n",
405 notmuch_path, strerror (errno));
409 xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
411 notmuch = talloc (NULL, notmuch_database_t);
412 notmuch->path = talloc_strdup (notmuch, path);
415 notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
416 Xapian::DB_CREATE_OR_OPEN);
417 notmuch->query_parser = new Xapian::QueryParser;
418 notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
419 notmuch->query_parser->set_database (*notmuch->xapian_db);
420 } catch (const Xapian::Error &error) {
421 fprintf (stderr, "A Xapian exception occurred: %s\n",
422 error.get_msg().c_str());
437 notmuch_database_close (notmuch_database_t *notmuch)
439 delete notmuch->query_parser;
440 delete notmuch->xapian_db;
441 talloc_free (notmuch);
445 notmuch_database_get_path (notmuch_database_t *notmuch)
447 return notmuch->path;
451 notmuch_database_add_message (notmuch_database_t *notmuch,
452 const char *filename)
454 notmuch_message_file_t *message_file;
455 notmuch_message_t *message;
456 notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
458 GPtrArray *parents, *thread_ids;
460 const char *refs, *in_reply_to, *date, *header;
461 const char *from, *to, *subject, *old_filename;
466 message_file = notmuch_message_file_open (filename);
467 if (message_file == NULL) {
468 ret = NOTMUCH_STATUS_FILE_ERROR;
472 notmuch_message_file_restrict_headers (message_file,
483 /* The first order of business is to find/create a message ID. */
485 header = notmuch_message_file_get_header (message_file, "message-id");
487 message_id = parse_message_id (header, NULL);
488 /* So the header value isn't RFC-compliant, but it's
489 * better than no message-id at all. */
490 if (message_id == NULL)
491 message_id = xstrdup (header);
493 /* No message-id at all, let's generate one by taking a
494 * hash over the file's contents. */
495 char *sha1 = notmuch_sha1_of_file (filename);
497 /* If that failed too, something is really wrong. Give up. */
499 ret = NOTMUCH_STATUS_FILE_ERROR;
503 message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
507 /* Now that we have a message ID, we get a message object,
508 * (which may or may not reference an existing document in the
511 /* Use NULL for owner since we want to free this locally. */
513 /* XXX: This call can fail by either out-of-memory or an
514 * "impossible" Xapian exception. We should rewrite it to
515 * allow us to propagate the error status. */
516 message = _notmuch_message_create_for_message_id (NULL, notmuch,
518 if (message == NULL) {
519 fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
520 fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
524 /* Has a message previously been added with the same ID? */
525 old_filename = notmuch_message_get_filename (message);
526 if (old_filename && strlen (old_filename)) {
527 /* XXX: This is too noisy to actually print, and what do we
528 * really expect the user to do? Go manually delete a
529 * redundant message or merge two similar messages?
530 * Instead we should handle this transparently.
532 * What we likely want to move to is adding both filenames
533 * to the database so that subsequent indexing will pick up
534 * terms from both files.
538 "Note: Attempting to add a message with a duplicate message ID:\n"
539 "Old: %s\n" "New: %s\n",
540 old_filename, filename);
541 fprintf (stderr, "The old filename will be used, but any new terms\n"
542 "from the new message will added to the database.\n");
545 _notmuch_message_set_filename (message, filename);
546 _notmuch_message_add_term (message, "type", "mail");
549 /* Next, find the thread(s) to which this message belongs. */
550 parents = g_ptr_array_new ();
552 refs = notmuch_message_file_get_header (message_file, "references");
553 parse_references (parents, refs);
555 in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
556 parse_references (parents, in_reply_to);
558 for (i = 0; i < parents->len; i++)
559 _notmuch_message_add_term (message, "ref",
560 (char *) g_ptr_array_index (parents, i));
562 thread_ids = find_thread_ids (notmuch, parents, message_id);
566 for (i = 0; i < parents->len; i++)
567 g_free (g_ptr_array_index (parents, i));
568 g_ptr_array_free (parents, TRUE);
570 if (thread_ids->len) {
575 for (i = 0; i < thread_ids->len; i++) {
576 id = (char *) thread_ids->pdata[i];
577 _notmuch_message_add_thread_id (message, id);
579 thread_id = g_string_new (id);
581 g_string_append_printf (thread_id, ",%s", id);
585 g_string_free (thread_id, TRUE);
587 _notmuch_message_ensure_thread_id (message);
590 g_ptr_array_free (thread_ids, TRUE);
592 date = notmuch_message_file_get_header (message_file, "date");
593 _notmuch_message_set_date (message, date);
595 from = notmuch_message_file_get_header (message_file, "from");
596 subject = notmuch_message_file_get_header (message_file, "subject");
597 to = notmuch_message_file_get_header (message_file, "to");
603 ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
606 _notmuch_message_sync (message);
608 } catch (const Xapian::Error &error) {
609 fprintf (stderr, "A Xapian exception occurred: %s.\n",
610 error.get_msg().c_str());
611 ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
617 notmuch_message_destroy (message);
619 notmuch_message_file_close (message_file);