* Author: Carl Worth <cworth@cworth.org>
*/
-#include "notmuch-private.h"
-
-#include <stdio.h>
-#include <errno.h>
-#include <time.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/time.h>
+#include "database-private.h"
#include <iostream>
-#include <gmime/gmime.h>
-
#include <xapian.h>
-using namespace std;
-
-struct _notmuch_database {
- char *path;
- Xapian::WritableDatabase *xapian_db;
- Xapian::TermGenerator *term_gen;
-};
-
-#define ARRAY_SIZE(arr) (sizeof (arr) / sizeof (arr[0]))
-
-/* Xapian complains if we provide a term longer than this. */
-#define NOTMUCH_MAX_TERM 245
-
-/* These prefix values are specifically chosen to be compatible
- * with sup, (http://sup.rubyforge.org), written by
- * William Morgan <wmorgan-sup@masanjin.net>, and released
- * under the GNU GPL v2.
- */
-
-typedef struct {
- const char *name;
- const char *prefix;
-} prefix_t;
-
-prefix_t NORMAL_PREFIX[] = {
- { "subject", "S" },
- { "body", "B" },
- { "from_name", "FN" },
- { "to_name", "TN" },
- { "name", "N" },
- { "attachment", "A" }
-};
-
-prefix_t BOOLEAN_PREFIX[] = {
- { "type", "K" },
- { "from_email", "FE" },
- { "to_email", "TE" },
- { "email", "E" },
- { "date", "D" },
- { "label", "L" },
- { "source_id", "I" },
- { "attachment_extension", "O" },
- { "msgid", "Q" },
- { "thread", "H" },
- { "ref", "R" }
-};
-
-/* Similarly, these value numbers are also chosen to be sup
- * compatible. */
-
-typedef enum {
- NOTMUCH_VALUE_MESSAGE_ID = 0,
- NOTMUCH_VALUE_THREAD = 1,
- NOTMUCH_VALUE_DATE = 2
-} notmuch_value_t;
-
-static const char *
-find_prefix (const char *name)
-{
- unsigned int i;
-
- for (i = 0; i < ARRAY_SIZE (NORMAL_PREFIX); i++)
- if (strcmp (name, NORMAL_PREFIX[i].name) == 0)
- return NORMAL_PREFIX[i].prefix;
-
- for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX); i++)
- if (strcmp (name, BOOLEAN_PREFIX[i].name) == 0)
- return BOOLEAN_PREFIX[i].prefix;
+#include <glib.h> /* g_strdup_printf, g_free, GPtrArray, GHashTable */
- return "";
-}
-
-/* "128 bits of thread-id ought to be enough for anybody" */
-#define NOTMUCH_THREAD_ID_BITS 128
-#define NOTMUCH_THREAD_ID_DIGITS (NOTMUCH_THREAD_ID_BITS / 4)
-typedef struct _thread_id {
- char str[NOTMUCH_THREAD_ID_DIGITS + 1];
-} thread_id_t;
+using namespace std;
-static void
-thread_id_generate (thread_id_t *thread_id)
+const char *
+notmuch_status_to_string (notmuch_status_t status)
{
- static int seeded = 0;
- FILE *dev_random;
- uint32_t value;
- char *s;
- int i;
-
- if (! seeded) {
- dev_random = fopen ("/dev/random", "r");
- if (dev_random == NULL) {
- srand (time (NULL));
- } else {
- fread ((void *) &value, sizeof (value), 1, dev_random);
- srand (value);
- fclose (dev_random);
- }
- seeded = 1;
- }
-
- s = thread_id->str;
- for (i = 0; i < NOTMUCH_THREAD_ID_DIGITS; i += 8) {
- value = rand ();
- sprintf (s, "%08x", value);
- s += 8;
+ switch (status) {
+ case NOTMUCH_STATUS_SUCCESS:
+ return "No error occurred";
+ case NOTMUCH_STATUS_XAPIAN_EXCEPTION:
+ return "A Xapian exception occurred";
+ case NOTMUCH_STATUS_FILE_ERROR:
+ return "Something went wrong trying to read or write a file";
+ case NOTMUCH_STATUS_FILE_NOT_EMAIL:
+ return "File is not an email";
+ case NOTMUCH_STATUS_NULL_POINTER:
+ return "Erroneous NULL pointer";
+ case NOTMUCH_STATUS_TAG_TOO_LONG:
+ return "Tag value is too long";
+ default:
+ case NOTMUCH_STATUS_LAST_STATUS:
+ return "Unknown error status value";
}
}
+/* XXX: We should drop this function and convert all callers to call
+ * _notmuch_message_add_term instead. */
static void
add_term (Xapian::Document doc,
const char *prefix_name,
if (value == NULL)
return;
- prefix = find_prefix (prefix_name);
+ prefix = _find_prefix (prefix_name);
term = g_strdup_printf ("%s%s", prefix, value);
- if (strlen (term) <= NOTMUCH_MAX_TERM)
+ if (strlen (term) <= NOTMUCH_TERM_MAX)
doc.add_term (term);
g_free (term);
}
static void
-find_messages_by_term (Xapian::Database *db,
- const char *prefix_name,
- const char *value,
- Xapian::PostingIterator *begin,
- Xapian::PostingIterator *end)
+find_doc_ids (notmuch_database_t *notmuch,
+ const char *prefix_name,
+ const char *value,
+ Xapian::PostingIterator *begin,
+ Xapian::PostingIterator *end)
{
Xapian::PostingIterator i;
char *term;
- term = g_strdup_printf ("%s%s", find_prefix (prefix_name), value);
+ term = g_strdup_printf ("%s%s", _find_prefix (prefix_name), value);
- *begin = db->postlist_begin (term);
+ *begin = notmuch->xapian_db->postlist_begin (term);
- if (end)
- *end = db->postlist_end (term);
+ *end = notmuch->xapian_db->postlist_end (term);
free (term);
}
-Xapian::Document
-find_message_by_docid (Xapian::Database *db, Xapian::docid docid)
+static notmuch_private_status_t
+find_unique_doc_id (notmuch_database_t *notmuch,
+ const char *prefix_name,
+ const char *value,
+ unsigned int *doc_id)
{
- return db->get_document (docid);
+ Xapian::PostingIterator i, end;
+
+ find_doc_ids (notmuch, prefix_name, value, &i, &end);
+
+ if (i == end) {
+ *doc_id = 0;
+ return NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND;
+ } else {
+ *doc_id = *i;
+ return NOTMUCH_PRIVATE_STATUS_SUCCESS;
+ }
}
-Xapian::Document
-find_message_by_message_id (Xapian::Database *db, const char *message_id)
+static Xapian::Document
+find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id)
{
- Xapian::PostingIterator i, end;
+ return notmuch->xapian_db->get_document (doc_id);
+}
+
+static notmuch_private_status_t
+find_unique_document (notmuch_database_t *notmuch,
+ const char *prefix_name,
+ const char *value,
+ Xapian::Document *document,
+ unsigned int *doc_id)
+{
+ notmuch_private_status_t status;
- find_messages_by_term (db, "msgid", message_id, &i, &end);
+ status = find_unique_doc_id (notmuch, prefix_name, value, doc_id);
- if (i != end)
- return find_message_by_docid (db, *i);
- else
- return Xapian::Document ();
+ if (status) {
+ *document = Xapian::Document ();
+ return status;
+ }
+
+ *document = find_document_for_doc_id (notmuch, *doc_id);
+ return NOTMUCH_PRIVATE_STATUS_SUCCESS;
}
static void
}
}
+notmuch_message_t *
+notmuch_database_find_message (notmuch_database_t *notmuch,
+ const char *message_id)
+{
+ notmuch_private_status_t status;
+ unsigned int doc_id;
+
+ status = find_unique_doc_id (notmuch, "msgid", message_id, &doc_id);
+
+ if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND)
+ return NULL;
+
+ return _notmuch_message_create (notmuch, notmuch, doc_id);
+}
+
/* Return one or more thread_ids, (as a GPtrArray of strings), for the
* given message based on looking into the database for any messages
* referenced in parents, and also for any messages in the database
* Caller should free all strings in the array and the array itself,
* (g_ptr_array_free) when done. */
static GPtrArray *
-find_thread_ids (Xapian::Database *db,
+find_thread_ids (notmuch_database_t *notmuch,
GPtrArray *parents,
const char *message_id)
{
thread_ids = g_hash_table_new_full (g_str_hash, g_str_equal,
free, NULL);
- find_messages_by_term (db, "ref", message_id, &child, &children_end);
+ find_doc_ids (notmuch, "ref", message_id, &child, &children_end);
for ( ; child != children_end; child++) {
- doc = find_message_by_docid (db, *child);
+ doc = find_document_for_doc_id (notmuch, *child);
insert_thread_id (thread_ids, doc);
}
for (i = 0; i < parents->len; i++) {
+ notmuch_message_t *parent;
+ notmuch_thread_ids_t *ids;
+
parent_message_id = (char *) g_ptr_array_index (parents, i);
- doc = find_message_by_message_id (db, parent_message_id);
- insert_thread_id (thread_ids, doc);
+ parent = notmuch_database_find_message (notmuch, parent_message_id);
+ if (parent == NULL)
+ continue;
+
+ for (ids = notmuch_message_get_thread_ids (parent);
+ notmuch_thread_ids_has_more (ids);
+ notmuch_thread_ids_advance (ids))
+ {
+ const char *id;
+
+ id = notmuch_thread_ids_get (ids);
+ g_hash_table_insert (thread_ids, strdup (id), NULL);
+ }
+
+ notmuch_message_destroy (parent);
}
result = g_ptr_array_new ();
return result;
}
-/* Add a term for each message-id in the References header of the
- * message. */
+/* Advance 'str' past any whitespace or RFC 822 comments. A comment is
+ * a (potentially nested) parenthesized sequence with '\' used to
+ * escape any character (including parentheses).
+ *
+ * If the sequence to be skipped continues to the end of the string,
+ * then 'str' will be left pointing at the final terminating '\0'
+ * character.
+ */
+static void
+skip_space_and_comments (const char **str)
+{
+ const char *s;
+
+ s = *str;
+ while (*s && (isspace (*s) || *s == '(')) {
+ while (*s && isspace (*s))
+ s++;
+ if (*s == '(') {
+ int nesting = 1;
+ s++;
+ while (*s && nesting) {
+ if (*s == '(')
+ nesting++;
+ else if (*s == ')')
+ nesting--;
+ else if (*s == '\\')
+ if (*(s+1))
+ s++;
+ s++;
+ }
+ }
+ }
+
+ *str = s;
+}
+
+/* Parse an RFC 822 message-id, discarding whitespace, any RFC 822
+ * comments, and the '<' and '>' delimeters.
+ *
+ * If not NULL, then *next will be made to point to the first character
+ * not parsed, (possibly pointing to the final '\0' terminator.
+ *
+ * Returns a newly allocated string which the caller should free()
+ * when done with it.
+ *
+ * Returns NULL if there is any error parsing the message-id. */
+static char *
+parse_message_id (const char *message_id, const char **next)
+{
+ const char *s, *end;
+ char *result;
+
+ if (message_id == NULL)
+ return NULL;
+
+ s = message_id;
+
+ skip_space_and_comments (&s);
+
+ /* Skip any unstructured text as well. */
+ while (*s && *s != '<')
+ s++;
+
+ if (*s == '<') {
+ s++;
+ } else {
+ if (next)
+ *next = s;
+ return NULL;
+ }
+
+ skip_space_and_comments (&s);
+
+ end = s;
+ while (*end && *end != '>')
+ end++;
+ if (next) {
+ if (*end)
+ *next = end + 1;
+ else
+ *next = end;
+ }
+
+ if (end > s && *end == '>')
+ end--;
+ if (end <= s)
+ return NULL;
+
+ result = strndup (s, end - s + 1);
+
+ /* Finally, collapse any whitespace that is within the message-id
+ * itself. */
+ {
+ char *r;
+ int len;
+
+ for (r = result, len = strlen (r); *r; r++, len--)
+ if (*r == ' ' || *r == '\t')
+ memmove (r, r+1, len);
+ }
+
+ return result;
+}
+
+/* Parse a References header value, putting a copy of each referenced
+ * message-id into 'array'. */
static void
parse_references (GPtrArray *array,
- const char *refs_str)
+ const char *refs)
{
- GMimeReferences *refs, *r;
- const char *message_id;
+ char *ref;
- if (refs_str == NULL)
+ if (refs == NULL)
return;
- refs = g_mime_references_decode (refs_str);
+ while (*refs) {
+ ref = parse_message_id (refs, &refs);
- for (r = refs; r; r = r->next) {
- message_id = g_mime_references_get_message_id (r);
- g_ptr_array_add (array, g_strdup (message_id));
+ if (ref)
+ g_ptr_array_add (array, ref);
}
+}
- g_mime_references_free (refs);
+char *
+notmuch_database_default_path (void)
+{
+ if (getenv ("NOTMUCH_BASE"))
+ return strdup (getenv ("NOTMUCH_BASE"));
+
+ return g_strdup_printf ("%s/mail", getenv ("HOME"));
}
notmuch_database_t *
notmuch_database_create (const char *path)
{
- char *notmuch_path;
+ notmuch_database_t *notmuch = NULL;
+ char *notmuch_path = NULL;
struct stat st;
int err;
+ char *local_path = NULL;
+
+ if (path == NULL)
+ path = local_path = notmuch_database_default_path ();
err = stat (path, &st);
if (err) {
fprintf (stderr, "Error: Cannot create database at %s: %s.\n",
path, strerror (errno));
- return NULL;
+ goto DONE;
}
if (! S_ISDIR (st.st_mode)) {
fprintf (stderr, "Error: Cannot create database at %s: Not a directory.\n",
path);
- return NULL;
+ goto DONE;
}
notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
if (err) {
fprintf (stderr, "Error: Cannot create directory %s: %s.\n",
notmuch_path, strerror (errno));
- free (notmuch_path);
- return NULL;
+ goto DONE;
}
- free (notmuch_path);
+ notmuch = notmuch_database_open (path);
- return notmuch_database_open (path);
+ DONE:
+ if (notmuch_path)
+ free (notmuch_path);
+ if (local_path)
+ free (local_path);
+
+ return notmuch;
}
notmuch_database_t *
notmuch_database_open (const char *path)
{
- notmuch_database_t *notmuch;
- char *notmuch_path, *xapian_path;
+ notmuch_database_t *notmuch = NULL;
+ char *notmuch_path = NULL, *xapian_path = NULL;
struct stat st;
int err;
+ char *local_path = NULL;
- g_mime_init (0);
+ if (path == NULL)
+ path = local_path = notmuch_database_default_path ();
notmuch_path = g_strdup_printf ("%s/%s", path, ".notmuch");
err = stat (notmuch_path, &st);
if (err) {
- fprintf (stderr, "Error: Cannot stat %s: %s\n",
- notmuch_path, strerror (err));
- free (notmuch_path);
- return NULL;
+ fprintf (stderr, "Error opening database at %s: %s\n",
+ notmuch_path, strerror (errno));
+ goto DONE;
}
xapian_path = g_strdup_printf ("%s/%s", notmuch_path, "xapian");
- free (notmuch_path);
- /* C++ is so nasty in requiring these casts. I'm almost tempted to
- * write a C wrapper for Xapian... */
- notmuch = (notmuch_database_t *) xmalloc (sizeof (notmuch_database_t));
- notmuch->path = xstrdup (path);
+ notmuch = talloc (NULL, notmuch_database_t);
+ notmuch->path = talloc_strdup (notmuch, path);
try {
notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
Xapian::DB_CREATE_OR_OPEN);
+ notmuch->query_parser = new Xapian::QueryParser;
+ notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
+ notmuch->query_parser->set_database (*notmuch->xapian_db);
} catch (const Xapian::Error &error) {
fprintf (stderr, "A Xapian exception occurred: %s\n",
error.get_msg().c_str());
}
- free (xapian_path);
+ DONE:
+ if (local_path)
+ free (local_path);
+ if (notmuch_path)
+ free (notmuch_path);
+ if (xapian_path)
+ free (xapian_path);
return notmuch;
}
void
notmuch_database_close (notmuch_database_t *notmuch)
{
+ delete notmuch->query_parser;
delete notmuch->xapian_db;
- free (notmuch->path);
- free (notmuch);
+ talloc_free (notmuch);
}
const char *
notmuch_database_add_message (notmuch_database_t *notmuch,
const char *filename)
{
- Xapian::WritableDatabase *db = notmuch->xapian_db;
- Xapian::Document doc;
+ notmuch_message_file_t *message_file;
+ notmuch_message_t *message;
+ notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
- GMimeStream *stream;
- GMimeParser *parser;
- GMimeMessage *message;
GPtrArray *parents, *thread_ids;
- FILE *file;
+ const char *refs, *in_reply_to, *date, *header;
+ const char *from, *to, *subject, *old_filename;
+ char *message_id;
- const char *refs, *in_reply_to;
- const char *message_id;
-
- time_t time;
unsigned int i;
- file = fopen (filename, "r");
- if (! file) {
- fprintf (stderr, "Error opening %s: %s\n", filename, strerror (errno));
- exit (1);
+ message_file = notmuch_message_file_open (filename);
+ if (message_file == NULL) {
+ ret = NOTMUCH_STATUS_FILE_ERROR;
+ goto DONE;
}
- stream = g_mime_stream_file_new (file);
+ notmuch_message_file_restrict_headers (message_file,
+ "date",
+ "from",
+ "in-reply-to",
+ "message-id",
+ "references",
+ "subject",
+ "to",
+ (char *) NULL);
- parser = g_mime_parser_new_with_stream (stream);
+ try {
+ /* The first order of business is to find/create a message ID. */
+
+ header = notmuch_message_file_get_header (message_file, "message-id");
+ if (header) {
+ message_id = parse_message_id (header, NULL);
+ /* So the header value isn't RFC-compliant, but it's
+ * better than no message-id at all. */
+ if (message_id == NULL)
+ message_id = xstrdup (header);
+ } else {
+ /* No message-id at all, let's generate one by taking a
+ * hash over the file's contents. */
+ char *sha1 = notmuch_sha1_of_file (filename);
+
+ /* If that failed too, something is really wrong. Give up. */
+ if (sha1 == NULL) {
+ ret = NOTMUCH_STATUS_FILE_ERROR;
+ goto DONE;
+ }
- message = g_mime_parser_construct_message (parser);
+ message_id = g_strdup_printf ("notmuch-sha1-%s", sha1);
+ free (sha1);
+ }
- try {
- doc = Xapian::Document ();
+ /* Now that we have a message ID, we get a message object,
+ * (which may or may not reference an existing document in the
+ * database). */
+
+ /* Use NULL for owner since we want to free this locally. */
+
+ /* XXX: This call can fail by either out-of-memory or an
+ * "impossible" Xapian exception. We should rewrite it to
+ * allow us to propagate the error status. */
+ message = _notmuch_message_create_for_message_id (NULL, notmuch,
+ message_id);
+ if (message == NULL) {
+ fprintf (stderr, "Internal error. This shouldn't happen.\n\n");
+ fprintf (stderr, "I mean, it's possible you ran out of memory, but then this code path is still an internal error since it should have detected that and propagated the status value up the stack.\n");
+ exit (1);
+ }
- doc.set_data (filename);
+ /* Has a message previously been added with the same ID? */
+ old_filename = notmuch_message_get_filename (message);
+ if (old_filename && strlen (old_filename)) {
+ /* XXX: This is too noisy to actually print, and what do we
+ * really expect the user to do? Go manually delete a
+ * redundant message or merge two similar messages?
+ * Instead we should handle this transparently.
+ *
+ * What we likely want to move to is adding both filenames
+ * to the database so that subsequent indexing will pick up
+ * terms from both files.
+ */
+#if 0
+ fprintf (stderr,
+ "Note: Attempting to add a message with a duplicate message ID:\n"
+ "Old: %s\n" "New: %s\n",
+ old_filename, filename);
+ fprintf (stderr, "The old filename will be used, but any new terms\n"
+ "from the new message will added to the database.\n");
+#endif
+ } else {
+ _notmuch_message_set_filename (message, filename);
+ _notmuch_message_add_term (message, "type", "mail");
+ }
+ /* Next, find the thread(s) to which this message belongs. */
parents = g_ptr_array_new ();
- refs = g_mime_object_get_header (GMIME_OBJECT (message), "references");
+ refs = notmuch_message_file_get_header (message_file, "references");
parse_references (parents, refs);
- in_reply_to = g_mime_object_get_header (GMIME_OBJECT (message),
- "in-reply-to");
+ in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
parse_references (parents, in_reply_to);
+
for (i = 0; i < parents->len; i++)
- add_term (doc, "ref", (char *) g_ptr_array_index (parents, i));
+ _notmuch_message_add_term (message, "ref",
+ (char *) g_ptr_array_index (parents, i));
- message_id = g_mime_message_get_message_id (message);
+ thread_ids = find_thread_ids (notmuch, parents, message_id);
- thread_ids = find_thread_ids (db, parents, message_id);
+ free (message_id);
for (i = 0; i < parents->len; i++)
g_free (g_ptr_array_index (parents, i));
g_ptr_array_free (parents, TRUE);
- if (message_id) {
- add_term (doc, "msgid", message_id);
- doc.add_value (NOTMUCH_VALUE_MESSAGE_ID, message_id);
- }
if (thread_ids->len) {
unsigned int i;
for (i = 0; i < thread_ids->len; i++) {
id = (char *) thread_ids->pdata[i];
- add_term (doc, "thread", id);
+ _notmuch_message_add_thread_id (message, id);
if (i == 0)
thread_id = g_string_new (id);
else
free (id);
}
- g_ptr_array_free (thread_ids, TRUE);
- doc.add_value (NOTMUCH_VALUE_THREAD, thread_id->str);
g_string_free (thread_id, TRUE);
- } else if (message_id) {
- /* If not part of any existing thread, generate a new thread_id. */
- thread_id_t thread_id;
-
- thread_id_generate (&thread_id);
- add_term (doc, "thread", thread_id.str);
- doc.add_value (NOTMUCH_VALUE_THREAD, thread_id.str);
+ } else {
+ _notmuch_message_ensure_thread_id (message);
}
- g_mime_message_get_date (message, &time, NULL);
- doc.add_value (NOTMUCH_VALUE_DATE, Xapian::sortable_serialise (time));
+ g_ptr_array_free (thread_ids, TRUE);
+
+ date = notmuch_message_file_get_header (message_file, "date");
+ _notmuch_message_set_date (message, date);
- db->add_document (doc);
+ from = notmuch_message_file_get_header (message_file, "from");
+ subject = notmuch_message_file_get_header (message_file, "subject");
+ to = notmuch_message_file_get_header (message_file, "to");
+
+ if (from == NULL &&
+ subject == NULL &&
+ to == NULL)
+ {
+ ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
+ goto DONE;
+ } else {
+ _notmuch_message_sync (message);
+ }
} catch (const Xapian::Error &error) {
fprintf (stderr, "A Xapian exception occurred: %s.\n",
error.get_msg().c_str());
- return NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+ ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
+ goto DONE;
}
- g_object_unref (message);
- g_object_unref (parser);
- g_object_unref (stream);
+ DONE:
+ if (message)
+ notmuch_message_destroy (message);
+ if (message_file)
+ notmuch_message_file_close (message_file);
- return NOTMUCH_STATUS_SUCCESS;
+ return ret;
}
projects / notmuch / blobdiff