X-Git-Url: https://git.notmuchmail.org/git?p=notmuch;a=blobdiff_plain;f=lib%2Fdatabase.cc;h=45c42607802d79db683f7cb77683d68aaa831a4a;hp=a021bf17253cd9ceecfa02a3b4c28d5f5583355d;hb=df8885f62c13f77b2b16cbb211e3a727945870b0;hpb=00d2ac2b41a6dba060837244ee87edce2fd1e465

diff --git a/lib/database.cc b/lib/database.cc
index a021bf17..45c42607 100644
--- a/lib/database.cc
+++ b/lib/database.cc
@@ -42,7 +42,7 @@ typedef struct {
     const char *prefix;
 } prefix_t;
 
-#define NOTMUCH_DATABASE_VERSION 1
+#define NOTMUCH_DATABASE_VERSION 2
 
 #define STRINGIFY(s) _SUB_STRINGIFY(s)
 #define _SUB_STRINGIFY(s) #s
@@ -100,8 +100,8 @@ typedef struct {
  * In addition, terms from the content of the message are added with
  * "from", "to", "attachment", and "subject" prefixes for use by the
  * user in searching. Similarly, terms from the path of the mail
- * message are added with a "folder" prefix. But the database doesn't
- * really care itself about any of these.
+ * message are added with "folder" and "path" prefixes. But the
+ * database doesn't really care itself about any of these.
  *
  * The data portion of a mail document is empty.
  *
@@ -208,7 +208,15 @@ static prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "thread",			"G" },
     { "tag",			"K" },
     { "is",			"K" },
-    { "id",			"Q" }
+    { "id",			"Q" },
+    { "path",			"P" },
+    /*
+     * Without the ":", since this is a multi-letter prefix, Xapian
+     * will add a colon itself if the first letter of the path is
+     * upper-case ASCII. Including the ":" forces there to always be a
+     * colon, which keeps our own logic simpler.
+     */
+    { "folder",			"XFOLDER:" },
 };
 
 static prefix_t PROBABILISTIC_PREFIX[]= {
@@ -216,7 +224,6 @@ static prefix_t PROBABILISTIC_PREFIX[]= {
     { "to",			"XTO" },
     { "attachment",		"XATTACHMENT" },
     { "subject",		"XSUBJECT"},
-    { "folder",			"XFOLDER"}
 };
 
 const char *
@@ -349,7 +356,7 @@ _message_id_compressed (void *ctx, const char *message_id)
 {
     char *sha1, *compressed;
 
-    sha1 = notmuch_sha1_of_string (message_id);
+    sha1 = _notmuch_sha1_of_string (message_id);
 
     compressed = talloc_asprintf (ctx, "notmuch-sha1-%s", sha1);
     free (sha1);
@@ -767,14 +774,17 @@ notmuch_database_open (const char *path,
     return status;
 }
 
-void
+notmuch_status_t
 notmuch_database_close (notmuch_database_t *notmuch)
 {
+    notmuch_status_t status = NOTMUCH_STATUS_SUCCESS;
+
     try {
 	if (notmuch->xapian_db != NULL &&
 	    notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE)
 	    (static_cast <Xapian::WritableDatabase *> (notmuch->xapian_db))->flush ();
     } catch (const Xapian::Error &error) {
+	status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 	if (! notmuch->exception_reported) {
 	    fprintf (stderr, "Error: A Xapian exception occurred flushing database: %s\n",
 		     error.get_msg().c_str());
@@ -788,7 +798,9 @@ notmuch_database_close (notmuch_database_t *notmuch)
 	try {
 	    notmuch->xapian_db->close();
 	} catch (const Xapian::Error &error) {
-	    /* do nothing */
+	    /* don't clobber previous error status */
+	    if (status == NOTMUCH_STATUS_SUCCESS)
+		status = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 	}
     }
 
@@ -802,20 +814,24 @@ notmuch_database_close (notmuch_database_t *notmuch)
     notmuch->value_range_processor = NULL;
     delete notmuch->date_range_processor;
     notmuch->date_range_processor = NULL;
+
+    return status;
 }
 
 #if HAVE_XAPIAN_COMPACT
-static int unlink_cb (const char *path,
-		      unused (const struct stat *sb),
-		      unused (int type),
-		      unused (struct FTW *ftw))
+static int
+unlink_cb (const char *path,
+	   unused (const struct stat *sb),
+	   unused (int type),
+	   unused (struct FTW *ftw))
 {
-    return remove(path);
+    return remove (path);
 }
 
-static int rmtree (const char *path)
+static int
+rmtree (const char *path)
 {
-    return nftw(path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
+    return nftw (path, unlink_cb, 64, FTW_DEPTH | FTW_PHYS);
 }
 
 class NotmuchCompactor : public Xapian::Compactor
@@ -825,17 +841,17 @@ class NotmuchCompactor : public Xapian::Compactor
 
 public:
     NotmuchCompactor(notmuch_compact_status_cb_t cb, void *closure) :
-	status_cb(cb), status_closure(closure) { }
+	status_cb (cb), status_closure (closure) { }
 
     virtual void
     set_status (const std::string &table, const std::string &status)
     {
-	char* msg;
+	char *msg;
 
 	if (status_cb == NULL)
 	    return;
 
-	if (status.length() == 0)
+	if (status.length () == 0)
 	    msg = talloc_asprintf (NULL, "compacting table %s", table.c_str());
 	else
 	    msg = talloc_asprintf (NULL, "     %s", status.c_str());
@@ -844,8 +860,8 @@ public:
 	    return;
 	}
 
-	status_cb(msg, status_closure);
-	talloc_free(msg);
+	status_cb (msg, status_closure);
+	talloc_free (msg);
     }
 };
 
@@ -861,8 +877,8 @@ public:
  * compaction process to protect data integrity.
  */
 notmuch_status_t
-notmuch_database_compact (const char* path,
-			  const char* backup_path,
+notmuch_database_compact (const char *path,
+			  const char *backup_path,
 			  notmuch_compact_status_cb_t status_cb,
 			  void *closure)
 {
@@ -871,12 +887,13 @@ notmuch_database_compact (const char* path,
     notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS;
     notmuch_database_t *notmuch = NULL;
     struct stat statbuf;
+    notmuch_bool_t keep_backup;
 
     local = talloc_new (NULL);
     if (! local)
 	return NOTMUCH_STATUS_OUT_OF_MEMORY;
 
-    ret = notmuch_database_open(path, NOTMUCH_DATABASE_MODE_READ_WRITE, &notmuch);
+    ret = notmuch_database_open (path, NOTMUCH_DATABASE_MODE_READ_WRITE, &notmuch);
     if (ret) {
 	goto DONE;
     }
@@ -896,60 +913,90 @@ notmuch_database_compact (const char* path,
 	goto DONE;
     }
 
-    if (backup_path != NULL) {
-	if (stat(backup_path, &statbuf) != -1) {
-	    fprintf (stderr, "Backup path already exists: %s\n", backup_path);
-	    ret = NOTMUCH_STATUS_FILE_ERROR;
-	    goto DONE;
-	}
-	if (errno != ENOENT) {
-	    fprintf (stderr, "Unknown error while stat()ing backup path: %s\n",
-		     strerror(errno));
+    if (backup_path == NULL) {
+	if (! (backup_path = talloc_asprintf (local, "%s.old", xapian_path))) {
+	    ret = NOTMUCH_STATUS_OUT_OF_MEMORY;
 	    goto DONE;
 	}
+	keep_backup = FALSE;
+    }
+    else {
+	keep_backup = TRUE;
+    }
+
+    if (stat (backup_path, &statbuf) != -1) {
+	fprintf (stderr, "Path already exists: %s\n", backup_path);
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
+    }
+    if (errno != ENOENT) {
+	fprintf (stderr, "Unknown error while stat()ing path: %s\n",
+		 strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
     }
 
+    /* Unconditionally attempt to remove old work-in-progress database (if
+     * any). This is "protected" by database lock. If this fails due to write
+     * errors (etc), the following code will fail and provide error message.
+     */
+    (void) rmtree (compact_xapian_path);
+
     try {
-	NotmuchCompactor compactor(status_cb, closure);
-
-	compactor.set_renumber(false);
-	compactor.add_source(xapian_path);
-	compactor.set_destdir(compact_xapian_path);
-	compactor.compact();
-    } catch (Xapian::InvalidArgumentError e) {
-	fprintf (stderr, "Error while compacting: %s\n", e.get_msg().c_str());
+	NotmuchCompactor compactor (status_cb, closure);
+
+	compactor.set_renumber (false);
+	compactor.add_source (xapian_path);
+	compactor.set_destdir (compact_xapian_path);
+	compactor.compact ();
+    } catch (const Xapian::Error &error) {
+	fprintf (stderr, "Error while compacting: %s\n", error.get_msg().c_str());
 	ret = NOTMUCH_STATUS_XAPIAN_EXCEPTION;
 	goto DONE;
     }
 
-    if (backup_path) {
-	if (rename(xapian_path, backup_path)) {
-	    fprintf (stderr, "Error moving old database out of the way\n");
-	    ret = NOTMUCH_STATUS_FILE_ERROR;
-	    goto DONE;
-	}
-    } else {
-	rmtree(xapian_path);
+    if (rename (xapian_path, backup_path)) {
+	fprintf (stderr, "Error moving %s to %s: %s\n",
+		 xapian_path, backup_path, strerror (errno));
+	ret = NOTMUCH_STATUS_FILE_ERROR;
+	goto DONE;
     }
 
-    if (rename(compact_xapian_path, xapian_path)) {
-	fprintf (stderr, "Error moving compacted database\n");
+    if (rename (compact_xapian_path, xapian_path)) {
+	fprintf (stderr, "Error moving %s to %s: %s\n",
+		 compact_xapian_path, xapian_path, strerror (errno));
 	ret = NOTMUCH_STATUS_FILE_ERROR;
 	goto DONE;
     }
 
-DONE:
-    if (notmuch)
-	notmuch_database_destroy (notmuch);
+    if (! keep_backup) {
+	if (rmtree (backup_path)) {
+	    fprintf (stderr, "Error removing old database %s: %s\n",
+		     backup_path, strerror (errno));
+	    ret = NOTMUCH_STATUS_FILE_ERROR;
+	    goto DONE;
+	}
+    }
+
+  DONE:
+    if (notmuch) {
+	notmuch_status_t ret2;
 
-    talloc_free(local);
+	ret2 = notmuch_database_destroy (notmuch);
+
+	/* don't clobber previous error status */
+	if (ret == NOTMUCH_STATUS_SUCCESS && ret2 != NOTMUCH_STATUS_SUCCESS)
+	    ret = ret2;
+    }
+
+    talloc_free (local);
 
     return ret;
 }
 #else
 notmuch_status_t
-notmuch_database_compact (unused (const char* path),
-			  unused (const char* backup_path),
+notmuch_database_compact (unused (const char *path),
+			  unused (const char *backup_path),
 			  unused (notmuch_compact_status_cb_t status_cb),
 			  unused (void *closure))
 {
@@ -958,11 +1005,15 @@ notmuch_database_compact (unused (const char* path),
 }
 #endif
 
-void
+notmuch_status_t
 notmuch_database_destroy (notmuch_database_t *notmuch)
 {
-    notmuch_database_close (notmuch);
+    notmuch_status_t status;
+
+    status = notmuch_database_close (notmuch);
     talloc_free (notmuch);
+
+    return status;
 }
 
 const char *
@@ -1141,6 +1192,40 @@ notmuch_database_upgrade (notmuch_database_t *notmuch,
 	}
     }
 
+    /*
+     * Prior to version 2, the "folder:" prefix was probabilistic and
+     * stemmed. Change it to the current boolean prefix. Add "path:"
+     * prefixes while at it.
+     */
+    if (version < 2) {
+	notmuch_query_t *query = notmuch_query_create (notmuch, "");
+	notmuch_messages_t *messages;
+	notmuch_message_t *message;
+
+	count = 0;
+	total = notmuch_query_count_messages (query);
+
+	for (messages = notmuch_query_search_messages (query);
+	     notmuch_messages_valid (messages);
+	     notmuch_messages_move_to_next (messages)) {
+	    if (do_progress_notify) {
+		progress_notify (closure, (double) count / total);
+		do_progress_notify = 0;
+	    }
+
+	    message = notmuch_messages_get (messages);
+
+	    _notmuch_message_upgrade_folder (message);
+	    _notmuch_message_sync (message);
+
+	    notmuch_message_destroy (message);
+
+	    count++;
+	}
+
+	notmuch_query_destroy (query);
+    }
+
     db->set_metadata ("version", STRINGIFY (NOTMUCH_DATABASE_VERSION));
     db->flush ();
 
@@ -1289,7 +1374,7 @@ _notmuch_database_get_directory_db_path (const char *path)
     int term_len = strlen (_find_prefix ("directory")) + strlen (path);
 
     if (term_len > NOTMUCH_TERM_MAX)
-	return notmuch_sha1_of_string (path);
+	return _notmuch_sha1_of_string (path);
     else
 	return path;
 }
@@ -1538,7 +1623,7 @@ _notmuch_database_generate_doc_id (notmuch_database_t *notmuch)
     notmuch->last_doc_id++;
 
     if (notmuch->last_doc_id == 0)
-	INTERNAL_ERROR ("Xapian document IDs are exhausted.\n");	
+	INTERNAL_ERROR ("Xapian document IDs are exhausted.\n");
 
     return notmuch->last_doc_id;
 }
@@ -1691,12 +1776,12 @@ _notmuch_database_link_message_to_parents (notmuch_database_t *notmuch,
 				     _my_talloc_free_for_g_hash, NULL);
     this_message_id = notmuch_message_get_message_id (message);
 
-    refs = notmuch_message_file_get_header (message_file, "references");
+    refs = _notmuch_message_file_get_header (message_file, "references");
     last_ref_message_id = parse_references (message,
 					    this_message_id,
 					    parents, refs);
 
-    in_reply_to = notmuch_message_file_get_header (message_file, "in-reply-to");
+    in_reply_to = _notmuch_message_file_get_header (message_file, "in-reply-to");
     in_reply_to_message_id = parse_references (message,
 					       this_message_id,
 					       parents, in_reply_to);
@@ -1894,7 +1979,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     if (ret)
 	return ret;
 
-    message_file = notmuch_message_file_open (filename);
+    message_file = _notmuch_message_file_open (filename);
     if (message_file == NULL)
 	return NOTMUCH_STATUS_FILE_ERROR;
 
@@ -1904,15 +1989,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     if (ret)
 	goto DONE;
 
-    notmuch_message_file_restrict_headers (message_file,
-					   "date",
-					   "from",
-					   "in-reply-to",
-					   "message-id",
-					   "references",
-					   "subject",
-					   "to",
-					   (char *) NULL);
+    /* Parse message up front to get better error status. */
+    ret = _notmuch_message_file_parse (message_file);
+    if (ret)
+	goto DONE;
 
     try {
 	/* Before we do any real work, (especially before doing a
@@ -1920,9 +2000,9 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	 * let's make sure that what we're looking at looks like an
 	 * actual email message.
 	 */
-	from = notmuch_message_file_get_header (message_file, "from");
-	subject = notmuch_message_file_get_header (message_file, "subject");
-	to = notmuch_message_file_get_header (message_file, "to");
+	from = _notmuch_message_file_get_header (message_file, "from");
+	subject = _notmuch_message_file_get_header (message_file, "subject");
+	to = _notmuch_message_file_get_header (message_file, "to");
 
 	if ((from == NULL || *from == '\0') &&
 	    (subject == NULL || *subject == '\0') &&
@@ -1935,7 +2015,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	/* Now that we're sure it's mail, the first order of business
 	 * is to find a message ID (or else create one ourselves). */
 
-	header = notmuch_message_file_get_header (message_file, "message-id");
+	header = _notmuch_message_file_get_header (message_file, "message-id");
 	if (header && *header != '\0') {
 	    message_id = _parse_message_id (message_file, header, NULL);
 
@@ -1956,7 +2036,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	if (message_id == NULL ) {
 	    /* No message-id at all, let's generate one by taking a
 	     * hash over the file's contents. */
-	    char *sha1 = notmuch_sha1_of_file (filename);
+	    char *sha1 = _notmuch_sha1_of_file (filename);
 
 	    /* If that failed too, something is really wrong. Give up. */
 	    if (sha1 == NULL) {
@@ -1996,10 +2076,10 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	    if (ret)
 		goto DONE;
 
-	    date = notmuch_message_file_get_header (message_file, "date");
+	    date = _notmuch_message_file_get_header (message_file, "date");
 	    _notmuch_message_set_header_values (message, date, from, subject);
 
-	    ret = _notmuch_message_index_file (message, filename);
+	    ret = _notmuch_message_index_file (message, message_file);
 	    if (ret)
 		goto DONE;
 	} else {
@@ -2025,7 +2105,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
     }
 
     if (message_file)
-	notmuch_message_file_close (message_file);
+	_notmuch_message_file_close (message_file);
 
     ret2 = notmuch_database_end_atomic (notmuch);
     if ((ret == NOTMUCH_STATUS_SUCCESS ||