cli/git-remote: handle message deletions

There are two main possibilities. One is explicit delete ('D' command
in the stream from git fast-export) and one is files disappearing
between commits. It is less clear the latter can happen in well formed
sequence of commits, but it could result e.g. from manual changes to
the repo.

2 files changed, 137 insertions, 7 deletions

diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index addf23c7..a81c969e 100644
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -49,6 +49,18 @@ typedef enum {
     MSG_STATE_DELETED
 } _message_state_t;
 
+static _message_state_t
+get_message_state (GHashTable *mid_state, const char *key)
+{
+    gpointer val = NULL;
+
+    if (! g_hash_table_lookup_extended (mid_state, key, NULL,
+					&val))
+	return MSG_STATE_UNKNOWN;
+    else
+	return GPOINTER_TO_INT (val);
+}
+
 static bool
 set_message_state (GHashTable *mid_state, const char *mid, _message_state_t state)
 {
@@ -262,7 +274,7 @@ cmd_import (notmuch_database_t *notmuch,
 	exit (EXIT_FAILURE);
 
     for (;
-	 notmuch_messages_valid (messages);
+	 ! notmuch_messages_status (messages);
 	 notmuch_messages_move_to_next (messages)) {
 	const char *tag_buf = "";
 	const char *mid;
@@ -293,6 +305,11 @@ cmd_import (notmuch_database_t *notmuch,
 	write_data (tag_buf);
 	notmuch_message_destroy (message);
     }
+    status = notmuch_messages_status (messages);
+    if (status != NOTMUCH_STATUS_ITERATOR_EXHAUSTED &&
+	print_status_database ("git-remote-notmuch", notmuch, status))
+	exit (EXIT_FAILURE);
+
     puts ("");
     puts ("done");
     fflush (stdout);
@@ -344,16 +361,112 @@ path_to_mid (notmuch_database_t *notmuch, const char *path, char **mid_p, size_t
     return true;
 }
 
+/* In order to force a message to be deleted from the database, we
+ * need to delete all of its filenames. XXX TODO Add to library
+ * API? */
+static notmuch_status_t
+remove_message_all (notmuch_database_t *notmuch,
+		    notmuch_message_t *message)
+{
+    notmuch_filenames_t *filenames = NULL;
+    notmuch_status_t status;
+
+    for (filenames = notmuch_message_get_filenames (message);
+	 notmuch_filenames_valid (filenames);
+	 notmuch_filenames_move_to_next (filenames)) {
+	const char *filename =  notmuch_filenames_get (filenames);
+	status = notmuch_database_remove_message (notmuch, filename);
+	if (status != NOTMUCH_STATUS_SUCCESS &&
+	    status != NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) {
+	    fprintf (stderr, "failed to remove %s from database\n", filename);
+	    return status;
+	}
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 static void
-mark_unseen (unused (notmuch_database_t *notmuch),
-	     unused (GHashTable *mid_state))
+mark_unseen (notmuch_database_t *notmuch,
+	     GHashTable *mid_state)
 {
+    notmuch_status_t status;
+    notmuch_messages_t *messages;
+    notmuch_query_t *query;
+
+    if (debug_flags && strchr (debug_flags, 'd')) {
+	flog ("total mids = %d\n", g_hash_table_size (mid_state));
+    }
+    status = notmuch_query_create_with_syntax (notmuch,
+					       "",
+					       NOTMUCH_QUERY_SYNTAX_XAPIAN,
+					       &query);
+
+    if (print_status_database ("git-remote-nm", notmuch, status))
+	exit (EXIT_FAILURE);
+
+    notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+
+    status = notmuch_query_search_messages (query, &messages);
+    if (print_status_query ("git-remote-nm", query, status))
+	exit (EXIT_FAILURE);
+
+    for (;
+	 ! notmuch_messages_status (messages);
+	 notmuch_messages_move_to_next (messages)) {
+	notmuch_message_t *message = notmuch_messages_get (messages);
+	const char *mid = notmuch_message_get_message_id (message);
+
+	switch (get_message_state (mid_state, mid)) {
+	case MSG_STATE_SEEN:
+	case MSG_STATE_DELETED:
+	    break;
+	case MSG_STATE_UNKNOWN:
+	    set_message_state (mid_state, mid, MSG_STATE_DELETED);
+	    break;
+	case MSG_STATE_MISSING:
+	    INTERNAL_ERROR ("found missing mid %s", mid);
+	}
+	notmuch_message_destroy (message);
+    }
+    status = notmuch_messages_status (messages);
+    if (status != NOTMUCH_STATUS_ITERATOR_EXHAUSTED &&
+	print_status_database ("git-remote-notmuch", notmuch, status))
+	exit (EXIT_FAILURE);
+
 }
 
 static void
-purge_database (unused (notmuch_database_t *notmuch),
-		unused (GHashTable *mid_state))
+purge_database (notmuch_database_t *notmuch, GHashTable *msg_state)
 {
+    gpointer key, value;
+
+    GHashTableIter iter;
+    int count = 0;
+
+    if (debug_flags && strchr (debug_flags, 'd'))
+	flog ("removing unseen messages from database\n");
+
+    g_hash_table_iter_init (&iter, msg_state);
+    while (g_hash_table_iter_next (&iter, &key, &value)) {
+	notmuch_message_t *message;
+	const char *mid = key;
+
+	if (GPOINTER_TO_INT (value) != MSG_STATE_DELETED)
+	    continue;
+
+	ASSERT (NOTMUCH_STATUS_SUCCESS ==
+		notmuch_database_find_message (notmuch,
+					       mid, &message));
+	/* If the message is in the database, clean up */
+	if (message) {
+	    remove_message_all (notmuch, message);
+	    if (debug_flags && strchr (debug_flags, 'd'))
+		flog ("removed from database %s\n", mid);
+	    count++;
+	}
+    }
+    if (debug_flags && strchr (debug_flags, 'd'))
+	flog ("removed %d messages from database\n", count);
 }
 
 static void
diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 3a1c128a..fa52c2b7 100755
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -32,7 +32,7 @@ export GIT_AUTHOR_NAME="Notmuch Test Suite"
 export GIT_AUTHOR_EMAIL="notmuch@example.com"
 export GIT_COMMITTER_NAME="Notmuch Test Suite"
 export GIT_COMMITTER_EMAIL="notmuch@example.com"
-export GIT_REMOTE_NM_DEBUG="s"
+export GIT_REMOTE_NM_DEBUG="sd"
 export GIT_REMOTE_NM_LOG=grn-log.txt
 EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
 MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
@@ -271,7 +271,6 @@ restore_state
 
 backup_state
 test_begin_subtest "removing message via repo"
-test_subtest_known_broken
 parent=$(dirname $TAG_FILE)
 # future proof this for when e.g. properties are stored
 git -C repo rm -r $parent
@@ -285,6 +284,24 @@ test_expect_equal_file EXPECTED OUTPUT
 restore_state
 
 backup_state
+test_begin_subtest "not removing later messages"
+add_message '[subject]="first new message"'
+git -C repo pull
+add_message '[subject]="second new message"'
+git -C repo pull
+notmuch dump | sort > EXPECTED
+git clone repo cloned_repo
+rm -rf ${MAIL_DIR}/.notmuch
+notmuch new --full-scan
+git -C cloned_repo remote add database notmuch::
+notmuch config set git.fail_on_missing false
+git -C cloned_repo push database master
+notmuch config set git.fail_on_missing true
+notmuch dump | sort > OUTPUT
+test_expect_equal_file_nonempty EXPECTED OUTPUT
+restore_state
+
+backup_state
 test_begin_subtest 'by default, missing messages are an error during export'
 test_subtest_known_broken
 sed s/4EFC743A.3060609@april.org/missing-message@example.com/ < export.in > missing.in