cli/git-remote: handle message deletions

There are two main possibilities. One is explicit delete ('D' command
in the stream from git fast-export) and one is files disappearing
between commits. It is less clear the latter can happen in well formed
sequence of commits, but it could result e.g. from manual changes to
the repo.

diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c
index addf23c7c4ad488cffe5143e63fa90ebc1afcd77..c5c46b1febaacfd95ec1f705c31623ad3d6294b9 100644 (file)
--- a/git-remote-notmuch.c
+++ b/git-remote-notmuch.c
@@ -49,6 +49,18 @@ typedef enum {
     MSG_STATE_DELETED
 } _message_state_t;
 
+static _message_state_t
+get_message_state (GHashTable *mid_state, const char *key)
+{
+    gpointer val = NULL;
+
+    if (! g_hash_table_lookup_extended (mid_state, key, NULL,
+                                       &val))
+       return MSG_STATE_UNKNOWN;
+    else
+       return GPOINTER_TO_INT (val);
+}
+
 static bool
 set_message_state (GHashTable *mid_state, const char *mid, _message_state_t state)
 {
@@ -344,16 +356,107 @@ path_to_mid (notmuch_database_t *notmuch, const char *path, char **mid_p, size_t
     return true;
 }
 
+/* In order to force a message to be deleted from the database, we
+ * need to delete all of its filenames. XXX TODO Add to library
+ * API? */
+static notmuch_status_t
+remove_message_all (notmuch_database_t *notmuch,
+                   notmuch_message_t *message)
+{
+    notmuch_filenames_t *filenames = NULL;
+    notmuch_status_t status;
+
+    for (filenames = notmuch_message_get_filenames (message);
+        notmuch_filenames_valid (filenames);
+        notmuch_filenames_move_to_next (filenames)) {
+       const char *filename =  notmuch_filenames_get (filenames);
+       status = notmuch_database_remove_message (notmuch, filename);
+       if (status != NOTMUCH_STATUS_SUCCESS &&
+           status != NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) {
+           fprintf (stderr, "failed to remove %s from database\n", filename);
+           return status;
+       }
+    }
+    return NOTMUCH_STATUS_SUCCESS;
+}
+
 static void
-mark_unseen (unused (notmuch_database_t *notmuch),
-            unused (GHashTable *mid_state))
+mark_unseen (notmuch_database_t *notmuch,
+            GHashTable *mid_state)
 {
+    notmuch_status_t status;
+    notmuch_messages_t *messages;
+    notmuch_query_t *query;
+
+    if (debug_flags && strchr (debug_flags, 'd')) {
+       flog ("total mids = %d\n", g_hash_table_size (mid_state));
+    }
+    status = notmuch_query_create_with_syntax (notmuch,
+                                              "",
+                                              NOTMUCH_QUERY_SYNTAX_XAPIAN,
+                                              &query);
+
+    if (print_status_database ("git-remote-nm", notmuch, status))
+       exit (EXIT_FAILURE);
+
+    notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED);
+
+    status = notmuch_query_search_messages (query, &messages);
+    if (print_status_query ("git-remote-nm", query, status))
+       exit (EXIT_FAILURE);
+
+    for (;
+        notmuch_messages_valid (messages);
+        notmuch_messages_move_to_next (messages)) {
+       notmuch_message_t *message = notmuch_messages_get (messages);
+       const char *mid = notmuch_message_get_message_id (message);
+
+       switch (get_message_state (mid_state, mid)) {
+       case MSG_STATE_SEEN:
+       case MSG_STATE_DELETED:
+           break;
+       case MSG_STATE_UNKNOWN:
+           set_message_state (mid_state, mid, MSG_STATE_DELETED);
+           break;
+       case MSG_STATE_MISSING:
+           INTERNAL_ERROR ("found missing mid %s", mid);
+       }
+       notmuch_message_destroy (message);
+    }
 }
 
 static void
-purge_database (unused (notmuch_database_t *notmuch),
-               unused (GHashTable *mid_state))
+purge_database (notmuch_database_t *notmuch, GHashTable *msg_state)
 {
+    gpointer key, value;
+
+    GHashTableIter iter;
+    int count = 0;
+
+    if (debug_flags && strchr (debug_flags, 'd'))
+       flog ("removing unseen messages from database\n");
+
+    g_hash_table_iter_init (&iter, msg_state);
+    while (g_hash_table_iter_next (&iter, &key, &value)) {
+       notmuch_message_t *message;
+       const char *mid = key;
+
+       if (GPOINTER_TO_INT (value) != MSG_STATE_DELETED)
+           continue;
+
+       ASSERT (NOTMUCH_STATUS_SUCCESS ==
+               notmuch_database_find_message (notmuch,
+                                              mid, &message));
+       /* If the message is in the database, clean up */
+       if (message) {
+           remove_message_all (notmuch, message);
+           if (debug_flags && strchr (debug_flags, 'd'))
+               flog ("removed from database %s\n", mid);
+           count++;
+       }
+    }
+    if (debug_flags && strchr (debug_flags, 'd'))
+       flog ("removed %d messages from database\n", count);
 }
 
 static void

diff --git a/test/T860-git-remote.sh b/test/T860-git-remote.sh
index 3a1c128ac53ae0b443e32f6ddb4a6a085daceb9f..fa52c2b788385242b36ffcfd1f5424f5624d6ed8 100755 (executable)
--- a/test/T860-git-remote.sh
+++ b/test/T860-git-remote.sh
@@ -32,7 +32,7 @@ export GIT_AUTHOR_NAME="Notmuch Test Suite"
 export GIT_AUTHOR_EMAIL="notmuch@example.com"
 export GIT_COMMITTER_NAME="Notmuch Test Suite"
 export GIT_COMMITTER_EMAIL="notmuch@example.com"
-export GIT_REMOTE_NM_DEBUG="s"
+export GIT_REMOTE_NM_DEBUG="sd"
 export GIT_REMOTE_NM_LOG=grn-log.txt
 EXPECTED=$NOTMUCH_SRCDIR/test/git-remote.expected-output
 MAKE_EXPORT_PY=$NOTMUCH_SRCDIR/test/make-export.py
@@ -271,7 +271,6 @@ restore_state
 
 backup_state
 test_begin_subtest "removing message via repo"
-test_subtest_known_broken
 parent=$(dirname $TAG_FILE)
 # future proof this for when e.g. properties are stored
 git -C repo rm -r $parent
@@ -284,6 +283,24 @@ EOF
 test_expect_equal_file EXPECTED OUTPUT
 restore_state
 
+backup_state
+test_begin_subtest "not removing later messages"
+add_message '[subject]="first new message"'
+git -C repo pull
+add_message '[subject]="second new message"'
+git -C repo pull
+notmuch dump | sort > EXPECTED
+git clone repo cloned_repo
+rm -rf ${MAIL_DIR}/.notmuch
+notmuch new --full-scan
+git -C cloned_repo remote add database notmuch::
+notmuch config set git.fail_on_missing false
+git -C cloned_repo push database master
+notmuch config set git.fail_on_missing true
+notmuch dump | sort > OUTPUT
+test_expect_equal_file_nonempty EXPECTED OUTPUT
+restore_state
+
 backup_state
 test_begin_subtest 'by default, missing messages are an error during export'
 test_subtest_known_broken