From f6526e6571847d1f27e0e4fead59a3b18c5fe455 Mon Sep 17 00:00:00 2001 From: David Bremner Date: Sat, 24 Aug 2024 08:13:46 -0700 Subject: [PATCH] cli/git-remote: add import command The output in default.import is based on a modified version of Felipe's git-remote-nm with Blake2 hashing replaced by SHA1 (for portability). This enable fetch/pull/clone, so test that as well. --- git-remote-notmuch.c | 111 +++++++++ performance-test/M07-git-remote.sh | 16 ++ performance-test/T08-git-remote.sh | 12 + test/T860-git-remote.sh | 72 ++++++ .../git-remote.expected-output/default.import | 228 ++++++++++++++++++ test/make-export.py | 2 +- 6 files changed, 440 insertions(+), 1 deletion(-) create mode 100755 performance-test/M07-git-remote.sh create mode 100755 performance-test/T08-git-remote.sh create mode 100644 test/git-remote.expected-output/default.import diff --git a/git-remote-notmuch.c b/git-remote-notmuch.c index fb5cfd55..bfd045a8 100644 --- a/git-remote-notmuch.c +++ b/git-remote-notmuch.c @@ -156,6 +156,30 @@ read_lastmod (const char *dir, char **uuid_out, unsigned long *counter_out) } +static void +store_lastmod (notmuch_database_t *notmuch, const char *dir) +{ + char *filename = NULL; + FILE *out; + unsigned long lastmod; + const char *uuid; + + ASSERT (filename = talloc_asprintf (notmuch, "%s/lastmod", dir)); + + out = fopen (filename, "w"); + ensure (out, "error opening %s for writing: %s", filename, strerror (errno)); + + lastmod = notmuch_database_get_revision (notmuch, &uuid); + ASSERT (fprintf (out, "%s\t%zu\n", uuid, lastmod) > 0); +} + +static void +write_data (const char *data) +{ + printf ("data %zu\n", strlen (data)); + fputs (data, stdout); +} + static void cmd_capabilities () { @@ -175,6 +199,91 @@ cmd_list (notmuch_database_t *db, const char *uuid, unsigned long lastmod) equal_lastmod (uuid, lastmod, db_uuid, db_lastmod) ? " unchanged" : ""); } +static void +cmd_import (notmuch_database_t *notmuch, + const char *nm_dir, + const char *uuid, + unsigned long lastmod) +{ + const char *ident = NULL; + const char *lastmod_str = NULL; + notmuch_messages_t *messages; + notmuch_status_t status; + notmuch_query_t *query; + char *mid_buf = NULL; + size_t mid_buf_len = 0; + + ident = talloc_asprintf (notmuch, "%s <%s> %zu +0000", + notmuch_config_get (notmuch, NOTMUCH_CONFIG_USER_NAME), + notmuch_config_get (notmuch, NOTMUCH_CONFIG_PRIMARY_EMAIL), + time (NULL)); + + + printf ("feature done\ncommit refs/notmuch/master\nmark :1\ncommitter %s\n", ident); + + ASSERT (lastmod_str = talloc_asprintf (notmuch, "lastmod: %zu\n", lastmod)); + write_data (lastmod_str); + if (uuid) + puts ("from refs/notmuch/master^0"); + + /* don't send deleteall here, as there may be other files in the + * repo outside the database prefix */ + + status = notmuch_query_create_with_syntax (notmuch, + "", + NOTMUCH_QUERY_SYNTAX_XAPIAN, + &query); + + if (print_status_database ("git-remote-nm", notmuch, status)) + exit (EXIT_FAILURE); + + if (debug_flags && strchr (debug_flags, 's')) + notmuch_query_set_sort (query, NOTMUCH_SORT_NEWEST_FIRST); + else + notmuch_query_set_sort (query, NOTMUCH_SORT_UNSORTED); + + status = notmuch_query_search_messages (query, &messages); + if (print_status_query ("git-remote-nm", query, status)) + exit (EXIT_FAILURE); + + for (; + notmuch_messages_valid (messages); + notmuch_messages_move_to_next (messages)) { + const char *tag_buf = ""; + const char *mid; + const char *hash; + const char *prefix = notmuch_config_get (notmuch, NOTMUCH_CONFIG_GIT_METADATA_PREFIX); + int ret; + + notmuch_message_t *message = notmuch_messages_get (messages); + mid = notmuch_message_get_message_id (message); + + ret = hex_encode (notmuch, mid, &mid_buf, &mid_buf_len); + ensure (ret == HEX_SUCCESS, "failed to hex-encode message-id %s\n", mid); + + /* we can't use _notmuch_sha1_from_string because we don't want + * to include the null terminator */ + g_autoptr (GChecksum) sha1 = NULL; + sha1 = g_checksum_new (G_CHECKSUM_SHA1); + g_checksum_update (sha1, (const guchar *) mid, strlen (mid)); + hash = g_checksum_get_string (sha1); + printf ("M 644 inline %s/%2.2s/%2.2s/%s/tags\n", prefix, hash, hash + 2, mid_buf); + + for (notmuch_tags_t *tags = notmuch_message_get_tags (message); + notmuch_tags_valid (tags); + notmuch_tags_move_to_next (tags)) { + const char *tag_str = notmuch_tags_get (tags); + ASSERT (tag_buf = talloc_asprintf (message, "%s%s\n", tag_buf, tag_str)); + } + write_data (tag_buf); + notmuch_message_destroy (message); + } + puts (""); + puts ("done"); + fflush (stdout); + store_lastmod (notmuch, nm_dir); +} + /* stubs since we cannot link with notmuch.o */ const notmuch_opt_desc_t notmuch_shared_options[] = { { } @@ -303,6 +412,8 @@ main (int argc, char *argv[]) if (STRNCMP_LITERAL (s, "capabilities") == 0) cmd_capabilities (); + else if (STRNCMP_LITERAL (s, "import") == 0) + cmd_import (db, nm_dir, uuid, lastmod); else if (STRNCMP_LITERAL (s, "list") == 0) cmd_list (db, uuid, lastmod); diff --git a/performance-test/M07-git-remote.sh b/performance-test/M07-git-remote.sh new file mode 100755 index 00000000..57b9ab32 --- /dev/null +++ b/performance-test/M07-git-remote.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +test_description='git remote helper' + +. $(dirname "$0")/perf-test-lib.sh || exit 1 + +mkdir repo +export GIT_DIR=`pwd`/repo + +memory_start + +echo "import refs/heads/master" > import.in + +memory_run "import" "git-remote-notmuch origin notmuch:// >import.out EXPECTED EOF test_expect_equal_file EXPECTED OUTPUT +test_begin_subtest 'import writes lastmod file' +echo import | run_helper dummy-alias dummy-url > /dev/null +lastmod=$(notmuch count --lastmod '*' | cut -f2-) +test_expect_equal "${lastmod}" "$(cat < ${git_tmp}/notmuch/lastmod)" + +# note that this test must not be the first time import is run, +# because it depends on the lastmod file +test_begin_subtest 'import produces expected output' +echo import | run_helper | notmuch_sanitize_git > OUTPUT +test_expect_equal_file $EXPECTED/default.import OUTPUT + +test_begin_subtest "clone notmuch://" +test_expect_success "git clone notmuch:// $(mktemp -d cloneXXXXXX)" + +test_begin_subtest "clone notmuch://?config=notmuch-config" +test_expect_success "git clone notmuch://?config=notmuch-config $(mktemp -d cloneXXXXXX)" + +test_begin_subtest "clone notmuch://?profile=default" +test_expect_success "git clone notmuch://?profile=default $(mktemp -d cloneXXXXXX)" + +test_begin_subtest "clone notmuch://?config=notmuch-config&profile=default" +test_expect_success "git clone notmuch://?config=notmuch-config\&profile=default $(mktemp -d cloneXXXXXX)" + +test_begin_subtest 'clone notmuch://`pwd`/mail' +test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail $(mktemp -d cloneXXXXXX)" + +test_begin_subtest 'clone notmuch://`pwd`/mail/?config=`pwd`/notmuch-config' +test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail?config=`pwd`/notmuch-config $(mktemp -d cloneXXXXXX)" + +test_begin_subtest 'clone notmuch://.../mail/?config=.../notmuch-config&profile=default' +test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://`pwd`/mail/?config=`pwd`/notmuch-config\&profile=default $(mktemp -d clone XXX)" + +test_begin_subtest 'clone notmuch://?path=.../mail/&config=.../notmuch-config&profile=default' +test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch://?path=`pwd`/mail\&config=notmuch-config\&profile=default $(mktemp -d cloneXXXXXX)" + +test_begin_subtest "clone notmuch::" +test_expect_success "git clone notmuch:: $(mktemp -d cloneXXXXXX)" + +test_begin_subtest 'clone notmuch::`pwd`/mail' +test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch::`pwd`/mail $(mktemp -d cloneXXXXXX)" + +test_begin_subtest 'clone notmuch::`pwd`/mail?config=`pwd`/notmuch-config' +test_expect_success "env -u NOTMUCH_CONFIG git clone notmuch::`pwd`/mail?config=`pwd`/notmuch-config $(mktemp -d cloneXXXXXX)" + +test_begin_subtest "clone has every message" +git clone notmuch:: repo +find repo -name tags -type f | sed -e s,repo/_notmuch_metadata/../../,id:, -e s,/tags$,, | sort > OUTPUT +notmuch search --output=messages '*' | sort > EXPECTED +test_expect_equal_file EXPECTED OUTPUT + +backup_state +test_begin_subtest "pull get new tag" +notmuch tag +zznew -- id:4EFC743A.3060609@april.org +git -C repo pull +cat<EXPECTED +inbox +unread +zznew +EOF +test_expect_equal_file EXPECTED repo/$TAG_FILE +restore_state + +backup_state +test_begin_subtest "pull sees deletion" +notmuch tag -unread -- id:4EFC743A.3060609@april.org +git -C repo pull +cat<EXPECTED +inbox +EOF +test_expect_equal_file EXPECTED repo/$TAG_FILE +restore_state + test_done diff --git a/test/git-remote.expected-output/default.import b/test/git-remote.expected-output/default.import new file mode 100644 index 00000000..ccbc8f8d --- /dev/null +++ b/test/git-remote.expected-output/default.import @@ -0,0 +1,228 @@ +feature done +commit refs/notmuch/master +mark :1 +committer Notmuch Test Suite TIMESTAMP TIMEZONE +data 12 +lastmod: 53 +from refs/notmuch/master^0 +M 644 inline _notmuch_metadata/87/b1/4EFC743A.3060609@april.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/9a/a0/877h1wv7mg.fsf@inf-8657.int-evry.fr/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/a8/61/1258544095-16616-1-git-send-email-chris@chris-wilson.co.uk/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/50/8c/877htoqdbo.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/50/69/878we4qdqf.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/94/67/87aaykqe24.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/bd/10/87bpj0qeng.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/27/84/87fx8cqf8v.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/3c/98/87hbssqfix.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/dd/a4/87iqd8qgiz.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/02/28/87k4xoqgnl.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/61/f6/87ocn0qh6d.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/9a/6b/87pr7gqidx.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/cc/ab/867hto2p0t.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/ba/76/1258532999-9316-1-git-send-email-keithp@keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/ec/65/86aayk2rbj.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/15/a0/86d43g2w3y.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/54/e7/ddd65cda0911172214t60d22b63hcfeb5a19ab54a39b@mail.gmail.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/60/cc/86einw2xof.fsf@fortitudo.i-did-not-set--mail-host-address--so-tickle-me/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/68/39/736613.51770.qm@web113505.mail.gq1.yahoo.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/1c/7b/1258520223-15328-1-git-send-email-jan@ryngle.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/46/60/ddd65cda0911171950o4eea4389v86de9525e46052d3@mail.gmail.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/78/87/1258510940-7018-1-git-send-email-stewart@flamingspork.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/8c/92/yunzl6kd1w0.fsf@aiko.keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/cf/e1/yun1vjwegii.fsf@aiko.keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/d3/55/yun3a4cegoa.fsf@aiko.keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/fa/e4/1258509400-32511-1-git-send-email-stewart@flamingspork.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/c2/7f/1258506353-20352-1-git-send-email-stewart@flamingspork.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/13/dd/20091118010116.GC25380@dottiness.seas.harvard.edu/tags +data 31 +attachment +inbox +signed +unread +M 644 inline _notmuch_metadata/97/cf/20091118005829.GB25380@dottiness.seas.harvard.edu/tags +data 31 +attachment +inbox +signed +unread +M 644 inline _notmuch_metadata/cf/81/20091118005040.GA25380@dottiness.seas.harvard.edu/tags +data 20 +inbox +signed +unread +M 644 inline _notmuch_metadata/ca/dc/cf0c4d610911171623q3e27a0adx802e47039b57604b@mail.gmail.com/tags +data 24 +attachment +inbox +unread +M 644 inline _notmuch_metadata/42/c5/1258500222-32066-1-git-send-email-ingmar@exherbo.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/47/c1/20091117232137.GA7669@griffis1.net/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/d7/7c/20091118002059.067214ed@hikari/tags +data 20 +inbox +signed +unread +M 644 inline _notmuch_metadata/ae/4b/1258498485-sup-142@elly/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/41/fa/f35dbb950911171438k5df6eb56k77b6c0944e2e79ae@mail.gmail.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/e4/81/f35dbb950911171435ieecd458o853c873e35f4be95@mail.gmail.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/fe/4b/1258496327-12086-1-git-send-email-jan@ryngle.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/2c/3b/1258493565-13508-1-git-send-email-keithp@keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/0c/a5/yunaayketfm.fsf@aiko.keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/21/a3/yunbpj0etua.fsf@aiko.keithp.com/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/b8/fa/1258491078-29658-1-git-send-email-dottedmag@dottedmag.net/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/08/50/87fx8can9z.fsf@vertex.dottedmag/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/d3/76/20091117203301.GV3165@dottiness.seas.harvard.edu/tags +data 20 +inbox +signed +unread +M 644 inline _notmuch_metadata/52/bb/87lji4lx9v.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/df/26/cf0c4d610911171136h1713aa59w9cf9aa31f052ad0a@mail.gmail.com/tags +data 24 +attachment +inbox +unread +M 644 inline _notmuch_metadata/ab/f7/87iqd9rn3l.fsf@vertex.dottedmag/tags +data 20 +inbox +signed +unread +M 644 inline _notmuch_metadata/c5/19/20091117190054.GU3165@dottiness.seas.harvard.edu/tags +data 20 +inbox +signed +unread +M 644 inline _notmuch_metadata/15/c2/87lji5cbwo.fsf@yoom.home.cworth.org/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/de/77/1258471718-6781-2-git-send-email-dottedmag@dottedmag.net/tags +data 13 +inbox +unread +M 644 inline _notmuch_metadata/77/76/1258471718-6781-1-git-send-email-dottedmag@dottedmag.net/tags +data 13 +inbox +unread + +done diff --git a/test/make-export.py b/test/make-export.py index 3837dc3a..0e6a1141 100644 --- a/test/make-export.py +++ b/test/make-export.py @@ -39,6 +39,6 @@ print (f"data 8\nignored") for msg in db.messages(""): digest = sha1(msg.messageid.encode('utf8')).hexdigest() filename = hexencode(msg.messageid) - print (f"M 100644 :{mark[msg.messageid]} {digest[0:2]}/{digest[2:4]}/{filename}/tags") + print (f"M 100644 :{mark[msg.messageid]} _notmuch_metadata/{digest[0:2]}/{digest[2:4]}/{filename}/tags") print("\ndone\n") -- 2.45.2