dump: support gzipped and atomic output

The main goal is to support gzipped output for future internal
calls (e.g. from notmuch-new) to notmuch_database_dump.

The additional dependency is not very heavy since xapian already pulls
in zlib.

We want the dump to be "atomic", in the sense that after running the
dump file is either present and complete, or not present.  This avoids
certain classes of mishaps involving overwriting a good backup with a
bad or partial one.

diff --git a/INSTALL b/INSTALL
index 690b0efd27eaa60b404d2ca72da11699d7dc9162..b543c5085e3278b494e501f3c991c1a56ff8d5a1 100644 (file)
--- a/INSTALL
+++ b/INSTALL
@@ -20,8 +20,8 @@ configure stage.
 
 Dependencies
 ------------
 
 Dependencies
 ------------

-Notmuch depends on three libraries: Xapian, GMime 2.4 or 2.6, and
-Talloc which are each described below:
+Notmuch depends on four libraries: Xapian, GMime 2.4 or 2.6,
+Talloc, and zlib which are each described below:

 
        Xapian
        ------
 
        Xapian
        ------


@@ -60,6 +60,18 @@ Talloc which are each described below:
 
        Talloc is available from http://talloc.samba.org/
 
 
        Talloc is available from http://talloc.samba.org/
 

+       zlib
+       ----
+
+       zlib is an extremely popular compression library. It is used
+       by Xapian, so if you installed that you will already have
+       zlib. You may need to install the zlib headers separately.
+
+       Notmuch needs the transparent write feature of zlib introduced
+       in version 1.2.5.2 (Dec. 2011).
+
+       zlib is available from http://zlib.net
+

 Building Documentation
 ----------------------
 
 Building Documentation
 ----------------------
 


@@ -79,11 +91,11 @@ dependencies with a simple simple command line. For example:
 
   For Debian and similar:
 
 
   For Debian and similar:
 

-        sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev python-sphinx
+        sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev python-sphinx

 
   For Fedora and similar:
 
 
   For Fedora and similar:
 

-       sudo yum install xapian-core-devel gmime-devel libtalloc-devel python-sphinx
+       sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel python-sphinx

 
 On other systems, a similar command can be used, but the details of
 the package names may be different.
 
 On other systems, a similar command can be used, but the details of
 the package names may be different.

diff --git a/Makefile.local b/Makefile.local
index 1856350efeec735975b0b875813168ffa392419e..877a97903a4ce12e4bb489d3a4c52785767788f0 100644 (file)
--- a/Makefile.local
+++ b/Makefile.local
@@ -46,7 +46,7 @@ PV_FILE=bindings/python/notmuch/version.py
 # Smash together user's values with our extra values
 FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS)
 FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS)
 # Smash together user's values with our extra values
 FINAL_CFLAGS = -DNOTMUCH_VERSION=$(VERSION) $(CPPFLAGS) $(CFLAGS) $(WARN_CFLAGS) $(extra_cflags) $(CONFIGURE_CFLAGS)
 FINAL_CXXFLAGS = $(CPPFLAGS) $(CXXFLAGS) $(WARN_CXXFLAGS) $(extra_cflags) $(extra_cxxflags) $(CONFIGURE_CXXFLAGS)

-FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS)
+FINAL_NOTMUCH_LDFLAGS = $(LDFLAGS) -Lutil -lutil -Llib -lnotmuch $(AS_NEEDED_LDFLAGS) $(GMIME_LDFLAGS) $(TALLOC_LDFLAGS) $(ZLIB_LDFLAGS)

 FINAL_NOTMUCH_LINKER = CC
 ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)
 FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS)
 FINAL_NOTMUCH_LINKER = CC
 ifneq ($(LINKER_RESOLVES_LIBRARY_DEPENDENCIES),1)
 FINAL_NOTMUCH_LDFLAGS += $(CONFIGURE_LDFLAGS)

diff --git a/configure b/configure
index 09f4650c4d8256cf49cff86810169d0210b98afb..b8a3ce768bc1ef44c4a4b56d5408879b2a42eab5 100755 (executable)
--- a/configure
+++ b/configure
@@ -340,6 +340,18 @@ else
     errors=$((errors + 1))
 fi
 
     errors=$((errors + 1))
 fi
 

+printf "Checking for zlib (>= 1.2.5.2)... "
+have_zlib=0
+if pkg-config --atleast-version=1.2.5.2 zlib; then
+    printf "Yes.\n"
+    have_zlib=1
+    zlib_cflags=$(pkg-config --cflags zlib)
+    zlib_ldflags=$(pkg-config --libs zlib)
+else
+    printf "No.\n"
+    errors=$((errors + 1))
+fi
+

 printf "Checking for talloc development files... "
 if pkg-config --exists talloc; then
     printf "Yes.\n"
 printf "Checking for talloc development files... "
 if pkg-config --exists talloc; then
     printf "Yes.\n"


@@ -496,6 +508,11 @@ EOF
        echo "  Xapian library (including development files such as headers)"
        echo "  http://xapian.org/"
     fi
        echo "  Xapian library (including development files such as headers)"
        echo "  http://xapian.org/"
     fi

+    if [ $have_zlib -eq 0 ]; then
+       echo "  zlib library (>= version 1.2.5.2, including development files such as headers)"
+       echo "  http://zlib.net/"
+       echo
+    fi

     if [ $have_gmime -eq 0 ]; then
        echo "  Either GMime 2.4 library" $GMIME_24_VERSION_CTR "or GMime 2.6 library" $GMIME_26_VERSION_CTR
        echo "  (including development files such as headers)"
     if [ $have_gmime -eq 0 ]; then
        echo "  Either GMime 2.4 library" $GMIME_24_VERSION_CTR "or GMime 2.6 library" $GMIME_26_VERSION_CTR
        echo "  (including development files such as headers)"


@@ -519,11 +536,11 @@ case a simple command will install everything you need. For example:
 
 On Debian and similar systems:
 
 
 On Debian and similar systems:
 

-       sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev
+       sudo apt-get install libxapian-dev libgmime-2.6-dev libtalloc-dev zlib1g-dev

 
 Or on Fedora and similar systems:
 
 
 Or on Fedora and similar systems:
 

-       sudo yum install xapian-core-devel gmime-devel libtalloc-devel
+       sudo yum install xapian-core-devel gmime-devel libtalloc-devel zlib-devel

 
 On other systems, similar commands can be used, but the details of the
 package names may be different.
 
 On other systems, similar commands can be used, but the details of the
 package names may be different.


@@ -860,6 +877,10 @@ XAPIAN_LDFLAGS = ${xapian_ldflags}
 GMIME_CFLAGS = ${gmime_cflags}
 GMIME_LDFLAGS = ${gmime_ldflags}
 
 GMIME_CFLAGS = ${gmime_cflags}
 GMIME_LDFLAGS = ${gmime_ldflags}
 

+# Flags needed to compile and link against zlib
+ZLIB_CFLAGS = ${zlib_cflags}
+ZLIB_LDFLAGS = ${zlib_ldflags}
+

 # Flags needed to compile and link against talloc
 TALLOC_CFLAGS = ${talloc_cflags}
 TALLOC_LDFLAGS = ${talloc_ldflags}
 # Flags needed to compile and link against talloc
 TALLOC_CFLAGS = ${talloc_cflags}
 TALLOC_LDFLAGS = ${talloc_ldflags}


@@ -898,6 +919,7 @@ CONFIGURE_CFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)      \\
                   -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
 CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)    \\
                   -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
 CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)    \\

+                    \$(ZLIB_CFLAGS)                                     \\

                     \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\
                     \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS)             \\
                     -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR)               \\
                     \$(TALLOC_CFLAGS) -DHAVE_VALGRIND=\$(HAVE_VALGRIND) \\
                     \$(VALGRIND_CFLAGS) \$(XAPIAN_CXXFLAGS)             \\
                     -DHAVE_STRCASESTR=\$(HAVE_STRCASESTR)               \\


@@ -908,5 +930,5 @@ CONFIGURE_CXXFLAGS = -DHAVE_GETLINE=\$(HAVE_GETLINE) \$(GMIME_CFLAGS)    \\
                     -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT)       \\
                     -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 
                     -DHAVE_XAPIAN_COMPACT=\$(HAVE_XAPIAN_COMPACT)       \\
                     -DUTIL_BYTE_ORDER=\$(UTIL_BYTE_ORDER)
 

-CONFIGURE_LDFLAGS =  \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(XAPIAN_LDFLAGS)
+CONFIGURE_LDFLAGS =  \$(GMIME_LDFLAGS) \$(TALLOC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(XAPIAN_LDFLAGS)

 EOF
 EOF

diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index 17d1da5c7d0884f67dcc8ad2280b6157a453f39f..d94cb4f84496ea3b1bd2d9b2c48982cbf70c775f 100644 (file)
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -19,6 +19,9 @@ recreated from the messages themselves. The output of notmuch dump is
 therefore the only critical thing to backup (and much more friendly to
 incremental backup than the native database files.)
 
 therefore the only critical thing to backup (and much more friendly to
 incremental backup than the native database files.)
 

+``--gzip``
+    Compress the output in a format compatible with **gzip(1)**.
+

 ``--format=(sup|batch-tag)``
     Notmuch restore supports two plain text dump formats, both with one
     message-id per line, followed by a list of tags.
 ``--format=(sup|batch-tag)``
     Notmuch restore supports two plain text dump formats, both with one
     message-id per line, followed by a list of tags.

diff --git a/notmuch-client.h b/notmuch-client.h
index d11064824ddf3ad0eed8fa5944a2e9b6e6477e19..e1efbe0c82520e1fc186c40f4a1eb3b9ef2634b3 100644 (file)
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -450,7 +450,9 @@ typedef enum dump_formats {
 int
 notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,
 int
 notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,

-                      const char *query_str, dump_format_t output_format);
+                      const char *query_str,
+                      dump_format_t output_format,
+                      notmuch_bool_t gzip_output);

 
 #include "command-line-arguments.h"
 #endif
 
 #include "command-line-arguments.h"
 #endif

diff --git a/notmuch-dump.c b/notmuch-dump.c
index 21702d793bcf1bd0ec7c379976bb040e5903d9de..2849eabaadb9b292b0eab8e4dab6c10d4b26398d 100644 (file)
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -21,9 +21,11 @@
 #include "notmuch-client.h"
 #include "hex-escape.h"
 #include "string-util.h"
 #include "notmuch-client.h"
 #include "hex-escape.h"
 #include "string-util.h"

+#include <zlib.h>
+

 
 static int
 
 static int

-database_dump_file (notmuch_database_t *notmuch, FILE *output,
+database_dump_file (notmuch_database_t *notmuch, gzFile output,

                    const char *query_str, int output_format)
 {
     notmuch_query_t *query;
                    const char *query_str, int output_format)
 {
     notmuch_query_t *query;


@@ -69,7 +71,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
        }
 
        if (output_format == DUMP_FORMAT_SUP) {
        }
 
        if (output_format == DUMP_FORMAT_SUP) {

-           fprintf (output, "%s (", message_id);
+           gzprintf (output, "%s (", message_id);

        }
 
        for (tags = notmuch_message_get_tags (message);
        }
 
        for (tags = notmuch_message_get_tags (message);


@@ -78,12 +80,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
            const char *tag_str = notmuch_tags_get (tags);
 
            if (! first)
            const char *tag_str = notmuch_tags_get (tags);
 
            if (! first)

-               fputs (" ", output);
+               gzputs (output, " ");

 
            first = 0;
 
            if (output_format == DUMP_FORMAT_SUP) {
 
            first = 0;
 
            if (output_format == DUMP_FORMAT_SUP) {

-               fputs (tag_str, output);
+               gzputs (output, tag_str);

            } else {
                if (hex_encode (notmuch, tag_str,
                                &buffer, &buffer_size) != HEX_SUCCESS) {
            } else {
                if (hex_encode (notmuch, tag_str,
                                &buffer, &buffer_size) != HEX_SUCCESS) {


@@ -91,12 +93,12 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
                             tag_str);
                    return EXIT_FAILURE;
                }
                             tag_str);
                    return EXIT_FAILURE;
                }

-               fprintf (output, "+%s", buffer);
+               gzprintf (output, "+%s", buffer);

            }
        }
 
        if (output_format == DUMP_FORMAT_SUP) {
            }
        }
 
        if (output_format == DUMP_FORMAT_SUP) {

-           fputs (")\n", output);
+           gzputs (output, ")\n");

        } else {
            if (make_boolean_term (notmuch, "id", message_id,
                                   &buffer, &buffer_size)) {
        } else {
            if (make_boolean_term (notmuch, "id", message_id,
                                   &buffer, &buffer_size)) {


@@ -104,7 +106,7 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
                             message_id, strerror (errno));
                    return EXIT_FAILURE;
            }
                             message_id, strerror (errno));
                    return EXIT_FAILURE;
            }

-           fprintf (output, " -- %s\n", buffer);
+           gzprintf (output, " -- %s\n", buffer);

        }
 
        notmuch_message_destroy (message);
        }
 
        notmuch_message_destroy (message);


@@ -121,24 +123,83 @@ database_dump_file (notmuch_database_t *notmuch, FILE *output,
 int
 notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,
 int
 notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,

-                      const char *query_str, dump_format_t output_format)
+                      const char *query_str,
+                      dump_format_t output_format,
+                      notmuch_bool_t gzip_output)

 {
 {

-    FILE *output = stdout;
-    int ret;
+    gzFile output = NULL;
+    const char *mode = gzip_output ? "w9" : "wT";
+    const char *name_for_error = output_file_name ? output_file_name : "stdout";
+
+    char *tempname = NULL;
+    int outfd = -1;
+
+    int ret = -1;

 
     if (output_file_name) {
 
     if (output_file_name) {

-       output = fopen (output_file_name, "w");
-       if (output == NULL) {
-           fprintf (stderr, "Error opening %s for writing: %s\n",
-                    output_file_name, strerror (errno));
-           return EXIT_FAILURE;
-       }
+       tempname = talloc_asprintf (notmuch, "%s.XXXXXX", output_file_name);
+       outfd = mkstemp (tempname);
+    } else {
+       outfd = dup (STDOUT_FILENO);
+    }
+
+    if (outfd < 0) {
+       fprintf (stderr, "Bad output file %s\n", name_for_error);
+       goto DONE;
+    }
+
+    output = gzdopen (outfd, mode);
+
+    if (output == NULL) {
+       fprintf (stderr, "Error opening %s for (gzip) writing: %s\n",
+                name_for_error, strerror (errno));
+       if (close (outfd))
+           fprintf (stderr, "Error closing %s during shutdown: %s\n",
+                name_for_error, strerror (errno));
+       goto DONE;

     }
 
     ret = database_dump_file (notmuch, output, query_str, output_format);
     }
 
     ret = database_dump_file (notmuch, output, query_str, output_format);

+    if (ret) goto DONE;
+
+    ret = gzflush (output, Z_FINISH);
+    if (ret) {
+       fprintf (stderr, "Error flushing output: %s\n", gzerror (output, NULL));
+       goto DONE;
+    }
+
+    if (output_file_name) {
+       ret = fdatasync (outfd);
+       if (ret) {
+           fprintf (stderr, "Error syncing %s to disk: %s\n",
+                    name_for_error, strerror (errno));
+           goto DONE;
+       }
+    }
+
+    if (gzclose_w (output) != Z_OK) {
+       fprintf (stderr, "Error closing %s: %s\n", name_for_error,
+                gzerror (output, NULL));
+       ret = EXIT_FAILURE;
+       output = NULL;
+       goto DONE;
+    }
+
+    if (output_file_name) {
+       ret = rename (tempname, output_file_name);
+       if (ret) {
+           fprintf (stderr, "Error renaming %s to %s: %s\n",
+                    tempname, output_file_name, strerror (errno));
+           goto DONE;
+       }
+
+    }
+ DONE:
+    if (ret != EXIT_SUCCESS && output)
+       (void) gzclose_w (output);

 
 

-    if (output != stdout)
-       fclose (output);
+    if (ret != EXIT_SUCCESS && output_file_name)
+       (void) unlink (tempname);

 
     return ret;
 }
 
     return ret;
 }


@@ -158,6 +219,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
     int opt_index;
 
     int output_format = DUMP_FORMAT_BATCH_TAG;
     int opt_index;
 
     int output_format = DUMP_FORMAT_BATCH_TAG;

+    notmuch_bool_t gzip_output = 0;

 
     notmuch_opt_desc_t options[] = {
        { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',
 
     notmuch_opt_desc_t options[] = {
        { NOTMUCH_OPT_KEYWORD, &output_format, "format", 'f',


@@ -165,6 +227,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
                                  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
                                  { 0, 0 } } },
        { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
                                  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
                                  { 0, 0 } } },
        { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },

+       { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },

        { 0, 0, 0, 0, 0 }
     };
 
        { 0, 0, 0, 0, 0 }
     };
 


@@ -181,7 +244,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
     }
 
     ret = notmuch_database_dump (notmuch, output_file_name, query_str,
     }
 
     ret = notmuch_database_dump (notmuch, output_file_name, query_str,

-                                output_format);
+                                output_format, gzip_output);

 
     notmuch_database_destroy (notmuch);
 
 
     notmuch_database_destroy (notmuch);
 

diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh
index 0004438db493514fed25c95fe244c0eab1767a06..d79aca86df3297b70763fe7613e47bdf96dcce44 100755 (executable)
--- a/test/T240-dump-restore.sh
+++ b/test/T240-dump-restore.sh
@@ -68,6 +68,18 @@ test_begin_subtest "dump --output=outfile --"
 notmuch dump --output=dump-1-arg-dash.actual --
 test_expect_equal_file dump.expected dump-1-arg-dash.actual
 
 notmuch dump --output=dump-1-arg-dash.actual --
 test_expect_equal_file dump.expected dump-1-arg-dash.actual
 

+# gzipped output
+
+test_begin_subtest "dump --gzip"
+notmuch dump --gzip > dump-gzip.gz
+gunzip dump-gzip.gz
+test_expect_equal_file dump.expected dump-gzip
+
+test_begin_subtest "dump --gzip --output=outfile"
+notmuch dump --gzip --output=dump-gzip-outfile.gz
+gunzip dump-gzip-outfile.gz
+test_expect_equal_file dump.expected dump-gzip-outfile
+

 # Note, we assume all messages from cworth have a message-id
 # containing cworth.org
 
 # Note, we assume all messages from cworth have a message-id
 # containing cworth.org