archival: support zstd in rpm and dpkgzstd

11 files changed, 78 insertions, 20 deletions

diff --git a/archival/Config.src b/archival/Config.src
index 1dbb0f3fa..bd75f0f65 100644
--- a/archival/Config.src
+++ b/archival/Config.src
@@ -27,7 +27,9 @@ config FEATURE_SEAMLESS_Z
 
 config FEATURE_SEAMLESS_ZSTD
         bool "Make tar, rpm, modprobe etc understand .zst data"
-        default y
+        default n
+        help
+        This requires external zstd and unzstd binaries.
 
 INSERT
 
diff --git a/archival/dpkg.c b/archival/dpkg.c
index a53da8df1..c007633dd 100644
--- a/archival/dpkg.c
+++ b/archival/dpkg.c
@@ -1510,6 +1510,9 @@ static void init_archive_deb_control(archive_handle_t *ar_handle)
 #if ENABLE_FEATURE_SEAMLESS_XZ
         llist_add_to(&(ar_handle->accept), (char*)"control.tar.xz");
 #endif
+#if ENABLE_FEATURE_SEAMLESS_ZSTD
+        llist_add_to(&(ar_handle->accept), (char*)"control.tar.zst");
+#endif
 
         /* Assign the tar handle as a subarchive of the ar handle */
         ar_handle->dpkg__sub_archive = tar_handle;
@@ -1537,6 +1540,9 @@ static void init_archive_deb_data(archive_handle_t *ar_handle)
 #if ENABLE_FEATURE_SEAMLESS_XZ
         llist_add_to(&(ar_handle->accept), (char*)"data.tar.xz");
 #endif
+#if ENABLE_FEATURE_SEAMLESS_ZSTD
+        llist_add_to(&(ar_handle->accept), (char*)"data.tar.zst");
+#endif
 
         /* Assign the tar handle as a subarchive of the ar handle */
         ar_handle->dpkg__sub_archive = tar_handle;
diff --git a/archival/dpkg_deb.c b/archival/dpkg_deb.c
index dda931169..0c60ae41a 100644
--- a/archival/dpkg_deb.c
+++ b/archival/dpkg_deb.c
@@ -77,6 +77,10 @@ int dpkg_deb_main(int argc UNUSED_PARAM, char **argv)
         llist_add_to(&ar_archive->accept, (char*)"data.tar.xz");
         llist_add_to(&control_tar_llist, (char*)"control.tar.xz");
 #endif
+#if ENABLE_FEATURE_SEAMLESS_ZSTD
+        llist_add_to(&ar_archive->accept, (char*)"data.tar.zst");
+        llist_add_to(&control_tar_llist, (char*)"control.tar.zst");
+#endif
 
         /* Must have 1 or 2 args */
         opt = getopt32(argv, "^" "cefXx"
diff --git a/archival/libarchive/Kbuild.src b/archival/libarchive/Kbuild.src
index 70c7fae5b..1af3cf61d 100644
--- a/archival/libarchive/Kbuild.src
+++ b/archival/libarchive/Kbuild.src
@@ -39,6 +39,7 @@ DPKG_FILES:= \
         get_header_tar_bz2.o \
         get_header_tar_lzma.o \
         get_header_tar_xz.o \
+        get_header_tar_zstd.o \
 
 INSERT
 
diff --git a/archival/libarchive/decompress_unzstd.c b/archival/libarchive/decompress_unzstd.c
index 08930a3e5..c55ab8575 100644
--- a/archival/libarchive/decompress_unzstd.c
+++ b/archival/libarchive/decompress_unzstd.c
@@ -4,16 +4,9 @@
 IF_DESKTOP(long long) int FAST_FUNC
 unpack_zstd_stream(transformer_state_t *xstate)
 {
-        char *argv[] = {(char *)"unzstd", NULL};
+        // FIXME Provide an internal implementation of zstd decompression.
+        char *argv[] = {(char *)"unzstd", (char *)"-cf", (char *)"-", NULL};
 
-        /* Exec external unzstd. */
-#if BB_MMU
-        if (xstate->signature_skipped) {
-                /* If called from fork_transformer_with_no_sig() signature_skipped
-                 * is just a flag, not the actual offset.  But we know it's 4. */
-                xlseek(xstate->src_fd, -4, SEEK_CUR);
-        }
-#endif
         xmove_fd(xstate->src_fd, STDIN_FILENO);
         xmove_fd(xstate->dst_fd, STDOUT_FILENO);
         BB_EXECVP_or_die(argv);
diff --git a/archival/libarchive/filter_accept_list_reassign.c b/archival/libarchive/filter_accept_list_reassign.c
index 826c5c29d..a78186a6e 100644
--- a/archival/libarchive/filter_accept_list_reassign.c
+++ b/archival/libarchive/filter_accept_list_reassign.c
@@ -55,6 +55,12 @@ char FAST_FUNC filter_accept_list_reassign(archive_handle_t *archive_handle)
                         archive_handle->dpkg__action_data_subarchive = get_header_tar_xz;
                         return EXIT_SUCCESS;
                 }
+                if (ENABLE_FEATURE_SEAMLESS_ZSTD
+                 && strcmp(name_ptr, "zst") == 0
+                ) {
+                        archive_handle->dpkg__action_data_subarchive = get_header_tar_zstd;
+                        return EXIT_SUCCESS;
+                }
         }
         return EXIT_FAILURE;
 }
diff --git a/archival/libarchive/get_header_tar_zstd.c b/archival/libarchive/get_header_tar_zstd.c
new file mode 100644
index 000000000..6f1b4b977
--- /dev/null
+++ b/archival/libarchive/get_header_tar_zstd.c
@@ -0,0 +1,20 @@
+/* vi: set sw=4 ts=4: */
+/*
+ * Licensed under GPLv2 or later, see file LICENSE in this source tree.
+ */
+#include "libbb.h"
+#include "bb_archive.h"
+
+char FAST_FUNC get_header_tar_zstd(archive_handle_t *archive_handle)
+{
+        /* Can't lseek over pipes */
+        archive_handle->seek = seek_by_read;
+
+        fork_transformer_with_sig(archive_handle->src_fd, unpack_zstd_stream, "unzstd");
+        archive_handle->offset = 0;
+        while (get_header_tar(archive_handle) == EXIT_SUCCESS)
+                continue;
+
+        /* Can only do one file at a time */
+        return EXIT_FAILURE;
+}
diff --git a/archival/libarchive/open_transformer.c b/archival/libarchive/open_transformer.c
index 9f41c7b4f..375e7b249 100644
--- a/archival/libarchive/open_transformer.c
+++ b/archival/libarchive/open_transformer.c
@@ -211,11 +211,14 @@ static transformer_state_t *setup_transformer_on_fd(int fd, int die_if_not_compr
         }
 
         if (ENABLE_FEATURE_SEAMLESS_ZSTD && xstate->magic.b16[0] == ZSTD_MAGIC1) {
-                xstate->signature_skipped = 4;
                 xread(fd, &xstate->magic.b16[1], 2);
                 if (xstate->magic.b16[1] == ZSTD_MAGIC2) {
                         xstate->xformer = unpack_zstd_stream;
                         USE_FOR_NOMMU(xstate->xformer_prog = "unzstd";)
+                        // FIXME We execute an external decompressor even on MMU.
+                        // Force a seek back to the signature.
+                        xstate->signature_skipped = 0;
+                        xlseek(xstate->src_fd, -4, SEEK_CUR);
                         goto found_magic;
                 }
         }
diff --git a/archival/tar.c b/archival/tar.c
index c3e9f0e0c..033e08105 100644
--- a/archival/tar.c
+++ b/archival/tar.c
@@ -856,8 +856,8 @@ enum {
         IF_FEATURE_TAR_NOPRESERVE_TIME(OPTBIT_NOPRESERVE_TIME,)
 #if ENABLE_FEATURE_TAR_LONG_OPTIONS
         OPTBIT_STRIP_COMPONENTS,
-        IF_FEATURE_SEAMLESS_LZMA(OPTBIT_LZMA        ,)
         IF_FEATURE_SEAMLESS_ZSTD(OPTBIT_ZSTD        ,)
+        IF_FEATURE_SEAMLESS_LZMA(OPTBIT_LZMA        ,)
         OPTBIT_NORECURSION,
         IF_FEATURE_TAR_TO_COMMAND(OPTBIT_2COMMAND   ,)
         OPTBIT_NUMERIC_OWNER,
@@ -884,8 +884,8 @@ enum {
         OPT_AUTOCOMPRESS_BY_EXT = 1 << OPTBIT_AUTOCOMPRESS_BY_EXT,                   // a
         OPT_NOPRESERVE_TIME  = IF_FEATURE_TAR_NOPRESERVE_TIME((1 << OPTBIT_NOPRESERVE_TIME)) + 0, // m
         OPT_STRIP_COMPONENTS = IF_FEATURE_TAR_LONG_OPTIONS((1 << OPTBIT_STRIP_COMPONENTS)) + 0, // strip-components
-        OPT_LZMA             = IF_FEATURE_TAR_LONG_OPTIONS(IF_FEATURE_SEAMLESS_LZMA((1 << OPTBIT_LZMA))) + 0, // lzma
         OPT_ZSTD             = IF_FEATURE_TAR_LONG_OPTIONS(IF_FEATURE_SEAMLESS_ZSTD((1 << OPTBIT_ZSTD))) + 0, // zstd
+        OPT_LZMA             = IF_FEATURE_TAR_LONG_OPTIONS(IF_FEATURE_SEAMLESS_LZMA((1 << OPTBIT_LZMA))) + 0, // lzma
         OPT_NORECURSION      = IF_FEATURE_TAR_LONG_OPTIONS((1 << OPTBIT_NORECURSION    )) + 0, // no-recursion
         OPT_2COMMAND         = IF_FEATURE_TAR_TO_COMMAND(  (1 << OPTBIT_2COMMAND       )) + 0, // to-command
         OPT_NUMERIC_OWNER    = IF_FEATURE_TAR_LONG_OPTIONS((1 << OPTBIT_NUMERIC_OWNER  )) + 0, // numeric-owner
@@ -932,13 +932,13 @@ static const char tar_longopts[] ALIGN1 =
 # if ENABLE_FEATURE_TAR_NOPRESERVE_TIME
         "touch\0"               No_argument       "m"
 # endif
-        "strip-components\0"    Required_argument "\xf8"
+        "strip-components\0"    Required_argument "\xf7"
+# if ENABLE_FEATURE_SEAMLESS_ZSTD
+        "zstd\0"                No_argument       "\xf8"
+# endif
 # if ENABLE_FEATURE_SEAMLESS_LZMA
         "lzma\0"                No_argument       "\xf9"
 # endif
-# if ENABLE_FEATURE_SEAMLESS_ZSTD
-        "zstd\0"                No_argument       "\xf7"
-# endif
         "no-recursion\0"        No_argument       "\xfa"
 # if ENABLE_FEATURE_TAR_TO_COMMAND
         "to-command\0"          Required_argument "\xfb"
@@ -1033,7 +1033,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv)
                 IF_FEATURE_SEAMLESS_Z(   "Z"     )
                 "a"
                 IF_FEATURE_TAR_NOPRESERVE_TIME("m")
-                IF_FEATURE_TAR_LONG_OPTIONS("\xf8:") // --strip-components
+                IF_FEATURE_TAR_LONG_OPTIONS("\xf7:") // --strip-components
                 "\0"
                 "tt:vv:" // count -t,-v
 #if ENABLE_FEATURE_TAR_LONG_OPTIONS && ENABLE_FEATURE_TAR_FROM
@@ -1043,7 +1043,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv)
                 IF_FEATURE_TAR_CREATE("c--tx:t--cx:x--ct") // mutually exclusive
                 IF_NOT_FEATURE_TAR_CREATE("t--x:x--t") // mutually exclusive
 #if ENABLE_FEATURE_TAR_LONG_OPTIONS
-                ":\xf8+" // --strip-components=NUM
+                ":\xf7+" // --strip-components=NUM
 #endif
                 LONGOPTS
                 , &base_dir // -C dir
@@ -1083,8 +1083,8 @@ int tar_main(int argc UNUSED_PARAM, char **argv)
         showopt(OPT_AUTOCOMPRESS_BY_EXT);
         showopt(OPT_NOPRESERVE_TIME );
         showopt(OPT_STRIP_COMPONENTS);
-        showopt(OPT_LZMA            );
         showopt(OPT_ZSTD            );
+        showopt(OPT_LZMA            );
         showopt(OPT_NORECURSION     );
         showopt(OPT_2COMMAND        );
         showopt(OPT_NUMERIC_OWNER   );
diff --git a/include/bb_archive.h b/include/bb_archive.h
index eb784d464..916c51bab 100644
--- a/include/bb_archive.h
+++ b/include/bb_archive.h
@@ -213,6 +213,7 @@ char get_header_tar_gz(archive_handle_t *archive_handle) FAST_FUNC;
 char get_header_tar_bz2(archive_handle_t *archive_handle) FAST_FUNC;
 char get_header_tar_lzma(archive_handle_t *archive_handle) FAST_FUNC;
 char get_header_tar_xz(archive_handle_t *archive_handle) FAST_FUNC;
+char get_header_tar_zstd(archive_handle_t *archive_handle) FAST_FUNC;
 
 void seek_by_jump(int fd, off_t amount) FAST_FUNC;
 void seek_by_read(int fd, off_t amount) FAST_FUNC;
diff --git a/testsuite/tar.tests b/testsuite/tar.tests
index 0f2e89112..d2155add7 100755
--- a/testsuite/tar.tests
+++ b/testsuite/tar.tests
@@ -245,6 +245,28 @@ SKIP=
 cd .. || exit 1; rm -rf tar.tempdir 2>/dev/null
 
 mkdir tar.tempdir && cd tar.tempdir || exit 1
+# Do we detect ZSTD-compressed data (even w/o .tar.zst extension)?
+# (uuencoded hello_world.tar.zst contains one empty file named "hello_world")
+# Currently we require an external unzstd program to be available.
+optional UUDECODE FEATURE_TAR_AUTODETECT FEATURE_SEAMLESS_ZSTD
+command -v unzstd >/dev/null || SKIP=1
+testing "tar extract tar.zst" "\
+uudecode -o input && tar tf input && echo Ok
+" "\
+hello_world
+Ok
+" \
+"" "\
+begin-base64 644 hello_world.tar.zst
+KLUv/QRYpQIA5ANoZWxsb193b3JsZAAwMDAwNjQ0ADAwMDE3NTAAMTUxNjA0
+NTQwMzAAMDExNDc3ACAwAHVzdGFyICAAcm15AAcAtAbwMBQOAJMCkBq8wApX
+awJ0ak0T/A==
+====
+"
+SKIP=
+cd .. || exit 1; rm -rf tar.tempdir 2>/dev/null
+
+mkdir tar.tempdir && cd tar.tempdir || exit 1
 # On extract, everything up to and including last ".." component is stripped
 optional FEATURE_TAR_CREATE
 testing "tar strips /../ on extract" "\