Support decompressing zstd tar archives for repository rules.

benjaminp · copybara-github · commit 00d74ff737cc · 2022-03-21T02:11:04.000-07:00
This is mostly a rebase of bazelbuild#11968 with a few tweaks of my own. Fixes bazelbuild#10342. Co-authored-by: Grzegorz Lukasik <glukasik@nuro.ai> RELNOTES[new]: `repository_ctx.extract`, and thus `http_archive`, can now decompress zstandard-compressed archives. Closes bazelbuild#15026. PiperOrigin-RevId: 436146779
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD
@@ -54,6 +54,7 @@ java_library(
         "//third_party:java-diff-utils",
         "//third_party:jsr305",
         "//third_party:xz",
+        "@zstd-jni",
     ],
 )
 
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
@@ -103,13 +103,15 @@ static Decompressor getDecompressor(Path archivePath)
       return TarGzFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.xz") || baseName.endsWith(".txz")) {
       return TarXzFunction.INSTANCE;
+    } else if (baseName.endsWith(".tar.zst") || baseName.endsWith(".tzst")) {
+      return TarZstFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.bz2")) {
       return TarBz2Function.INSTANCE;
     } else {
       throw new RepositoryFunctionException(
           Starlark.errorf(
-              "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz, "
-                  + "or .tar.bz2 suffix (got %s)",
+              "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz,"
+                  + " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)",
               archivePath),
           Transience.PERSISTENT);
     }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstFunction.java
@@ -0,0 +1,38 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/** Creates a repository by unarchiving a zstandard-compressed .tar file. */
+final class TarZstFunction extends CompressedTarFunction {
+  static final Decompressor INSTANCE = new TarZstFunction();
+  private static final int BUFFER_SIZE = 32 * 1024;
+
+  private TarZstFunction() {}
+
+  @Override
+  protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
+      throws IOException {
+    return new ZstdInputStreamNoFinalizer(
+        new BufferedInputStream(
+            new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE));
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java
@@ -50,6 +50,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception {
     DecompressorDescriptor.builder().setArchivePath(path).build();
     path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.txz");
     DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.zst");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tzst");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
     path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2");
     DecompressorDescriptor.builder().setArchivePath(path).build();
   }
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
@@ -657,7 +657,10 @@ sh_test(
     name = "external_integration_test",
     size = "large",
     srcs = ["external_integration_test.sh"],
-    data = [":test-deps"],
+    data = [
+        ":test-deps",
+        "//src/test/shell/bazel/testdata:zstd_test_archive.tar.zst",
+    ],
     shard_count = 15,
     tags = [
         "no_windows",
diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh
@@ -205,6 +205,22 @@ function test_http_archive_tar_xz() {
   http_archive_helper tar_xz_up
 }
 
+function test_http_archive_tar_zstd() {
+  cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+    name = 'test_zstd_repo',
+    url = 'file://$(rlocation io_bazel/src/test/shell/bazel/testdata/zstd_test_archive.tar.zst)',
+    sha256 = '12b0116f2a3c804859438e102a8a1d5f494c108d1b026da9f6ca55fb5107c7e9',
+    build_file_content = 'filegroup(name="x", srcs=glob(["*"]))',
+)
+EOF
+  bazel build @test_zstd_repo//...
+
+  base_external_path=bazel-out/../external/test_zstd_repo
+  assert_contains "test content" "${base_external_path}/test_dir/test_file"
+}
+
 function test_http_archive_no_server() {
   cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
diff --git a/src/test/shell/bazel/testdata/BUILD b/src/test/shell/bazel/testdata/BUILD
@@ -1,14 +1,16 @@
 load("//:distdir_deps.bzl", "gen_workspace_stanza")
 
+package(default_testonly = True)
+
 filegroup(
     name = "srcs",
+    testonly = False,
     srcs = glob(["**"]),
     visibility = ["//src/test/shell/bazel:__pkg__"],
 )
 
 filegroup(
     name = "git-repos",
-    testonly = 1,
     srcs = [
         "outer-planets-repo.tar.gz",
         "pluto-repo.tar.gz",
@@ -20,7 +22,6 @@ filegroup(
 
 filegroup(
     name = "embedded_tools_deps_test_data",
-    testonly = 1,
     srcs = [
         "//src/test/shell/bazel/testdata:embedded_tools_srcs_deps",
     ],
@@ -29,7 +30,6 @@ filegroup(
 
 filegroup(
     name = "jdk_http_archives_filegroup",
-    testonly = 1,
     srcs = [
         "//src/test/shell/bazel/testdata:jdk_http_archives",
     ],
@@ -64,3 +64,5 @@ gen_workspace_stanza(
     template = "jdk_http_archives.tmpl",
     visibility = ["//:__pkg__"],
 )
+
+exports_files(["zstd_test_archive.tar.zst"])
diff --git a/src/test/shell/bazel/testdata/zstd_test_archive.tar.zst b/src/test/shell/bazel/testdata/zstd_test_archive.tar.zst

Original file line number	Diff line number	Diff line change
`@@ -54,6 +54,7 @@ java_library(`
`54`	`54`	`"//third_party:java-diff-utils",`
`55`	`55`	`"//third_party:jsr305",`
`56`	`56`	`"//third_party:xz",`
	`57`	`+ "@zstd-jni",`
`57`	`58`	`],`
`58`	`59`	`)`
`59`	`60`