allowedIpCidrRanges) {
+ this.network = network;
+ this.allowedIpCidrRanges = allowedIpCidrRanges;
+ }
+
+ /**
+ * Name of the network.
+ *
+ * Format: `projects/PROJECT_ID/global/networks/NETWORK_NAME`
+ *
+ * @since 2.54.0
+ * @see Builder#setNetwork(String)
+ */
+ public @Nullable String getNetwork() {
+ return network;
+ }
+
+ /**
+ * Optional. The list of public or private IPv4 and IPv6 CIDR ranges that can access the
+ * bucket. In the CIDR IP address block, the specified IP address must be properly truncated,
+ * meaning all the host bits must be zero or else the input is considered malformed. For
+ * example, `192.0.2.0/24` is accepted but `192.0.2.1/24` is not. Similarly, for IPv6,
+ * `2001:db8::/32` is accepted whereas `2001:db8::1/32` is not.
+ *
+ * @since 2.54.0
+ * @see Builder#setAllowedIpCidrRanges(List)
+ */
+ public @Nullable List getAllowedIpCidrRanges() {
+ return allowedIpCidrRanges;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof VpcNetworkSource)) {
+ return false;
+ }
+ VpcNetworkSource that = (VpcNetworkSource) o;
+ return Objects.equals(network, that.network)
+ && Objects.equals(allowedIpCidrRanges, that.allowedIpCidrRanges);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(network, allowedIpCidrRanges);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("network", network)
+ .add("allowedIpCidrRanges", allowedIpCidrRanges)
+ .toString();
+ }
+
+ /**
+ * @since 2.54.0
+ */
+ public Builder toBuilder() {
+ return new Builder(this);
+ }
+
+ /**
+ * @since 2.54.0
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /**
+ * @since 2.54.0
+ */
+ public static final class Builder {
+ private @Nullable String network;
+ private @Nullable List allowedIpCidrRanges;
+
+ private Builder() {}
+
+ public Builder(VpcNetworkSource vpcNetworksource) {
+ this.network = vpcNetworksource.network;
+ this.allowedIpCidrRanges = vpcNetworksource.allowedIpCidrRanges;
+ }
+
+ /**
+ * Name of the network.
+ *
+ * Format: `projects/PROJECT_ID/global/networks/NETWORK_NAME`
+ *
+ * @since 2.54.0
+ * @see VpcNetworkSource#getNetwork()
+ */
+ public Builder setNetwork(@Nullable String network) {
+ this.network = network;
+ return this;
+ }
+
+ /**
+ * Optional. The list of public or private IPv4 and IPv6 CIDR ranges that can access the
+ * bucket. In the CIDR IP address block, the specified IP address must be properly
+ * truncated, meaning all the host bits must be zero or else the input is considered
+ * malformed. For example, `192.0.2.0/24` is accepted but `192.0.2.1/24` is not. Similarly,
+ * for IPv6, `2001:db8::/32` is accepted whereas `2001:db8::1/32` is not.
+ *
+ * @since 2.54.0
+ * @see VpcNetworkSource#getAllowedIpCidrRanges()
+ */
+ public Builder setAllowedIpCidrRanges(@Nullable List allowedIpCidrRanges) {
+ this.allowedIpCidrRanges = allowedIpCidrRanges;
+ return this;
+ }
+
+ /**
+ * @since 2.54.0
+ */
+ public VpcNetworkSource build() {
+ return new VpcNetworkSource(network, allowedIpCidrRanges);
+ }
+ }
+ }
+ }
+
/** Builder for {@code BucketInfo}. */
public abstract static class Builder {
Builder() {}
@@ -1857,6 +2285,11 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {
public abstract Builder setSoftDeletePolicy(SoftDeletePolicy softDeletePolicy);
+ /**
+ * @since 2.54.0
+ */
+ public abstract Builder setIpFilter(IpFilter ipFilter);
+
/** Creates a {@code BucketInfo} object. */
public abstract BucketInfo build();
@@ -1919,6 +2352,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {
abstract Builder clearLogging();
abstract Builder clearCustomPlacementConfig();
+
+ abstract Builder clearIpFilter();
}
static final class BuilderImpl extends Builder {
@@ -1958,6 +2393,7 @@ static final class BuilderImpl extends Builder {
private SoftDeletePolicy softDeletePolicy;
private HierarchicalNamespace hierarchicalNamespace;
+ private IpFilter ipFilter;
private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder();
BuilderImpl(String name) {
@@ -1999,6 +2435,7 @@ static final class BuilderImpl extends Builder {
objectRetention = bucketInfo.objectRetention;
softDeletePolicy = bucketInfo.softDeletePolicy;
hierarchicalNamespace = bucketInfo.hierarchicalNamespace;
+ ipFilter = bucketInfo.ipFilter;
}
@Override
@@ -2404,6 +2841,15 @@ Builder setLocationType(String locationType) {
return this;
}
+ @Override
+ public Builder setIpFilter(IpFilter ipFilter) {
+ if (!Objects.equals(this.ipFilter, ipFilter)) {
+ modifiedFields.add(BucketField.IP_FILTER);
+ }
+ this.ipFilter = ipFilter;
+ return this;
+ }
+
@Override
public BucketInfo build() {
checkNotNull(name);
@@ -2590,6 +3036,12 @@ BuilderImpl clearCustomPlacementConfig() {
return this;
}
+ @Override
+ BuilderImpl clearIpFilter() {
+ this.ipFilter = null;
+ return this;
+ }
+
private Builder clearDeleteLifecycleRules() {
if (lifecycleRules != null && !lifecycleRules.isEmpty()) {
ImmutableList nonDeleteRules =
@@ -2638,6 +3090,7 @@ private Builder clearDeleteLifecycleRules() {
objectRetention = builder.objectRetention;
softDeletePolicy = builder.softDeletePolicy;
hierarchicalNamespace = builder.hierarchicalNamespace;
+ ipFilter = builder.ipFilter;
modifiedFields = builder.modifiedFields.build();
}
@@ -2989,6 +3442,13 @@ public HierarchicalNamespace getHierarchicalNamespace() {
return hierarchicalNamespace;
}
+ /**
+ * @since 2.54.0
+ */
+ public @Nullable IpFilter getIpFilter() {
+ return ipFilter;
+ }
+
/** Returns a builder for the current bucket. */
public Builder toBuilder() {
return new BuilderImpl(this);
@@ -3030,7 +3490,8 @@ public int hashCode() {
softDeletePolicy,
customPlacementConfig,
hierarchicalNamespace,
- logging);
+ logging,
+ ipFilter);
}
@Override
@@ -3075,7 +3536,8 @@ public boolean equals(Object o) {
&& Objects.equals(objectRetention, that.objectRetention)
&& Objects.equals(softDeletePolicy, that.softDeletePolicy)
&& Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace)
- && Objects.equals(logging, that.logging);
+ && Objects.equals(logging, that.logging)
+ && Objects.equals(ipFilter, that.ipFilter);
}
@Override
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java
index 4d9d5e31d..a530c3546 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java
@@ -16,6 +16,7 @@
package com.google.cloud.storage;
+import static com.google.cloud.storage.Storage.BucketField.IP_FILTER;
import static com.google.cloud.storage.Storage.BucketField.SOFT_DELETE_POLICY;
import static com.google.cloud.storage.Utils.bucketNameCodec;
import static com.google.cloud.storage.Utils.ifNonNull;
@@ -30,6 +31,7 @@
import com.google.cloud.storage.Acl.Role;
import com.google.cloud.storage.BlobInfo.CustomerEncryption;
import com.google.cloud.storage.BucketInfo.CustomPlacementConfig;
+import com.google.cloud.storage.BucketInfo.IpFilter;
import com.google.cloud.storage.BucketInfo.LifecycleRule;
import com.google.cloud.storage.BucketInfo.LifecycleRule.AbortIncompleteMPUAction;
import com.google.cloud.storage.BucketInfo.Logging;
@@ -44,6 +46,8 @@
import com.google.protobuf.Timestamp;
import com.google.storage.v2.Bucket;
import com.google.storage.v2.Bucket.Billing;
+import com.google.storage.v2.Bucket.IpFilter.PublicNetworkSource;
+import com.google.storage.v2.Bucket.IpFilter.VpcNetworkSource;
import com.google.storage.v2.Bucket.Website;
import com.google.storage.v2.BucketAccessControl;
import com.google.storage.v2.CryptoKeyName;
@@ -107,6 +111,14 @@ final class GrpcConversions {
hierarchicalNamespaceCodec =
Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);
+ private final Codec ipFilterCodec =
+ Codec.of(this::ipFilterEncode, this::ipFilterDecode);
+ private final Codec
+ publicNetworkSourceCodec =
+ Codec.of(this::publicNetworkSourceEncode, this::publicNetworkSourceDecode);
+ private final Codec
+ vpcNetworkSourceCodec = Codec.of(this::vpcNetworkSourceEncode, this::vpcNetworkSourceDecode);
+
private final Codec byteStringB64StringCodec =
Codec.of(
bs -> Base64.getEncoder().encodeToString(bs.toByteArray()),
@@ -296,7 +308,9 @@ private BucketInfo bucketInfoDecode(Bucket from) {
to.setHierarchicalNamespace(
hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace()));
}
- // TODO(frankyn): Add SelfLink when the field is available
+ if (from.hasIpFilter()) {
+ to.setIpFilter(ipFilterCodec.decode(from.getIpFilter()));
+ }
if (!from.getEtag().isEmpty()) {
to.setEtag(from.getEtag());
}
@@ -390,7 +404,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
from.getHierarchicalNamespace(),
hierarchicalNamespaceCodec::encode,
to::setHierarchicalNamespace);
- // TODO(frankyn): Add SelfLink when the field is available
+ ifNonNull(from.getIpFilter(), ipFilterCodec::encode, to::setIpFilter);
+ if (from.getModifiedFields().contains(IP_FILTER) && from.getIpFilter() == null) {
+ to.clearIpFilter();
+ }
ifNonNull(from.getEtag(), to::setEtag);
return to.build();
}
@@ -1037,8 +1054,68 @@ private String removeKmsVersion(String from) {
return from;
}
- private static T todo() {
- throw new IllegalStateException("Not yet implemented");
+ private Bucket.IpFilter ipFilterEncode(IpFilter from) {
+ Bucket.IpFilter.Builder to = Bucket.IpFilter.newBuilder();
+ ifNonNull(from.getMode(), to::setMode);
+ ifNonNull(
+ from.getPublicNetworkSource(),
+ publicNetworkSourceCodec::encode,
+ to::setPublicNetworkSource);
+ ifNonNull(
+ from.getVpcNetworkSources(),
+ toImmutableListOf(vpcNetworkSourceCodec::encode),
+ to::addAllVpcNetworkSources);
+ ifNonNull(from.getAllowCrossOrgVpcs(), to::setAllowCrossOrgVpcs);
+ ifNonNull(from.getAllowAllServiceAgentAccess(), to::setAllowAllServiceAgentAccess);
+ return to.build();
+ }
+
+ private IpFilter ipFilterDecode(Bucket.IpFilter from) {
+ IpFilter.Builder to = IpFilter.newBuilder();
+ if (!from.getMode().isEmpty()) {
+ to.setMode(from.getMode());
+ }
+ ifNonNull(
+ from.getPublicNetworkSource(),
+ publicNetworkSourceCodec::decode,
+ to::setPublicNetworkSource);
+ ifNonNull(
+ from.getVpcNetworkSourcesList(),
+ toImmutableListOf(vpcNetworkSourceCodec::decode),
+ to::setVpcNetworkSources);
+ ifNonNull(from.getAllowCrossOrgVpcs(), to::setAllowCrossOrgVpcs);
+ if (from.hasAllowAllServiceAgentAccess()) {
+ to.setAllowAllServiceAgentAccess(from.getAllowAllServiceAgentAccess());
+ }
+ return to.build();
+ }
+
+ private PublicNetworkSource publicNetworkSourceEncode(IpFilter.PublicNetworkSource from) {
+ PublicNetworkSource.Builder to = PublicNetworkSource.newBuilder();
+ ifNonNull(from.getAllowedIpCidrRanges(), to::addAllAllowedIpCidrRanges);
+ return to.build();
+ }
+
+ private IpFilter.PublicNetworkSource publicNetworkSourceDecode(PublicNetworkSource from) {
+ return IpFilter.PublicNetworkSource.of(from.getAllowedIpCidrRangesList());
+ }
+
+ private VpcNetworkSource vpcNetworkSourceEncode(IpFilter.VpcNetworkSource from) {
+ VpcNetworkSource.Builder to = VpcNetworkSource.newBuilder();
+ ifNonNull(from.getNetwork(), to::setNetwork);
+ ifNonNull(from.getAllowedIpCidrRanges(), to::addAllAllowedIpCidrRanges);
+ return to.build();
+ }
+
+ private IpFilter.VpcNetworkSource vpcNetworkSourceDecode(VpcNetworkSource from) {
+ IpFilter.VpcNetworkSource.Builder to = IpFilter.VpcNetworkSource.newBuilder();
+ if (from.hasNetwork()) {
+ to.setNetwork(from.getNetwork());
+ }
+ if (!from.getAllowedIpCidrRangesList().isEmpty()) {
+ to.setAllowedIpCidrRanges(from.getAllowedIpCidrRangesList());
+ }
+ return to.build();
}
/**
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java
index 7f9da1f58..c79fa288e 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java
@@ -16,6 +16,7 @@
package com.google.cloud.storage;
+import static com.google.cloud.storage.Storage.BucketField.IP_FILTER;
import static com.google.cloud.storage.Storage.BucketField.SOFT_DELETE_POLICY;
import static com.google.cloud.storage.Utils.dateTimeCodec;
import static com.google.cloud.storage.Utils.durationSecondsCodec;
@@ -31,6 +32,7 @@
import com.google.api.services.storage.model.Bucket.Billing;
import com.google.api.services.storage.model.Bucket.Encryption;
import com.google.api.services.storage.model.Bucket.IamConfiguration.UniformBucketLevelAccess;
+import com.google.api.services.storage.model.Bucket.IpFilter.VpcNetworkSources;
import com.google.api.services.storage.model.Bucket.Lifecycle;
import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
import com.google.api.services.storage.model.Bucket.Lifecycle.Rule.Action;
@@ -57,6 +59,7 @@
import com.google.cloud.storage.BucketInfo.Autoclass;
import com.google.cloud.storage.BucketInfo.CustomPlacementConfig;
import com.google.cloud.storage.BucketInfo.IamConfiguration;
+import com.google.cloud.storage.BucketInfo.IpFilter;
import com.google.cloud.storage.BucketInfo.LifecycleRule;
import com.google.cloud.storage.BucketInfo.LifecycleRule.AbortIncompleteMPUAction;
import com.google.cloud.storage.BucketInfo.LifecycleRule.DeleteLifecycleAction;
@@ -149,6 +152,14 @@ final class JsonConversions {
private final Codec
iamConditionCodec = Codec.of(this::conditionEncode, this::conditionDecode);
+ private final Codec ipFilterCodec =
+ Codec.of(this::ipFilterEncode, this::ipFilterDecode);
+ private final Codec
+ publicNetworkSourceCodec =
+ Codec.of(this::publicNetworkSourceEncode, this::publicNetworkSourceDecode);
+ private final Codec
+ vpcNetworkSourceCodec = Codec.of(this::vpcNetworkSourceEncode, this::vpcNetworkSourceDecode);
+
private JsonConversions() {}
Codec entity() {
@@ -466,6 +477,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
from.getHierarchicalNamespace(),
this::hierarchicalNamespaceEncode,
to::setHierarchicalNamespace);
+ ifNonNull(from.getIpFilter(), ipFilterCodec::encode, to::setIpFilter);
+ if (from.getModifiedFields().contains(IP_FILTER) && from.getIpFilter() == null) {
+ to.setIpFilter(Data.nullOf(Bucket.IpFilter.class));
+ }
return to;
}
@@ -519,6 +534,7 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket
to::setHierarchicalNamespace);
ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention);
ifNonNull(from.getSoftDeletePolicy(), this::softDeletePolicyDecode, to::setSoftDeletePolicy);
+ ifNonNull(from.getIpFilter(), ipFilterCodec::decode, to::setIpFilter);
return to.build();
}
@@ -1006,6 +1022,64 @@ private CustomPlacementConfig customPlacementConfigDecode(Bucket.CustomPlacement
return CustomPlacementConfig.newBuilder().setDataLocations(from.getDataLocations()).build();
}
+ private Bucket.IpFilter ipFilterEncode(IpFilter from) {
+ Bucket.IpFilter to = new Bucket.IpFilter();
+ ifNonNull(from.getMode(), to::setMode);
+ ifNonNull(
+ from.getPublicNetworkSource(),
+ publicNetworkSourceCodec::encode,
+ to::setPublicNetworkSource);
+ ifNonNull(
+ from.getVpcNetworkSources(),
+ toListOf(vpcNetworkSourceCodec::encode),
+ to::setVpcNetworkSources);
+ ifNonNull(from.getAllowCrossOrgVpcs(), to::setAllowCrossOrgVpcs);
+ ifNonNull(from.getAllowAllServiceAgentAccess(), to::setAllowAllServiceAgentAccess);
+ return to;
+ }
+
+ private IpFilter ipFilterDecode(Bucket.IpFilter from) {
+ IpFilter.Builder to = IpFilter.newBuilder();
+ ifNonNull(from.getMode(), to::setMode);
+ ifNonNull(
+ from.getPublicNetworkSource(),
+ publicNetworkSourceCodec::decode,
+ to::setPublicNetworkSource);
+ ifNonNull(
+ from.getVpcNetworkSources(),
+ toListOf(vpcNetworkSourceCodec::decode),
+ to::setVpcNetworkSources);
+ ifNonNull(from.getAllowCrossOrgVpcs(), to::setAllowCrossOrgVpcs);
+ ifNonNull(from.getAllowAllServiceAgentAccess(), to::setAllowAllServiceAgentAccess);
+ return to.build();
+ }
+
+ private Bucket.IpFilter.PublicNetworkSource publicNetworkSourceEncode(
+ IpFilter.PublicNetworkSource from) {
+ Bucket.IpFilter.PublicNetworkSource to = new Bucket.IpFilter.PublicNetworkSource();
+ ifNonNull(from.getAllowedIpCidrRanges(), to::setAllowedIpCidrRanges);
+ return to;
+ }
+
+ private IpFilter.PublicNetworkSource publicNetworkSourceDecode(
+ Bucket.IpFilter.PublicNetworkSource from) {
+ return IpFilter.PublicNetworkSource.of(from.getAllowedIpCidrRanges());
+ }
+
+ private Bucket.IpFilter.VpcNetworkSources vpcNetworkSourceEncode(IpFilter.VpcNetworkSource from) {
+ VpcNetworkSources to = new VpcNetworkSources();
+ ifNonNull(from.getNetwork(), to::setNetwork);
+ ifNonNull(from.getAllowedIpCidrRanges(), to::setAllowedIpCidrRanges);
+ return to;
+ }
+
+ private IpFilter.VpcNetworkSource vpcNetworkSourceDecode(Bucket.IpFilter.VpcNetworkSources from) {
+ IpFilter.VpcNetworkSource.Builder to = IpFilter.VpcNetworkSource.newBuilder();
+ ifNonNull(from.getNetwork(), to::setNetwork);
+ ifNonNull(from.getAllowedIpCidrRanges(), to::setAllowedIpCidrRanges);
+ return to.build();
+ }
+
private static void maybeEncodeRetentionPolicy(BucketInfo from, Bucket to) {
if (from.getRetentionPeriodDuration() != null
|| from.retentionPolicyIsLocked() != null
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
index d2c38763e..61597307f 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
@@ -194,7 +194,13 @@ enum BucketField implements FieldSelector, NamedField {
com.google.api.services.storage.model.Bucket.SoftDeletePolicy.class),
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
- PROJECT("projectNumber", "project", BigInteger.class);
+ PROJECT("projectNumber", "project", BigInteger.class),
+ /**
+ * @since 2.54.0
+ */
+ @TransportCompatibility({Transport.HTTP, Transport.GRPC})
+ IP_FILTER("ipFilter", "ip_filter", com.google.api.services.storage.model.Bucket.IpFilter.class),
+ ;
static final List REQUIRED_FIELDS = ImmutableList.of(NAME);
private static final Map JSON_FIELD_NAME_INDEX;
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
index 77cb1344a..29444ad13 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
@@ -1084,6 +1084,7 @@ private Set getPaths() {
BucketField.IAMCONFIGURATION.getGrpcName(),
BucketInfo.Builder::clearIamConfiguration)
.put(BucketField.ID.getGrpcName(), BucketInfo.Builder::clearGeneratedId)
+ .put(BucketField.IP_FILTER.getGrpcName(), BucketInfo.Builder::clearIpFilter)
.put(BucketField.LABELS.getGrpcName(), BucketInfo.Builder::clearLabels)
.put(BucketField.LIFECYCLE.getGrpcName(), BucketInfo.Builder::clearLifecycleRules)
.put(BucketField.LOCATION.getGrpcName(), BucketInfo.Builder::clearLocation)
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java
index 36991f1b8..cb9b5515c 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java
@@ -115,6 +115,7 @@ public ImmutableList parameters() {
new Args<>(BucketField.ETAG, LazyAssertion.equal()),
new Args<>(BucketField.IAMCONFIGURATION, LazyAssertion.equal()),
new Args<>(BucketField.ID, LazyAssertion.equal()),
+ new Args<>(BucketField.IP_FILTER, LazyAssertion.equal()),
new Args<>(BucketField.LABELS, LazyAssertion.equal()),
new Args<>(BucketField.LIFECYCLE, LazyAssertion.equal()),
new Args<>(BucketField.LOCATION, LazyAssertion.equal()),
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java
index 905fe5478..e30fc3125 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java
@@ -61,6 +61,7 @@
import com.google.common.collect.ImmutableSet;
import java.util.Locale;
import java.util.Set;
+import java.util.TreeSet;
import java.util.function.Function;
import org.junit.Before;
import org.junit.Test;
@@ -298,7 +299,7 @@ public void storage_BucketGetOption_userProject_String() {
@Test
public void storage_BucketGetOption_fields_BucketField() {
Set expected =
- ImmutableSet.of(
+ treeSet(
"acl",
"autoclass",
"billing",
@@ -310,6 +311,7 @@ public void storage_BucketGetOption_fields_BucketField() {
"etag",
"iamConfiguration",
"id",
+ "ipFilter",
"labels",
"lifecycle",
"location",
@@ -705,7 +707,7 @@ public void storage_BlobGetOption_metagenerationNotMatch_long() {
@Test
public void storage_BlobGetOption_fields_BlobField() {
Set expected =
- ImmutableSet.of(
+ treeSet(
"acl",
"bucket",
"cacheControl",
@@ -790,7 +792,7 @@ public void storage_BucketListOption_userProject_String() {
@Test
public void storage_BucketListOption_fields_BucketField() {
Set expected =
- ImmutableSet.of(
+ treeSet(
"nextPageToken",
"items/acl",
"items/autoclass",
@@ -803,6 +805,7 @@ public void storage_BucketListOption_fields_BucketField() {
"items/etag",
"items/iamConfiguration",
"items/id",
+ "items/ipFilter",
"items/labels",
"items/lifecycle",
"items/location",
@@ -883,7 +886,7 @@ public void storage_BlobListOption_versions_boolean() {
@Test
public void storage_BlobListOption_fields_BlobField() {
Set expected =
- ImmutableSet.of(
+ treeSet(
"nextPageToken",
"prefixes",
"items/acl",
@@ -1194,6 +1197,15 @@ private static String objectName() {
}
private static Function> splitOnCommaToSet() {
- return s -> ImmutableSet.copyOf(s.split(","));
+ return s -> treeSet(s.split(","));
+ }
+
+ /**
+ * Util method to make a TreeSet easily. This makes failed assertions easier to read by sorting
+ * the values.
+ */
+ @SafeVarargs
+ private static > Set treeSet(T... ts) {
+ return new TreeSet<>(ImmutableSet.copyOf(ts));
}
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/BucketArbitraryProvider.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/BucketArbitraryProvider.java
index 1839c32b4..9b71a59ac 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/BucketArbitraryProvider.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/BucketArbitraryProvider.java
@@ -77,9 +77,7 @@ public Set> provideFor(TypeUsage targetType, SubtypeProvider subtyp
.as(Tuple::of),
Combinators.combine(
StorageArbitraries.projectNumber().map(ProjectNumber::toProjectName),
- StorageArbitraries
- .alnum() // ignored for now, tuples can't be a single element
- )
+ StorageArbitraries.buckets().ipFilter().injectNull(0.75))
.as(Tuple::of))
.as(
(t1, t2, t3, t4) -> {
@@ -109,6 +107,7 @@ public Set> provideFor(TypeUsage targetType, SubtypeProvider subtyp
ifNonNull(t3.get7(), b::putAllLabels);
ifNonNull(t3.get8(), b::setEtag);
ifNonNull(t4.get1(), ProjectName::toString, b::setProject);
+ ifNonNull(t4.get2(), b::setIpFilter);
// TODO: add CustomPlacementConfig
return b.build();
});
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/StorageArbitraries.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/StorageArbitraries.java
index f5ee93ce0..0b6e21d98 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/StorageArbitraries.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/jqwik/StorageArbitraries.java
@@ -26,6 +26,9 @@
import com.google.storage.v2.Bucket;
import com.google.storage.v2.Bucket.Billing;
import com.google.storage.v2.Bucket.Encryption;
+import com.google.storage.v2.Bucket.IpFilter;
+import com.google.storage.v2.Bucket.IpFilter.PublicNetworkSource;
+import com.google.storage.v2.Bucket.IpFilter.VpcNetworkSource;
import com.google.storage.v2.Bucket.Lifecycle.Rule.Condition;
import com.google.storage.v2.Bucket.Logging;
import com.google.storage.v2.Bucket.RetentionPolicy;
@@ -58,6 +61,7 @@
import net.jqwik.api.providers.TypeUsage;
import net.jqwik.time.api.DateTimes;
import net.jqwik.web.api.Web;
+import org.checkerframework.checker.nullness.qual.Nullable;
public final class StorageArbitraries {
@@ -172,6 +176,19 @@ public static Arbitrary etag() {
.edgeCases(config -> config.add(""));
}
+ /** Arbitrary of ipv4 and ipv6 cidr ranges */
+ public static Arbitrary cidr() {
+ return Arbitraries.oneOf(cidrIpv4(), cidrIpv6());
+ }
+
+ public static Arbitrary cidrIpv4() {
+ return Arbitraries.of("182.0.2.0/24");
+ }
+
+ public static Arbitrary cidrIpv6() {
+ return Arbitraries.of("2001:db8::/32");
+ }
+
public static final class Buckets {
private static final Buckets INSTANCE = new Buckets();
@@ -429,6 +446,66 @@ public Arbitrary locationType() {
public Arbitrary> labels() {
return objects().customMetadata();
}
+
+ public Arbitrary ipFilter() {
+ return Combinators.combine(
+ Arbitraries.of("Enabled", "Disabled").injectNull(0.33), // mode
+ publicNetworkSource(),
+ vpcNetworkSource().list().ofMinSize(1).ofMaxSize(3).injectNull(0.5),
+ bool().injectNull(0.5), // allow_cross_org_vpcs
+ bool().injectNull(0.5) // allow_all_service_agent_access
+ )
+ .as(
+ (mode, pns, vnss, allowCrossOrgVpcs, allowAllServiceAgentAccess) -> {
+ IpFilter.Builder b = IpFilter.newBuilder();
+ ifNonNull(mode, b::setMode);
+ ifNonNull(pns, b::setPublicNetworkSource);
+ ifNonNull(vnss, b::addAllVpcNetworkSources);
+ ifNonNull(allowCrossOrgVpcs, b::setAllowCrossOrgVpcs);
+ ifNonNull(allowAllServiceAgentAccess, b::setAllowAllServiceAgentAccess);
+ return b.build();
+ });
+ }
+
+ public Arbitrary publicNetworkSource() {
+ return Arbitraries.oneOf(cidr().list().ofMinSize(1).ofMaxSize(3).injectNull(0.5))
+ .map(
+ ranges -> {
+ PublicNetworkSource.Builder b = PublicNetworkSource.newBuilder();
+ ifNonNull(ranges, b::addAllAllowedIpCidrRanges);
+ return b.build();
+ });
+ }
+
+ public Arbitrary vpcNetworkSource() {
+ return Combinators.combine(
+ networkResource().injectNull(0.25),
+ cidr().list().ofMinSize(1).ofMaxSize(3).injectNull(0.5))
+ .as(
+ (network, ranges) -> {
+ VpcNetworkSource.Builder b = VpcNetworkSource.newBuilder();
+ ifNonNull(network, b::setNetwork);
+ ifNonNull(ranges, b::addAllAllowedIpCidrRanges);
+ return b.build();
+ });
+ }
+
+ Arbitrary networkResource() {
+ return Combinators.combine(projectID(), networkName())
+ .as(
+ (projectId, networkName) ->
+ String.format(
+ Locale.US, "projects/%s/global/networks/%s", projectId, networkName));
+ }
+
+ Arbitrary<@Nullable String> networkName() {
+ return Arbitraries.strings()
+ .withCharRange('a', 'z')
+ .numeric()
+ .withChars('-')
+ .ofMinLength(1)
+ .ofMaxLength(10);
+ }
}
public static final class ProjectID {
From fa0f6a03380af78e239bd0079267649ba4138f38 Mon Sep 17 00:00:00 2001
From: BenWhitehead
Date: Wed, 23 Jul 2025 12:20:34 -0400
Subject: [PATCH 11/15] chore: merge train for e2e-crc32c-media related upload
PRs (#3197)
Merge train for
* chore: cleanup Crc32cValue and associated Hasher operations #3167
* chore: add Hasher#defaultHasher() #3173
* chore: add UnifiedOpts.Opts#getHasher() to return a hasher relative to the set of Opts defined for an operation. #3174
* feat: add default end-to-end crc32c checksumming for several upload methods via grpc transport #3176
* feat: add default end-to-end checksumming for JournalingBlobWriteSessionConfig #3180
* fix: give user provided checksum precondition priority for Storage#create methods that accept byte[] #3182
* fix: move crc32c computation before writing to disk for BufferToDiskThenUpload BlobWriteSession config #3187
---
.../cloud/storage/BufferToDiskThenUpload.java | 75 ++++--
.../com/google/cloud/storage/Crc32cValue.java | 14 ++
.../DefaultBlobWriteSessionConfig.java | 2 +-
...apicWritableByteChannelSessionBuilder.java | 38 +++-
.../google/cloud/storage/GrpcStorageImpl.java | 10 +-
.../java/com/google/cloud/storage/Hasher.java | 66 +++++-
.../JournalingBlobWriteSessionConfig.java | 5 +-
.../com/google/cloud/storage/StorageImpl.java | 82 ++-----
.../com/google/cloud/storage/UnifiedOpts.java | 97 +++++---
.../com/google/cloud/storage/WriteCtx.java | 14 +-
.../storage/BufferToDiskThenUploadTest.java | 98 ++++++++
.../google/cloud/storage/Crc32cValueTest.java | 28 +++
...unkedResumableWritableByteChannelTest.java | 54 ++---
...bufferedDirectWritableByteChannelTest.java | 29 ++-
...CloseResumableWritableByteChannelTest.java | 20 +-
...apicUnbufferedWritableByteChannelTest.java | 8 +-
...fferedWritableByteChannelPropertyTest.java | 59 +++--
...ompositeUploadWritableByteChannelTest.java | 3 +-
.../cloud/storage/StorageImplMockitoTest.java | 213 +-----------------
.../com/google/cloud/storage/TestUtils.java | 23 ++
.../java/com/google/cloud/storage/TmpDir.java | 59 +++++
.../google/cloud/storage/UnifiedOptsTest.java | 29 ++-
.../storage/it/ChecksummedTestContent.java | 14 +-
...ITBlobWriteSessionCommonSemanticsTest.java | 47 ++++
.../it/ITObjectChecksumSupportTest.java | 72 ++++++
25 files changed, 722 insertions(+), 437 deletions(-)
create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/TmpDir.java
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BufferToDiskThenUpload.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BufferToDiskThenUpload.java
index bfd85c1d8..d831f2737 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BufferToDiskThenUpload.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BufferToDiskThenUpload.java
@@ -28,6 +28,8 @@
import com.google.cloud.storage.UnifiedOpts.Opts;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.MoreExecutors;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -42,6 +44,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collector;
import javax.annotation.concurrent.Immutable;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@@ -63,6 +66,7 @@
@TransportCompatibility({Transport.GRPC, Transport.HTTP})
public final class BufferToDiskThenUpload extends BlobWriteSessionConfig
implements BlobWriteSessionConfig.HttpCompatible, BlobWriteSessionConfig.GrpcCompatible {
+
private static final long serialVersionUID = 9059242302276891867L;
/**
@@ -211,41 +215,78 @@ public ApiFuture getResult() {
return result;
}
+ @SuppressWarnings("UnstableApiUsage")
private final class Flusher implements WritableByteChannel {
private final WritableByteChannel delegate;
+ private final Hasher cumulativeCrc32c;
+ private final ReentrantLock lock;
private Flusher(WritableByteChannel delegate) {
this.delegate = delegate;
+ this.cumulativeCrc32c =
+ opts.getHasher().initialValue() == null ? null : Hashing.crc32c().newHasher();
+ this.lock = new ReentrantLock();
}
@Override
public int write(ByteBuffer src) throws IOException {
- return delegate.write(src);
+ lock.lock();
+ try {
+ if (cumulativeCrc32c != null) {
+ cumulativeCrc32c.putBytes(src.duplicate());
+ }
+ return delegate.write(src);
+ } finally {
+ lock.unlock();
+ }
}
@Override
public boolean isOpen() {
- return delegate.isOpen();
+ lock.lock();
+ try {
+ return delegate.isOpen();
+ } finally {
+ lock.unlock();
+ }
}
@Override
public void close() throws IOException {
- delegate.close();
- try (RecoveryFile rf = Factory.WriteToFileThenUpload.this.rf) {
- Path path = rf.getPath();
- long size = Files.size(path);
- ThroughputSink.computeThroughput(
- clock,
- gcs,
- size,
- () -> {
- BlobInfo blob = storage.internalCreateFrom(path, info, opts);
- result.set(blob);
- });
- } catch (StorageException | IOException e) {
- result.setException(e);
- throw e;
+ lock.lock();
+ try {
+
+ delegate.close();
+ try (RecoveryFile rf = Factory.WriteToFileThenUpload.this.rf) {
+ Path path = rf.getPath();
+ long size = Files.size(path);
+ ThroughputSink.computeThroughput(
+ clock,
+ gcs,
+ size,
+ () -> {
+ BlobInfo pendingInfo = info;
+ Opts pendingOpts = opts;
+ if (cumulativeCrc32c != null) {
+ int hashCodeInt = cumulativeCrc32c.hash().asInt();
+ pendingInfo =
+ pendingInfo.toBuilder()
+ .clearMd5()
+ .clearCrc32c()
+ .setCrc32c(Utils.crc32cCodec.encode(hashCodeInt))
+ .build();
+ pendingOpts = opts.prepend(Opts.from(UnifiedOpts.crc32cMatch(hashCodeInt)));
+ }
+ BlobInfo blob = storage.internalCreateFrom(path, pendingInfo, pendingOpts);
+ result.set(blob);
+ });
+ } catch (StorageException | IOException e) {
+ result.setException(e);
+ throw e;
+ }
+ } finally {
+ lock.unlock();
}
}
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Crc32cValue.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Crc32cValue.java
index 245999dd7..5da5a2037 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Crc32cValue.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Crc32cValue.java
@@ -16,6 +16,7 @@
package com.google.cloud.storage;
+import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.Objects;
@@ -56,6 +57,10 @@ public boolean eqValue(Crc32cValue other) {
return this.getValue() == other.getValue();
}
+ static Crc32cLengthKnown zero() {
+ return Crc32cLengthKnown.ZERO;
+ }
+
static Crc32cLengthUnknown of(int value) {
return new Crc32cLengthUnknown(value);
}
@@ -81,6 +86,9 @@ public int getValue() {
@Override
public Crc32cLengthUnknown concat(Crc32cLengthKnown other) {
+ if (other == Crc32cLengthKnown.ZERO) {
+ return this;
+ }
int combined = Crc32cUtility.concatCrc32c(value, other.value, other.length);
return new Crc32cLengthUnknown(combined);
}
@@ -118,6 +126,7 @@ public int hashCode() {
}
static final class Crc32cLengthKnown extends Crc32cValue {
+ private static final Crc32cLengthKnown ZERO = Hasher.enabled().hash(ByteBuffer.allocate(0));
private final int value;
private final long length;
@@ -137,6 +146,11 @@ public long getLength() {
@Override
public Crc32cLengthKnown concat(Crc32cLengthKnown other) {
+ if (other == ZERO) {
+ return this;
+ } else if (this == ZERO) {
+ return other;
+ }
int combined = Crc32cUtility.concatCrc32c(value, other.value, other.length);
return new Crc32cLengthKnown(combined, length + other.length);
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultBlobWriteSessionConfig.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultBlobWriteSessionConfig.java
index 67360ad17..53a14ca26 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultBlobWriteSessionConfig.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultBlobWriteSessionConfig.java
@@ -160,7 +160,7 @@ public WritableByteChannelSession writeSession(
grpc.storageClient
.writeObjectCallable()
.withDefaultCallContext(grpcCallContext))
- .setHasher(Hasher.noop())
+ .setHasher(opts.getHasher())
.setByteStringStrategy(ByteStringStrategy.copy())
.resumable()
.withRetryConfig(
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java
index ff73dc689..a256d39b7 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java
@@ -50,14 +50,14 @@ final class GapicWritableByteChannelSessionBuilder {
GapicWritableByteChannelSessionBuilder(
ClientStreamingCallable write) {
this.write = write;
- this.hasher = Hasher.noop();
+ this.hasher = Hasher.defaultHasher();
this.byteStringStrategy = ByteStringStrategy.copy();
}
/**
* Set the {@link Hasher} to apply to the bytes passing through the built session's channel.
*
- * Default: {@link Hasher#noop()}
+ *
Default: {@link Hasher#defaultHasher()}
*
* @see Hasher#enabled()
* @see Hasher#noop()
@@ -179,14 +179,17 @@ UnbufferedDirectUploadBuilder setRequest(WriteObjectRequest req) {
}
UnbufferedWritableByteChannelSession build() {
+ ChunkSegmenter chunkSegmenter = getChunkSegmenter();
return new UnbufferedWriteSession<>(
ApiFutures.immediateFuture(requireNonNull(req, "req must be non null")),
lift((WriteObjectRequest start, SettableApiFuture resultFuture) ->
new GapicUnbufferedDirectWritableByteChannel(
resultFuture,
- getChunkSegmenter(),
+ chunkSegmenter,
write,
- new WriteCtx<>(WriteObjectRequestBuilderFactory.simple(start))))
+ WriteCtx.of(
+ WriteObjectRequestBuilderFactory.simple(start),
+ chunkSegmenter.getHasher())))
.andThen(StorageByteChannels.writable()::createSynchronized));
}
}
@@ -207,14 +210,17 @@ BufferedDirectUploadBuilder setRequest(WriteObjectRequest req) {
}
BufferedWritableByteChannelSession build() {
+ ChunkSegmenter chunkSegmenter = getChunkSegmenter();
return new BufferedWriteSession<>(
ApiFutures.immediateFuture(requireNonNull(req, "req must be non null")),
lift((WriteObjectRequest start, SettableApiFuture resultFuture) ->
new GapicUnbufferedDirectWritableByteChannel(
resultFuture,
- getChunkSegmenter(),
+ chunkSegmenter,
write,
- new WriteCtx<>(WriteObjectRequestBuilderFactory.simple(start))))
+ WriteCtx.of(
+ WriteObjectRequestBuilderFactory.simple(start),
+ chunkSegmenter.getHasher())))
.andThen(c -> new DefaultBufferedWritableByteChannel(bufferHandle, c))
.andThen(StorageByteChannels.writable()::createSynchronized));
}
@@ -290,20 +296,24 @@ UnbufferedResumableUploadBuilder setStartAsync(ApiFuture start)
UnbufferedWritableByteChannelSession build() {
RetrierWithAlg boundRetrier = retrier;
+ ChunkSegmenter chunkSegmenter = getChunkSegmenter();
return new UnbufferedWriteSession<>(
requireNonNull(start, "start must be non null"),
lift((ResumableWrite start, SettableApiFuture result) -> {
if (fsyncEvery) {
return new GapicUnbufferedChunkedResumableWritableByteChannel(
result,
- getChunkSegmenter(),
+ chunkSegmenter,
write,
- new WriteCtx<>(start),
+ WriteCtx.of(start, chunkSegmenter.getHasher()),
boundRetrier,
Retrying::newCallContext);
} else {
return new GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel(
- result, getChunkSegmenter(), write, new WriteCtx<>(start));
+ result,
+ chunkSegmenter,
+ write,
+ WriteCtx.of(start, chunkSegmenter.getHasher()));
}
})
.andThen(StorageByteChannels.writable()::createSynchronized));
@@ -330,20 +340,24 @@ BufferedResumableUploadBuilder setStartAsync(ApiFuture start) {
}
BufferedWritableByteChannelSession build() {
+ ChunkSegmenter chunkSegmenter = getChunkSegmenter();
return new BufferedWriteSession<>(
requireNonNull(start, "start must be non null"),
lift((ResumableWrite start, SettableApiFuture result) -> {
if (fsyncEvery) {
return new GapicUnbufferedChunkedResumableWritableByteChannel(
result,
- getChunkSegmenter(),
+ chunkSegmenter,
write,
- new WriteCtx<>(start),
+ WriteCtx.of(start, chunkSegmenter.getHasher()),
retrier,
Retrying::newCallContext);
} else {
return new GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel(
- result, getChunkSegmenter(), write, new WriteCtx<>(start));
+ result,
+ chunkSegmenter,
+ write,
+ WriteCtx.of(start, chunkSegmenter.getHasher()));
}
})
.andThen(c -> new DefaultBufferedWritableByteChannel(bufferHandle, c))
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
index 50b3596c9..7d1192464 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
@@ -263,7 +263,7 @@ public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... op
GrpcCallContext grpcCallContext =
optsWithDefaults.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
WriteObjectRequest req = getWriteObjectRequest(blobInfo, optsWithDefaults);
- Hasher hasher = Hasher.enabled();
+ Hasher hasher = optsWithDefaults.getHasher();
GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
UnbufferedWritableByteChannelSession session =
ResumableMedia.gapic()
@@ -324,7 +324,7 @@ public Blob internalCreateFrom(Path path, BlobInfo info, Opts o
write,
storageClient.queryWriteStatusCallable(),
rw,
- Hasher.noop()),
+ opts.getHasher()),
MoreExecutors.directExecutor());
try {
GrpcResumableSession got = session2.get();
@@ -365,7 +365,7 @@ public Blob createFrom(
.write()
.byteChannel(
storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext))
- .setHasher(Hasher.noop())
+ .setHasher(opts.getHasher())
.setByteStringStrategy(ByteStringStrategy.noCopy())
.resumable()
.withRetryConfig(retrier.withAlg(retryAlgorithmManager.idempotent()))
@@ -779,7 +779,7 @@ public GrpcBlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options
GrpcCallContext grpcCallContext =
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
- Hasher hasher = Hasher.noop();
+ Hasher hasher = opts.getHasher();
// in JSON, the starting of the resumable session happens before the invocation of write can
// happen. Emulate the same thing here.
// 1. create the future
@@ -804,7 +804,7 @@ public BlobInfo internalDirectUpload(
GrpcCallContext grpcCallContext =
optsWithDefaults.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
WriteObjectRequest req = getWriteObjectRequest(blobInfo, optsWithDefaults);
- Hasher hasher = Hasher.enabled();
+ Hasher hasher = opts.getHasher();
GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
RewindableContent content = RewindableContent.of(buf);
return retrier.run(
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java
index 3342fd8c3..47a7b029e 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Hasher.java
@@ -27,11 +27,35 @@
import java.util.List;
import java.util.Locale;
import java.util.function.Supplier;
+import java.util.logging.Logger;
+import javax.annotation.ParametersAreNonnullByDefault;
import javax.annotation.concurrent.Immutable;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
+@SuppressWarnings("ClassEscapesDefinedScope")
+@ParametersAreNonnullByDefault
interface Hasher {
+ final class DefaultInstanceHolder {
+ private static final Logger LOGGER = Logger.getLogger(Hasher.class.getName());
+ private static final String PROPERTY_NAME = "com.google.cloud.storage.Hasher.default";
+ private static final String PROPERTY_VALUE = System.getProperty(PROPERTY_NAME, "enabled");
+ static final Hasher DEFAULT_HASHER;
+
+ static {
+ LOGGER.fine(String.format(Locale.US, "-D%s=%s", PROPERTY_NAME, PROPERTY_VALUE));
+ if ("disabled".equalsIgnoreCase(PROPERTY_VALUE)) {
+ DEFAULT_HASHER = noop();
+ } else {
+ try {
+ Class.forName("java.util.zip.CRC32C");
+ } catch (ClassNotFoundException e) {
+ LOGGER.fine("Fast CRC32C implementation (Java 9+) is not available.");
+ }
+ DEFAULT_HASHER = enabled();
+ }
+ }
+ }
@Nullable
default Crc32cLengthKnown hash(Supplier b) {
@@ -49,16 +73,30 @@ default Crc32cLengthKnown hash(Supplier b) {
void validateUnchecked(Crc32cValue expected, ByteString byteString)
throws UncheckedChecksumMismatchException;
- @Nullable Crc32cLengthKnown nullSafeConcat(Crc32cLengthKnown r1, Crc32cLengthKnown r2);
+ @Nullable Crc32cLengthKnown nullSafeConcat(
+ @Nullable Crc32cLengthKnown r1, @NonNull Crc32cLengthKnown r2);
- static Hasher noop() {
+ /**
+ * The initial value to use for this hasher.
+ *
+ * Not ideal, really we should always start with {@link Crc32cValue#zero()} but this saves us
+ * from having to plumb the initial value along with the actual hasher to the constructor of the
+ * WriteCtx when hashing is disabled because of user provided crc32c/md5 preconditions.
+ */
+ @Nullable Crc32cLengthKnown initialValue();
+
+ static NoOpHasher noop() {
return NoOpHasher.INSTANCE;
}
- static Hasher enabled() {
+ static GuavaHasher enabled() {
return GuavaHasher.INSTANCE;
}
+ static Hasher defaultHasher() {
+ return DefaultInstanceHolder.DEFAULT_HASHER;
+ }
+
@Immutable
class NoOpHasher implements Hasher {
private static final NoOpHasher INSTANCE = new NoOpHasher();
@@ -85,7 +123,13 @@ public void validate(Crc32cValue expected, ByteString b) {}
public void validateUnchecked(Crc32cValue expected, ByteString byteString) {}
@Override
- public @Nullable Crc32cLengthKnown nullSafeConcat(Crc32cLengthKnown r1, Crc32cLengthKnown r2) {
+ public @Nullable Crc32cLengthKnown nullSafeConcat(
+ @Nullable Crc32cLengthKnown r1, @NonNull Crc32cLengthKnown r2) {
+ return null;
+ }
+
+ @Override
+ public @Nullable Crc32cLengthKnown initialValue() {
return null;
}
}
@@ -107,7 +151,7 @@ private GuavaHasher() {}
return Crc32cValue.of(Hashing.crc32c().hashBytes(b).asInt(), remaining);
}
- @SuppressWarnings({"ConstantConditions", "UnstableApiUsage"})
+ @SuppressWarnings({"UnstableApiUsage"})
@Override
public @NonNull Crc32cLengthKnown hash(ByteString byteString) {
List buffers = byteString.asReadOnlyByteBufferList();
@@ -118,7 +162,6 @@ private GuavaHasher() {}
return Crc32cValue.of(crc32c.hash().asInt(), byteString.size());
}
- @SuppressWarnings({"ConstantConditions"})
@Override
public void validate(Crc32cValue expected, ByteString byteString)
throws ChecksumMismatchException {
@@ -137,7 +180,6 @@ public void validate(Crc32cValue expected, Supplier b)
}
}
- @SuppressWarnings({"ConstantConditions"})
@Override
public void validateUnchecked(Crc32cValue expected, ByteString byteString)
throws UncheckedChecksumMismatchException {
@@ -149,13 +191,19 @@ public void validateUnchecked(Crc32cValue expected, ByteString byteString)
@Override
@Nullable
- public Crc32cLengthKnown nullSafeConcat(Crc32cLengthKnown r1, Crc32cLengthKnown r2) {
+ public Crc32cLengthKnown nullSafeConcat(
+ @Nullable Crc32cLengthKnown r1, @NonNull Crc32cLengthKnown r2) {
if (r1 == null) {
- return r2;
+ return null;
} else {
return r1.concat(r2);
}
}
+
+ @Override
+ public @NonNull Crc32cLengthKnown initialValue() {
+ return Crc32cValue.zero();
+ }
}
final class ChecksumMismatchException extends IOException {
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JournalingBlobWriteSessionConfig.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JournalingBlobWriteSessionConfig.java
index b770a1b4e..d8db80a65 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JournalingBlobWriteSessionConfig.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JournalingBlobWriteSessionConfig.java
@@ -192,7 +192,8 @@ public WritableByteChannelSession writeSession(
grpcStorage.startResumableWrite(
grpcCallContext, grpcStorage.getWriteObjectRequest(info, opts), opts);
ApiFuture> start =
- ApiFutures.transform(f, WriteCtx::new, MoreExecutors.directExecutor());
+ ApiFutures.transform(
+ f, s -> WriteCtx.of(s, opts.getHasher()), MoreExecutors.directExecutor());
ClientStreamingCallable write =
grpcStorage.storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext);
@@ -200,7 +201,7 @@ public WritableByteChannelSession writeSession(
ResumableMedia.gapic()
.write()
.byteChannel(write)
- .setHasher(Hasher.noop())
+ .setHasher(opts.getHasher())
.setByteStringStrategy(ByteStringStrategy.copy())
.journaling()
.withRetryConfig(
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
index bfa956011..1974183ac 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
@@ -41,6 +41,7 @@
import com.google.cloud.storage.BlobReadChannelV2.BlobReadChannelContext;
import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory;
import com.google.cloud.storage.Conversions.Decoder;
+import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
import com.google.cloud.storage.PostPolicyV4.ConditionV4Type;
import com.google.cloud.storage.PostPolicyV4.PostConditionsV4;
@@ -55,7 +56,6 @@
import com.google.cloud.storage.spi.v1.StorageRpc;
import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest;
import com.google.common.base.CharMatcher;
-import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -63,10 +63,8 @@
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.CountingOutputStream;
-import com.google.common.primitives.Ints;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -97,7 +95,6 @@
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import java.util.stream.Stream;
-import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
final class StorageImpl extends BaseService implements Storage, StorageInternal {
@@ -152,46 +149,23 @@ public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
@Override
public Blob create(BlobInfo blobInfo, BlobTargetOption... options) {
- BlobInfo updatedInfo =
- blobInfo.toBuilder()
- .setMd5(EMPTY_BYTE_ARRAY_MD5)
- .setCrc32c(EMPTY_BYTE_ARRAY_CRC32C)
- .build();
- final Opts objectTargetOptOpts = Opts.unwrap(options).resolveFrom(updatedInfo);
- return internalCreate(updatedInfo, EMPTY_BYTE_ARRAY, 0, 0, objectTargetOptOpts);
+ Opts opts = Opts.unwrap(options).resolveFrom(blobInfo);
+ return internalDirectUpload(blobInfo, opts, Buffers.allocate(0)).asBlob(this);
}
@Override
public Blob create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) {
content = firstNonNull(content, EMPTY_BYTE_ARRAY);
- BlobInfo updatedInfo =
- blobInfo.toBuilder()
- .setMd5(BaseEncoding.base64().encode(Hashing.md5().hashBytes(content).asBytes()))
- .setCrc32c(
- BaseEncoding.base64()
- .encode(Ints.toByteArray(Hashing.crc32c().hashBytes(content).asInt())))
- .build();
- final Opts objectTargetOptOpts = Opts.unwrap(options).resolveFrom(updatedInfo);
- return internalCreate(updatedInfo, content, 0, content.length, objectTargetOptOpts);
+ Opts opts = Opts.unwrap(options).resolveFrom(blobInfo);
+ return internalDirectUpload(blobInfo, opts, ByteBuffer.wrap(content)).asBlob(this);
}
@Override
public Blob create(
BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
- content = firstNonNull(content, EMPTY_BYTE_ARRAY);
- BlobInfo updatedInfo =
- blobInfo.toBuilder()
- .setMd5(
- BaseEncoding.base64()
- .encode(Hashing.md5().hashBytes(content, offset, length).asBytes()))
- .setCrc32c(
- BaseEncoding.base64()
- .encode(
- Ints.toByteArray(
- Hashing.crc32c().hashBytes(content, offset, length).asInt())))
- .build();
- final Opts objectTargetOptOpts = Opts.unwrap(options).resolveFrom(updatedInfo);
- return internalCreate(updatedInfo, content, offset, length, objectTargetOptOpts);
+ Opts opts = Opts.unwrap(options).resolveFrom(blobInfo);
+ return internalDirectUpload(blobInfo, opts, ByteBuffer.wrap(content, offset, length))
+ .asBlob(this);
}
@Override
@@ -212,30 +186,6 @@ public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... op
return info.asBlob(this);
}
- private Blob internalCreate(
- BlobInfo info,
- final byte @NonNull [] content,
- final int offset,
- final int length,
- Opts opts) {
- Preconditions.checkNotNull(content);
- final Map optionsMap = opts.getRpcOptions();
-
- BlobInfo updated = opts.blobInfoMapper().apply(info.toBuilder()).build();
- final StorageObject blobPb = codecs.blobInfo().encode(updated);
- ResultRetryAlgorithm algorithm =
- retryAlgorithmManager.getForObjectsCreate(blobPb, optionsMap);
- return run(
- algorithm,
- () ->
- storageRpc.create(
- blobPb, new ByteArrayInputStream(content, offset, length), optionsMap),
- (x) -> {
- BlobInfo info1 = Conversions.json().blobInfo().decode(x);
- return info1.asBlob(this);
- });
- }
-
@Override
public Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options)
throws IOException {
@@ -1731,7 +1681,7 @@ public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts optionsMap = opts.getRpcOptions();
BlobInfo.Builder builder = info.toBuilder().setMd5(null).setCrc32c(null);
@@ -1767,17 +1717,15 @@ public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts opts, ByteBuffer buf) {
-
BlobInfo.Builder builder =
- info.toBuilder()
- .setMd5(
- BaseEncoding.base64().encode(Hashing.md5().hashBytes(buf.duplicate()).asBytes()))
- .setCrc32c(
- BaseEncoding.base64()
- .encode(Ints.toByteArray(Hashing.crc32c().hashBytes(buf.duplicate()).asInt())));
+ opts.blobInfoMapper().apply(info.toBuilder().clearMd5().clearCrc32c());
+ @Nullable Crc32cLengthKnown hash = opts.getHasher().hash(buf.duplicate());
+ if (hash != null) {
+ builder.setCrc32c(Utils.crc32cCodec.encode(hash.getValue()));
+ }
final Map optionsMap = opts.getRpcOptions();
- BlobInfo updated = opts.blobInfoMapper().apply(builder).build();
+ BlobInfo updated = builder.build();
final StorageObject encoded = codecs.blobInfo().encode(updated);
ResultRetryAlgorithm algorithm =
retryAlgorithmManager.getForObjectsCreate(encoded, optionsMap);
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
index 29444ad13..3444323cf 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
@@ -281,6 +281,10 @@ interface ProjectAsSource {
O asSource();
}
+ interface HasherSelector extends BucketObjectHmacKeyAllOpt {
+ Hasher getHasher();
+ }
+
/**
* This class extends off {@link ObjectSourceOpt} and {@link ObjectTargetOpt} in order to satisfy
* some the shimming constraints of the subclasses of {@link OptionShim}.
@@ -589,7 +593,11 @@ static Headers extraHeaders(ImmutableMap extraHeaders) {
return new Headers(extraHeaders);
}
- static final class Crc32cMatch implements ObjectTargetOpt {
+ static DefaultHasherSelector defaultHasherSelector() {
+ return DefaultHasherSelector.INSTANCE;
+ }
+
+ static final class Crc32cMatch implements ObjectTargetOpt, HasherSelector {
private static final long serialVersionUID = 8172282701777561769L;
private final int val;
@@ -630,6 +638,11 @@ public Mapper bidiWriteObject() {
};
}
+ @Override
+ public Hasher getHasher() {
+ return Hasher.noop();
+ }
+
@Override
public int hashCode() {
return Objects.hash(val);
@@ -1316,7 +1329,7 @@ public Mapper listObjects() {
}
@Deprecated
- static final class Md5Match implements ObjectTargetOpt {
+ static final class Md5Match implements ObjectTargetOpt, HasherSelector {
private static final long serialVersionUID = 5237207911268363887L;
private final String val;
@@ -1359,6 +1372,11 @@ public Mapper bidiWriteObject() {
};
}
+ @Override
+ public Hasher getHasher() {
+ return Hasher.noop();
+ }
+
@Override
public int hashCode() {
return Objects.hash(val);
@@ -1984,8 +2002,8 @@ public Mapper listObjects() {
}
}
- static final class UserProject extends RpcOptVal
- implements BucketSourceOpt,
+ interface BucketObjectHmacKeyAllOpt
+ extends BucketSourceOpt,
BucketTargetOpt,
BucketListOpt,
ObjectSourceOpt,
@@ -1994,6 +2012,18 @@ static final class UserProject extends RpcOptVal
HmacKeySourceOpt,
HmacKeyTargetOpt,
HmacKeyListOpt {
+ @Override
+ default Mapper rewriteObject() {
+ return Mapper.identity();
+ }
+
+ @Override
+ default Mapper moveObject() {
+ return Mapper.identity();
+ }
+ }
+
+ static final class UserProject extends RpcOptVal implements BucketObjectHmacKeyAllOpt {
private static final long serialVersionUID = 3962499996741180460L;
private UserProject(String val) {
@@ -2005,28 +2035,10 @@ public Mapper getGrpcMetadataMapper() {
return ctx ->
ctx.withExtraHeaders(ImmutableMap.of("X-Goog-User-Project", ImmutableList.of(val)));
}
-
- @Override
- public Mapper rewriteObject() {
- return Mapper.identity();
- }
-
- @Override
- public Mapper moveObject() {
- return Mapper.identity();
- }
}
static final class Headers extends RpcOptVal>
- implements BucketSourceOpt,
- BucketTargetOpt,
- BucketListOpt,
- ObjectSourceOpt,
- ObjectTargetOpt,
- ObjectListOpt,
- HmacKeySourceOpt,
- HmacKeyTargetOpt,
- HmacKeyListOpt {
+ implements BucketObjectHmacKeyAllOpt {
/**
* The set of header names which are blocked from being able to be provided for an instance of
@@ -2182,16 +2194,6 @@ private void copyEntries(
}
}
}
-
- @Override
- public Mapper rewriteObject() {
- return Mapper.identity();
- }
-
- @Override
- public Mapper moveObject() {
- return Mapper.identity();
- }
}
static final class VersionsFilter extends RpcOptVal<@NonNull Boolean> implements ObjectListOpt {
@@ -2607,6 +2609,17 @@ public String toString() {
}
}
+ static final class DefaultHasherSelector implements HasherSelector, Opt {
+ private static final DefaultHasherSelector INSTANCE = new DefaultHasherSelector();
+
+ private DefaultHasherSelector() {}
+
+ @Override
+ public Hasher getHasher() {
+ return Hasher.defaultHasher();
+ }
+ }
+
/**
* Internal "collection" class to represent a set of {@link Opt}s, and to provide useful
* transformations to individual mappers or to resolve any extractors providing a new instance
@@ -2703,6 +2716,22 @@ Opts projectAsSource() {
return Utils.mapBuild(builder);
}
+ @VisibleForTesting
+ HasherSelector getHasherSelector() {
+ HasherSelector search = defaultHasherSelector();
+ Predicate p = isInstanceOf(HasherSelector.class);
+ for (T opt : opts) {
+ if (p.test(opt)) {
+ search = (HasherSelector) opt;
+ }
+ }
+ return search;
+ }
+
+ Hasher getHasher() {
+ return getHasherSelector().getHasher();
+ }
+
Mapper grpcMetadataMapper() {
return fuseMappers(GrpcMetadataMapper.class, GrpcMetadataMapper::getGrpcMetadataMapper);
}
@@ -2755,7 +2784,7 @@ Mapper readObjectRequest() {
return fuseMappers(ObjectSourceOpt.class, ObjectSourceOpt::readObject);
}
- public Mapper bidiReadObjectRequest() {
+ Mapper bidiReadObjectRequest() {
return fuseMappers(ObjectSourceOpt.class, ObjectSourceOpt::bidiReadObject);
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteCtx.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteCtx.java
index 9816d4dc0..5539c04ba 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteCtx.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/WriteCtx.java
@@ -23,6 +23,7 @@
import com.google.storage.v2.WriteObjectRequest;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
final class WriteCtx {
@@ -31,13 +32,18 @@ final class WriteCtx {
private final AtomicLong totalSentBytes;
private final AtomicLong confirmedBytes;
- private final AtomicReference cumulativeCrc32c;
+ private final AtomicReference<@Nullable Crc32cLengthKnown> cumulativeCrc32c;
- WriteCtx(RequestFactoryT requestFactory) {
+ private WriteCtx(RequestFactoryT requestFactory, @Nullable Crc32cLengthKnown initialValue) {
this.requestFactory = requestFactory;
this.totalSentBytes = new AtomicLong(0);
this.confirmedBytes = new AtomicLong(0);
- this.cumulativeCrc32c = new AtomicReference<>();
+ this.cumulativeCrc32c = new AtomicReference<>(initialValue);
+ }
+
+ static WriteCtx of(
+ RFT rft, @NonNull Hasher hasher) {
+ return new WriteCtx<>(rft, hasher.initialValue());
}
public RequestFactoryT getRequestFactory() {
@@ -56,7 +62,7 @@ public AtomicLong getConfirmedBytes() {
return confirmedBytes;
}
- public AtomicReference getCumulativeCrc32c() {
+ public AtomicReference<@Nullable Crc32cLengthKnown> getCumulativeCrc32c() {
return cumulativeCrc32c;
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BufferToDiskThenUploadTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BufferToDiskThenUploadTest.java
index 1c9635227..04ddbd6c1 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BufferToDiskThenUploadTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BufferToDiskThenUploadTest.java
@@ -22,6 +22,7 @@
import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory;
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
import com.google.cloud.storage.UnifiedOpts.Opts;
+import com.google.cloud.storage.it.ChecksummedTestContent;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
@@ -73,4 +74,101 @@ public BlobInfo internalCreateFrom(
String xxdExpected = xxd(bytes);
assertThat(xxdActual).isEqualTo(xxdExpected);
}
+
+ @Test
+ public void crc32c_default() throws IOException {
+ Path tempDir = temporaryFolder.newFolder(testName.getMethodName()).toPath();
+
+ BufferToDiskThenUpload btdtu = BlobWriteSessionConfigs.bufferToDiskThenUpload(tempDir);
+ TestClock clock = TestClock.tickBy(Instant.EPOCH, Duration.ofSeconds(1));
+ WriterFactory factory = btdtu.createFactory(clock);
+
+ BlobInfo blobInfo = BlobInfo.newBuilder("bucket", "object").build();
+ ChecksummedTestContent testContent =
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(128));
+ WritableByteChannelSession writeSession =
+ factory.writeSession(
+ new StorageInternal() {
+ @Override
+ public BlobInfo internalCreateFrom(
+ Path path, BlobInfo info, Opts opts) {
+ assertThat(info.getCrc32c()).isEqualTo(testContent.getCrc32cBase64());
+ assertThat(opts)
+ .isEqualTo(Opts.from(UnifiedOpts.crc32cMatch(testContent.getCrc32c())));
+ return info;
+ }
+ },
+ blobInfo,
+ Opts.empty());
+
+ try (WritableByteChannel open = writeSession.open()) {
+ open.write(ByteBuffer.wrap(testContent.getBytes()));
+ }
+ }
+
+ @Test
+ public void userProvidedCrc32cTakesPriority() throws IOException {
+ Path tempDir = temporaryFolder.newFolder(testName.getMethodName()).toPath();
+
+ BufferToDiskThenUpload btdtu = BlobWriteSessionConfigs.bufferToDiskThenUpload(tempDir);
+ TestClock clock = TestClock.tickBy(Instant.EPOCH, Duration.ofSeconds(1));
+ WriterFactory factory = btdtu.createFactory(clock);
+
+ BlobInfo blobInfo =
+ BlobInfo.newBuilder("bucket", "object")
+ .setCrc32c(Utils.crc32cCodec.encode(737))
+ .setMd5("something")
+ .build();
+ ChecksummedTestContent testContent =
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(128));
+ Opts origOpts = Opts.from(UnifiedOpts.crc32cMatch(737));
+ WritableByteChannelSession writeSession =
+ factory.writeSession(
+ new StorageInternal() {
+ @Override
+ public BlobInfo internalCreateFrom(
+ Path path, BlobInfo info, Opts opts) {
+ assertThat(Utils.crc32cCodec.decode(info.getCrc32c())).isEqualTo(737);
+ assertThat(opts).isEqualTo(origOpts);
+ return info;
+ }
+ },
+ blobInfo,
+ origOpts);
+
+ try (WritableByteChannel open = writeSession.open()) {
+ open.write(ByteBuffer.wrap(testContent.getBytes()));
+ }
+ }
+
+ @Test
+ public void userProvidedMd5TakesPriority() throws IOException {
+ Path tempDir = temporaryFolder.newFolder(testName.getMethodName()).toPath();
+
+ BufferToDiskThenUpload btdtu = BlobWriteSessionConfigs.bufferToDiskThenUpload(tempDir);
+ TestClock clock = TestClock.tickBy(Instant.EPOCH, Duration.ofSeconds(1));
+ WriterFactory factory = btdtu.createFactory(clock);
+
+ BlobInfo blobInfo = BlobInfo.newBuilder("bucket", "object").setMd5("something").build();
+ ChecksummedTestContent testContent =
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(128));
+ Opts origOpts = Opts.from(UnifiedOpts.md5Match("something"));
+ WritableByteChannelSession writeSession =
+ factory.writeSession(
+ new StorageInternal() {
+ @Override
+ public BlobInfo internalCreateFrom(
+ Path path, BlobInfo info, Opts opts) {
+ assertThat(info.getMd5()).isEqualTo("something");
+ assertThat(opts).isEqualTo(origOpts);
+ return info;
+ }
+ },
+ blobInfo,
+ origOpts);
+
+ try (WritableByteChannel open = writeSession.open()) {
+ open.write(ByteBuffer.wrap(testContent.getBytes()));
+ }
+ }
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/Crc32cValueTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/Crc32cValueTest.java
index 600b1337b..1565e926c 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/Crc32cValueTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/Crc32cValueTest.java
@@ -20,9 +20,11 @@
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
import com.google.cloud.storage.Crc32cValue.Crc32cLengthUnknown;
+import com.google.cloud.storage.it.ChecksummedTestContent;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import net.jqwik.api.Example;
+import org.checkerframework.checker.nullness.qual.NonNull;
final class Crc32cValueTest {
@@ -67,4 +69,30 @@ public void ensureConcatSatisfiesTheLeftDistributedProperty() {
assertThat(nesting.getValue()).isEqualTo(expected);
assertThat(mixed.getValue()).isEqualTo(expected);
}
+
+ @Example
+ void zeroDoesNotTransform() {
+ Crc32cLengthKnown base =
+ Hasher.enabled().hash(DataGenerator.base64Characters().genByteBuffer(64));
+
+ assertThat(base.concat(Crc32cValue.zero())).isSameInstanceAs(base);
+ assertThat(Crc32cValue.zero().concat(base)).isSameInstanceAs(base);
+ }
+
+ @Example
+ void nullSafeConcat_isAlwaysNull() {
+ ChecksummedTestContent testContent =
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(2 * 1024 * 1024));
+
+ Crc32cLengthKnown actual =
+ testContent.chunkup(373).stream()
+ .map(Crc32cValueTest::toCrc32cValue)
+ .reduce(null, Hasher.enabled()::nullSafeConcat);
+
+ assertThat(actual).isNull();
+ }
+
+ private static @NonNull Crc32cLengthKnown toCrc32cValue(ChecksummedTestContent testContent) {
+ return Crc32cValue.of(testContent.getCrc32c(), testContent.getBytes().length);
+ }
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedChunkedResumableWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedChunkedResumableWritableByteChannelTest.java
index 6108a0b0b..473e73a9c 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedChunkedResumableWritableByteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedChunkedResumableWritableByteChannelTest.java
@@ -26,13 +26,16 @@
import com.google.api.core.SettableApiFuture;
import com.google.api.gax.grpc.GrpcCallContext;
+import com.google.cloud.storage.Hasher.GuavaHasher;
import com.google.cloud.storage.ITGapicUnbufferedWritableByteChannelTest.DirectWriteService;
import com.google.cloud.storage.Retrying.RetrierWithAlg;
+import com.google.cloud.storage.it.ChecksummedTestContent;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.protobuf.ByteString;
import com.google.storage.v2.ChecksummedData;
import com.google.storage.v2.Object;
+import com.google.storage.v2.ObjectChecksums;
import com.google.storage.v2.StartResumableWriteRequest;
import com.google.storage.v2.StartResumableWriteResponse;
import com.google.storage.v2.StorageClient;
@@ -48,8 +51,9 @@
public final class ITGapicUnbufferedChunkedResumableWritableByteChannelTest {
+ public static final GuavaHasher HASHER = Hasher.enabled();
private static final ChunkSegmenter CHUNK_SEGMENTER =
- new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), _256KiB, _256KiB);
+ new ChunkSegmenter(HASHER, ByteStringStrategy.copy(), _256KiB, _256KiB);
/**
*
@@ -94,10 +98,8 @@ public void scenario1() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setChecksummedData(
- ChecksummedData.newBuilder()
- .setContent(
- ByteString.copyFrom(DataGenerator.base64Characters().genBytes(_256KiB)))
- .build())
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(_256KiB))
+ .asChecksummedData())
.build();
WriteObjectResponse resp1 =
WriteObjectResponse.newBuilder()
@@ -115,7 +117,7 @@ public void scenario1() throws Exception {
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
SettableApiFuture done = SettableApiFuture.create();
//noinspection resource
GapicUnbufferedChunkedResumableWritableByteChannel channel =
@@ -179,6 +181,7 @@ public void scenario2() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_256KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 = WriteObjectResponse.newBuilder().setPersistedSize(_512KiB).build();
@@ -194,7 +197,7 @@ public void scenario2() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_256KiB);
writeCtx.getConfirmedBytes().set(_256KiB);
@@ -259,6 +262,7 @@ public void scenario3() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 = WriteObjectResponse.newBuilder().setPersistedSize(_256KiB).build();
@@ -274,7 +278,7 @@ public void scenario3() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(_512KiB);
@@ -339,6 +343,7 @@ public void scenario4() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_256KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 =
@@ -357,7 +362,7 @@ public void scenario4() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_256KiB);
writeCtx.getConfirmedBytes().set(_256KiB);
@@ -419,6 +424,7 @@ public void scenario4_1() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 =
@@ -437,7 +443,7 @@ public void scenario4_1() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(_512KiB);
@@ -502,6 +508,7 @@ public void scenario4_2() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 =
@@ -520,7 +527,7 @@ public void scenario4_2() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(_512KiB);
@@ -586,9 +593,8 @@ public void scenario5() throws Exception {
.setUploadId(uploadId)
.setWriteOffset(_256KiB)
.setChecksummedData(
- ChecksummedData.newBuilder()
- .setContent(
- ByteString.copyFrom(DataGenerator.base64Characters().genBytes(_256KiB))))
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(_256KiB))
+ .asChecksummedData())
.build();
StorageImplBase service1 =
new DirectWriteService(
@@ -611,7 +617,7 @@ public void scenario5() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_256KiB);
writeCtx.getConfirmedBytes().set(_256KiB);
@@ -656,10 +662,8 @@ public void scenario7() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setChecksummedData(
- ChecksummedData.newBuilder()
- .setContent(
- ByteString.copyFrom(DataGenerator.base64Characters().genBytes(_256KiB)))
- .build())
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(_256KiB))
+ .asChecksummedData())
.build();
WriteObjectResponse resp1 = WriteObjectResponse.newBuilder().setPersistedSize(_512KiB).build();
@@ -674,7 +678,7 @@ public void scenario7() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
//noinspection resource
GapicUnbufferedChunkedResumableWritableByteChannel channel =
@@ -703,10 +707,8 @@ public void incremental_success() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setChecksummedData(
- ChecksummedData.newBuilder()
- .setContent(
- ByteString.copyFrom(DataGenerator.base64Characters().genBytes(_256KiB)))
- .build())
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(_256KiB))
+ .asChecksummedData())
.build();
WriteObjectResponse resp1 = WriteObjectResponse.newBuilder().setPersistedSize(_256KiB).build();
@@ -721,7 +723,7 @@ public void incremental_success() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
//noinspection resource
GapicUnbufferedChunkedResumableWritableByteChannel channel =
@@ -768,7 +770,7 @@ public void incremental_partialSuccess() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
ChunkSegmenter chunkSegmenter =
new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), _512KiB, _256KiB);
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedDirectWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedDirectWritableByteChannelTest.java
index 4a70e9a2e..e8b9cd7d3 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedDirectWritableByteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedDirectWritableByteChannelTest.java
@@ -24,12 +24,14 @@
import static org.junit.Assert.assertThrows;
import com.google.api.core.SettableApiFuture;
+import com.google.cloud.storage.Hasher.GuavaHasher;
import com.google.cloud.storage.ITGapicUnbufferedWritableByteChannelTest.DirectWriteService;
import com.google.cloud.storage.WriteCtx.SimpleWriteObjectRequestBuilderFactory;
import com.google.cloud.storage.WriteCtx.WriteObjectRequestBuilderFactory;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.storage.v2.Object;
+import com.google.storage.v2.ObjectChecksums;
import com.google.storage.v2.StorageClient;
import com.google.storage.v2.WriteObjectRequest;
import com.google.storage.v2.WriteObjectResponse;
@@ -39,14 +41,19 @@
public final class ITGapicUnbufferedDirectWritableByteChannelTest {
+ public static final GuavaHasher HASHER = Hasher.enabled();
private static final ChunkSegmenter CHUNK_SEGMENTER =
- new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), _256KiB, _256KiB);
+ new ChunkSegmenter(HASHER, ByteStringStrategy.copy(), _256KiB, _256KiB);
/** Attempting to finalize, ack equals expected */
@Test
public void ack_eq() throws Exception {
WriteObjectRequest req1 =
- WriteObjectRequest.newBuilder().setWriteOffset(_256KiB).setFinishWrite(true).build();
+ WriteObjectRequest.newBuilder()
+ .setWriteOffset(_256KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
+ .setFinishWrite(true)
+ .build();
WriteObjectResponse resp1 =
WriteObjectResponse.newBuilder()
.setResource(Object.newBuilder().setName("name").setSize(_256KiB).build())
@@ -63,7 +70,7 @@ public void ack_eq() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
WriteCtx writeCtx =
- new WriteCtx<>(WriteObjectRequestBuilderFactory.simple(req1));
+ WriteCtx.of(WriteObjectRequestBuilderFactory.simple(req1), HASHER);
writeCtx.getTotalSentBytes().set(_256KiB);
writeCtx.getConfirmedBytes().set(0);
@@ -85,7 +92,11 @@ public void ack_eq() throws Exception {
@Test
public void ack_lt() throws Exception {
WriteObjectRequest req1 =
- WriteObjectRequest.newBuilder().setWriteOffset(_512KiB).setFinishWrite(true).build();
+ WriteObjectRequest.newBuilder()
+ .setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
+ .setFinishWrite(true)
+ .build();
WriteObjectResponse resp1 =
WriteObjectResponse.newBuilder()
.setResource(Object.newBuilder().setName("name").setSize(_256KiB).build())
@@ -102,7 +113,7 @@ public void ack_lt() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
WriteCtx writeCtx =
- new WriteCtx<>(WriteObjectRequestBuilderFactory.simple(req1));
+ WriteCtx.of(WriteObjectRequestBuilderFactory.simple(req1), HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(0);
@@ -124,7 +135,11 @@ public void ack_lt() throws Exception {
@Test
public void ack_gt() throws Exception {
WriteObjectRequest req1 =
- WriteObjectRequest.newBuilder().setWriteOffset(_512KiB).setFinishWrite(true).build();
+ WriteObjectRequest.newBuilder()
+ .setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
+ .setFinishWrite(true)
+ .build();
WriteObjectResponse resp1 =
WriteObjectResponse.newBuilder()
.setResource(Object.newBuilder().setName("name").setSize(_768KiB).build())
@@ -141,7 +156,7 @@ public void ack_gt() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
WriteCtx writeCtx =
- new WriteCtx<>(WriteObjectRequestBuilderFactory.simple(req1));
+ WriteCtx.of(WriteObjectRequestBuilderFactory.simple(req1), HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(0);
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedFinalizeOnCloseResumableWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedFinalizeOnCloseResumableWritableByteChannelTest.java
index 8181bd2bc..e9dd60f7f 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedFinalizeOnCloseResumableWritableByteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedFinalizeOnCloseResumableWritableByteChannelTest.java
@@ -24,10 +24,12 @@
import static org.junit.Assert.assertThrows;
import com.google.api.core.SettableApiFuture;
+import com.google.cloud.storage.Hasher.GuavaHasher;
import com.google.cloud.storage.ITGapicUnbufferedWritableByteChannelTest.DirectWriteService;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.storage.v2.Object;
+import com.google.storage.v2.ObjectChecksums;
import com.google.storage.v2.StartResumableWriteRequest;
import com.google.storage.v2.StartResumableWriteResponse;
import com.google.storage.v2.StorageClient;
@@ -40,8 +42,9 @@
public final class ITGapicUnbufferedFinalizeOnCloseResumableWritableByteChannelTest {
+ public static final GuavaHasher HASHER = Hasher.enabled();
private static final ChunkSegmenter CHUNK_SEGMENTER =
- new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), _256KiB, _256KiB);
+ new ChunkSegmenter(HASHER, ByteStringStrategy.copy(), _256KiB, _256KiB);
@Test
public void incrementalResponseForFinalizingRequest() throws Exception {
@@ -50,6 +53,7 @@ public void incrementalResponseForFinalizingRequest() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 = WriteObjectResponse.newBuilder().setPersistedSize(_256KiB).build();
@@ -65,7 +69,7 @@ public void incrementalResponseForFinalizingRequest() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(0);
@@ -75,7 +79,6 @@ public void incrementalResponseForFinalizingRequest() throws Exception {
done, CHUNK_SEGMENTER, storageClient.writeObjectCallable(), writeCtx);
StorageException se = assertThrows(StorageException.class, channel::close);
- se.printStackTrace(System.out);
assertAll(
() -> assertThat(se.getCode()).isEqualTo(0),
() -> assertThat(se.getReason()).isEqualTo("invalid"),
@@ -127,6 +130,7 @@ public void scenario4() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_256KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 =
@@ -145,7 +149,7 @@ public void scenario4() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_256KiB);
writeCtx.getConfirmedBytes().set(0);
@@ -203,6 +207,7 @@ public void scenario4_1() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 =
@@ -221,7 +226,7 @@ public void scenario4_1() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(0);
@@ -231,7 +236,6 @@ public void scenario4_1() throws Exception {
done, CHUNK_SEGMENTER, storageClient.writeObjectCallable(), writeCtx);
StorageException se = assertThrows(StorageException.class, channel::close);
- se.printStackTrace(System.out);
assertAll(
() -> assertThat(se.getCode()).isEqualTo(0),
() -> assertThat(se.getReason()).isEqualTo("dataLoss"),
@@ -282,6 +286,7 @@ public void scenario4_2() throws Exception {
WriteObjectRequest.newBuilder()
.setUploadId(uploadId)
.setWriteOffset(_512KiB)
+ .setObjectChecksums(ObjectChecksums.newBuilder().setCrc32C(0).build())
.setFinishWrite(true)
.build();
WriteObjectResponse resp1 =
@@ -300,7 +305,7 @@ public void scenario4_2() throws Exception {
SettableApiFuture done = SettableApiFuture.create();
ResumableWrite resumableWrite = getResumableWrite(uploadId);
- WriteCtx writeCtx = new WriteCtx<>(resumableWrite);
+ WriteCtx writeCtx = WriteCtx.of(resumableWrite, HASHER);
writeCtx.getTotalSentBytes().set(_512KiB);
writeCtx.getConfirmedBytes().set(0);
@@ -310,7 +315,6 @@ public void scenario4_2() throws Exception {
done, CHUNK_SEGMENTER, storageClient.writeObjectCallable(), writeCtx);
StorageException se = assertThrows(StorageException.class, channel::close);
- se.printStackTrace(System.out);
assertAll(
() -> assertThat(se.getCode()).isEqualTo(0),
() -> assertThat(se.getReason()).isEqualTo("dataLoss"),
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java
index a31fc0c33..e23b2a57f 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGapicUnbufferedWritableByteChannelTest.java
@@ -159,7 +159,7 @@ public void directUpload() throws IOException, InterruptedException, ExecutionEx
SettableApiFuture result = SettableApiFuture.create();
try (GapicUnbufferedDirectWritableByteChannel c =
new GapicUnbufferedDirectWritableByteChannel(
- result, segmenter, sc.writeObjectCallable(), new WriteCtx<>(reqFactory))) {
+ result, segmenter, sc.writeObjectCallable(), WriteCtx.of(reqFactory, HASHER))) {
c.write(ByteBuffer.wrap(bytes));
}
assertThat(result.get()).isEqualTo(resp);
@@ -185,7 +185,7 @@ public void resumableUpload() throws IOException, InterruptedException, Executio
result,
segmenter,
sc.writeObjectCallable(),
- new WriteCtx<>(reqFactory),
+ WriteCtx.of(reqFactory, HASHER),
RetrierWithAlg.attemptOnce(),
Retrying::newCallContext);
ArrayList debugMessages = new ArrayList<>();
@@ -267,7 +267,7 @@ public void resumableUpload_chunkAutomaticRetry()
result,
segmenter,
sc.writeObjectCallable(),
- new WriteCtx<>(reqFactory),
+ WriteCtx.of(reqFactory, HASHER),
TestUtils.retrierFromStorageOptions(fake.getGrpcStorageOptions())
.withAlg(Retrying.alwaysRetry()),
Retrying::newCallContext)) {
@@ -319,7 +319,7 @@ public void resumableUpload_finalizeWhenWriteAndCloseCalledEvenWhenQuantumAligne
result,
segmenter,
sc.writeObjectCallable(),
- new WriteCtx<>(reqFactory),
+ WriteCtx.of(reqFactory, HASHER),
RetrierWithAlg.attemptOnce(),
Retrying::newCallContext);
try {
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest.java
index fd4f0fd58..2d55a26cb 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITSyncAndUploadUnbufferedWritableByteChannelPropertyTest.java
@@ -47,6 +47,7 @@
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
import com.google.protobuf.TextFormat;
+import com.google.storage.v2.ChecksummedData;
import com.google.storage.v2.Object;
import com.google.storage.v2.ObjectChecksums;
import com.google.storage.v2.QueryWriteStatusRequest;
@@ -63,11 +64,8 @@
import io.grpc.stub.StreamObserver;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
@@ -124,23 +122,7 @@ static void beforeContainer() throws IOException {
@AfterContainer
static void afterContainer() throws IOException {
if (tmpFolder != null) {
- Files.walkFileTree(
- tmpFolder,
- new SimpleFileVisitor() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
- throws IOException {
- Files.deleteIfExists(file);
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult postVisitDirectory(Path dir, IOException exc)
- throws IOException {
- Files.deleteIfExists(dir);
- return FileVisitResult.CONTINUE;
- }
- });
+ TestUtils.rmDashRf(tmpFolder);
}
}
@@ -356,7 +338,7 @@ void testUploads(@ForAll("scenario") Scenario s) throws Exception {
// TestUtils.defaultRetrier(),
new DefaultRetrier(UnaryOperator.identity(), defaultRetryingDeps()),
StorageRetryStrategy.getDefaultStorageRetryStrategy().getIdempotentHandler(),
- new WriteCtx<>(resumableWrite),
+ WriteCtx.of(resumableWrite, s.chunkSegmenter.getHasher()),
rf,
s.copyBuffer);
try (BufferedWritableByteChannel w = s.buffered(syncAndUpload)) {
@@ -550,19 +532,22 @@ private void unexpected(StreamObserver respond, Message msg) {
BlobInfo info = BlobInfo.newBuilder("b", "o").build();
SettableApiFuture resultFuture = SettableApiFuture.create();
BufferHandle recoverBufferHandle = BufferHandle.allocate(2);
+ ChunkSegmenter chunkSegmenter =
+ new ChunkSegmenter(Hasher.enabled(), ByteStringStrategy.copy(), 2, 2);
SyncAndUploadUnbufferedWritableByteChannel syncAndUpload =
new SyncAndUploadUnbufferedWritableByteChannel(
storage.storageClient.writeObjectCallable(),
storage.storageClient.queryWriteStatusCallable(),
resultFuture,
- new ChunkSegmenter(Hasher.enabled(), ByteStringStrategy.copy(), 2, 2),
+ chunkSegmenter,
new DefaultRetrier(UnaryOperator.identity(), storage.getOptions()),
StorageRetryStrategy.getDefaultStorageRetryStrategy().getIdempotentHandler(),
- new WriteCtx<>(
+ WriteCtx.of(
new ResumableWrite(
reqStart,
resStart,
- id -> reqWrite0.toBuilder().clearWriteObjectSpec().setUploadId(id).build())),
+ id -> reqWrite0.toBuilder().clearWriteObjectSpec().setUploadId(id).build()),
+ chunkSegmenter.getHasher()),
recoveryFileManager.newRecoveryFile(info),
recoverBufferHandle);
try (BufferedWritableByteChannel w =
@@ -751,7 +736,7 @@ public static Scenario of(
.toString(),
objectName,
objectSize,
- new ChunkSegmenter(Hasher.noop(), ByteStringStrategy.copy(), segmentSize, quantum),
+ new ChunkSegmenter(Hasher.enabled(), ByteStringStrategy.copy(), segmentSize, quantum),
BufferHandle.allocate(segmentSize),
BufferHandle.allocate(segmentSize),
failuresQueue,
@@ -1008,6 +993,25 @@ public FailureInducingWriteObjectRequestObserver(
@Override
public void onNext(WriteObjectRequest writeObjectRequest) {
+ if (writeObjectRequest.hasChecksummedData()) {
+ ChecksummedData checksummedData = writeObjectRequest.getChecksummedData();
+ if (!checksummedData.hasCrc32C()) {
+ errored = true;
+ sendFailure("no crc32c value specified");
+ return;
+ }
+ if (!checksummedData.getContent().isEmpty() && checksummedData.getCrc32C() == 0) {
+ errored = true;
+ sendFailure("crc32c value of 0 with non-empty content");
+ return;
+ }
+ }
+ if (writeObjectRequest.hasObjectChecksums()
+ && !writeObjectRequest.getObjectChecksums().hasCrc32C()) {
+ errored = true;
+ sendFailure("missing object_checksums.crc32c");
+ return;
+ }
if (ctx == null) {
UploadId uploadId = UploadId.of(writeObjectRequest.getUploadId());
if (data.containsKey(uploadId)) {
@@ -1050,6 +1054,11 @@ public void onCompleted() {
responseObserver.onNext(resp);
responseObserver.onCompleted();
}
+
+ private void sendFailure(String description) {
+ responseObserver.onError(
+ Code.INVALID_ARGUMENT.toStatus().withDescription(description).asRuntimeException());
+ }
}
@FunctionalInterface
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
index a4902111f..14e9ef1ec 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ParallelCompositeUploadWritableByteChannelTest.java
@@ -892,7 +892,8 @@ public BlobInfo compose(ComposeRequest composeRequest) {
.map(Data::getCrc32c)
.collect(ImmutableList.toImmutableList());
- Crc32cLengthKnown reduce = crc32cs.stream().reduce(null, HASHER::nullSafeConcat);
+ Crc32cLengthKnown reduce =
+ crc32cs.stream().reduce(Crc32cValue.zero(), Crc32cLengthKnown::concat);
Preconditions.checkState(reduce != null, "unable to compute crc32c for compose request");
b.setCrc32c(Utils.crc32cCodec.encode(reduce.getValue()));
BlobInfo gen1 = b.build();
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
index 036f8203a..bae429597 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
@@ -31,7 +31,6 @@
import com.google.api.services.storage.model.StorageObject;
import com.google.cloud.ServiceOptions;
import com.google.cloud.Tuple;
-import com.google.cloud.storage.Storage.BlobTargetOption;
import com.google.cloud.storage.spi.StorageRpcFactory;
import com.google.cloud.storage.spi.v1.StorageRpc;
import com.google.common.collect.ImmutableList;
@@ -103,15 +102,14 @@ public class StorageImplMockitoTest {
BlobInfo.newBuilder(BUCKET_NAME1, BLOB_NAME1, 24L)
.setMetageneration(PAGE_SIZE)
.setContentType("application/json")
- .setMd5("md5string")
.build();
private static final BlobInfo BLOB_INFO2 = BlobInfo.newBuilder(BUCKET_NAME1, BLOB_NAME2).build();
private static final BlobInfo BLOB_INFO3 = BlobInfo.newBuilder(BUCKET_NAME1, BLOB_NAME3).build();
private static final BlobInfo BLOB_INFO_WITH_HASHES =
- BLOB_INFO1.toBuilder().setMd5(CONTENT_MD5).setCrc32c(CONTENT_CRC32C).build();
+ BLOB_INFO1.toBuilder().setCrc32c(CONTENT_CRC32C).build();
private static final BlobInfo BLOB_INFO_WITHOUT_HASHES =
- BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build();
+ BLOB_INFO1.toBuilder().setCrc32c(null).build();
// Empty StorageRpc options
private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of();
@@ -616,213 +614,6 @@ private void verifyCreateBlobCapturedStream(ArgumentCaptor
assertEquals(-1, byteStream.read(streamBytes));
}
- @Test
- public void testCreateBlob() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(Conversions.json().blobInfo().encode(BLOB_INFO_WITH_HASHES)),
- capturedStream.capture(),
- Mockito.eq(EMPTY_RPC_OPTIONS));
- initializeService();
-
- Blob blob = storage.create(BLOB_INFO1, BLOB_CONTENT);
-
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- }
-
- @Test
- public void testCreateBlobWithSubArrayFromByteArray() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(
- Conversions.json()
- .blobInfo()
- .encode(
- BLOB_INFO1.toBuilder()
- .setMd5(SUB_CONTENT_MD5)
- .setCrc32c(SUB_CONTENT_CRC32C)
- .build())),
- capturedStream.capture(),
- Mockito.eq(EMPTY_RPC_OPTIONS));
- initializeService();
-
- Blob blob = storage.create(BLOB_INFO1, BLOB_CONTENT, 1, 2);
-
- assertEquals(expectedBlob1, blob);
- ByteArrayInputStream byteStream = capturedStream.getValue();
- byte[] streamBytes = new byte[BLOB_SUB_CONTENT.length];
- assertEquals(BLOB_SUB_CONTENT.length, byteStream.read(streamBytes));
- assertArrayEquals(BLOB_SUB_CONTENT, streamBytes);
- assertEquals(-1, byteStream.read(streamBytes));
- }
-
- @Test
- public void testCreateBlobRetry() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
-
- StorageObject storageObject = Conversions.json().blobInfo().encode(BLOB_INFO_WITH_HASHES);
-
- doThrow(new StorageException(500, "internalError"))
- .doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(storageObject),
- capturedStream.capture(),
- Mockito.eq(BLOB_INFO1_RPC_OPTIONS_WITH_GENERATION));
-
- storage =
- options.toBuilder()
- .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
- .build()
- .getService();
- initializeServiceDependentObjects();
-
- Blob blob = storage.create(BLOB_INFO1, BLOB_CONTENT, BlobTargetOption.generationMatch());
-
- assertEquals(expectedBlob1, blob);
-
- byte[] streamBytes = new byte[BLOB_CONTENT.length];
- for (ByteArrayInputStream byteStream : capturedStream.getAllValues()) {
- assertEquals(BLOB_CONTENT.length, byteStream.read(streamBytes));
- assertArrayEquals(BLOB_CONTENT, streamBytes);
- assertEquals(-1, byteStream.read(streamBytes));
- }
- }
-
- @Test
- public void testCreateEmptyBlob() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
-
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(
- Conversions.json()
- .blobInfo()
- .encode(
- BLOB_INFO1.toBuilder()
- .setMd5("1B2M2Y8AsgTpgAmY7PhCfg==")
- .setCrc32c("AAAAAA==")
- .build())),
- capturedStream.capture(),
- Mockito.eq(EMPTY_RPC_OPTIONS));
- initializeService();
-
- Blob blob = storage.create(BLOB_INFO1);
- assertEquals(expectedBlob1, blob);
- ByteArrayInputStream byteStream = capturedStream.getValue();
- byte[] streamBytes = new byte[BLOB_CONTENT.length];
- assertEquals(-1, byteStream.read(streamBytes));
- }
-
- @Test
- public void testCreateBlobWithOptions() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
-
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(Conversions.json().blobInfo().encode(BLOB_INFO_WITH_HASHES)),
- capturedStream.capture(),
- Mockito.eq(BLOB_TARGET_OPTIONS_CREATE));
- initializeService();
-
- Blob blob =
- storage.create(
- BLOB_INFO1,
- BLOB_CONTENT,
- BLOB_TARGET_METAGENERATION,
- BLOB_TARGET_NOT_EXIST,
- BLOB_TARGET_PREDEFINED_ACL);
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- }
-
- @Test
- public void testCreateBlobWithDisabledGzipContent() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
-
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(Conversions.json().blobInfo().encode(BLOB_INFO_WITH_HASHES)),
- capturedStream.capture(),
- Mockito.eq(BLOB_TARGET_OPTIONS_CREATE_DISABLE_GZIP_CONTENT));
- initializeService();
-
- Blob blob = storage.create(BLOB_INFO1, BLOB_CONTENT, BLOB_TARGET_DISABLE_GZIP_CONTENT);
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- }
-
- @Test
- public void testCreateBlobWithEncryptionKey() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
-
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(Conversions.json().blobInfo().encode(BLOB_INFO_WITH_HASHES)),
- capturedStream.capture(),
- Mockito.eq(ENCRYPTION_KEY_OPTIONS));
- initializeService();
-
- Blob blob =
- storage.create(BLOB_INFO1, BLOB_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY));
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- blob =
- storage.create(
- BLOB_INFO1, BLOB_CONTENT, Storage.BlobTargetOption.encryptionKey(BASE64_KEY));
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- }
-
- @Test
- public void testCreateBlobWithKmsKeyName() throws IOException {
- ArgumentCaptor capturedStream =
- ArgumentCaptor.forClass(ByteArrayInputStream.class);
-
- doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doReturn(Conversions.json().blobInfo().encode(BLOB_INFO1))
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .create(
- Mockito.eq(Conversions.json().blobInfo().encode(BLOB_INFO_WITH_HASHES)),
- capturedStream.capture(),
- Mockito.eq(KMS_KEY_NAME_OPTIONS));
- initializeService();
-
- Blob blob =
- storage.create(BLOB_INFO1, BLOB_CONTENT, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME));
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- blob =
- storage.create(BLOB_INFO1, BLOB_CONTENT, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME));
- assertEquals(expectedBlob1, blob);
- verifyCreateBlobCapturedStream(capturedStream);
- }
-
@Test
@SuppressWarnings({"unchecked", "deprecation"})
public void testCreateBlobFromStream() throws IOException {
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java
index c501d11b0..5fd68dfc8 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java
@@ -53,6 +53,10 @@
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -374,4 +378,23 @@ private static String messagesToText(Throwable t, String indent) {
.flatMap(s -> s)
.collect(Collectors.joining("\n"));
}
+
+ public static void rmDashRf(Path path) throws IOException {
+ java.nio.file.Files.walkFileTree(
+ path,
+ new SimpleFileVisitor() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+ throws IOException {
+ java.nio.file.Files.deleteIfExists(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ java.nio.file.Files.deleteIfExists(dir);
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpDir.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpDir.java
new file mode 100644
index 000000000..4cc29eac8
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpDir.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.common.base.MoreObjects;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public final class TmpDir implements AutoCloseable {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TmpDir.class);
+
+ private final Path path;
+
+ private TmpDir(Path path) {
+ this.path = path;
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ /** Delete the TmpFile this handle is holding */
+ @Override
+ public void close() throws IOException {
+ TestUtils.rmDashRf(path);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("path", path).toString();
+ }
+
+ /**
+ * Create a temporary file, which will be deleted when close is called on the returned {@link
+ * TmpDir}
+ */
+ public static TmpDir of(Path baseDir, String prefix) throws IOException {
+ LOGGER.trace("of(baseDir : {}, prefix : {})", baseDir, prefix);
+ Path path = Files.createTempDirectory(baseDir, prefix);
+ return new TmpDir(path);
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/UnifiedOptsTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/UnifiedOptsTest.java
index 73d88580a..e5a38b436 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/UnifiedOptsTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/UnifiedOptsTest.java
@@ -20,6 +20,10 @@
import static com.google.common.truth.Truth.assertWithMessage;
import static org.junit.Assert.assertThrows;
+import com.google.cloud.storage.UnifiedOpts.Crc32cMatch;
+import com.google.cloud.storage.UnifiedOpts.DefaultHasherSelector;
+import com.google.cloud.storage.UnifiedOpts.HasherSelector;
+import com.google.cloud.storage.UnifiedOpts.Md5Match;
import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt;
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
import com.google.cloud.storage.UnifiedOpts.Opt;
@@ -146,6 +150,27 @@ public void validateFactoryMethodEnforceNonNull_storage_updateHmacKeyOption() th
validateFactoryMethodEnforceNonNull(Storage.UpdateHmacKeyOption.class);
}
+ @Test
+ public void getHasher_selectsLastValue() {
+ DefaultHasherSelector first = UnifiedOpts.defaultHasherSelector();
+ Md5Match second = UnifiedOpts.md5Match("asdf");
+ Crc32cMatch third = UnifiedOpts.crc32cMatch(3);
+ Opts hasherOpts = Opts.from(first, second, third);
+
+ HasherSelector actual = hasherOpts.getHasherSelector();
+ assertThat(actual).isSameInstanceAs(third);
+ }
+
+ @Test
+ public void hasher_md5Match_noop() {
+ assertThat(UnifiedOpts.md5Match("xyz").getHasher()).isEqualTo(Hasher.noop());
+ }
+
+ @Test
+ public void hasher_crc32cMatch_noop() {
+ assertThat(UnifiedOpts.crc32cMatch(77).getHasher()).isEqualTo(Hasher.noop());
+ }
+
@Test
public void transformTo() {
SecretKey key =
@@ -172,8 +197,8 @@ public byte[] getEncoded() {
UnifiedOpts.encryptionKey(key),
// userProject implements both target and source
UnifiedOpts.userProject("user-project"),
- // crc32c is not a source opt or a ProjectToSource opt, it should be excluded
- UnifiedOpts.crc32cMatch(1));
+ // contentType is not a source opt or a ProjectToSource opt, it should be excluded
+ UnifiedOpts.setContentType("application/octet-stream"));
Opts sourceOpts = targetOpts.transformTo(ObjectSourceOpt.class);
Opts expected =
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ChecksummedTestContent.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ChecksummedTestContent.java
index aa87e6478..6ccb05f52 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ChecksummedTestContent.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ChecksummedTestContent.java
@@ -20,6 +20,7 @@
import static com.google.common.base.Preconditions.checkPositionIndexes;
import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.primitives.Ints;
@@ -28,8 +29,10 @@
import com.google.storage.v2.ChecksummedData;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
+import java.util.List;
public final class ChecksummedTestContent {
@@ -100,12 +103,19 @@ public ChecksummedTestContent slice(int begin, int length) {
return of(bytes, begin, Math.min(length, bytes.length - begin));
}
+ public List chunkup(int chunkSize) {
+ List elements = new ArrayList<>();
+ for (int i = 0; i < bytes.length; i += chunkSize) {
+ elements.add(slice(i, chunkSize));
+ }
+ return ImmutableList.copyOf(elements);
+ }
+
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("byteCount", bytes.length)
- .add("crc32c", crc32c)
- .add("md5Base64", md5Base64)
+ .add("crc32c", Integer.toUnsignedString(crc32c))
.toString();
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteSessionCommonSemanticsTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteSessionCommonSemanticsTest.java
index dc02c3fb9..663a3b1de 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteSessionCommonSemanticsTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteSessionCommonSemanticsTest.java
@@ -19,6 +19,7 @@
import static com.google.cloud.storage.TestUtils.assertAll;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
+import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import com.google.cloud.storage.BlobInfo;
@@ -26,6 +27,7 @@
import com.google.cloud.storage.BlobWriteSessionConfig;
import com.google.cloud.storage.BlobWriteSessionConfigs;
import com.google.cloud.storage.BucketInfo;
+import com.google.cloud.storage.DataGenerator;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.BufferAllocationStrategy;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.ExecutorSupplier;
import com.google.cloud.storage.ParallelCompositeUploadBlobWriteSessionConfig.PartCleanupStrategy;
@@ -179,6 +181,51 @@ public void getResultErrorsWhenTheSessionErrors() throws Exception {
() -> assertThat(((StorageException) resultSe.getCause()).getCode()).isEqualTo(412));
}
+ @Test
+ public void userProvidedCrc32cValueIsRespected() throws IOException {
+ assumeFalse("b/226975500", params.desc.startsWith("p"));
+ ChecksummedTestContent testContent =
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(5 * 1024 * 1024 + 17));
+ ChecksummedTestContent missingAByte = testContent.slice(0, testContent.getBytes().length - 1);
+
+ BlobInfo info =
+ BlobInfo.newBuilder(bucket, generator.randomObjectName())
+ .setCrc32c(missingAByte.getCrc32cBase64())
+ .build();
+ BlobWriteSession session =
+ storage.blobWriteSession(
+ info, BlobWriteOption.crc32cMatch(), BlobWriteOption.doesNotExist());
+ WritableByteChannel open = session.open();
+ try {
+ open.write(ByteBuffer.wrap(testContent.getBytes()));
+ } finally {
+ StorageException se = assertThrows(StorageException.class, () -> open.close());
+ assertThat(se.getCode()).isEqualTo(400);
+ }
+ }
+
+ @Test
+ public void userProvidedMd5ValueIsRespected() throws IOException {
+ assumeFalse("b/226975500", params.desc.startsWith("p"));
+ ChecksummedTestContent testContent =
+ ChecksummedTestContent.of(DataGenerator.base64Characters().genBytes(5 * 1024 * 1024 + 17));
+ ChecksummedTestContent missingAByte = testContent.slice(0, testContent.getBytes().length - 1);
+
+ BlobInfo info =
+ BlobInfo.newBuilder(bucket, generator.randomObjectName())
+ .setMd5(missingAByte.getMd5Base64())
+ .build();
+ BlobWriteSession session =
+ storage.blobWriteSession(info, BlobWriteOption.md5Match(), BlobWriteOption.doesNotExist());
+ WritableByteChannel open = session.open();
+ try {
+ open.write(ByteBuffer.wrap(testContent.getBytes()));
+ } finally {
+ StorageException se = assertThrows(StorageException.class, () -> open.close());
+ assertThat(se.getCode()).isEqualTo(400);
+ }
+ }
+
public static final class ParamsProvider implements ParametersProvider {
@Override
public ImmutableList parameters() {
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java
index da0c50452..5153ea127 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java
@@ -31,6 +31,7 @@
import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.TmpDir;
import com.google.cloud.storage.TmpFile;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.it.ITObjectChecksumSupportTest.ChecksummedTestContentProvider;
@@ -54,7 +55,9 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.TimeUnit;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.junit.runner.RunWith;
@RunWith(StorageITRunner.class)
@@ -75,6 +78,8 @@ public final class ITObjectChecksumSupportTest {
@Parameter public ChecksummedTestContent content;
+ @Rule public final TestName testName = new TestName();
+
public static final class ChecksummedTestContentProvider implements ParametersProvider {
@Override
@@ -85,6 +90,8 @@ public ImmutableList parameters() {
int _24MiB = 24 * 1024 * 1024;
return ImmutableList.of(
+ // empty object content
+ ChecksummedTestContent.of(new byte[0]),
// small, single message single stream when resumable
ChecksummedTestContent.of(gen.genBytes(15)),
// med, multiple messages single stream when resumable
@@ -349,4 +356,69 @@ public void testCrc32cValidated_bidiWrite_expectFailure() throws Exception {
assertThat(expected.getCode()).isEqualTo(400);
}
}
+
+ @Test
+ @CrossRun.Exclude(transports = Transport.HTTP)
+ public void testCrc32cValidated_journaling_expectSuccess() throws Exception {
+ String blobName = generator.randomObjectName();
+ BlobId blobId = BlobId.of(bucket.getName(), blobName);
+ BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(content.getCrc32cBase64()).build();
+
+ byte[] bytes = content.getBytes();
+
+ try (TmpDir journalingDir = TmpDir.of(tmpDir, testName.getMethodName())) {
+ StorageOptions options =
+ this.storage.getOptions().toBuilder()
+ .setBlobWriteSessionConfig(
+ BlobWriteSessionConfigs.journaling(ImmutableList.of(journalingDir.getPath())))
+ .build();
+
+ try (Storage storage = options.getService()) {
+ BlobWriteSession session =
+ storage.blobWriteSession(
+ blobInfo, BlobWriteOption.doesNotExist(), BlobWriteOption.crc32cMatch());
+
+ try (ReadableByteChannel src = Channels.newChannel(new ByteArrayInputStream(bytes));
+ WritableByteChannel dst = session.open()) {
+ ByteStreams.copy(src, dst);
+ }
+
+ BlobInfo gen1 = session.getResult().get(5, TimeUnit.SECONDS);
+ assertThat(gen1.getCrc32c()).isEqualTo(content.getCrc32cBase64());
+ }
+ }
+ }
+
+ @Test
+ @CrossRun.Exclude(transports = Transport.HTTP)
+ public void testCrc32cValidated_journaling_expectFailure() throws Exception {
+ String blobName = generator.randomObjectName();
+ BlobId blobId = BlobId.of(bucket.getName(), blobName);
+ BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(content.getCrc32cBase64()).build();
+
+ byte[] bytes = content.concat('x');
+
+ try (TmpDir journalingDir = TmpDir.of(tmpDir, generator.randomObjectName())) {
+ StorageOptions options =
+ this.storage.getOptions().toBuilder()
+ .setBlobWriteSessionConfig(
+ BlobWriteSessionConfigs.journaling(ImmutableList.of(journalingDir.getPath())))
+ .build();
+
+ try (Storage storage = options.getService()) {
+ BlobWriteSession session =
+ storage.blobWriteSession(
+ blobInfo, BlobWriteOption.doesNotExist(), BlobWriteOption.crc32cMatch());
+
+ WritableByteChannel dst = session.open();
+ try (ReadableByteChannel src = Channels.newChannel(new ByteArrayInputStream(bytes))) {
+ ByteStreams.copy(src, dst);
+ }
+
+ StorageException expected = assertThrows(StorageException.class, dst::close);
+
+ assertThat(expected.getCode()).isEqualTo(400);
+ }
+ }
+ }
}
From a1dda0249e76c3eaa33de65427a7db6151401564 Mon Sep 17 00:00:00 2001
From: Diego Marquez
Date: Thu, 24 Jul 2025 13:12:52 -0400
Subject: [PATCH 12/15] chore: manual update of update_generation_config.sh
(#3205)
Unfortunately, https://github.com/googleapis/sdk-platform-java/pull/3853 cannot be automatically propagated.
This PR manually updates this script with the latest.
---
.github/scripts/update_generation_config.sh | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/.github/scripts/update_generation_config.sh b/.github/scripts/update_generation_config.sh
index fff56bf5d..92efcf881 100644
--- a/.github/scripts/update_generation_config.sh
+++ b/.github/scripts/update_generation_config.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -e
+set -ex
# This script should be run at the root of the repository.
# This script is used to update googleapis_commitish, gapic_generator_version,
# and libraries_bom_version in generation configuration at the time of running
@@ -15,8 +15,10 @@ set -e
function get_latest_released_version() {
local group_id=$1
local artifact_id=$2
- json_content=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json")
- latest=$(jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' <<< "${json_content}" | sort -V | tail -n 1)
+ group_id_url_path="$(sed 's|\.|/|g' <<< "${group_id}")"
+ url="https://repo1.maven.org/maven2/${group_id_url_path}/${artifact_id}/maven-metadata.xml"
+ xml_content=$(curl -s --fail "${url}")
+ latest=$(xmllint --xpath 'metadata/versioning/latest/text()' - <<< "${xml_content}")
if [[ -z "${latest}" ]]; then
echo "The latest version of ${group_id}:${artifact_id} is empty."
echo "The returned json from maven.org is invalid: ${json_content}"
From 1ac0ba41947e638afd3926d8fbe0ae64fd9b7740 Mon Sep 17 00:00:00 2001
From: Diego Marquez
Date: Thu, 24 Jul 2025 14:05:23 -0400
Subject: [PATCH 13/15] build: include update config yaml template (#3207)
alternative to https://github.com/googleapis/java-storage/pull/3206
---
.github/workflows/update_generation_config.yaml | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml
index cd2d5fd5a..7b14069ce 100644
--- a/.github/workflows/update_generation_config.yaml
+++ b/.github/workflows/update_generation_config.yaml
@@ -18,7 +18,6 @@ on:
schedule:
- cron: '0 2 * * *'
workflow_dispatch:
-
jobs:
update-generation-config:
runs-on: ubuntu-24.04
@@ -30,6 +29,9 @@ jobs:
with:
fetch-depth: 0
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
+ - name: Install Dependencies
+ shell: bash
+ run: sudo apt-get update && sudo apt-get install -y libxml2-utils
- name: Update params in generation config to latest
shell: bash
run: |
@@ -40,5 +42,4 @@ jobs:
--base_branch "${base_branch}" \
--repo ${{ github.repository }}
env:
- GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
-
+ GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
\ No newline at end of file
From bd1f199cf57c2b8039c303586d5beac64aeca0ba Mon Sep 17 00:00:00 2001
From: cloud-java-bot <122572305+cloud-java-bot@users.noreply.github.com>
Date: Thu, 24 Jul 2025 15:20:26 -0400
Subject: [PATCH 14/15] chore: Update generation configuration at Thu Jul 24
18:07:08 UTC 2025 (#3198)
* chore: Update generation configuration at Sat Jul 19 02:33:14 UTC 2025
* chore: Update generation configuration at Tue Jul 22 02:37:15 UTC 2025
* chore: generate libraries at Tue Jul 22 02:37:45 UTC 2025
* chore: Update generation configuration at Wed Jul 23 02:38:05 UTC 2025
* chore: Update generation configuration at Thu Jul 24 02:37:39 UTC 2025
* chore: Update generation configuration at Thu Jul 24 17:57:25 UTC 2025
* chore: generate libraries at Thu Jul 24 17:57:53 UTC 2025
* chore: generate libraries at Thu Jul 24 18:07:39 UTC 2025
---
.github/scripts/update_generation_config.sh | 2 +-
.github/workflows/hermetic_library_generation.yaml | 2 +-
.github/workflows/update_generation_config.yaml | 4 +++-
.kokoro/presubmit/graalvm-native-a.cfg | 2 +-
.kokoro/presubmit/graalvm-native-b.cfg | 2 +-
.kokoro/presubmit/graalvm-native-c.cfg | 2 +-
README.md | 8 ++++----
generation_config.yaml | 6 +++---
8 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/.github/scripts/update_generation_config.sh b/.github/scripts/update_generation_config.sh
index 92efcf881..3b890a76b 100644
--- a/.github/scripts/update_generation_config.sh
+++ b/.github/scripts/update_generation_config.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-set -ex
+set -e
# This script should be run at the root of the repository.
# This script is used to update googleapis_commitish, gapic_generator_version,
# and libraries_bom_version in generation configuration at the time of running
diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml
index 4ce1c236a..876d040f3 100644
--- a/.github/workflows/hermetic_library_generation.yaml
+++ b/.github/workflows/hermetic_library_generation.yaml
@@ -43,7 +43,7 @@ jobs:
with:
fetch-depth: 0
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
- - uses: googleapis/sdk-platform-java/.github/scripts@v2.60.1
+ - uses: googleapis/sdk-platform-java/.github/scripts@v2.60.2
if: env.SHOULD_RUN == 'true'
with:
base_ref: ${{ github.base_ref }}
diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml
index 7b14069ce..a7e14bb48 100644
--- a/.github/workflows/update_generation_config.yaml
+++ b/.github/workflows/update_generation_config.yaml
@@ -18,6 +18,7 @@ on:
schedule:
- cron: '0 2 * * *'
workflow_dispatch:
+
jobs:
update-generation-config:
runs-on: ubuntu-24.04
@@ -42,4 +43,5 @@ jobs:
--base_branch "${base_branch}" \
--repo ${{ github.repository }}
env:
- GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
\ No newline at end of file
+ GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
+
diff --git a/.kokoro/presubmit/graalvm-native-a.cfg b/.kokoro/presubmit/graalvm-native-a.cfg
index 5cc939cf9..96b4fed86 100644
--- a/.kokoro/presubmit/graalvm-native-a.cfg
+++ b/.kokoro/presubmit/graalvm-native-a.cfg
@@ -3,7 +3,7 @@
# Configure the docker image for kokoro-trampoline.
env_vars: {
key: "TRAMPOLINE_IMAGE"
- value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.49.0" # {x-version-update:google-cloud-shared-dependencies:current}
+ value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.50.2" # {x-version-update:google-cloud-shared-dependencies:current}
}
env_vars: {
diff --git a/.kokoro/presubmit/graalvm-native-b.cfg b/.kokoro/presubmit/graalvm-native-b.cfg
index fd0015918..76f048331 100644
--- a/.kokoro/presubmit/graalvm-native-b.cfg
+++ b/.kokoro/presubmit/graalvm-native-b.cfg
@@ -3,7 +3,7 @@
# Configure the docker image for kokoro-trampoline.
env_vars: {
key: "TRAMPOLINE_IMAGE"
- value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.49.0" # {x-version-update:google-cloud-shared-dependencies:current}
+ value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.50.2" # {x-version-update:google-cloud-shared-dependencies:current}
}
env_vars: {
diff --git a/.kokoro/presubmit/graalvm-native-c.cfg b/.kokoro/presubmit/graalvm-native-c.cfg
index 424494b02..277aa2338 100644
--- a/.kokoro/presubmit/graalvm-native-c.cfg
+++ b/.kokoro/presubmit/graalvm-native-c.cfg
@@ -3,7 +3,7 @@
# Configure the docker image for kokoro-trampoline.
env_vars: {
key: "TRAMPOLINE_IMAGE"
- value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.49.0" # {x-version-update:google-cloud-shared-dependencies:current}
+ value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_c:3.50.2" # {x-version-update:google-cloud-shared-dependencies:current}
}
env_vars: {
diff --git a/README.md b/README.md
index 60b2a6358..b75087fa2 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
com.google.cloud
libraries-bom
- 26.63.0
+ 26.64.0
pom
import
@@ -46,12 +46,12 @@ If you are using Maven without the BOM, add this to your dependencies:
com.google.cloud
google-cloud-storage
- 2.53.2
+ 2.53.3
com.google.cloud
google-cloud-storage-control
- 2.53.2
+ 2.53.3
```
@@ -59,7 +59,7 @@ If you are using Maven without the BOM, add this to your dependencies:
If you are using Gradle 5.x or later, add this to your dependencies:
```Groovy
-implementation platform('com.google.cloud:libraries-bom:26.61.0')
+implementation platform('com.google.cloud:libraries-bom:26.64.0')
implementation 'com.google.cloud:google-cloud-storage'
```
diff --git a/generation_config.yaml b/generation_config.yaml
index 559faca1f..295ed2340 100644
--- a/generation_config.yaml
+++ b/generation_config.yaml
@@ -1,6 +1,6 @@
-gapic_generator_version: 2.59.0
-googleapis_commitish: 56f26a7d042d41ca5da9e7cdf16f427c7aee255d
-libraries_bom_version: 26.61.0
+gapic_generator_version: 2.60.2
+googleapis_commitish: fb0d8ac09cd1c2d94e9da572f0ad4692c8781410
+libraries_bom_version: 26.64.0
libraries:
- api_shortname: storage
name_pretty: Cloud Storage
From 4017026d17e96a49d78cb4d7b0cc0eeb07bb1002 Mon Sep 17 00:00:00 2001
From: "release-please[bot]"
<55107282+release-please[bot]@users.noreply.github.com>
Date: Mon, 28 Jul 2025 12:21:10 -0400
Subject: [PATCH 15/15] chore(main): release 2.54.0 (#3200)
* chore(main): release 2.54.0
* chore: generate libraries at Thu Jul 24 19:21:13 UTC 2025
---------
Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com>
Co-authored-by: cloud-java-bot
---
CHANGELOG.md | 23 +++++++++++++++++++
README.md | 6 ++---
gapic-google-cloud-storage-v2/pom.xml | 4 ++--
google-cloud-storage-bom/pom.xml | 16 ++++++-------
google-cloud-storage-control/pom.xml | 4 ++--
google-cloud-storage/pom.xml | 4 ++--
grpc-google-cloud-storage-control-v2/pom.xml | 4 ++--
grpc-google-cloud-storage-v2/pom.xml | 4 ++--
pom.xml | 16 ++++++-------
proto-google-cloud-storage-control-v2/pom.xml | 4 ++--
proto-google-cloud-storage-v2/pom.xml | 4 ++--
samples/snapshot/pom.xml | 6 ++---
storage-shared-benchmarking/pom.xml | 4 ++--
versions.txt | 14 +++++------
14 files changed, 68 insertions(+), 45 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d5f0ec1a..740eacae7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,28 @@
# Changelog
+## [2.54.0](https://github.com/googleapis/java-storage/compare/v2.53.3...v2.54.0) (2025-07-24)
+
+
+### Features
+
+* Add BucketInfo.IpFilter ([#3177](https://github.com/googleapis/java-storage/issues/3177)) ([14a91ec](https://github.com/googleapis/java-storage/commit/14a91ec208067e6afc55923cffda5f69aa33d8b4))
+* Add default end-to-end checksumming for JournalingBlobWriteSessionConfig [#3180](https://github.com/googleapis/java-storage/issues/3180) ([fa0f6a0](https://github.com/googleapis/java-storage/commit/fa0f6a03380af78e239bd0079267649ba4138f38))
+* Add default end-to-end crc32c checksumming for several upload methods via grpc transport [#3176](https://github.com/googleapis/java-storage/issues/3176) ([fa0f6a0](https://github.com/googleapis/java-storage/commit/fa0f6a03380af78e239bd0079267649ba4138f38))
+
+
+### Bug Fixes
+
+* **deps:** Update the Java code generator (gapic-generator-java) to 2.60.2 ([bd1f199](https://github.com/googleapis/java-storage/commit/bd1f199cf57c2b8039c303586d5beac64aeca0ba))
+* Give user provided checksum precondition priority for Storage#create methods that accept byte[] [#3182](https://github.com/googleapis/java-storage/issues/3182) ([fa0f6a0](https://github.com/googleapis/java-storage/commit/fa0f6a03380af78e239bd0079267649ba4138f38))
+* Move crc32c computation before writing to disk for BufferToDiskThenUpload BlobWriteSession config [#3187](https://github.com/googleapis/java-storage/issues/3187) ([fa0f6a0](https://github.com/googleapis/java-storage/commit/fa0f6a03380af78e239bd0079267649ba4138f38))
+
+
+### Dependencies
+
+* Update dependency com.google.apis:google-api-services-storage to v1-rev20250718-2.0.0 ([#3203](https://github.com/googleapis/java-storage/issues/3203)) ([18978e4](https://github.com/googleapis/java-storage/commit/18978e4ec54790df2939490ef76fc19b9f72eb04))
+* Update dependency com.google.cloud:sdk-platform-java-config to v3.50.2 ([#3201](https://github.com/googleapis/java-storage/issues/3201)) ([782c3c4](https://github.com/googleapis/java-storage/commit/782c3c416583704a196b17f23e9c12c33659f67d))
+* Update googleapis/sdk-platform-java action to v2.60.1 ([#3196](https://github.com/googleapis/java-storage/issues/3196)) ([6ba56e5](https://github.com/googleapis/java-storage/commit/6ba56e5a4b86a75a9f48beccf79ff6d5fdd3e19f))
+
## [2.53.3](https://github.com/googleapis/java-storage/compare/v2.53.2...v2.53.3) (2025-07-09)
diff --git a/README.md b/README.md
index b75087fa2..121e15591 100644
--- a/README.md
+++ b/README.md
@@ -66,13 +66,13 @@ implementation 'com.google.cloud:google-cloud-storage'
If you are using Gradle without BOM, add this to your dependencies:
```Groovy
-implementation 'com.google.cloud:google-cloud-storage:2.53.3'
+implementation 'com.google.cloud:google-cloud-storage:2.54.0'
```
If you are using SBT, add this to your dependencies:
```Scala
-libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.53.3"
+libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.54.0"
```
## Authentication
@@ -522,7 +522,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html
[stability-image]: https://img.shields.io/badge/stability-stable-green
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg
-[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.53.3
+[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.54.0
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles
diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml
index 3a601c38a..b531898a2 100644
--- a/gapic-google-cloud-storage-v2/pom.xml
+++ b/gapic-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
gapic-google-cloud-storage-v2
GRPC library for gapic-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/google-cloud-storage-bom/pom.xml b/google-cloud-storage-bom/pom.xml
index 0d51f2f59..d2e9a9f95 100644
--- a/google-cloud-storage-bom/pom.xml
+++ b/google-cloud-storage-bom/pom.xml
@@ -19,7 +19,7 @@
4.0.0
com.google.cloud
google-cloud-storage-bom
- 2.53.4-SNAPSHOT
+ 2.54.0
pom
com.google.cloud
@@ -69,37 +69,37 @@
com.google.cloud
google-cloud-storage
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.cloud
google-cloud-storage-control
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
grpc-google-cloud-storage-control-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
proto-google-cloud-storage-control-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/google-cloud-storage-control/pom.xml b/google-cloud-storage-control/pom.xml
index 416ffa54e..9a2320e07 100644
--- a/google-cloud-storage-control/pom.xml
+++ b/google-cloud-storage-control/pom.xml
@@ -5,13 +5,13 @@
4.0.0
com.google.cloud
google-cloud-storage-control
- 2.53.4-SNAPSHOT
+ 2.54.0
google-cloud-storage-control
GRPC library for google-cloud-storage-control
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml
index 13e44fd0b..245bb6ef0 100644
--- a/google-cloud-storage/pom.xml
+++ b/google-cloud-storage/pom.xml
@@ -2,7 +2,7 @@
4.0.0
google-cloud-storage
- 2.53.4-SNAPSHOT
+ 2.54.0
jar
Google Cloud Storage
https://github.com/googleapis/java-storage
@@ -12,7 +12,7 @@
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
google-cloud-storage
diff --git a/grpc-google-cloud-storage-control-v2/pom.xml b/grpc-google-cloud-storage-control-v2/pom.xml
index 9e1627edd..5a3413ed0 100644
--- a/grpc-google-cloud-storage-control-v2/pom.xml
+++ b/grpc-google-cloud-storage-control-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
grpc-google-cloud-storage-control-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
grpc-google-cloud-storage-control-v2
GRPC library for google-cloud-storage
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml
index a744a8f4a..49beaac99 100644
--- a/grpc-google-cloud-storage-v2/pom.xml
+++ b/grpc-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
grpc-google-cloud-storage-v2
GRPC library for grpc-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/pom.xml b/pom.xml
index 072cdcb17..02ce7678b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
com.google.cloud
google-cloud-storage-parent
pom
- 2.53.4-SNAPSHOT
+ 2.54.0
Storage Parent
https://github.com/googleapis/java-storage
@@ -82,7 +82,7 @@
com.google.cloud
google-cloud-storage
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.apis
@@ -104,32 +104,32 @@
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
grpc-google-cloud-storage-control-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.api.grpc
proto-google-cloud-storage-control-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.cloud
google-cloud-storage-control
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.cloud
diff --git a/proto-google-cloud-storage-control-v2/pom.xml b/proto-google-cloud-storage-control-v2/pom.xml
index cca819358..164d36794 100644
--- a/proto-google-cloud-storage-control-v2/pom.xml
+++ b/proto-google-cloud-storage-control-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
proto-google-cloud-storage-control-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
proto-google-cloud-storage-control-v2
Proto library for proto-google-cloud-storage-control-v2
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml
index 6fe5a4cbf..02953fb8e 100644
--- a/proto-google-cloud-storage-v2/pom.xml
+++ b/proto-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.53.4-SNAPSHOT
+ 2.54.0
proto-google-cloud-storage-v2
PROTO library for proto-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml
index 40c96caaa..312b98666 100644
--- a/samples/snapshot/pom.xml
+++ b/samples/snapshot/pom.xml
@@ -28,12 +28,12 @@
com.google.cloud
google-cloud-storage
- 2.53.4-SNAPSHOT
+ 2.54.0
com.google.cloud
google-cloud-storage-control
- 2.53.4-SNAPSHOT
+ 2.54.0
compile
@@ -70,7 +70,7 @@
com.google.cloud
google-cloud-storage
- 2.53.4-SNAPSHOT
+ 2.54.0
tests
test
diff --git a/storage-shared-benchmarking/pom.xml b/storage-shared-benchmarking/pom.xml
index b55be369d..89e8fd00a 100644
--- a/storage-shared-benchmarking/pom.xml
+++ b/storage-shared-benchmarking/pom.xml
@@ -10,7 +10,7 @@
com.google.cloud
google-cloud-storage-parent
- 2.53.4-SNAPSHOT
+ 2.54.0
@@ -31,7 +31,7 @@
com.google.cloud
google-cloud-storage
- 2.53.4-SNAPSHOT
+ 2.54.0
tests
diff --git a/versions.txt b/versions.txt
index 70560e648..edce9b20a 100644
--- a/versions.txt
+++ b/versions.txt
@@ -1,10 +1,10 @@
# Format:
# module:released-version:current-version
-google-cloud-storage:2.53.3:2.53.4-SNAPSHOT
-gapic-google-cloud-storage-v2:2.53.3:2.53.4-SNAPSHOT
-grpc-google-cloud-storage-v2:2.53.3:2.53.4-SNAPSHOT
-proto-google-cloud-storage-v2:2.53.3:2.53.4-SNAPSHOT
-google-cloud-storage-control:2.53.3:2.53.4-SNAPSHOT
-proto-google-cloud-storage-control-v2:2.53.3:2.53.4-SNAPSHOT
-grpc-google-cloud-storage-control-v2:2.53.3:2.53.4-SNAPSHOT
+google-cloud-storage:2.54.0:2.54.0
+gapic-google-cloud-storage-v2:2.54.0:2.54.0
+grpc-google-cloud-storage-v2:2.54.0:2.54.0
+proto-google-cloud-storage-v2:2.54.0:2.54.0
+google-cloud-storage-control:2.54.0:2.54.0
+proto-google-cloud-storage-control-v2:2.54.0:2.54.0
+grpc-google-cloud-storage-control-v2:2.54.0:2.54.0