Skip to content

feat: Set CRC32C as the default checksum option #2635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
thiyaguk09 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat: Set CRC32C as the default checksum option #2635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
da1ec6e
fix-Set crc32c as the default option
thiyaguk09 Aug 13, 2025
7bb00e5
added test cases
thiyaguk09 Aug 18, 2025
76c67c3
Merge branch 'googleapis:main' into feat/default-crc32c
thiyaguk09 Aug 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions src/transfer-manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ export interface UploadFileInChunksOptions {
uploadId?: string;
autoAbortFailure?: boolean;
partsMap?: Map<number, string>;
validation?: 'md5' | false;
validation?: 'md5' | 'crc32c' | false;
headers?: {[key: string]: string};
}

Expand All @@ -140,7 +140,7 @@ export interface MultiPartUploadHelper {
uploadPart(
partNumber: number,
chunk: Buffer,
validation?: 'md5' | false
validation?: 'md5' | 'crc32c' | false
): Promise<void>;
completeUpload(): Promise<GaxiosResponse | undefined>;
abortUpload(): Promise<void>;
Expand Down Expand Up @@ -289,7 +289,7 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper {
async uploadPart(
partNumber: number,
chunk: Buffer,
validation?: 'md5' | false
validation?: 'md5' | 'crc32c' | false
): Promise<void> {
const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`;
let headers: Headers = this.#setGoogApiClientHeaders();
Expand All @@ -299,6 +299,10 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper {
headers = {
'Content-MD5': hash,
};
} else if (validation === 'crc32c') {
const crc = new CRC32C();
crc.update(chunk);
headers['x-goog-hash'] = `crc32c=${crc.toString()}`;
}

return AsyncRetry(async bail => {
Expand Down Expand Up @@ -806,6 +810,8 @@ export class TransferManager {
);
let partNumber = 1;
let promises: Promise<void>[] = [];
const validation =
options.validation === undefined ? 'crc32c' : options.validation;
try {
if (options.uploadId === undefined) {
await mpuHelper.initiateUpload(options.headers);
Expand All @@ -823,9 +829,7 @@ export class TransferManager {
promises = [];
}
promises.push(
limit(() =>
mpuHelper.uploadPart(partNumber++, curChunk, options.validation)
)
limit(() => mpuHelper.uploadPart(partNumber++, curChunk, validation))
);
}
await Promise.all(promises);
Expand Down
57 changes: 57 additions & 0 deletions test/transfer-manager.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,5 +730,62 @@ describe('Transfer Manager', () => {

assert(called);
});

it('should use CRC32C validation when specified', async () => {
mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => {
fakeHelper = sandbox.createStubInstance(FakeXMLHelper);
fakeHelper.uploadId = uploadId || '';
fakeHelper.partsMap = partsMap || new Map<number, string>();
fakeHelper.initiateUpload.resolves();
fakeHelper.uploadPart.callsFake((partNumber, chunk, validation) => {
assert.strictEqual(validation, 'crc32c');

return Promise.resolve();
});
fakeHelper.completeUpload.resolves();
fakeHelper.abortUpload.resolves();
return fakeHelper;
};

await transferManager.uploadFileInChunks(
filePath,
{validation: 'crc32c'},
mockGeneratorFunction
);

assert.strictEqual(fakeHelper.uploadPart.calledOnce, true);
});

it('should apply crc32c validation by default', async () => {
let assertionMade = false;

mockGeneratorFunction = (bucket, fileName, uploadId, partsMap) => {
fakeHelper = sandbox.createStubInstance(FakeXMLHelper);
fakeHelper.uploadId = uploadId || '';
fakeHelper.partsMap = partsMap || new Map<number, string>();
fakeHelper.initiateUpload.resolves();

fakeHelper.uploadPart.callsFake((partNumber, chunk, validation) => {
// Confirm the validation is set to 'crc32c' by default.
assert.strictEqual(validation, 'crc32c');
assertionMade = true;
return Promise.resolve();
});

fakeHelper.completeUpload.resolves();
fakeHelper.abortUpload.resolves();
return fakeHelper;
};

// Call the function without specifying any validation option.
await transferManager.uploadFileInChunks(
filePath,
{},
mockGeneratorFunction
);

assert.strictEqual(fakeHelper.uploadPart.calledOnce, true);
assert.strictEqual(assertionMade, true);
});
});
});
Loading