add transfer manager samples and samples tests

googleapis · ddelgrosso1 · Dec 5, 2022 · Aug 15, 2022 · Sep 15, 2022 · Sep 15, 2022
commit aad8f2bfd724ac3aa4ea7588b7e5cbaa15309aa2
diff --git a/samples/downloadFileInChunksWithTransferManager.js b/samples/downloadFileInChunksWithTransferManager.js
@@ -0,0 +1,77 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * @experimental
+ */
+
+const path = require('path');
+const cwd = path.join(__dirname, '..');
+
+// sample-metadata:
+//   title: Download a File in Chunks Utilzing Transfer Manager
+//   description: Downloads a single file in in chunks in parallel utilizing transfer manager.
+//   usage: node downloadFileInChunksWithTransferManager.js <BUCKET_NAME> <FILE_NAME> <DESTINATION_FILE_NAME> <CHUNK_SIZE>
+
+function main(
+  bucketName = 'my-bucket',
+  fileName = 'file1.txt',
+  destFileName = path.join(cwd, fileName),
+  chunkSize = 1024
+) {
+  // [START storage_download_many_files_transfer_manager]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the GCS file to download
+  // const fileName = 'your-file-name';
+
+  // The path to which the file should be downloaded
+  // const destFileName = '/local/path/to/file.txt';
+
+  // The size of each chunk to be downloaded
+  // const chunkSize = 1024;
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager instance
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function downloadFileInChunksWithTransferManager() {
+    // Downloads the files
+    await transferManager.downloadFileInChunks(fileName, {
+      destination: destFileName,
+      chunkSizeBytes: chunkSize,
+    });
+
+    console.log(
+      `gs://${bucketName}/${fileName} downloaded to ${destFileName}.`
+    );
+  }
+
+  downloadFileInChunksWithTransferManager().catch(console.error);
+  // [END storage_download_many_files_transfer_manager]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+main(...process.argv.slice(2));
diff --git a/samples/downloadManyFilesWithTransferManager.js b/samples/downloadManyFilesWithTransferManager.js
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * @experimental
+ */
+
+// sample-metadata:
+//   title: Download Many Files With Transfer Manager
+//   description: Downloads many files in parallel utilizing transfer manager.
+//   usage: node downloadManyFilesWithTransferManager.js <BUCKET_NAME> <FIRST_FILE_NAME> <SECOND_FILE_NAME>
+
+function main(
+  bucketName = 'my-bucket',
+  firstFileName = 'file1.txt',
+  secondFileName = 'file2.txt'
+) {
+  // [START storage_download_many_files_transfer_manager]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the first GCS file to download
+  // const firstFileName = 'your-first-file-name';
+
+  // The ID of the second GCS file to download
+  // const secondFileName = 'your-second-file-name;
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager instance
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function downloadManyFilesWithTransferManager() {
+    // Downloads the files
+    await transferManager.downloadManyFiles([firstFileName, secondFileName]);
+
+    for (const fileName of [firstFileName, secondFileName]) {
+      console.log(`gs://${bucketName}/${fileName} downloaded to ${fileName}.`);
+    }
+  }
+
+  downloadManyFilesWithTransferManager().catch(console.error);
+  // [END storage_download_many_files_transfer_manager]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+main(...process.argv.slice(2));
diff --git a/samples/downloaded.txt b/samples/downloaded.txt
@@ -0,0 +1 @@
+Hello World!
diff --git a/samples/resources/test2.txt b/samples/resources/test2.txt
@@ -0,0 +1 @@
+Hello World 2!
diff --git a/samples/system-test/transfer-manager.test.js b/samples/system-test/transfer-manager.test.js
@@ -0,0 +1,85 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const path = require('path');
+const {Storage} = require('@google-cloud/storage');
+const {before, after, it, describe} = require('mocha');
+const uuid = require('uuid');
+const cp = require('child_process');
+const {assert} = require('chai');
+
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+const storage = new Storage();
+const cwd = path.join(__dirname, '..');
+const bucketName = generateName();
+const bucket = storage.bucket(bucketName);
+const firstFileName = 'test.txt';
+const secondFileName = 'test2.txt';
+const firstFilePath = path.join(cwd, 'resources', firstFileName);
+const secondFilePath = path.join(cwd, 'resources', secondFileName);
+const downloadFilePath = path.join(cwd, 'downloaded.txt');
+const chunkSize = 1024;
+
+describe('transfer manager', () => {
+  before(async () => {
+    await bucket.create();
+  });
+
+  after(async () => {
+    await bucket.deleteFiles({force: true}).catch(console.error);
+    await bucket.delete().catch(console.error);
+  });
+
+  it('should upload multiple files', async () => {
+    const output = execSync(
+      `node uploadManyFilesWithTransferManager.js ${bucketName} ${firstFilePath} ${secondFilePath}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `${firstFilePath} uploaded to ${bucketName}.\n${secondFilePath} uploaded to ${bucketName}`
+      )
+    );
+  });
+
+  it('should download mulitple files', async () => {
+    const output = execSync(
+      `node downloadManyFilesWithTransferManager.js ${bucketName} ${firstFilePath} ${secondFilePath}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `gs://${bucketName}/${firstFilePath} downloaded to ${firstFilePath}.\ngs://${bucketName}/${secondFilePath} downloaded to ${secondFilePath}.`
+      )
+    );
+  });
+
+  it('should download a file utilizing chunked download', async () => {
+    const output = execSync(
+      `node downloadFileInChunksWithTransferManager.js ${bucketName} ${firstFilePath} ${downloadFilePath} ${chunkSize}`
+    );
+    assert.match(
+      output,
+      new RegExp(
+        `gs://${bucketName}/${firstFilePath} downloaded to ${downloadFilePath}.`
+      )
+    );
+  });
+});
+
+function generateName() {
+  return `nodejs-storage-samples-${uuid.v4()}`;
+}
diff --git a/samples/uploadManyFilesWithTransferManager.js b/samples/uploadManyFilesWithTransferManager.js
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * @experimental
+ */
+
+// sample-metadata:
+//   title: Upload Many Files With Transfer Manager
+//   description: Uploads many files in parallel utilizing transfer manager.
+//   usage: node uploadManyFilesWithTransferManager.js <BUCKET_NAME> <FIRST_FILE_NAME> <SECOND_FILE_NAME>
+
+function main(
+  bucketName = 'my-bucket',
+  firstFilePath = './local/path/to/file1.txt',
+  secondFilePath = './local/path/to/file2.txt'
+) {
+  // [START storage_upload_many_files_transfer_manager]
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of your GCS bucket
+  // const bucketName = 'your-unique-bucket-name';
+
+  // The ID of the first GCS file to download
+  // const firstFileName = 'your-first-file-name';
+
+  // The ID of the second GCS file to download
+  // const secondFileName = 'your-second-file-name;
+
+  // Imports the Google Cloud client library
+  const {Storage, TransferManager} = require('@google-cloud/storage');
+
+  // Creates a client
+  const storage = new Storage();
+
+  // Creates a transfer manager instance
+  const transferManager = new TransferManager(storage.bucket(bucketName));
+
+  async function uploadManyFilesWithTransferManager() {
+    // Uploads the files
+    await transferManager.uploadManyFiles([firstFilePath, secondFilePath]);
+
+    for (const filePath of [firstFilePath, secondFilePath]) {
+      console.log(`${filePath} uploaded to ${bucketName}.`);
+    }
+  }
+
+  uploadManyFilesWithTransferManager().catch(console.error);
+  // [END storage_upload_many_files_transfer_manager]
+}
+
+process.on('unhandledRejection', err => {
+  console.error(err.message);
+  process.exitCode = 1;
+});
+main(...process.argv.slice(2));
diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts
@@ -270,7 +270,7 @@ export class TransferManager {
    * Download a large file in chunks utilizing parallel download operations. This is a convenience method
    * that utilizes {@link File#download} to perform the download.
    *
-   * @param {object} [file] {@link File} to download.
+   * @param {object} [file | string] {@link File} to download.
    * @param {DownloadFileInChunksOptions} [options] Configuration options.
    * @returns {Promise<DownloadResponse>}
    *
@@ -291,7 +291,7 @@ export class TransferManager {
    * @experimental
    */
   async downloadFileInChunks(
-    file: File,
+    fileOrName: File | string,
     options: DownloadFileInChunksOptions = {}
   ): Promise<void | DownloadResponse> {
     let chunkSize =
@@ -300,6 +300,10 @@ export class TransferManager {
       options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT
     );
     const promises = [];
+    const file: File =
+      typeof fileOrName === 'string'
+        ? this.bucket.file(fileOrName)
+        : fileOrName;
 
     const fileInfo = await file.get();
     const size = parseInt(fileInfo[0].metadata.size);