Skip to content

Commit c73d995

Browse files
sudarshan-SpringMLandrewferlitsch
sudarshan-SpringML
and
andrewferlitsch
authored
Made minor changes to sdk_automl_image_object_detection_batch.ipynb file (#513)
* modified notebook * ran linter test Co-authored-by: Andrew Ferlitsch <[email protected]>
1 parent 3fe5028 commit c73d995

File tree

1 file changed

+97
-48
lines changed

1 file changed

+97
-48
lines changed

‎notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb‎

Lines changed: 97 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -32,23 +32,24 @@
3232
"# Vertex AI SDK : AutoML training image object detection model for batch prediction\n",
3333
"\n",
3434
"<table align=\"left\">\n",
35+
" \n",
3536
" <td>\n",
36-
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb\">\n",
37+
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb\">\n",
3738
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
3839
" </a>\n",
3940
" </td>\n",
4041
" <td>\n",
41-
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb\">\n",
42+
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb\">\n",
4243
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
4344
" View on GitHub\n",
4445
" </a>\n",
4546
" </td>\n",
4647
" <td>\n",
47-
" <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb\">\n",
48-
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
48+
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb\">\n",
49+
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
4950
" Open in Vertex AI Workbench\n",
5051
" </a>\n",
51-
" </td>\n",
52+
" </td> \n",
5253
"</table>\n",
5354
"<br/><br/><br/>"
5455
]
@@ -168,7 +169,7 @@
168169
"cell_type": "code",
169170
"execution_count": null,
170171
"metadata": {
171-
"id": "install_aip:mbsdk"
172+
"id": "hw7H6ADSv5mI"
172173
},
173174
"outputs": [],
174175
"source": [
@@ -189,20 +190,29 @@
189190
"id": "install_storage"
190191
},
191192
"source": [
192-
"Install the latest GA version of *google-cloud-storage* library as well."
193+
"Install the latest GA version of *google-cloud-storage* library."
193194
]
194195
},
195196
{
196197
"cell_type": "code",
197198
"execution_count": null,
198199
"metadata": {
199-
"id": "install_storage"
200+
"id": "d6Pa6Sybv5mK"
200201
},
201202
"outputs": [],
202203
"source": [
203204
"! pip3 install -U google-cloud-storage $USER_FLAG"
204205
]
205206
},
207+
{
208+
"cell_type": "markdown",
209+
"metadata": {
210+
"id": "9_zWlX10v5mL"
211+
},
212+
"source": [
213+
"Install the latest version of *tensorflow* library."
214+
]
215+
},
206216
{
207217
"cell_type": "code",
208218
"execution_count": null,
@@ -211,8 +221,7 @@
211221
},
212222
"outputs": [],
213223
"source": [
214-
"if os.getenv(\"IS_TESTING\"):\n",
215-
" ! pip3 install --upgrade tensorflow $USER_FLAG"
224+
"! pip3 install --upgrade tensorflow $USER_FLAG"
216225
]
217226
},
218227
{
@@ -230,7 +239,7 @@
230239
"cell_type": "code",
231240
"execution_count": null,
232241
"metadata": {
233-
"id": "restart"
242+
"id": "D-ZBOjErv5mM"
234243
},
235244
"outputs": [],
236245
"source": [
@@ -282,7 +291,9 @@
282291
},
283292
"outputs": [],
284293
"source": [
285-
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
294+
"import os\n",
295+
"\n",
296+
"PROJECT_ID = \"\""
286297
]
287298
},
288299
{
@@ -293,11 +304,32 @@
293304
},
294305
"outputs": [],
295306
"source": [
296-
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
297-
" # Get your GCP project id from gcloud\n",
298-
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
307+
"# Get your Google Cloud project ID from gcloud\n",
308+
"if not os.getenv(\"IS_TESTING\"):\n",
309+
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
299310
" PROJECT_ID = shell_output[0]\n",
300-
" print(\"Project ID:\", PROJECT_ID)"
311+
" print(\"Project ID: \", PROJECT_ID)"
312+
]
313+
},
314+
{
315+
"cell_type": "markdown",
316+
"metadata": {
317+
"id": "W2F5WRyhv5mO"
318+
},
319+
"source": [
320+
"Otherwise, set your project ID here."
321+
]
322+
},
323+
{
324+
"cell_type": "code",
325+
"execution_count": null,
326+
"metadata": {
327+
"id": "d7-MjQafv5mO"
328+
},
329+
"outputs": [],
330+
"source": [
331+
"if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
332+
" PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
301333
]
302334
},
303335
{
@@ -335,11 +367,14 @@
335367
"cell_type": "code",
336368
"execution_count": null,
337369
"metadata": {
338-
"id": "region"
370+
"id": "kAfG6tDAv5mQ"
339371
},
340372
"outputs": [],
341373
"source": [
342-
"REGION = \"us-central1\" # @param {type: \"string\"}"
374+
"REGION = \"[your-region]\" # @param {type:\"string\"}\n",
375+
"\n",
376+
"if REGION == \"[your-region]\":\n",
377+
" REGION = \"us-central1\""
343378
]
344379
},
345380
{
@@ -357,7 +392,7 @@
357392
"cell_type": "code",
358393
"execution_count": null,
359394
"metadata": {
360-
"id": "timestamp"
395+
"id": "PzKW-zT_v5mR"
361396
},
362397
"outputs": [],
363398
"source": [
@@ -397,7 +432,7 @@
397432
"cell_type": "code",
398433
"execution_count": null,
399434
"metadata": {
400-
"id": "gcp_authenticate"
435+
"id": "FvQeFm3Gv5mR"
401436
},
402437
"outputs": [],
403438
"source": [
@@ -475,7 +510,7 @@
475510
"cell_type": "code",
476511
"execution_count": null,
477512
"metadata": {
478-
"id": "create_bucket"
513+
"id": "09kHSsKmv5mT"
479514
},
480515
"outputs": [],
481516
"source": [
@@ -495,7 +530,7 @@
495530
"cell_type": "code",
496531
"execution_count": null,
497532
"metadata": {
498-
"id": "validate_bucket"
533+
"id": "N9JY-esPv5mU"
499534
},
500535
"outputs": [],
501536
"source": [
@@ -540,7 +575,7 @@
540575
"cell_type": "code",
541576
"execution_count": null,
542577
"metadata": {
543-
"id": "init_aip:mbsdk"
578+
"id": "w2Oa_jZSv5mV"
544579
},
545580
"outputs": [],
546581
"source": [
@@ -597,7 +632,7 @@
597632
"cell_type": "code",
598633
"execution_count": null,
599634
"metadata": {
600-
"id": "quick_peek:csv"
635+
"id": "ITshYFagv5mZ"
601636
},
602637
"outputs": [],
603638
"source": [
@@ -629,7 +664,7 @@
629664
"cell_type": "code",
630665
"execution_count": null,
631666
"metadata": {
632-
"id": "create_dataset:image,iod"
667+
"id": "FYyaPzfRv5mc"
633668
},
634669
"outputs": [],
635670
"source": [
@@ -677,7 +712,7 @@
677712
"cell_type": "code",
678713
"execution_count": null,
679714
"metadata": {
680-
"id": "create_automl_pipeline:image,iod"
715+
"id": "vtAgY1Nmv5md"
681716
},
682717
"outputs": [],
683718
"source": [
@@ -719,7 +754,7 @@
719754
"cell_type": "code",
720755
"execution_count": null,
721756
"metadata": {
722-
"id": "run_automl_pipeline:image"
757+
"id": "SvV1nFDTv5md"
723758
},
724759
"outputs": [],
725760
"source": [
@@ -750,7 +785,7 @@
750785
"cell_type": "code",
751786
"execution_count": null,
752787
"metadata": {
753-
"id": "evaluate_the_model:mbsdk"
788+
"id": "RcpDJMgev5me"
754789
},
755790
"outputs": [],
756791
"source": [
@@ -833,7 +868,7 @@
833868
"cell_type": "code",
834869
"execution_count": null,
835870
"metadata": {
836-
"id": "copy_test_items:batch_prediction"
871+
"id": "JXpg67zjv5mf"
837872
},
838873
"outputs": [],
839874
"source": [
@@ -869,7 +904,7 @@
869904
"cell_type": "code",
870905
"execution_count": null,
871906
"metadata": {
872-
"id": "make_batch_file:automl,image"
907+
"id": "hFjX62hvv5mg"
873908
},
874909
"outputs": [],
875910
"source": [
@@ -901,21 +936,37 @@
901936
"- `job_display_name`: The human readable name for the batch prediction job.\n",
902937
"- `gcs_source`: A list of one or more batch request input files.\n",
903938
"- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
939+
"- `machine_type`: The type of machine for running batch prediction on dedicated resources. Not specifying machine type will result in batch prediction job being run with automatic resources.\n",
940+
"- `starting_replica_count`: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set.\n",
941+
"- `max_replica_count`: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. Default is 10.\n",
904942
"- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
905943
]
906944
},
945+
{
946+
"cell_type": "markdown",
947+
"metadata": {
948+
"id": "axn-CW7xv5mg"
949+
},
950+
"source": [
951+
"For AutoML models, only manual scaling is supported. In manual scaling both starting_replica_count and max_replica_count have the same value.\n",
952+
"For this batch job we are using manual scaling. Here we are setting both starting_replica_count and max_replica_count to the same value that is 1. "
953+
]
954+
},
907955
{
908956
"cell_type": "code",
909957
"execution_count": null,
910958
"metadata": {
911-
"id": "batch_request:mbsdk"
959+
"id": "5VMMaJhbv5mh"
912960
},
913961
"outputs": [],
914962
"source": [
915963
"batch_predict_job = model.batch_predict(\n",
916964
" job_display_name=\"salads_\" + TIMESTAMP,\n",
917965
" gcs_source=gcs_input_uri,\n",
918966
" gcs_destination_prefix=BUCKET_URI,\n",
967+
" machine_type=\"n1-standard-4\",\n",
968+
" starting_replica_count=1,\n",
969+
" max_replica_count=1,\n",
919970
" sync=False,\n",
920971
")\n",
921972
"\n",
@@ -937,7 +988,7 @@
937988
"cell_type": "code",
938989
"execution_count": null,
939990
"metadata": {
940-
"id": "batch_request_wait:mbsdk"
991+
"id": "Yc3YaEqGv5mh"
941992
},
942993
"outputs": [],
943994
"source": [
@@ -967,7 +1018,7 @@
9671018
"cell_type": "code",
9681019
"execution_count": null,
9691020
"metadata": {
970-
"id": "get_batch_prediction:mbsdk,iod"
1021+
"id": "UGGJxFjEv5mh"
9711022
},
9721023
"outputs": [],
9731024
"source": [
@@ -986,8 +1037,7 @@
9861037
" with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
9871038
" for line in gfile.readlines():\n",
9881039
" line = json.loads(line)\n",
989-
" print(line)\n",
990-
" break"
1040+
" print(line)"
9911041
]
9921042
},
9931043
{
@@ -1014,27 +1064,26 @@
10141064
"cell_type": "code",
10151065
"execution_count": null,
10161066
"metadata": {
1017-
"id": "cleanup:mbsdk"
1067+
"id": "_Olkhs6xv5mi"
10181068
},
10191069
"outputs": [],
10201070
"source": [
1021-
"delete_all = True\n",
1071+
"delete_bucket = False\n",
10221072
"\n",
1023-
"if delete_all:\n",
1024-
" # Delete the dataset using the Vertex dataset object\n",
1025-
" dataset.delete()\n",
1073+
"# Delete the dataset using the Vertex dataset object\n",
1074+
"dataset.delete()\n",
10261075
"\n",
1027-
" # Delete the model using the Vertex model object\n",
1028-
" model.delete()\n",
1076+
"# Delete the model using the Vertex model object\n",
1077+
"model.delete()\n",
10291078
"\n",
1030-
" # Delete the AutoML or Pipeline trainig job\n",
1031-
" job.delete()\n",
1079+
"# Delete the AutoML or Pipeline trainig job\n",
1080+
"job.delete()\n",
10321081
"\n",
1033-
" # Delete the batch prediction job using the Vertex batch prediction object\n",
1034-
" batch_predict_job.delete()\n",
1082+
"# Delete the batch prediction job using the Vertex batch prediction object\n",
1083+
"batch_predict_job.delete()\n",
10351084
"\n",
1036-
" if os.getenv(\"IS_TESTING\"):\n",
1037-
" ! gsutil rm -r $BUCKET_URI"
1085+
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1086+
" ! gsutil rm -r $BUCKET_URI"
10381087
]
10391088
}
10401089
],

0 commit comments

Comments
 (0)