google-cloud-pipeline-components 2.15.0__py3-none-any.whl → 2.16.1__py3-none-any.whl

google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml

@@ -1074,6 +1074,8 @@ components:
                 componentInputParameter: pipelinechannel--dataflow_subnetwork
               dataflow_use_public_ips:
                 componentInputParameter: pipelinechannel--dataflow_use_public_ips
+              dataflow_workers_num:
+                componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers
               encryption_spec_key_name:
                 componentInputParameter: pipelinechannel--encryption_spec_key_name
               forecasting_quantiles:
@@ -1795,6 +1797,8 @@ components:
                 componentInputParameter: pipelinechannel--dataflow_subnetwork
               dataflow_use_public_ips:
                 componentInputParameter: pipelinechannel--dataflow_use_public_ips
+              dataflow_workers_num:
+                componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers
               encryption_spec_key_name:
                 componentInputParameter: pipelinechannel--encryption_spec_key_name
               forecasting_quantiles:
@@ -5573,7 +5577,7 @@ deploymentSpec:
         - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
           "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"},
           "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec":
-          {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625",
+          {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625",
           "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}",
           "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb",
           "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}",
@@ -5607,7 +5611,7 @@ deploymentSpec:
         - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
           "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"},
           "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec":
-          {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625",
+          {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625",
           "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}",
           "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb",
           "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}",
@@ -5642,11 +5646,11 @@ deploymentSpec:
           \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
           "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
           {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
-          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625",
+          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625",
           "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=",
           "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=",
           "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=",
-          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625",
+          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625",
           "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}",
           "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=",
           "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train",
@@ -5685,11 +5689,11 @@ deploymentSpec:
           \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
           "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
           {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
-          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625",
+          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625",
           "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=",
           "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=",
           "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=",
-          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625",
+          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240808_0625",
           "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=",
           "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train",
           "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}",
@@ -5728,7 +5732,7 @@ deploymentSpec:
           \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}",
           "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\":
           {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"",
-          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240419_0625", "\",
+          "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625", "\",
           \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}",
           "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=",
           "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}'
@@ -5793,7 +5797,7 @@ deploymentSpec:
           \          'stage_2_single_run_max_secs',\n      ],\n  )(\n      stage_1_deadline_hours,\n\
           \      stage_1_single_run_max_secs,\n      stage_2_deadline_hours,\n   \
           \   stage_2_single_run_max_secs,\n  )\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-calculate-training-parameters-2:
       container:
         args:
@@ -5849,7 +5853,7 @@ deploymentSpec:
           \          'stage_2_single_run_max_secs',\n      ],\n  )(\n      stage_1_deadline_hours,\n\
           \      stage_1_single_run_max_secs,\n      stage_2_deadline_hours,\n   \
           \   stage_2_single_run_max_secs,\n  )\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-feature-attribution:
       container:
         args:
@@ -6040,8 +6044,8 @@ deploymentSpec:
           "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}'
         - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}'
         - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}'
-        - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240419_0625
-        - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240419_0625
+        - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
+        - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
         - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}'
         - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}'
         - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}'
@@ -6058,7 +6062,7 @@ deploymentSpec:
         - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
           ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
         - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}'
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
         resources:
           cpuLimit: 8.0
           memoryLimit: 30.0
@@ -6089,7 +6093,7 @@ deploymentSpec:
           \ = 'point'\n  else:\n    forecasting_type = 'quantile'\n\n  return collections.namedtuple(\n\
           \      'Outputs',\n      (\n          'forecasting_type',\n          'quantiles',\n\
           \      ),\n  )(forecasting_type, quantiles)\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-finalize-eval-quantile-parameters-2:
       container:
         args:
@@ -6117,7 +6121,7 @@ deploymentSpec:
           \ = 'point'\n  else:\n    forecasting_type = 'quantile'\n\n  return collections.namedtuple(\n\
           \      'Outputs',\n      (\n          'forecasting_type',\n          'quantiles',\n\
           \      ),\n  )(forecasting_type, quantiles)\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-get-or-create-model-description:
       container:
         args:
@@ -6146,7 +6150,7 @@ deploymentSpec:
           \    return f'{original_description} From: {pipeline_url}'\n\n  # The pipeline\
           \ url contains KFP placeholders injected at runtime.\n  return f'Vertex\
           \ forecasting model trained in the pipeline: {pipeline_url}'\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-get-or-create-model-description-2:
       container:
         args:
@@ -6175,7 +6179,7 @@ deploymentSpec:
           \    return f'{original_description} From: {pipeline_url}'\n\n  # The pipeline\
           \ url contains KFP placeholders injected at runtime.\n  return f'Vertex\
           \ forecasting model trained in the pipeline: {pipeline_url}'\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-get-prediction-image-uri:
       container:
         args:
@@ -6198,14 +6202,14 @@ deploymentSpec:
           Returns the prediction image corresponding to the given model type.\"\"\"\
           \n  # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n  # The URIs\
           \ must be hardcoded without any breaks in the code so string\n  # replacement\
-          \ will work correctly.\n  images = {\n      'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240419_0625',\n\
-          \      'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240419_0625',\n\
-          \      'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240419_0625',\n\
-          \      'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240419_0625',\n\
+          \ will work correctly.\n  images = {\n      'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\
+          \      'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\
+          \      'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\
+          \      'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\
           \  }\n  if model_type not in images:\n    raise ValueError(\n        f'Invalid\
           \ forecasting model type: {model_type}. Valid options are: '\n        f'{images.keys()}.'\n\
           \    )\n  return images[model_type]\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-get-prediction-image-uri-2:
       container:
         args:
@@ -6228,14 +6232,14 @@ deploymentSpec:
           Returns the prediction image corresponding to the given model type.\"\"\"\
           \n  # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n  # The URIs\
           \ must be hardcoded without any breaks in the code so string\n  # replacement\
-          \ will work correctly.\n  images = {\n      'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240419_0625',\n\
-          \      'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240419_0625',\n\
-          \      'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240419_0625',\n\
-          \      'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240419_0625',\n\
+          \ will work correctly.\n  images = {\n      'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20240808_0625',\n\
+          \      'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20240808_0625',\n\
+          \      'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20240808_0625',\n\
+          \      'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20240808_0625',\n\
           \  }\n  if model_type not in images:\n    raise ValueError(\n        f'Invalid\
           \ forecasting model type: {model_type}. Valid options are: '\n        f'{images.keys()}.'\n\
           \    )\n  return images[model_type]\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-get-predictions-column:
       container:
         args:
@@ -6258,7 +6262,7 @@ deploymentSpec:
           \ str) -> str:\n  \"\"\"Generates the BP output's target column name.\"\"\
           \"\n  if forecasting_type == 'quantile':\n    return f'predicted_{target_column}.quantile_predictions'\n\
           \  return f'predicted_{target_column}.value'\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-get-predictions-column-2:
       container:
         args:
@@ -6281,7 +6285,7 @@ deploymentSpec:
           \ str) -> str:\n  \"\"\"Generates the BP output's target column name.\"\"\
           \"\n  if forecasting_type == 'quantile':\n    return f'predicted_{target_column}.quantile_predictions'\n\
           \  return f'predicted_{target_column}.value'\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-importer:
       importer:
         artifactUri:
@@ -6330,7 +6334,7 @@ deploymentSpec:
         - -u
         - -m
         - launcher
-        image: gcr.io/ml-pipeline/automl-tables-private:1.0.13
+        image: gcr.io/ml-pipeline/automl-tables-private:1.0.18
     exec-model-batch-explanation-2:
       container:
         args:
@@ -6372,7 +6376,7 @@ deploymentSpec:
         - -u
         - -m
         - launcher
-        image: gcr.io/ml-pipeline/automl-tables-private:1.0.13
+        image: gcr.io/ml-pipeline/automl-tables-private:1.0.18
     exec-model-batch-predict:
       container:
         args:
@@ -6727,7 +6731,7 @@ deploymentSpec:
         - -u
         - -m
         - launcher
-        image: gcr.io/ml-pipeline/automl-tables-private:1.0.17
+        image: gcr.io/ml-pipeline/automl-tables-private:1.0.18
     exec-model-upload-2:
       container:
         args:
@@ -6756,7 +6760,7 @@ deploymentSpec:
         - -u
         - -m
         - launcher
-        image: gcr.io/ml-pipeline/automl-tables-private:1.0.17
+        image: gcr.io/ml-pipeline/automl-tables-private:1.0.18
     exec-set-optional-inputs:
       container:
         args:
@@ -6813,7 +6817,7 @@ deploymentSpec:
           \          'model_display_name',\n          'transformations',\n      ],\n\
           \  )(\n      data_source_csv_filenames,\n      data_source_bigquery_table_path,\n\
           \      model_display_name,\n      transformations,\n  )\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-split-materialized-data:
       container:
         args:
@@ -6859,7 +6863,7 @@ deploymentSpec:
           \ 'w') as f:\n    f.write(file_patterns[0])\n\n  with tf.io.gfile.GFile(materialized_eval_split,\
           \ 'w') as f:\n    f.write(file_patterns[1])\n\n  with tf.io.gfile.GFile(materialized_test_split,\
           \ 'w') as f:\n    f.write(file_patterns[2])\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20240808_0625
     exec-string-not-empty:
       container:
         args:
@@ -6883,7 +6887,7 @@ deploymentSpec:
           \n  Returns:\n    Boolean value. -> 'true' if empty, 'false' if not empty.\
           \ We need to use str\n    instead of bool due to a limitation in KFP compiler.\n\
           \  \"\"\"\n  return 'true' if value else 'false'\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-table-to-uri:
       container:
         args:
@@ -6913,7 +6917,7 @@ deploymentSpec:
           \ if use_bq_prefix:\n    bq_uri = 'bq://' + bq_uri\n  outputs.append(bq_uri)\n\
           \  return collections.namedtuple(\n      'Outputs',\n      ['project_id',\
           \ 'dataset_id', 'table_id', 'uri'],\n  )(*outputs)\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-table-to-uri-2:
       container:
         args:
@@ -6943,7 +6947,7 @@ deploymentSpec:
           \ if use_bq_prefix:\n    bq_uri = 'bq://' + bq_uri\n  outputs.append(bq_uri)\n\
           \  return collections.namedtuple(\n      'Outputs',\n      ['project_id',\
           \ 'dataset_id', 'table_id', 'uri'],\n  )(*outputs)\n\n"
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20240808_0625
     exec-training-configurator-and-validator:
       container:
         args:
@@ -6988,7 +6992,7 @@ deploymentSpec:
           ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}'
         - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat":
           ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}'
-        image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240419_0625
+        image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20240808_0625
 pipelineInfo:
   description: The Timeseries Dense Encoder (TiDE) Forecasting pipeline.
   name: time-series-dense-encoder-forecasting

google_cloud_pipeline_components/preview/automl/forecasting/utils.py

@@ -13,6 +13,20 @@ _RETAIL_MODEL_DISABLED_OPTIONS = frozenset([
 ])
 
 
+def _validate_start_max_parameters(
+    starting_worker_count: int,
+    max_worker_count: int,
+    starting_count_name: str,
+    max_count_name: str,
+):
+  if starting_worker_count > max_worker_count:
+    raise ValueError(
+        'Starting count must be less than or equal to max count.'
+        f' {starting_count_name}: {starting_worker_count}, {max_count_name}:'
+        f' {max_worker_count}'
+    )
+
+
 def _get_base_forecasting_parameters(
     *,
     project: str,
@@ -59,6 +73,7 @@ def _get_base_forecasting_parameters(
     evaluation_batch_predict_max_replica_count: int = 25,
     evaluation_dataflow_machine_type: str = 'n1-standard-16',
     evaluation_dataflow_max_num_workers: int = 25,
+    evaluation_dataflow_starting_num_workers: int = 22,
     evaluation_dataflow_disk_size_gb: int = 50,
     study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None,
     stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None,
@@ -91,6 +106,20 @@ def _get_base_forecasting_parameters(
     )
     time_series_identifier_columns = [time_series_identifier_column]
 
+  _validate_start_max_parameters(
+      starting_worker_count=evaluation_batch_predict_starting_replica_count,
+      max_worker_count=evaluation_batch_predict_max_replica_count,
+      starting_count_name='evaluation_batch_predict_starting_replica_count',
+      max_count_name='evaluation_batch_predict_max_replica_count',
+  )
+
+  _validate_start_max_parameters(
+      starting_worker_count=evaluation_dataflow_starting_num_workers,
+      max_worker_count=evaluation_dataflow_max_num_workers,
+      starting_count_name='evaluation_dataflow_starting_num_workers',
+      max_count_name='evaluation_dataflow_max_num_workers',
+  )
+
   parameter_values = {}
   parameters = {
       'project': project,
@@ -152,6 +181,9 @@ def _get_base_forecasting_parameters(
       'evaluation_dataflow_max_num_workers': (
           evaluation_dataflow_max_num_workers
       ),
+      'evaluation_dataflow_starting_num_workers': (
+          evaluation_dataflow_starting_num_workers
+      ),
       'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb,
       'study_spec_parameters_override': study_spec_parameters_override,
       'stage_1_tuner_worker_pool_specs_override': (
@@ -174,13 +206,11 @@ def _get_base_forecasting_parameters(
 
   # Filter out empty values and those excluded from the particular pipeline.
   # (example: TFT and Seq2Seq don't support `quantiles`.)
-  parameter_values.update(
-      {
-          param: value
-          for param, value in parameters.items()
-          if value is not None and param not in fields_to_exclude
-      }
-  )
+  parameter_values.update({
+      param: value
+      for param, value in parameters.items()
+      if value is not None and param not in fields_to_exclude
+  })
   return parameter_values
 
 
@@ -229,6 +259,7 @@ def get_learn_to_learn_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: int = 25,
     evaluation_dataflow_machine_type: str = 'n1-standard-16',
     evaluation_dataflow_max_num_workers: int = 25,
+    evaluation_dataflow_starting_num_workers: int = 22,
     evaluation_dataflow_disk_size_gb: int = 50,
     study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None,
     stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None,
@@ -291,6 +322,7 @@ def get_learn_to_learn_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: The maximum count of replicas the batch prediction job can scale to.
     evaluation_dataflow_machine_type: Machine type for the dataflow job in evaluation, such as 'n1-standard-16'.
     evaluation_dataflow_max_num_workers: Maximum number of dataflow workers.
+    evaluation_dataflow_starting_num_workers: Starting number of dataflow workers.
     evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow.
     study_spec_parameters_override: The list for overriding study spec.
     stage_1_tuner_worker_pool_specs_override: The dictionary for overriding stage 1 tuner worker pool spec.
@@ -354,6 +386,7 @@ def get_learn_to_learn_forecasting_pipeline_and_parameters(
       evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count,
       evaluation_dataflow_machine_type=evaluation_dataflow_machine_type,
       evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers,
+      evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers,
       evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb,
       study_spec_parameters_override=study_spec_parameters_override,
       stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override,
@@ -423,6 +456,7 @@ def get_time_series_dense_encoder_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: int = 25,
     evaluation_dataflow_machine_type: str = 'n1-standard-16',
     evaluation_dataflow_max_num_workers: int = 25,
+    evaluation_dataflow_starting_num_workers: int = 22,
     evaluation_dataflow_disk_size_gb: int = 50,
     study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None,
     stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None,
@@ -485,6 +519,7 @@ def get_time_series_dense_encoder_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: The maximum count of replicas the batch prediction job can scale to.
     evaluation_dataflow_machine_type: Machine type for the dataflow job in evaluation, such as 'n1-standard-16'.
     evaluation_dataflow_max_num_workers: Maximum number of dataflow workers.
+    evaluation_dataflow_starting_num_workers: Starting number of dataflow workers.
     evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow.
     study_spec_parameters_override: The list for overriding study spec.
     stage_1_tuner_worker_pool_specs_override: The dictionary for overriding stage 1 tuner worker pool spec.
@@ -548,6 +583,7 @@ def get_time_series_dense_encoder_forecasting_pipeline_and_parameters(
       evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count,
       evaluation_dataflow_machine_type=evaluation_dataflow_machine_type,
       evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers,
+      evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers,
       evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb,
       study_spec_parameters_override=study_spec_parameters_override,
       stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override,
@@ -616,6 +652,7 @@ def get_temporal_fusion_transformer_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: int = 25,
     evaluation_dataflow_machine_type: str = 'n1-standard-16',
     evaluation_dataflow_max_num_workers: int = 25,
+    evaluation_dataflow_starting_num_workers: int = 22,
     evaluation_dataflow_disk_size_gb: int = 50,
     study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None,
     stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None,
@@ -671,6 +708,7 @@ def get_temporal_fusion_transformer_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: The maximum count of replicas the batch prediction job can scale to.
     evaluation_dataflow_machine_type: Machine type for the dataflow job in evaluation, such as 'n1-standard-16'.
     evaluation_dataflow_max_num_workers: Maximum number of dataflow workers.
+    evaluation_dataflow_starting_num_workers: Starting number of dataflow workers.
     evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow.
     study_spec_parameters_override: The list for overriding study spec.
     stage_1_tuner_worker_pool_specs_override: The dictionary for overriding stage 1 tuner worker pool spec.
@@ -731,6 +769,7 @@ def get_temporal_fusion_transformer_forecasting_pipeline_and_parameters(
       evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count,
       evaluation_dataflow_machine_type=evaluation_dataflow_machine_type,
       evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers,
+      evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers,
       evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb,
       study_spec_parameters_override=study_spec_parameters_override,
       stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override,
@@ -795,6 +834,7 @@ def get_sequence_to_sequence_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: int = 25,
     evaluation_dataflow_machine_type: str = 'n1-standard-16',
     evaluation_dataflow_max_num_workers: int = 25,
+    evaluation_dataflow_starting_num_workers: int = 22,
     evaluation_dataflow_disk_size_gb: int = 50,
     study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None,
     stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None,
@@ -851,6 +891,7 @@ def get_sequence_to_sequence_forecasting_pipeline_and_parameters(
     evaluation_batch_predict_max_replica_count: The maximum count of replicas the batch prediction job can scale to.
     evaluation_dataflow_machine_type: Machine type for the dataflow job in evaluation, such as 'n1-standard-16'.
     evaluation_dataflow_max_num_workers: Maximum number of dataflow workers.
+    evaluation_dataflow_starting_num_workers: Starting number of dataflow workers.
     evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow.
     study_spec_parameters_override: The list for overriding study spec.
     stage_1_tuner_worker_pool_specs_override: The dictionary for overriding stage 1 tuner worker pool spec.
@@ -908,6 +949,7 @@ def get_sequence_to_sequence_forecasting_pipeline_and_parameters(
       evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count,
       evaluation_dataflow_machine_type=evaluation_dataflow_machine_type,
       evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers,
+      evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers,
       evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb,
       study_spec_parameters_override=study_spec_parameters_override,
       stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override,

google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py

@@ -65,7 +65,7 @@ def automated_feature_engineering(
                       ' 1, "machine_spec": {"machine_type": "n1-standard-16"},'
                       ' "container_spec": {"image_uri":"'
                   ),
-                  'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240419_0625',
+                  'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20240808_0625',
                   '", "args": ["feature_engineering", "--project=', project,
                   '", "--location=', location, '", "--data_source_bigquery_table_path=',
                   data_source_bigquery_table_path,