google-cloud-pipeline-components 2.14.1__py3-none-any.whl → 2.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of google-cloud-pipeline-components might be problematic. Click here for more details.

Files changed (88) hide show
  1. google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py +1 -1
  2. google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py +24 -0
  3. google_cloud_pipeline_components/_implementation/starry_net/__init__.py +41 -0
  4. google_cloud_pipeline_components/_implementation/{model_evaluation/import_evaluation → starry_net/dataprep}/__init__.py +1 -2
  5. google_cloud_pipeline_components/_implementation/starry_net/dataprep/component.py +173 -0
  6. google_cloud_pipeline_components/_implementation/starry_net/evaluation/__init__.py +13 -0
  7. google_cloud_pipeline_components/_implementation/starry_net/evaluation/component.py +23 -0
  8. google_cloud_pipeline_components/_implementation/starry_net/evaluation/evaluation.yaml +197 -0
  9. google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/__init__.py +13 -0
  10. google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/component.py +62 -0
  11. google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/__init__.py +13 -0
  12. google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/component.py +77 -0
  13. google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/__init__.py +13 -0
  14. google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/component.py +97 -0
  15. google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/__init__.py +13 -0
  16. google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/component.py +76 -0
  17. google_cloud_pipeline_components/_implementation/starry_net/set_test_set/__init__.py +13 -0
  18. google_cloud_pipeline_components/_implementation/starry_net/set_test_set/component.py +48 -0
  19. google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/__init__.py +13 -0
  20. google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/component.py +70 -0
  21. google_cloud_pipeline_components/_implementation/starry_net/set_train_args/__init__.py +13 -0
  22. google_cloud_pipeline_components/_implementation/starry_net/set_train_args/component.py +90 -0
  23. google_cloud_pipeline_components/_implementation/starry_net/train/__init__.py +13 -0
  24. google_cloud_pipeline_components/_implementation/starry_net/train/component.py +220 -0
  25. google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/__init__.py +13 -0
  26. google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py +64 -0
  27. google_cloud_pipeline_components/_implementation/starry_net/upload_model/__init__.py +13 -0
  28. google_cloud_pipeline_components/_implementation/starry_net/upload_model/component.py +23 -0
  29. google_cloud_pipeline_components/_implementation/starry_net/upload_model/upload_model.yaml +37 -0
  30. google_cloud_pipeline_components/_implementation/starry_net/version.py +18 -0
  31. google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py +22 -0
  32. google_cloud_pipeline_components/container/utils/error_surfacing.py +45 -0
  33. google_cloud_pipeline_components/container/v1/model/get_model/remote_runner.py +36 -7
  34. google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py +1 -1
  35. google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py +2 -2
  36. google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py +2 -2
  37. google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml +38 -34
  38. google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml +38 -34
  39. google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml +38 -34
  40. google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml +38 -34
  41. google_cloud_pipeline_components/preview/automl/forecasting/utils.py +49 -7
  42. google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py +1 -1
  43. google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml +39 -39
  44. google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml +41 -41
  45. google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py +2 -2
  46. google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py +2 -2
  47. google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml +4 -4
  48. google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py +3 -3
  49. google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py +2 -2
  50. google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml +15 -15
  51. google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py +2 -2
  52. google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml +13 -13
  53. google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py +2 -2
  54. google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml +14 -14
  55. google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py +2 -2
  56. google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml +13 -13
  57. google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml +14 -14
  58. google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml +13 -13
  59. google_cloud_pipeline_components/preview/custom_job/utils.py +45 -6
  60. google_cloud_pipeline_components/preview/llm/rlhf/component.py +3 -6
  61. google_cloud_pipeline_components/preview/starry_net/__init__.py +19 -0
  62. google_cloud_pipeline_components/preview/starry_net/component.py +469 -0
  63. google_cloud_pipeline_components/proto/task_error_pb2.py +0 -1
  64. google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml +10 -10
  65. google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml +31 -31
  66. google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml +13 -13
  67. google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py +3 -3
  68. google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml +14 -14
  69. google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml +37 -37
  70. google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py +2 -2
  71. google_cloud_pipeline_components/v1/automl/tabular/ensemble.py +2 -2
  72. google_cloud_pipeline_components/v1/automl/tabular/finalizer.py +1 -1
  73. google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py +1 -1
  74. google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py +1 -1
  75. google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py +2 -2
  76. google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py +2 -2
  77. google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py +1 -1
  78. google_cloud_pipeline_components/v1/automl/tabular/transform.py +2 -2
  79. google_cloud_pipeline_components/v1/custom_job/component.py +3 -0
  80. google_cloud_pipeline_components/v1/custom_job/utils.py +4 -0
  81. google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +21 -0
  82. google_cloud_pipeline_components/version.py +1 -1
  83. {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/METADATA +17 -20
  84. {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/RECORD +87 -58
  85. {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/WHEEL +1 -1
  86. google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py +0 -208
  87. {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/LICENSE +0 -0
  88. {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,220 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Container Component for training STARRY-Net."""
15
+
16
+ from google_cloud_pipeline_components import _placeholders
17
+ from google_cloud_pipeline_components import utils
18
+ from google_cloud_pipeline_components._implementation.starry_net import version
19
+
20
+ from kfp import dsl
21
+
22
+
23
+ @dsl.container_component
24
+ def train(
25
+ gcp_resources: dsl.OutputPath(str),
26
+ trainer_dir: dsl.Output[dsl.Artifact], # pytype: disable=unsupported-operands
27
+ num_epochs: int,
28
+ backcast_length: int,
29
+ forecast_length: int,
30
+ train_end_date: str,
31
+ csv_data_path: str,
32
+ csv_static_covariates_path: str,
33
+ static_covariates_vocab_path: str,
34
+ train_tf_record_patterns: str,
35
+ val_tf_record_patterns: str,
36
+ test_tf_record_patterns: str,
37
+ n_decomposition_plots: int,
38
+ n_val_windows: int,
39
+ n_test_windows: int,
40
+ test_set_stride: int,
41
+ nan_threshold: float,
42
+ zero_threshold: float,
43
+ cleaning_activation_regularizer_coeff: float,
44
+ change_point_activation_regularizer_coeff: float,
45
+ change_point_output_regularizer_coeff: float,
46
+ alpha_upper_bound: float,
47
+ beta_upper_bound: float,
48
+ phi_lower_bound: float,
49
+ b_fixed_val: int,
50
+ b0_fixed_val: int,
51
+ phi_fixed_val: int,
52
+ quantiles: str,
53
+ use_static_covariates: bool,
54
+ static_covariate_names: str,
55
+ model_blocks: str,
56
+ freeze_point_forecasts: bool,
57
+ machine_type: str,
58
+ accelerator_type: str,
59
+ docker_region: str,
60
+ location: str,
61
+ job_id: str,
62
+ project: str,
63
+ encryption_spec_key_name: str,
64
+ ):
65
+ # fmt: off
66
+ """Trains a STARRY-Net model.
67
+
68
+ Args:
69
+ gcp_resources: Serialized JSON of ``gcp_resources`` which tracks the
70
+ CustomJob.
71
+ trainer_dir: The gcp bucket path where training artifacts are saved.
72
+ num_epochs: The number of epochs to train for.
73
+ backcast_length: The length of the input window to feed into the model.
74
+ forecast_length: The length of the forecast horizon.
75
+ train_end_date: The last date of data to use in the training set. All
76
+ subsequent dates are part of the test set.
77
+ csv_data_path: The path to the training data csv.
78
+ csv_static_covariates_path: The path to the static covariates csv.
79
+ static_covariates_vocab_path: The path to the master static covariates vocab
80
+ json.
81
+ train_tf_record_patterns: The glob patterns to the tf records to use for
82
+ training.
83
+ val_tf_record_patterns: The glob patterns to the tf records to use for
84
+ validation.
85
+ test_tf_record_patterns: The glob patterns to the tf records to use for
86
+ testing.
87
+ n_decomposition_plots: How many decomposition plots to save to tensorboard.
88
+ n_val_windows: The number of windows to use for the val set. If 0, no
89
+ validation set is used.
90
+ n_test_windows: The number of windows to use for the test set. Must be >= 1.
91
+ test_set_stride: The number of timestamps to roll forward when
92
+ constructing the val and test sets.
93
+ nan_threshold: Series having more nan / missing values than
94
+ nan_threshold (inclusive) in percentage for either backtest or forecast
95
+ will not be sampled in the training set (including missing due to
96
+ train_start and train_end). All existing nans are replaced by zeros.
97
+ zero_threshold: Series having more 0.0 values than zero_threshold
98
+ (inclusive) in percentage for either backtest or forecast will not be
99
+ sampled in the training set.
100
+ cleaning_activation_regularizer_coeff: The regularization coefficient for
101
+ the cleaning param estimator's final layer's activation in the cleaning
102
+ block.
103
+ change_point_activation_regularizer_coeff: The regularization coefficient
104
+ for the change point param estimator's final layer's activation in the
105
+ change_point block.
106
+ change_point_output_regularizer_coeff: The regularization coefficient
107
+ for the change point param estimator's output in the change_point block.
108
+ alpha_upper_bound: The upper bound for data smooth parameter alpha in the
109
+ trend block.
110
+ beta_upper_bound: The upper bound for data smooth parameter beta in the
111
+ trend block.
112
+ phi_lower_bound: The lower bound for damping param phi in the trend block.
113
+ b_fixed_val: The fixed value for b in the trend block. If set to anything
114
+ other than -1, the trend block will not learn to provide estimates
115
+ but use the fixed value directly.
116
+ b0_fixed_val: The fixed value for b0 in the trend block. If set to
117
+ anything other than -1, the trend block will not learn to provide
118
+ estimates but use the fixed value directly.
119
+ phi_fixed_val: The fixed value for phi in the trend block. If set to
120
+ anything other than -1, the trend block will not learn to provide
121
+ estimates but use the fixed value directly.
122
+ quantiles: The stringified tuple of quantiles to learn in the quantile
123
+ block, e.g., 0.5,0.9,0.95. This should always start with 0.5,
124
+ representing the point forecasts.
125
+ use_static_covariates: Whether to use static covariates.
126
+ static_covariate_names: The stringified tuple of names of the static
127
+ covariates.
128
+ model_blocks: The stringified tuple of blocks to use in the order
129
+ that they appear in the model. Possible values are `cleaning`,
130
+ `change_point`, `trend`, `hour_of_week-hybrid`, `day_of_week-hybrid`,
131
+ `day_of_year-hybrid`, `week_of_year-hybrid`, `month_of_year-hybrid`,
132
+ `residual`, `quantile`.
133
+ freeze_point_forecasts: Whether or not to do two pass training, where
134
+ first the point forecast model is trained, then the quantile block is,
135
+ added, all preceding blocks are frozen, and the quantile block is trained.
136
+ This should always be True if quantiles != [0.5].
137
+ machine_type: The machine type.
138
+ accelerator_type: The accelerator type.
139
+ docker_region: The docker region, used to determine which image to use.
140
+ location: Location for creating the custom training job. If not set,
141
+ defaults to us-central1.
142
+ job_id: The pipeline job id.
143
+ project: Project to create the custom training job in. Defaults to
144
+ the project in which the PipelineJob is run.
145
+ encryption_spec_key_name: Customer-managed encryption key options for the
146
+ CustomJob. If this is set, then all resources created by the CustomJob
147
+ will be encrypted with the provided encryption key.
148
+
149
+ Returns:
150
+ gcp_resources: Serialized JSON of ``gcp_resources`` which tracks the
151
+ CustomJob.
152
+ trainer_dir: The gcp bucket path where training artifacts are saved.
153
+ """
154
+ job_name = f'trainer-{job_id}'
155
+ payload = {
156
+ 'display_name': job_name,
157
+ 'encryption_spec': {
158
+ 'kms_key_name': str(encryption_spec_key_name),
159
+ },
160
+ 'job_spec': {
161
+ 'worker_pool_specs': [{
162
+ 'replica_count': '1',
163
+ 'machine_spec': {
164
+ 'machine_type': str(machine_type),
165
+ 'accelerator_type': str(accelerator_type),
166
+ 'accelerator_count': 1,
167
+ },
168
+ 'disk_spec': {
169
+ 'boot_disk_type': 'pd-ssd',
170
+ 'boot_disk_size_gb': 100,
171
+ },
172
+ 'container_spec': {
173
+ 'image_uri': f'{docker_region}-docker.pkg.dev/vertex-ai-restricted/starryn/trainer:{version.TRAINER_VERSION}',
174
+ 'args': [
175
+ f'--vertex_experiment_dir={trainer_dir.path}',
176
+ f'--vertex_job_id={job_id}',
177
+ '--config=analysis/trafficforecast/starryn/experiments/configs/vertex.py',
178
+ f'--config.num_epochs={num_epochs}',
179
+ f'--config.freeze_point_forecasts={freeze_point_forecasts}',
180
+ f'--config.callbacks.tensorboard.n_decomposition_plots={n_decomposition_plots}',
181
+ f'--config.datasets.backcast_length={backcast_length}',
182
+ f'--config.datasets.forecast_length={forecast_length}',
183
+ f'--config.datasets.train_end_date={train_end_date}',
184
+ f'--config.datasets.train_path={csv_data_path}',
185
+ f'--config.datasets.static_covariates_path={csv_static_covariates_path}',
186
+ f'--config.datasets.static_covariates_vocab_path={static_covariates_vocab_path}',
187
+ f'--config.datasets.train_tf_record_patterns={train_tf_record_patterns}',
188
+ f'--config.datasets.val_tf_record_patterns={val_tf_record_patterns}',
189
+ f'--config.datasets.test_tf_record_patterns={test_tf_record_patterns}',
190
+ f'--config.datasets.n_val_windows={n_val_windows}',
191
+ f'--config.datasets.val_rolling_window_size={test_set_stride}',
192
+ f'--config.datasets.n_test_windows={n_test_windows}',
193
+ f'--config.datasets.test_rolling_window_size={test_set_stride}',
194
+ f'--config.datasets.nan_threshold={nan_threshold}',
195
+ f'--config.datasets.zero_threshold={zero_threshold}',
196
+ f'--config.model.regularizer_coeff={cleaning_activation_regularizer_coeff}',
197
+ f'--config.model.activation_regularizer_coeff={change_point_activation_regularizer_coeff}',
198
+ f'--config.model.output_regularizer_coeff={change_point_output_regularizer_coeff}',
199
+ f'--config.model.alpha_upper_bound={alpha_upper_bound}',
200
+ f'--config.model.beta_upper_bound={beta_upper_bound}',
201
+ f'--config.model.phi_lower_bound={phi_lower_bound}',
202
+ f'--config.model.b_fixed_val={b_fixed_val}',
203
+ f'--config.model.b0_fixed_val={b0_fixed_val}',
204
+ f'--config.model.phi_fixed_val={phi_fixed_val}',
205
+ f'--config.model.quantiles={quantiles}',
206
+ f'--config.model.use_static_covariates_trend={use_static_covariates}',
207
+ f'--config.model.use_static_covariates_calendar={use_static_covariates}',
208
+ f'--config.model.static_cov_names={static_covariate_names}',
209
+ f'--config.model.blocks_list={model_blocks}',
210
+ ],
211
+ },
212
+ }]
213
+ }
214
+ }
215
+ return utils.build_serverless_customjob_container_spec(
216
+ project=project,
217
+ location=location,
218
+ custom_job_payload=payload,
219
+ gcp_resources=gcp_resources,
220
+ )
@@ -0,0 +1,13 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
@@ -0,0 +1,64 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Starry Net upload decomposition plots component."""
15
+
16
+ from kfp import dsl
17
+
18
+
19
+ @dsl.component(
20
+ packages_to_install=[
21
+ 'google-cloud-aiplatform[tensorboard]',
22
+ 'protobuf==3.20.*',
23
+ ]
24
+ )
25
+ def upload_decomposition_plots(
26
+ project: str,
27
+ location: str,
28
+ tensorboard_id: str,
29
+ display_name: str,
30
+ trainer_dir: dsl.InputPath(),
31
+ ) -> dsl.Artifact:
32
+ # fmt: off
33
+ """Uploads decomposition plots to Tensorboard.
34
+
35
+ Args:
36
+ project: The project where the pipeline is run. Defaults to current project.
37
+ location: The location where the pipeline components are run.
38
+ tensorboard_id: The tensorboard instance ID.
39
+ display_name: The diplay name of the job.
40
+ trainer_dir: The directory where training artifacts where stored.
41
+
42
+ Returns:
43
+ A dsl.Artifact where the URI is the URI where decomposition plots can be
44
+ viewed.
45
+ """
46
+ import os # pylint: disable=g-import-not-at-top
47
+ from google.cloud import aiplatform # pylint: disable=g-import-not-at-top
48
+
49
+ log_dir = os.path.join(trainer_dir, 'tensorboard', 'r=1:gc=0')
50
+ project_number = os.environ['CLOUD_ML_PROJECT_ID']
51
+ aiplatform.init(project=project, location=location)
52
+ aiplatform.upload_tb_log(
53
+ tensorboard_id=tensorboard_id,
54
+ tensorboard_experiment_name=display_name,
55
+ logdir=log_dir,
56
+ experiment_display_name=display_name,
57
+ description=f'Tensorboard for {display_name}',
58
+ )
59
+ uri = (
60
+ f'https://{location}.tensorboard.googleusercontent.com/experiment/'
61
+ f'projects+{project_number}+locations+{location}+tensorboards+'
62
+ f'{tensorboard_id}+experiments+{display_name}/#images'
63
+ )
64
+ return dsl.Artifact(uri=uri)
@@ -0,0 +1,13 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
@@ -0,0 +1,23 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Starry Net Upload Model Component."""
15
+
16
+ import os
17
+
18
+ from kfp import components
19
+
20
+ # TODO(b/346580764)
21
+ upload_model = components.load_component_from_file(
22
+ os.path.join(os.path.dirname(__file__), 'upload_model.yaml')
23
+ )
@@ -0,0 +1,37 @@
1
+ name: model_upload
2
+ inputs:
3
+ - {name: project, type: String}
4
+ - {name: location, type: String, default: "us-central1"}
5
+ - {name: display_name, type: String}
6
+ - {name: description, type: String, optional: true, default: ''}
7
+ - {name: unmanaged_container_model, type: google.UnmanagedContainerModel, optional: true}
8
+ - {name: encryption_spec_key_name, type: String, optional: true, default: ''}
9
+ - {name: labels, type: JsonObject, optional: true, default: '{}'}
10
+ - {name: parent_model, type: google.VertexModel, optional: true}
11
+ outputs:
12
+ - {name: model, type: google.VertexModel}
13
+ - {name: gcp_resources, type: String}
14
+ implementation:
15
+ container:
16
+ image: gcr.io/ml-pipeline/automl-tables-private:1.0.17
17
+ command: [python3, -u, -m, launcher]
18
+ args: [
19
+ --type, UploadModel,
20
+ --payload,
21
+ concat: [
22
+ '{',
23
+ '"display_name": "', {inputValue: display_name}, '"',
24
+ ', "description": "', {inputValue: description}, '"',
25
+ ', "encryption_spec": {"kms_key_name":"', {inputValue: encryption_spec_key_name}, '"}',
26
+ ', "labels": ', {inputValue: labels},
27
+ '}'
28
+ ],
29
+ --project, {inputValue: project},
30
+ --location, {inputValue: location},
31
+ --gcp_resources, {outputPath: gcp_resources},
32
+ --executor_input, "{{$}}",
33
+ {if: {
34
+ cond: {isPresent: parent_model},
35
+ then: ["--parent_model_name", "{{$.inputs.artifacts['parent_model'].metadata['resourceName']}}",]
36
+ }},
37
+ ]
@@ -0,0 +1,18 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Version constants for starry net components."""
15
+
16
+ DATAPREP_VERSION = '20240722_2225_RC00'
17
+ PREDICTOR_VERSION = '20240723_0542_RC00'
18
+ TRAINER_VERSION = '20240723_0542_RC00'
@@ -32,6 +32,27 @@ def insert_system_labels_into_payload(payload):
32
32
  return json.dumps(job_spec)
33
33
 
34
34
 
35
+ def cast_accelerator_count_to_int(payload):
36
+ """Casts accelerator_count from string to an int."""
37
+
38
+ job_spec = json.loads(payload)
39
+ # TODO(b/353577594): accelerator_count placeholder is not resolved to int.
40
+ # Need to typecast to int to avoid type mismatch error. Can remove when fix
41
+ # placeholder resolution.
42
+ if (
43
+ 'accelerator_count'
44
+ in job_spec['job_spec']['worker_pool_specs'][0]['machine_spec']
45
+ ):
46
+ job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'][
47
+ 'accelerator_count'
48
+ ] = int(
49
+ job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'][
50
+ 'accelerator_count'
51
+ ]
52
+ )
53
+ return json.dumps(job_spec)
54
+
55
+
35
56
  def create_custom_job_with_client(job_client, parent, job_spec):
36
57
  create_custom_job_fn = None
37
58
  try:
@@ -86,6 +107,7 @@ def create_custom_job(
86
107
  # Create custom job if it does not exist
87
108
  job_name = remote_runner.check_if_job_exists()
88
109
  if job_name is None:
110
+ payload = cast_accelerator_count_to_int(payload)
89
111
  job_name = remote_runner.create_job(
90
112
  create_custom_job_with_client,
91
113
  insert_system_labels_into_payload(payload),
@@ -0,0 +1,45 @@
1
+ # Copyright 2024 The Kubeflow Authors. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Utilities for surface user defined error messages."""
16
+
17
+ import json
18
+ import os
19
+ from google.protobuf import json_format
20
+ from google_cloud_pipeline_components.proto import task_error_pb2
21
+
22
+
23
+ def write_customized_error(
24
+ executor_input: str, error: task_error_pb2.TaskError
25
+ ):
26
+ """Writes a TaskError customized by the author of the pipelines to a JSON file ('executor_error.json') in the output directory specified in the executor input.
27
+
28
+ Args:
29
+ executor_input: JSON string containing executor input data.
30
+ error: TaskError protocol buffer message.
31
+ """
32
+ executor_input_json = json.loads(executor_input)
33
+ os.makedirs(
34
+ os.path.dirname(executor_input_json['outputs']['outputFile']),
35
+ exist_ok=True,
36
+ )
37
+ executor_out_path = executor_input_json['outputs']['outputFile']
38
+ directory_path = os.path.dirname(executor_out_path)
39
+ executor_error_path = os.path.join(directory_path, 'executor_error.json')
40
+ error_dict = json_format.MessageToDict(error)
41
+ with open(
42
+ executor_error_path,
43
+ 'w',
44
+ ) as f:
45
+ json.dump(error_dict, f)
@@ -12,13 +12,33 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  """Remote runner for Get Model based on the Vertex AI SDK."""
15
-
15
+ import contextlib
16
+ from typing import Tuple, Type, Union
16
17
  from google.api_core.client_options import ClientOptions
17
18
  from google.cloud import aiplatform_v1 as aip_v1
18
19
  from google_cloud_pipeline_components.container.utils import artifact_utils
20
+ from google_cloud_pipeline_components.container.utils import error_surfacing
21
+ from google_cloud_pipeline_components.proto import task_error_pb2
19
22
  from google_cloud_pipeline_components.types import artifact_types
20
23
 
21
24
 
25
+ @contextlib.contextmanager
26
+ def catch_write_and_raise(
27
+ executor_input: str,
28
+ exception_types: Union[
29
+ Type[Exception], Tuple[Type[Exception], ...]
30
+ ] = Exception,
31
+ ):
32
+ """Context manager to catch specified exceptions, log them using error_surfacing, and then re-raise."""
33
+ try:
34
+ yield
35
+ except exception_types as e:
36
+ task_error = task_error_pb2.TaskError()
37
+ task_error.error_message = str(e)
38
+ error_surfacing.write_customized_error(executor_input, task_error)
39
+ raise
40
+
41
+
22
42
  def get_model(
23
43
  executor_input,
24
44
  model_name: str,
@@ -26,11 +46,16 @@ def get_model(
26
46
  location: str,
27
47
  ) -> None:
28
48
  """Get model."""
29
- if not location or not project:
30
- raise ValueError(
31
- 'Model resource name must be in the format'
32
- ' projects/{project}/locations/{location}/models/{model_name}'
33
- )
49
+ with catch_write_and_raise(
50
+ executor_input=executor_input,
51
+ exception_types=ValueError,
52
+ ):
53
+ if not location or not project:
54
+ model_name_error_message = (
55
+ 'Model resource name must be in the format'
56
+ ' projects/{project}/locations/{location}/models/{model_name}'
57
+ )
58
+ raise ValueError(model_name_error_message)
34
59
  api_endpoint = location + '-aiplatform.googleapis.com'
35
60
  vertex_uri_prefix = f'https://{api_endpoint}/v1/'
36
61
  model_resource_name = (
@@ -40,7 +65,11 @@ def get_model(
40
65
  client_options = ClientOptions(api_endpoint=api_endpoint)
41
66
  client = aip_v1.ModelServiceClient(client_options=client_options)
42
67
  request = aip_v1.GetModelRequest(name=model_resource_name)
43
- get_model_response = client.get_model(request)
68
+ with catch_write_and_raise(
69
+ executor_input=executor_input,
70
+ exception_types=Exception,
71
+ ):
72
+ get_model_response = client.get_model(request)
44
73
  resp_model_name_without_version = get_model_response.name.split('@', 1)[0]
45
74
  model_resource_name = (
46
75
  f'{resp_model_name_without_version}@{get_model_response.version_id}'
@@ -72,7 +72,7 @@ def automl_forecasting_ensemble(
72
72
  # fmt: on
73
73
  job_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER
74
74
  task_id = dsl.PIPELINE_TASK_ID_PLACEHOLDER
75
- image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625'
75
+ image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625'
76
76
  display_name = f'automl-forecasting-ensemble-{job_id}-{task_id}'
77
77
 
78
78
  error_file_path = f'{root_dir}/{job_id}/{task_id}/error.pb'
@@ -99,14 +99,14 @@ def automl_forecasting_stage_1_tuner(
99
99
  ' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
100
100
  ' "container_spec": {"image_uri":"'
101
101
  ),
102
- 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625',
102
+ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
103
103
  '", "args": ["forecasting_mp_l2l_stage_1_tuner',
104
104
  '", "--region=',
105
105
  location,
106
106
  '", "--transform_output_path=',
107
107
  transform_output.uri,
108
108
  '", "--training_docker_uri=',
109
- 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625',
109
+ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
110
110
  '", "--reduce_search_space_mode=',
111
111
  reduce_search_space_mode,
112
112
  f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
@@ -97,14 +97,14 @@ def automl_forecasting_stage_2_tuner(
97
97
  ' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
98
98
  ' "container_spec": {"image_uri":"'
99
99
  ),
100
- 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625',
100
+ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
101
101
  '", "args": ["forecasting_mp_l2l_stage_2_tuner',
102
102
  '", "--region=',
103
103
  location,
104
104
  '", "--transform_output_path=',
105
105
  transform_output.uri,
106
106
  '", "--training_docker_uri=',
107
- 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240419_0625',
107
+ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
108
108
  f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
109
109
  '", "--training_base_dir=',
110
110
  root_dir,