google-cloud-pipeline-components 2.14.1__py3-none-any.whl → 2.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of google-cloud-pipeline-components might be problematic. Click here for more details.
- google_cloud_pipeline_components/_implementation/llm/generated/refined_image_versions.py +1 -1
- google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation_preprocessor/component.py +24 -0
- google_cloud_pipeline_components/_implementation/starry_net/__init__.py +41 -0
- google_cloud_pipeline_components/_implementation/{model_evaluation/import_evaluation → starry_net/dataprep}/__init__.py +1 -2
- google_cloud_pipeline_components/_implementation/starry_net/dataprep/component.py +173 -0
- google_cloud_pipeline_components/_implementation/starry_net/evaluation/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/evaluation/component.py +23 -0
- google_cloud_pipeline_components/_implementation/starry_net/evaluation/evaluation.yaml +197 -0
- google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/get_training_artifacts/component.py +62 -0
- google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/maybe_set_tfrecord_args/component.py +77 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_dataprep_args/component.py +97 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_eval_args/component.py +76 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_test_set/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_test_set/component.py +48 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_tfrecord_args/component.py +70 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_train_args/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/set_train_args/component.py +90 -0
- google_cloud_pipeline_components/_implementation/starry_net/train/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/train/component.py +220 -0
- google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py +64 -0
- google_cloud_pipeline_components/_implementation/starry_net/upload_model/__init__.py +13 -0
- google_cloud_pipeline_components/_implementation/starry_net/upload_model/component.py +23 -0
- google_cloud_pipeline_components/_implementation/starry_net/upload_model/upload_model.yaml +37 -0
- google_cloud_pipeline_components/_implementation/starry_net/version.py +18 -0
- google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py +22 -0
- google_cloud_pipeline_components/container/utils/error_surfacing.py +45 -0
- google_cloud_pipeline_components/container/v1/model/get_model/remote_runner.py +36 -7
- google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py +1 -1
- google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py +2 -2
- google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py +2 -2
- google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml +38 -34
- google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml +38 -34
- google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml +38 -34
- google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml +38 -34
- google_cloud_pipeline_components/preview/automl/forecasting/utils.py +49 -7
- google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py +1 -1
- google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml +39 -39
- google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml +41 -41
- google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py +2 -2
- google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py +2 -2
- google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml +4 -4
- google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py +3 -3
- google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py +2 -2
- google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml +15 -15
- google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py +2 -2
- google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml +13 -13
- google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py +2 -2
- google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml +14 -14
- google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py +2 -2
- google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml +13 -13
- google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml +14 -14
- google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml +13 -13
- google_cloud_pipeline_components/preview/custom_job/utils.py +45 -6
- google_cloud_pipeline_components/preview/llm/rlhf/component.py +3 -6
- google_cloud_pipeline_components/preview/starry_net/__init__.py +19 -0
- google_cloud_pipeline_components/preview/starry_net/component.py +469 -0
- google_cloud_pipeline_components/proto/task_error_pb2.py +0 -1
- google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml +10 -10
- google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml +31 -31
- google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml +13 -13
- google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py +3 -3
- google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml +14 -14
- google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml +37 -37
- google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py +2 -2
- google_cloud_pipeline_components/v1/automl/tabular/ensemble.py +2 -2
- google_cloud_pipeline_components/v1/automl/tabular/finalizer.py +1 -1
- google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py +1 -1
- google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py +1 -1
- google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py +2 -2
- google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py +2 -2
- google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py +1 -1
- google_cloud_pipeline_components/v1/automl/tabular/transform.py +2 -2
- google_cloud_pipeline_components/v1/custom_job/component.py +3 -0
- google_cloud_pipeline_components/v1/custom_job/utils.py +4 -0
- google_cloud_pipeline_components/v1/model_evaluation/evaluation_llm_text_generation_pipeline.py +21 -0
- google_cloud_pipeline_components/version.py +1 -1
- {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/METADATA +17 -20
- {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/RECORD +87 -58
- {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/WHEEL +1 -1
- google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py +0 -208
- {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/LICENSE +0 -0
- {google_cloud_pipeline_components-2.14.1.dist-info → google_cloud_pipeline_components-2.16.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Container Component for training STARRY-Net."""
|
|
15
|
+
|
|
16
|
+
from google_cloud_pipeline_components import _placeholders
|
|
17
|
+
from google_cloud_pipeline_components import utils
|
|
18
|
+
from google_cloud_pipeline_components._implementation.starry_net import version
|
|
19
|
+
|
|
20
|
+
from kfp import dsl
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dsl.container_component
|
|
24
|
+
def train(
|
|
25
|
+
gcp_resources: dsl.OutputPath(str),
|
|
26
|
+
trainer_dir: dsl.Output[dsl.Artifact], # pytype: disable=unsupported-operands
|
|
27
|
+
num_epochs: int,
|
|
28
|
+
backcast_length: int,
|
|
29
|
+
forecast_length: int,
|
|
30
|
+
train_end_date: str,
|
|
31
|
+
csv_data_path: str,
|
|
32
|
+
csv_static_covariates_path: str,
|
|
33
|
+
static_covariates_vocab_path: str,
|
|
34
|
+
train_tf_record_patterns: str,
|
|
35
|
+
val_tf_record_patterns: str,
|
|
36
|
+
test_tf_record_patterns: str,
|
|
37
|
+
n_decomposition_plots: int,
|
|
38
|
+
n_val_windows: int,
|
|
39
|
+
n_test_windows: int,
|
|
40
|
+
test_set_stride: int,
|
|
41
|
+
nan_threshold: float,
|
|
42
|
+
zero_threshold: float,
|
|
43
|
+
cleaning_activation_regularizer_coeff: float,
|
|
44
|
+
change_point_activation_regularizer_coeff: float,
|
|
45
|
+
change_point_output_regularizer_coeff: float,
|
|
46
|
+
alpha_upper_bound: float,
|
|
47
|
+
beta_upper_bound: float,
|
|
48
|
+
phi_lower_bound: float,
|
|
49
|
+
b_fixed_val: int,
|
|
50
|
+
b0_fixed_val: int,
|
|
51
|
+
phi_fixed_val: int,
|
|
52
|
+
quantiles: str,
|
|
53
|
+
use_static_covariates: bool,
|
|
54
|
+
static_covariate_names: str,
|
|
55
|
+
model_blocks: str,
|
|
56
|
+
freeze_point_forecasts: bool,
|
|
57
|
+
machine_type: str,
|
|
58
|
+
accelerator_type: str,
|
|
59
|
+
docker_region: str,
|
|
60
|
+
location: str,
|
|
61
|
+
job_id: str,
|
|
62
|
+
project: str,
|
|
63
|
+
encryption_spec_key_name: str,
|
|
64
|
+
):
|
|
65
|
+
# fmt: off
|
|
66
|
+
"""Trains a STARRY-Net model.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
gcp_resources: Serialized JSON of ``gcp_resources`` which tracks the
|
|
70
|
+
CustomJob.
|
|
71
|
+
trainer_dir: The gcp bucket path where training artifacts are saved.
|
|
72
|
+
num_epochs: The number of epochs to train for.
|
|
73
|
+
backcast_length: The length of the input window to feed into the model.
|
|
74
|
+
forecast_length: The length of the forecast horizon.
|
|
75
|
+
train_end_date: The last date of data to use in the training set. All
|
|
76
|
+
subsequent dates are part of the test set.
|
|
77
|
+
csv_data_path: The path to the training data csv.
|
|
78
|
+
csv_static_covariates_path: The path to the static covariates csv.
|
|
79
|
+
static_covariates_vocab_path: The path to the master static covariates vocab
|
|
80
|
+
json.
|
|
81
|
+
train_tf_record_patterns: The glob patterns to the tf records to use for
|
|
82
|
+
training.
|
|
83
|
+
val_tf_record_patterns: The glob patterns to the tf records to use for
|
|
84
|
+
validation.
|
|
85
|
+
test_tf_record_patterns: The glob patterns to the tf records to use for
|
|
86
|
+
testing.
|
|
87
|
+
n_decomposition_plots: How many decomposition plots to save to tensorboard.
|
|
88
|
+
n_val_windows: The number of windows to use for the val set. If 0, no
|
|
89
|
+
validation set is used.
|
|
90
|
+
n_test_windows: The number of windows to use for the test set. Must be >= 1.
|
|
91
|
+
test_set_stride: The number of timestamps to roll forward when
|
|
92
|
+
constructing the val and test sets.
|
|
93
|
+
nan_threshold: Series having more nan / missing values than
|
|
94
|
+
nan_threshold (inclusive) in percentage for either backtest or forecast
|
|
95
|
+
will not be sampled in the training set (including missing due to
|
|
96
|
+
train_start and train_end). All existing nans are replaced by zeros.
|
|
97
|
+
zero_threshold: Series having more 0.0 values than zero_threshold
|
|
98
|
+
(inclusive) in percentage for either backtest or forecast will not be
|
|
99
|
+
sampled in the training set.
|
|
100
|
+
cleaning_activation_regularizer_coeff: The regularization coefficient for
|
|
101
|
+
the cleaning param estimator's final layer's activation in the cleaning
|
|
102
|
+
block.
|
|
103
|
+
change_point_activation_regularizer_coeff: The regularization coefficient
|
|
104
|
+
for the change point param estimator's final layer's activation in the
|
|
105
|
+
change_point block.
|
|
106
|
+
change_point_output_regularizer_coeff: The regularization coefficient
|
|
107
|
+
for the change point param estimator's output in the change_point block.
|
|
108
|
+
alpha_upper_bound: The upper bound for data smooth parameter alpha in the
|
|
109
|
+
trend block.
|
|
110
|
+
beta_upper_bound: The upper bound for data smooth parameter beta in the
|
|
111
|
+
trend block.
|
|
112
|
+
phi_lower_bound: The lower bound for damping param phi in the trend block.
|
|
113
|
+
b_fixed_val: The fixed value for b in the trend block. If set to anything
|
|
114
|
+
other than -1, the trend block will not learn to provide estimates
|
|
115
|
+
but use the fixed value directly.
|
|
116
|
+
b0_fixed_val: The fixed value for b0 in the trend block. If set to
|
|
117
|
+
anything other than -1, the trend block will not learn to provide
|
|
118
|
+
estimates but use the fixed value directly.
|
|
119
|
+
phi_fixed_val: The fixed value for phi in the trend block. If set to
|
|
120
|
+
anything other than -1, the trend block will not learn to provide
|
|
121
|
+
estimates but use the fixed value directly.
|
|
122
|
+
quantiles: The stringified tuple of quantiles to learn in the quantile
|
|
123
|
+
block, e.g., 0.5,0.9,0.95. This should always start with 0.5,
|
|
124
|
+
representing the point forecasts.
|
|
125
|
+
use_static_covariates: Whether to use static covariates.
|
|
126
|
+
static_covariate_names: The stringified tuple of names of the static
|
|
127
|
+
covariates.
|
|
128
|
+
model_blocks: The stringified tuple of blocks to use in the order
|
|
129
|
+
that they appear in the model. Possible values are `cleaning`,
|
|
130
|
+
`change_point`, `trend`, `hour_of_week-hybrid`, `day_of_week-hybrid`,
|
|
131
|
+
`day_of_year-hybrid`, `week_of_year-hybrid`, `month_of_year-hybrid`,
|
|
132
|
+
`residual`, `quantile`.
|
|
133
|
+
freeze_point_forecasts: Whether or not to do two pass training, where
|
|
134
|
+
first the point forecast model is trained, then the quantile block is,
|
|
135
|
+
added, all preceding blocks are frozen, and the quantile block is trained.
|
|
136
|
+
This should always be True if quantiles != [0.5].
|
|
137
|
+
machine_type: The machine type.
|
|
138
|
+
accelerator_type: The accelerator type.
|
|
139
|
+
docker_region: The docker region, used to determine which image to use.
|
|
140
|
+
location: Location for creating the custom training job. If not set,
|
|
141
|
+
defaults to us-central1.
|
|
142
|
+
job_id: The pipeline job id.
|
|
143
|
+
project: Project to create the custom training job in. Defaults to
|
|
144
|
+
the project in which the PipelineJob is run.
|
|
145
|
+
encryption_spec_key_name: Customer-managed encryption key options for the
|
|
146
|
+
CustomJob. If this is set, then all resources created by the CustomJob
|
|
147
|
+
will be encrypted with the provided encryption key.
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
gcp_resources: Serialized JSON of ``gcp_resources`` which tracks the
|
|
151
|
+
CustomJob.
|
|
152
|
+
trainer_dir: The gcp bucket path where training artifacts are saved.
|
|
153
|
+
"""
|
|
154
|
+
job_name = f'trainer-{job_id}'
|
|
155
|
+
payload = {
|
|
156
|
+
'display_name': job_name,
|
|
157
|
+
'encryption_spec': {
|
|
158
|
+
'kms_key_name': str(encryption_spec_key_name),
|
|
159
|
+
},
|
|
160
|
+
'job_spec': {
|
|
161
|
+
'worker_pool_specs': [{
|
|
162
|
+
'replica_count': '1',
|
|
163
|
+
'machine_spec': {
|
|
164
|
+
'machine_type': str(machine_type),
|
|
165
|
+
'accelerator_type': str(accelerator_type),
|
|
166
|
+
'accelerator_count': 1,
|
|
167
|
+
},
|
|
168
|
+
'disk_spec': {
|
|
169
|
+
'boot_disk_type': 'pd-ssd',
|
|
170
|
+
'boot_disk_size_gb': 100,
|
|
171
|
+
},
|
|
172
|
+
'container_spec': {
|
|
173
|
+
'image_uri': f'{docker_region}-docker.pkg.dev/vertex-ai-restricted/starryn/trainer:{version.TRAINER_VERSION}',
|
|
174
|
+
'args': [
|
|
175
|
+
f'--vertex_experiment_dir={trainer_dir.path}',
|
|
176
|
+
f'--vertex_job_id={job_id}',
|
|
177
|
+
'--config=analysis/trafficforecast/starryn/experiments/configs/vertex.py',
|
|
178
|
+
f'--config.num_epochs={num_epochs}',
|
|
179
|
+
f'--config.freeze_point_forecasts={freeze_point_forecasts}',
|
|
180
|
+
f'--config.callbacks.tensorboard.n_decomposition_plots={n_decomposition_plots}',
|
|
181
|
+
f'--config.datasets.backcast_length={backcast_length}',
|
|
182
|
+
f'--config.datasets.forecast_length={forecast_length}',
|
|
183
|
+
f'--config.datasets.train_end_date={train_end_date}',
|
|
184
|
+
f'--config.datasets.train_path={csv_data_path}',
|
|
185
|
+
f'--config.datasets.static_covariates_path={csv_static_covariates_path}',
|
|
186
|
+
f'--config.datasets.static_covariates_vocab_path={static_covariates_vocab_path}',
|
|
187
|
+
f'--config.datasets.train_tf_record_patterns={train_tf_record_patterns}',
|
|
188
|
+
f'--config.datasets.val_tf_record_patterns={val_tf_record_patterns}',
|
|
189
|
+
f'--config.datasets.test_tf_record_patterns={test_tf_record_patterns}',
|
|
190
|
+
f'--config.datasets.n_val_windows={n_val_windows}',
|
|
191
|
+
f'--config.datasets.val_rolling_window_size={test_set_stride}',
|
|
192
|
+
f'--config.datasets.n_test_windows={n_test_windows}',
|
|
193
|
+
f'--config.datasets.test_rolling_window_size={test_set_stride}',
|
|
194
|
+
f'--config.datasets.nan_threshold={nan_threshold}',
|
|
195
|
+
f'--config.datasets.zero_threshold={zero_threshold}',
|
|
196
|
+
f'--config.model.regularizer_coeff={cleaning_activation_regularizer_coeff}',
|
|
197
|
+
f'--config.model.activation_regularizer_coeff={change_point_activation_regularizer_coeff}',
|
|
198
|
+
f'--config.model.output_regularizer_coeff={change_point_output_regularizer_coeff}',
|
|
199
|
+
f'--config.model.alpha_upper_bound={alpha_upper_bound}',
|
|
200
|
+
f'--config.model.beta_upper_bound={beta_upper_bound}',
|
|
201
|
+
f'--config.model.phi_lower_bound={phi_lower_bound}',
|
|
202
|
+
f'--config.model.b_fixed_val={b_fixed_val}',
|
|
203
|
+
f'--config.model.b0_fixed_val={b0_fixed_val}',
|
|
204
|
+
f'--config.model.phi_fixed_val={phi_fixed_val}',
|
|
205
|
+
f'--config.model.quantiles={quantiles}',
|
|
206
|
+
f'--config.model.use_static_covariates_trend={use_static_covariates}',
|
|
207
|
+
f'--config.model.use_static_covariates_calendar={use_static_covariates}',
|
|
208
|
+
f'--config.model.static_cov_names={static_covariate_names}',
|
|
209
|
+
f'--config.model.blocks_list={model_blocks}',
|
|
210
|
+
],
|
|
211
|
+
},
|
|
212
|
+
}]
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return utils.build_serverless_customjob_container_spec(
|
|
216
|
+
project=project,
|
|
217
|
+
location=location,
|
|
218
|
+
custom_job_payload=payload,
|
|
219
|
+
gcp_resources=gcp_resources,
|
|
220
|
+
)
|
google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
google_cloud_pipeline_components/_implementation/starry_net/upload_decomposition_plots/component.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Starry Net upload decomposition plots component."""
|
|
15
|
+
|
|
16
|
+
from kfp import dsl
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dsl.component(
|
|
20
|
+
packages_to_install=[
|
|
21
|
+
'google-cloud-aiplatform[tensorboard]',
|
|
22
|
+
'protobuf==3.20.*',
|
|
23
|
+
]
|
|
24
|
+
)
|
|
25
|
+
def upload_decomposition_plots(
|
|
26
|
+
project: str,
|
|
27
|
+
location: str,
|
|
28
|
+
tensorboard_id: str,
|
|
29
|
+
display_name: str,
|
|
30
|
+
trainer_dir: dsl.InputPath(),
|
|
31
|
+
) -> dsl.Artifact:
|
|
32
|
+
# fmt: off
|
|
33
|
+
"""Uploads decomposition plots to Tensorboard.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
project: The project where the pipeline is run. Defaults to current project.
|
|
37
|
+
location: The location where the pipeline components are run.
|
|
38
|
+
tensorboard_id: The tensorboard instance ID.
|
|
39
|
+
display_name: The diplay name of the job.
|
|
40
|
+
trainer_dir: The directory where training artifacts where stored.
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
A dsl.Artifact where the URI is the URI where decomposition plots can be
|
|
44
|
+
viewed.
|
|
45
|
+
"""
|
|
46
|
+
import os # pylint: disable=g-import-not-at-top
|
|
47
|
+
from google.cloud import aiplatform # pylint: disable=g-import-not-at-top
|
|
48
|
+
|
|
49
|
+
log_dir = os.path.join(trainer_dir, 'tensorboard', 'r=1:gc=0')
|
|
50
|
+
project_number = os.environ['CLOUD_ML_PROJECT_ID']
|
|
51
|
+
aiplatform.init(project=project, location=location)
|
|
52
|
+
aiplatform.upload_tb_log(
|
|
53
|
+
tensorboard_id=tensorboard_id,
|
|
54
|
+
tensorboard_experiment_name=display_name,
|
|
55
|
+
logdir=log_dir,
|
|
56
|
+
experiment_display_name=display_name,
|
|
57
|
+
description=f'Tensorboard for {display_name}',
|
|
58
|
+
)
|
|
59
|
+
uri = (
|
|
60
|
+
f'https://{location}.tensorboard.googleusercontent.com/experiment/'
|
|
61
|
+
f'projects+{project_number}+locations+{location}+tensorboards+'
|
|
62
|
+
f'{tensorboard_id}+experiments+{display_name}/#images'
|
|
63
|
+
)
|
|
64
|
+
return dsl.Artifact(uri=uri)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Starry Net Upload Model Component."""
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
|
|
18
|
+
from kfp import components
|
|
19
|
+
|
|
20
|
+
# TODO(b/346580764)
|
|
21
|
+
upload_model = components.load_component_from_file(
|
|
22
|
+
os.path.join(os.path.dirname(__file__), 'upload_model.yaml')
|
|
23
|
+
)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
name: model_upload
|
|
2
|
+
inputs:
|
|
3
|
+
- {name: project, type: String}
|
|
4
|
+
- {name: location, type: String, default: "us-central1"}
|
|
5
|
+
- {name: display_name, type: String}
|
|
6
|
+
- {name: description, type: String, optional: true, default: ''}
|
|
7
|
+
- {name: unmanaged_container_model, type: google.UnmanagedContainerModel, optional: true}
|
|
8
|
+
- {name: encryption_spec_key_name, type: String, optional: true, default: ''}
|
|
9
|
+
- {name: labels, type: JsonObject, optional: true, default: '{}'}
|
|
10
|
+
- {name: parent_model, type: google.VertexModel, optional: true}
|
|
11
|
+
outputs:
|
|
12
|
+
- {name: model, type: google.VertexModel}
|
|
13
|
+
- {name: gcp_resources, type: String}
|
|
14
|
+
implementation:
|
|
15
|
+
container:
|
|
16
|
+
image: gcr.io/ml-pipeline/automl-tables-private:1.0.17
|
|
17
|
+
command: [python3, -u, -m, launcher]
|
|
18
|
+
args: [
|
|
19
|
+
--type, UploadModel,
|
|
20
|
+
--payload,
|
|
21
|
+
concat: [
|
|
22
|
+
'{',
|
|
23
|
+
'"display_name": "', {inputValue: display_name}, '"',
|
|
24
|
+
', "description": "', {inputValue: description}, '"',
|
|
25
|
+
', "encryption_spec": {"kms_key_name":"', {inputValue: encryption_spec_key_name}, '"}',
|
|
26
|
+
', "labels": ', {inputValue: labels},
|
|
27
|
+
'}'
|
|
28
|
+
],
|
|
29
|
+
--project, {inputValue: project},
|
|
30
|
+
--location, {inputValue: location},
|
|
31
|
+
--gcp_resources, {outputPath: gcp_resources},
|
|
32
|
+
--executor_input, "{{$}}",
|
|
33
|
+
{if: {
|
|
34
|
+
cond: {isPresent: parent_model},
|
|
35
|
+
then: ["--parent_model_name", "{{$.inputs.artifacts['parent_model'].metadata['resourceName']}}",]
|
|
36
|
+
}},
|
|
37
|
+
]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Version constants for starry net components."""
|
|
15
|
+
|
|
16
|
+
DATAPREP_VERSION = '20240722_2225_RC00'
|
|
17
|
+
PREDICTOR_VERSION = '20240723_0542_RC00'
|
|
18
|
+
TRAINER_VERSION = '20240723_0542_RC00'
|
|
@@ -32,6 +32,27 @@ def insert_system_labels_into_payload(payload):
|
|
|
32
32
|
return json.dumps(job_spec)
|
|
33
33
|
|
|
34
34
|
|
|
35
|
+
def cast_accelerator_count_to_int(payload):
|
|
36
|
+
"""Casts accelerator_count from string to an int."""
|
|
37
|
+
|
|
38
|
+
job_spec = json.loads(payload)
|
|
39
|
+
# TODO(b/353577594): accelerator_count placeholder is not resolved to int.
|
|
40
|
+
# Need to typecast to int to avoid type mismatch error. Can remove when fix
|
|
41
|
+
# placeholder resolution.
|
|
42
|
+
if (
|
|
43
|
+
'accelerator_count'
|
|
44
|
+
in job_spec['job_spec']['worker_pool_specs'][0]['machine_spec']
|
|
45
|
+
):
|
|
46
|
+
job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'][
|
|
47
|
+
'accelerator_count'
|
|
48
|
+
] = int(
|
|
49
|
+
job_spec['job_spec']['worker_pool_specs'][0]['machine_spec'][
|
|
50
|
+
'accelerator_count'
|
|
51
|
+
]
|
|
52
|
+
)
|
|
53
|
+
return json.dumps(job_spec)
|
|
54
|
+
|
|
55
|
+
|
|
35
56
|
def create_custom_job_with_client(job_client, parent, job_spec):
|
|
36
57
|
create_custom_job_fn = None
|
|
37
58
|
try:
|
|
@@ -86,6 +107,7 @@ def create_custom_job(
|
|
|
86
107
|
# Create custom job if it does not exist
|
|
87
108
|
job_name = remote_runner.check_if_job_exists()
|
|
88
109
|
if job_name is None:
|
|
110
|
+
payload = cast_accelerator_count_to_int(payload)
|
|
89
111
|
job_name = remote_runner.create_job(
|
|
90
112
|
create_custom_job_with_client,
|
|
91
113
|
insert_system_labels_into_payload(payload),
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright 2024 The Kubeflow Authors. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Utilities for surface user defined error messages."""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from google.protobuf import json_format
|
|
20
|
+
from google_cloud_pipeline_components.proto import task_error_pb2
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def write_customized_error(
|
|
24
|
+
executor_input: str, error: task_error_pb2.TaskError
|
|
25
|
+
):
|
|
26
|
+
"""Writes a TaskError customized by the author of the pipelines to a JSON file ('executor_error.json') in the output directory specified in the executor input.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
executor_input: JSON string containing executor input data.
|
|
30
|
+
error: TaskError protocol buffer message.
|
|
31
|
+
"""
|
|
32
|
+
executor_input_json = json.loads(executor_input)
|
|
33
|
+
os.makedirs(
|
|
34
|
+
os.path.dirname(executor_input_json['outputs']['outputFile']),
|
|
35
|
+
exist_ok=True,
|
|
36
|
+
)
|
|
37
|
+
executor_out_path = executor_input_json['outputs']['outputFile']
|
|
38
|
+
directory_path = os.path.dirname(executor_out_path)
|
|
39
|
+
executor_error_path = os.path.join(directory_path, 'executor_error.json')
|
|
40
|
+
error_dict = json_format.MessageToDict(error)
|
|
41
|
+
with open(
|
|
42
|
+
executor_error_path,
|
|
43
|
+
'w',
|
|
44
|
+
) as f:
|
|
45
|
+
json.dump(error_dict, f)
|
|
@@ -12,13 +12,33 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
"""Remote runner for Get Model based on the Vertex AI SDK."""
|
|
15
|
-
|
|
15
|
+
import contextlib
|
|
16
|
+
from typing import Tuple, Type, Union
|
|
16
17
|
from google.api_core.client_options import ClientOptions
|
|
17
18
|
from google.cloud import aiplatform_v1 as aip_v1
|
|
18
19
|
from google_cloud_pipeline_components.container.utils import artifact_utils
|
|
20
|
+
from google_cloud_pipeline_components.container.utils import error_surfacing
|
|
21
|
+
from google_cloud_pipeline_components.proto import task_error_pb2
|
|
19
22
|
from google_cloud_pipeline_components.types import artifact_types
|
|
20
23
|
|
|
21
24
|
|
|
25
|
+
@contextlib.contextmanager
|
|
26
|
+
def catch_write_and_raise(
|
|
27
|
+
executor_input: str,
|
|
28
|
+
exception_types: Union[
|
|
29
|
+
Type[Exception], Tuple[Type[Exception], ...]
|
|
30
|
+
] = Exception,
|
|
31
|
+
):
|
|
32
|
+
"""Context manager to catch specified exceptions, log them using error_surfacing, and then re-raise."""
|
|
33
|
+
try:
|
|
34
|
+
yield
|
|
35
|
+
except exception_types as e:
|
|
36
|
+
task_error = task_error_pb2.TaskError()
|
|
37
|
+
task_error.error_message = str(e)
|
|
38
|
+
error_surfacing.write_customized_error(executor_input, task_error)
|
|
39
|
+
raise
|
|
40
|
+
|
|
41
|
+
|
|
22
42
|
def get_model(
|
|
23
43
|
executor_input,
|
|
24
44
|
model_name: str,
|
|
@@ -26,11 +46,16 @@ def get_model(
|
|
|
26
46
|
location: str,
|
|
27
47
|
) -> None:
|
|
28
48
|
"""Get model."""
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
49
|
+
with catch_write_and_raise(
|
|
50
|
+
executor_input=executor_input,
|
|
51
|
+
exception_types=ValueError,
|
|
52
|
+
):
|
|
53
|
+
if not location or not project:
|
|
54
|
+
model_name_error_message = (
|
|
55
|
+
'Model resource name must be in the format'
|
|
56
|
+
' projects/{project}/locations/{location}/models/{model_name}'
|
|
57
|
+
)
|
|
58
|
+
raise ValueError(model_name_error_message)
|
|
34
59
|
api_endpoint = location + '-aiplatform.googleapis.com'
|
|
35
60
|
vertex_uri_prefix = f'https://{api_endpoint}/v1/'
|
|
36
61
|
model_resource_name = (
|
|
@@ -40,7 +65,11 @@ def get_model(
|
|
|
40
65
|
client_options = ClientOptions(api_endpoint=api_endpoint)
|
|
41
66
|
client = aip_v1.ModelServiceClient(client_options=client_options)
|
|
42
67
|
request = aip_v1.GetModelRequest(name=model_resource_name)
|
|
43
|
-
|
|
68
|
+
with catch_write_and_raise(
|
|
69
|
+
executor_input=executor_input,
|
|
70
|
+
exception_types=Exception,
|
|
71
|
+
):
|
|
72
|
+
get_model_response = client.get_model(request)
|
|
44
73
|
resp_model_name_without_version = get_model_response.name.split('@', 1)[0]
|
|
45
74
|
model_resource_name = (
|
|
46
75
|
f'{resp_model_name_without_version}@{get_model_response.version_id}'
|
|
@@ -72,7 +72,7 @@ def automl_forecasting_ensemble(
|
|
|
72
72
|
# fmt: on
|
|
73
73
|
job_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER
|
|
74
74
|
task_id = dsl.PIPELINE_TASK_ID_PLACEHOLDER
|
|
75
|
-
image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:
|
|
75
|
+
image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625'
|
|
76
76
|
display_name = f'automl-forecasting-ensemble-{job_id}-{task_id}'
|
|
77
77
|
|
|
78
78
|
error_file_path = f'{root_dir}/{job_id}/{task_id}/error.pb'
|
|
@@ -99,14 +99,14 @@ def automl_forecasting_stage_1_tuner(
|
|
|
99
99
|
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
|
|
100
100
|
' "container_spec": {"image_uri":"'
|
|
101
101
|
),
|
|
102
|
-
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:
|
|
102
|
+
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
|
|
103
103
|
'", "args": ["forecasting_mp_l2l_stage_1_tuner',
|
|
104
104
|
'", "--region=',
|
|
105
105
|
location,
|
|
106
106
|
'", "--transform_output_path=',
|
|
107
107
|
transform_output.uri,
|
|
108
108
|
'", "--training_docker_uri=',
|
|
109
|
-
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:
|
|
109
|
+
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
|
|
110
110
|
'", "--reduce_search_space_mode=',
|
|
111
111
|
reduce_search_space_mode,
|
|
112
112
|
f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
|
|
@@ -97,14 +97,14 @@ def automl_forecasting_stage_2_tuner(
|
|
|
97
97
|
' 1, "machine_spec": {"machine_type": "n1-standard-8"},'
|
|
98
98
|
' "container_spec": {"image_uri":"'
|
|
99
99
|
),
|
|
100
|
-
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:
|
|
100
|
+
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
|
|
101
101
|
'", "args": ["forecasting_mp_l2l_stage_2_tuner',
|
|
102
102
|
'", "--region=',
|
|
103
103
|
location,
|
|
104
104
|
'", "--transform_output_path=',
|
|
105
105
|
transform_output.uri,
|
|
106
106
|
'", "--training_docker_uri=',
|
|
107
|
-
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:
|
|
107
|
+
'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20240710_0625',
|
|
108
108
|
f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}',
|
|
109
109
|
'", "--training_base_dir=',
|
|
110
110
|
root_dir,
|