apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +193 -246
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +17 -17
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +8 -8
- airflow/providers/google/cloud/hooks/dataproc.py +88 -0
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +25 -23
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +7 -7
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +9 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +298 -65
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +2 -2
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +29 -22
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +121 -120
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -80,7 +80,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
80
80
|
metadata: Sequence[tuple[str, str]] = (),
|
81
81
|
) -> Any:
|
82
82
|
"""
|
83
|
-
|
83
|
+
Fetch all the Alert Policies identified by the filter passed as filter parameter.
|
84
84
|
|
85
85
|
The desired return type can be specified by the format parameter, the supported formats
|
86
86
|
are "dict", "json" and None which returns python dictionary, stringified JSON and protobuf
|
@@ -161,7 +161,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
161
161
|
metadata: Sequence[tuple[str, str]] = (),
|
162
162
|
) -> None:
|
163
163
|
"""
|
164
|
-
|
164
|
+
Enable one or more disabled alerting policies identified by filter parameter.
|
165
165
|
|
166
166
|
Inoperative in case the policy is already enabled.
|
167
167
|
|
@@ -229,7 +229,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
229
229
|
metadata: Sequence[tuple[str, str]] = (),
|
230
230
|
) -> None:
|
231
231
|
"""
|
232
|
-
|
232
|
+
Create a new alert or updates an existing policy identified the name field in the alerts parameter.
|
233
233
|
|
234
234
|
:param project_id: The project in which alert needs to be created/updated.
|
235
235
|
:param alerts: A JSON string or file that specifies all the alerts that needs
|
@@ -322,7 +322,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
322
322
|
metadata: Sequence[tuple[str, str]] = (),
|
323
323
|
) -> None:
|
324
324
|
"""
|
325
|
-
|
325
|
+
Delete an alerting policy.
|
326
326
|
|
327
327
|
:param name: The alerting policy to delete. The format is:
|
328
328
|
``projects/[PROJECT_ID]/alertPolicies/[ALERT_POLICY_ID]``.
|
@@ -354,7 +354,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
354
354
|
metadata: Sequence[tuple[str, str]] = (),
|
355
355
|
) -> Any:
|
356
356
|
"""
|
357
|
-
|
357
|
+
Fetch all the Notification Channels identified by the filter passed as filter parameter.
|
358
358
|
|
359
359
|
The desired return type can be specified by the format parameter, the supported formats are
|
360
360
|
"dict", "json" and None which returns python dictionary, stringified JSON and protobuf
|
@@ -437,7 +437,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
437
437
|
metadata: Sequence[tuple[str, str]] = (),
|
438
438
|
) -> None:
|
439
439
|
"""
|
440
|
-
|
440
|
+
Enable one or more disabled alerting policies identified by filter parameter.
|
441
441
|
|
442
442
|
Inoperative in case the policy is already enabled.
|
443
443
|
|
@@ -567,7 +567,7 @@ class StackdriverHook(GoogleBaseHook):
|
|
567
567
|
metadata: Sequence[tuple[str, str]] = (),
|
568
568
|
) -> None:
|
569
569
|
"""
|
570
|
-
|
570
|
+
Delete a notification channel.
|
571
571
|
|
572
572
|
:param name: The alerting policy to delete. The format is:
|
573
573
|
``projects/[PROJECT_ID]/notificationChannels/[CHANNEL_ID]``.
|
@@ -73,7 +73,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
73
73
|
|
74
74
|
def get_conn(self) -> CloudTasksClient:
|
75
75
|
"""
|
76
|
-
|
76
|
+
Provide a client for interacting with the Google Cloud Tasks API.
|
77
77
|
|
78
78
|
:return: Google Cloud Tasks API Client
|
79
79
|
"""
|
@@ -93,7 +93,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
93
93
|
metadata: Sequence[tuple[str, str]] = (),
|
94
94
|
) -> Queue:
|
95
95
|
"""
|
96
|
-
|
96
|
+
Create a queue in Cloud Tasks.
|
97
97
|
|
98
98
|
:param location: The location name in which the queue will be created.
|
99
99
|
:param task_queue: The task queue to create.
|
@@ -141,7 +141,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
141
141
|
metadata: Sequence[tuple[str, str]] = (),
|
142
142
|
) -> Queue:
|
143
143
|
"""
|
144
|
-
|
144
|
+
Update a queue in Cloud Tasks.
|
145
145
|
|
146
146
|
:param task_queue: The task queue to update.
|
147
147
|
This method creates the queue if it does not exist and updates the queue if
|
@@ -190,7 +190,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
190
190
|
metadata: Sequence[tuple[str, str]] = (),
|
191
191
|
) -> Queue:
|
192
192
|
"""
|
193
|
-
|
193
|
+
Get a queue from Cloud Tasks.
|
194
194
|
|
195
195
|
:param location: The location name in which the queue was created.
|
196
196
|
:param queue_name: The queue's name.
|
@@ -225,7 +225,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
225
225
|
metadata: Sequence[tuple[str, str]] = (),
|
226
226
|
) -> list[Queue]:
|
227
227
|
"""
|
228
|
-
|
228
|
+
List queues from Cloud Tasks.
|
229
229
|
|
230
230
|
:param location: The location name in which the queues were created.
|
231
231
|
:param project_id: (Optional) The ID of the Google Cloud project that owns the Cloud Tasks.
|
@@ -262,7 +262,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
262
262
|
metadata: Sequence[tuple[str, str]] = (),
|
263
263
|
) -> None:
|
264
264
|
"""
|
265
|
-
|
265
|
+
Delete a queue from Cloud Tasks, even if it has tasks in it.
|
266
266
|
|
267
267
|
:param location: The location name in which the queue will be deleted.
|
268
268
|
:param queue_name: The queue's name.
|
@@ -401,7 +401,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
401
401
|
metadata: Sequence[tuple[str, str]] = (),
|
402
402
|
) -> Task:
|
403
403
|
"""
|
404
|
-
|
404
|
+
Create a task in Cloud Tasks.
|
405
405
|
|
406
406
|
:param location: The location name in which the task will be created.
|
407
407
|
:param queue_name: The queue's name.
|
@@ -453,7 +453,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
453
453
|
metadata: Sequence[tuple[str, str]] = (),
|
454
454
|
) -> Task:
|
455
455
|
"""
|
456
|
-
|
456
|
+
Get a task from Cloud Tasks.
|
457
457
|
|
458
458
|
:param location: The location name in which the task was created.
|
459
459
|
:param queue_name: The queue's name.
|
@@ -492,7 +492,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
492
492
|
metadata: Sequence[tuple[str, str]] = (),
|
493
493
|
) -> list[Task]:
|
494
494
|
"""
|
495
|
-
|
495
|
+
List the tasks in Cloud Tasks.
|
496
496
|
|
497
497
|
:param location: The location name in which the tasks were created.
|
498
498
|
:param queue_name: The queue's name.
|
@@ -531,7 +531,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
531
531
|
metadata: Sequence[tuple[str, str]] = (),
|
532
532
|
) -> None:
|
533
533
|
"""
|
534
|
-
|
534
|
+
Delete a task from Cloud Tasks.
|
535
535
|
|
536
536
|
:param location: The location name in which the task will be deleted.
|
537
537
|
:param queue_name: The queue's name.
|
@@ -568,7 +568,7 @@ class CloudTasksHook(GoogleBaseHook):
|
|
568
568
|
metadata: Sequence[tuple[str, str]] = (),
|
569
569
|
) -> Task:
|
570
570
|
"""
|
571
|
-
|
571
|
+
Force run a task in Cloud Tasks.
|
572
572
|
|
573
573
|
:param location: The location name in which the task was created.
|
574
574
|
:param queue_name: The queue's name.
|
@@ -73,7 +73,7 @@ class CloudTextToSpeechHook(GoogleBaseHook):
|
|
73
73
|
|
74
74
|
def get_conn(self) -> TextToSpeechClient:
|
75
75
|
"""
|
76
|
-
|
76
|
+
Retrieve connection to Cloud Text to Speech.
|
77
77
|
|
78
78
|
:return: Google Cloud Text to Speech client object.
|
79
79
|
"""
|
@@ -74,7 +74,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
74
74
|
self,
|
75
75
|
region: str | None = None,
|
76
76
|
) -> PipelineServiceClient:
|
77
|
-
"""
|
77
|
+
"""Return PipelineServiceClient."""
|
78
78
|
if region and region != "global":
|
79
79
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
80
80
|
else:
|
@@ -88,7 +88,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
88
88
|
self,
|
89
89
|
region: str | None = None,
|
90
90
|
) -> JobServiceClient:
|
91
|
-
"""
|
91
|
+
"""Return JobServiceClient."""
|
92
92
|
if region and region != "global":
|
93
93
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
94
94
|
else:
|
@@ -113,7 +113,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
113
113
|
training_encryption_spec_key_name: str | None = None,
|
114
114
|
model_encryption_spec_key_name: str | None = None,
|
115
115
|
) -> AutoMLTabularTrainingJob:
|
116
|
-
"""
|
116
|
+
"""Return AutoMLTabularTrainingJob object."""
|
117
117
|
return AutoMLTabularTrainingJob(
|
118
118
|
display_name=display_name,
|
119
119
|
optimization_prediction_type=optimization_prediction_type,
|
@@ -142,7 +142,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
142
142
|
training_encryption_spec_key_name: str | None = None,
|
143
143
|
model_encryption_spec_key_name: str | None = None,
|
144
144
|
) -> AutoMLForecastingTrainingJob:
|
145
|
-
"""
|
145
|
+
"""Return AutoMLForecastingTrainingJob object."""
|
146
146
|
return AutoMLForecastingTrainingJob(
|
147
147
|
display_name=display_name,
|
148
148
|
optimization_objective=optimization_objective,
|
@@ -169,7 +169,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
169
169
|
training_encryption_spec_key_name: str | None = None,
|
170
170
|
model_encryption_spec_key_name: str | None = None,
|
171
171
|
) -> AutoMLImageTrainingJob:
|
172
|
-
"""
|
172
|
+
"""Return AutoMLImageTrainingJob object."""
|
173
173
|
return AutoMLImageTrainingJob(
|
174
174
|
display_name=display_name,
|
175
175
|
prediction_type=prediction_type,
|
@@ -196,7 +196,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
196
196
|
training_encryption_spec_key_name: str | None = None,
|
197
197
|
model_encryption_spec_key_name: str | None = None,
|
198
198
|
) -> AutoMLTextTrainingJob:
|
199
|
-
"""
|
199
|
+
"""Return AutoMLTextTrainingJob object."""
|
200
200
|
return AutoMLTextTrainingJob(
|
201
201
|
display_name=display_name,
|
202
202
|
prediction_type=prediction_type,
|
@@ -221,7 +221,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
221
221
|
training_encryption_spec_key_name: str | None = None,
|
222
222
|
model_encryption_spec_key_name: str | None = None,
|
223
223
|
) -> AutoMLVideoTrainingJob:
|
224
|
-
"""
|
224
|
+
"""Return AutoMLVideoTrainingJob object."""
|
225
225
|
return AutoMLVideoTrainingJob(
|
226
226
|
display_name=display_name,
|
227
227
|
prediction_type=prediction_type,
|
@@ -236,16 +236,16 @@ class AutoMLHook(GoogleBaseHook):
|
|
236
236
|
|
237
237
|
@staticmethod
|
238
238
|
def extract_model_id(obj: dict) -> str:
|
239
|
-
"""
|
239
|
+
"""Return unique id of the Model."""
|
240
240
|
return obj["name"].rpartition("/")[-1]
|
241
241
|
|
242
242
|
@staticmethod
|
243
243
|
def extract_training_id(resource_name: str) -> str:
|
244
|
-
"""
|
244
|
+
"""Return unique id of the Training pipeline."""
|
245
245
|
return resource_name.rpartition("/")[-1]
|
246
246
|
|
247
247
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
248
|
-
"""
|
248
|
+
"""Wait for long-lasting operation to complete."""
|
249
249
|
try:
|
250
250
|
return operation.result(timeout=timeout)
|
251
251
|
except Exception:
|
@@ -1301,7 +1301,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
1301
1301
|
metadata: Sequence[tuple[str, str]] = (),
|
1302
1302
|
) -> Operation:
|
1303
1303
|
"""
|
1304
|
-
|
1304
|
+
Delete a TrainingPipeline.
|
1305
1305
|
|
1306
1306
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1307
1307
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1334,7 +1334,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
1334
1334
|
metadata: Sequence[tuple[str, str]] = (),
|
1335
1335
|
) -> TrainingPipeline:
|
1336
1336
|
"""
|
1337
|
-
|
1337
|
+
Get a TrainingPipeline.
|
1338
1338
|
|
1339
1339
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1340
1340
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1370,7 +1370,7 @@ class AutoMLHook(GoogleBaseHook):
|
|
1370
1370
|
metadata: Sequence[tuple[str, str]] = (),
|
1371
1371
|
) -> ListTrainingPipelinesPager:
|
1372
1372
|
"""
|
1373
|
-
|
1373
|
+
List TrainingPipelines in a Location.
|
1374
1374
|
|
1375
1375
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1376
1376
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -55,7 +55,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
55
55
|
self._batch_prediction_job: BatchPredictionJob | None = None
|
56
56
|
|
57
57
|
def get_job_service_client(self, region: str | None = None) -> JobServiceClient:
|
58
|
-
"""
|
58
|
+
"""Return JobServiceClient object."""
|
59
59
|
if region and region != "global":
|
60
60
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
61
61
|
else:
|
@@ -66,7 +66,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
66
66
|
)
|
67
67
|
|
68
68
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
69
|
-
"""
|
69
|
+
"""Wait for long-lasting operation to complete."""
|
70
70
|
try:
|
71
71
|
return operation.result(timeout=timeout)
|
72
72
|
except Exception:
|
@@ -75,7 +75,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
75
75
|
|
76
76
|
@staticmethod
|
77
77
|
def extract_batch_prediction_job_id(obj: dict) -> str:
|
78
|
-
"""
|
78
|
+
"""Return unique id of the batch_prediction_job."""
|
79
79
|
return obj["name"].rpartition("/")[-1]
|
80
80
|
|
81
81
|
def cancel_batch_prediction_job(self) -> None:
|
@@ -252,7 +252,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
252
252
|
metadata: Sequence[tuple[str, str]] = (),
|
253
253
|
) -> Operation:
|
254
254
|
"""
|
255
|
-
|
255
|
+
Delete a BatchPredictionJob. Can only be called on jobs that already finished.
|
256
256
|
|
257
257
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
258
258
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -285,7 +285,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
285
285
|
metadata: Sequence[tuple[str, str]] = (),
|
286
286
|
) -> BatchPredictionJob:
|
287
287
|
"""
|
288
|
-
|
288
|
+
Get a BatchPredictionJob.
|
289
289
|
|
290
290
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
291
291
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -321,7 +321,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
321
321
|
metadata: Sequence[tuple[str, str]] = (),
|
322
322
|
) -> ListBatchPredictionJobsPager:
|
323
323
|
"""
|
324
|
-
|
324
|
+
List BatchPredictionJobs in a Location.
|
325
325
|
|
326
326
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
327
327
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -18,9 +18,9 @@
|
|
18
18
|
"""This module contains a Google Cloud Vertex AI hook."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
23
|
+
from deprecated import deprecated
|
24
24
|
from google.api_core.client_options import ClientOptions
|
25
25
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
26
|
from google.cloud.aiplatform import (
|
@@ -73,7 +73,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
73
73
|
self,
|
74
74
|
region: str | None = None,
|
75
75
|
) -> PipelineServiceClient:
|
76
|
-
"""
|
76
|
+
"""Return PipelineServiceClient object."""
|
77
77
|
if region and region != "global":
|
78
78
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
79
79
|
else:
|
@@ -86,7 +86,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
86
86
|
self,
|
87
87
|
region: str | None = None,
|
88
88
|
) -> JobServiceClient:
|
89
|
-
"""
|
89
|
+
"""Return JobServiceClient object."""
|
90
90
|
if region and region != "global":
|
91
91
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
92
92
|
else:
|
@@ -119,7 +119,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
119
119
|
model_encryption_spec_key_name: str | None = None,
|
120
120
|
staging_bucket: str | None = None,
|
121
121
|
) -> CustomContainerTrainingJob:
|
122
|
-
"""
|
122
|
+
"""Return CustomContainerTrainingJob object."""
|
123
123
|
return CustomContainerTrainingJob(
|
124
124
|
display_name=display_name,
|
125
125
|
container_uri=container_uri,
|
@@ -168,7 +168,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
168
168
|
model_encryption_spec_key_name: str | None = None,
|
169
169
|
staging_bucket: str | None = None,
|
170
170
|
):
|
171
|
-
"""
|
171
|
+
"""Return CustomPythonPackageTrainingJob object."""
|
172
172
|
return CustomPythonPackageTrainingJob(
|
173
173
|
display_name=display_name,
|
174
174
|
container_uri=container_uri,
|
@@ -218,7 +218,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
218
218
|
model_encryption_spec_key_name: str | None = None,
|
219
219
|
staging_bucket: str | None = None,
|
220
220
|
):
|
221
|
-
"""
|
221
|
+
"""Return CustomTrainingJob object."""
|
222
222
|
return CustomTrainingJob(
|
223
223
|
display_name=display_name,
|
224
224
|
script_path=script_path,
|
@@ -246,21 +246,21 @@ class CustomJobHook(GoogleBaseHook):
|
|
246
246
|
|
247
247
|
@staticmethod
|
248
248
|
def extract_model_id(obj: dict) -> str:
|
249
|
-
"""
|
249
|
+
"""Return unique id of the Model."""
|
250
250
|
return obj["name"].rpartition("/")[-1]
|
251
251
|
|
252
252
|
@staticmethod
|
253
253
|
def extract_training_id(resource_name: str) -> str:
|
254
|
-
"""
|
254
|
+
"""Return unique id of the Training pipeline."""
|
255
255
|
return resource_name.rpartition("/")[-1]
|
256
256
|
|
257
257
|
@staticmethod
|
258
258
|
def extract_custom_job_id(custom_job_name: str) -> str:
|
259
|
-
"""
|
259
|
+
"""Return unique id of the Custom Job pipeline."""
|
260
260
|
return custom_job_name.rpartition("/")[-1]
|
261
261
|
|
262
262
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
263
|
-
"""
|
263
|
+
"""Wait for long-lasting operation to complete."""
|
264
264
|
try:
|
265
265
|
return operation.result(timeout=timeout)
|
266
266
|
except Exception:
|
@@ -358,6 +358,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
358
358
|
return model, training_id, custom_job_id
|
359
359
|
|
360
360
|
@GoogleBaseHook.fallback_to_default_project_id
|
361
|
+
@deprecated(
|
362
|
+
reason="Please use `PipelineJobHook.cancel_pipeline_job`",
|
363
|
+
category=AirflowProviderDeprecationWarning,
|
364
|
+
)
|
361
365
|
def cancel_pipeline_job(
|
362
366
|
self,
|
363
367
|
project_id: str,
|
@@ -368,7 +372,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
368
372
|
metadata: Sequence[tuple[str, str]] = (),
|
369
373
|
) -> None:
|
370
374
|
"""
|
371
|
-
|
375
|
+
Cancel a PipelineJob.
|
372
376
|
|
373
377
|
Starts asynchronous cancellation on the PipelineJob. The server makes the best
|
374
378
|
effort to cancel the pipeline, but success is not guaranteed. Clients can use
|
@@ -388,11 +392,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
388
392
|
:param timeout: The timeout for this request.
|
389
393
|
:param metadata: Strings which should be sent along with the request as metadata.
|
390
394
|
"""
|
391
|
-
warnings.warn(
|
392
|
-
"This method is deprecated, please use `PipelineJobHook.cancel_pipeline_job` method.",
|
393
|
-
AirflowProviderDeprecationWarning,
|
394
|
-
stacklevel=2,
|
395
|
-
)
|
396
395
|
client = self.get_pipeline_service_client(region)
|
397
396
|
name = client.pipeline_job_path(project_id, region, pipeline_job)
|
398
397
|
|
@@ -416,7 +415,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
416
415
|
metadata: Sequence[tuple[str, str]] = (),
|
417
416
|
) -> None:
|
418
417
|
"""
|
419
|
-
|
418
|
+
Cancel a TrainingPipeline.
|
420
419
|
|
421
420
|
Starts asynchronous cancellation on the TrainingPipeline. The server makes
|
422
421
|
the best effort to cancel the pipeline, but success is not guaranteed. Clients can use
|
@@ -457,7 +456,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
457
456
|
metadata: Sequence[tuple[str, str]] = (),
|
458
457
|
) -> None:
|
459
458
|
"""
|
460
|
-
|
459
|
+
Cancel a CustomJob.
|
461
460
|
|
462
461
|
Starts asynchronous cancellation on the CustomJob. The server makes the best effort
|
463
462
|
to cancel the job, but success is not guaranteed. Clients can use
|
@@ -488,6 +487,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
488
487
|
)
|
489
488
|
|
490
489
|
@GoogleBaseHook.fallback_to_default_project_id
|
490
|
+
@deprecated(
|
491
|
+
reason="Please use `PipelineJobHook.create_pipeline_job`",
|
492
|
+
category=AirflowProviderDeprecationWarning,
|
493
|
+
)
|
491
494
|
def create_pipeline_job(
|
492
495
|
self,
|
493
496
|
project_id: str,
|
@@ -499,7 +502,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
499
502
|
metadata: Sequence[tuple[str, str]] = (),
|
500
503
|
) -> PipelineJob:
|
501
504
|
"""
|
502
|
-
|
505
|
+
Create a PipelineJob. A PipelineJob will run immediately when created.
|
503
506
|
|
504
507
|
This method is deprecated, please use `PipelineJobHook.create_pipeline_job` method.
|
505
508
|
|
@@ -514,11 +517,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
514
517
|
:param timeout: The timeout for this request.
|
515
518
|
:param metadata: Strings which should be sent along with the request as metadata.
|
516
519
|
"""
|
517
|
-
warnings.warn(
|
518
|
-
"This method is deprecated, please use `PipelineJobHook.create_pipeline_job` method.",
|
519
|
-
AirflowProviderDeprecationWarning,
|
520
|
-
stacklevel=2,
|
521
|
-
)
|
522
520
|
client = self.get_pipeline_service_client(region)
|
523
521
|
parent = client.common_location_path(project_id, region)
|
524
522
|
|
@@ -545,7 +543,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
545
543
|
metadata: Sequence[tuple[str, str]] = (),
|
546
544
|
) -> TrainingPipeline:
|
547
545
|
"""
|
548
|
-
|
546
|
+
Create a TrainingPipeline. A created TrainingPipeline right away will be attempted to be run.
|
549
547
|
|
550
548
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
551
549
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -579,7 +577,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
579
577
|
metadata: Sequence[tuple[str, str]] = (),
|
580
578
|
) -> CustomJob:
|
581
579
|
"""
|
582
|
-
|
580
|
+
Create a CustomJob. A created CustomJob right away will be attempted to be run.
|
583
581
|
|
584
582
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
585
583
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1755,6 +1753,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
1755
1753
|
return model, training_id, custom_job_id
|
1756
1754
|
|
1757
1755
|
@GoogleBaseHook.fallback_to_default_project_id
|
1756
|
+
@deprecated(
|
1757
|
+
reason="Please use `PipelineJobHook.delete_pipeline_job`",
|
1758
|
+
category=AirflowProviderDeprecationWarning,
|
1759
|
+
)
|
1758
1760
|
def delete_pipeline_job(
|
1759
1761
|
self,
|
1760
1762
|
project_id: str,
|
@@ -1765,7 +1767,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1765
1767
|
metadata: Sequence[tuple[str, str]] = (),
|
1766
1768
|
) -> Operation:
|
1767
1769
|
"""
|
1768
|
-
|
1770
|
+
Delete a PipelineJob.
|
1769
1771
|
|
1770
1772
|
This method is deprecated, please use `PipelineJobHook.delete_pipeline_job` method.
|
1771
1773
|
|
@@ -1776,11 +1778,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
1776
1778
|
:param timeout: The timeout for this request.
|
1777
1779
|
:param metadata: Strings which should be sent along with the request as metadata.
|
1778
1780
|
"""
|
1779
|
-
warnings.warn(
|
1780
|
-
"This method is deprecated, please use `PipelineJobHook.delete_pipeline_job` method.",
|
1781
|
-
AirflowProviderDeprecationWarning,
|
1782
|
-
stacklevel=2,
|
1783
|
-
)
|
1784
1781
|
client = self.get_pipeline_service_client(region)
|
1785
1782
|
name = client.pipeline_job_path(project_id, region, pipeline_job)
|
1786
1783
|
|
@@ -1805,7 +1802,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1805
1802
|
metadata: Sequence[tuple[str, str]] = (),
|
1806
1803
|
) -> Operation:
|
1807
1804
|
"""
|
1808
|
-
|
1805
|
+
Delete a TrainingPipeline.
|
1809
1806
|
|
1810
1807
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1811
1808
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1838,7 +1835,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1838
1835
|
metadata: Sequence[tuple[str, str]] = (),
|
1839
1836
|
) -> Operation:
|
1840
1837
|
"""
|
1841
|
-
|
1838
|
+
Delete a CustomJob.
|
1842
1839
|
|
1843
1840
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1844
1841
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1861,6 +1858,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
1861
1858
|
return result
|
1862
1859
|
|
1863
1860
|
@GoogleBaseHook.fallback_to_default_project_id
|
1861
|
+
@deprecated(
|
1862
|
+
reason="Please use `PipelineJobHook.get_pipeline_job`",
|
1863
|
+
category=AirflowProviderDeprecationWarning,
|
1864
|
+
)
|
1864
1865
|
def get_pipeline_job(
|
1865
1866
|
self,
|
1866
1867
|
project_id: str,
|
@@ -1871,7 +1872,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1871
1872
|
metadata: Sequence[tuple[str, str]] = (),
|
1872
1873
|
) -> PipelineJob:
|
1873
1874
|
"""
|
1874
|
-
|
1875
|
+
Get a PipelineJob.
|
1875
1876
|
|
1876
1877
|
This method is deprecated, please use `PipelineJobHook.get_pipeline_job` method.
|
1877
1878
|
|
@@ -1882,11 +1883,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
1882
1883
|
:param timeout: The timeout for this request.
|
1883
1884
|
:param metadata: Strings which should be sent along with the request as metadata.
|
1884
1885
|
"""
|
1885
|
-
warnings.warn(
|
1886
|
-
"This method is deprecated, please use `PipelineJobHook.get_pipeline_job` method.",
|
1887
|
-
AirflowProviderDeprecationWarning,
|
1888
|
-
stacklevel=2,
|
1889
|
-
)
|
1890
1886
|
client = self.get_pipeline_service_client(region)
|
1891
1887
|
name = client.pipeline_job_path(project_id, region, pipeline_job)
|
1892
1888
|
|
@@ -1911,7 +1907,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1911
1907
|
metadata: Sequence[tuple[str, str]] = (),
|
1912
1908
|
) -> TrainingPipeline:
|
1913
1909
|
"""
|
1914
|
-
|
1910
|
+
Get a TrainingPipeline.
|
1915
1911
|
|
1916
1912
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1917
1913
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1944,7 +1940,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1944
1940
|
metadata: Sequence[tuple[str, str]] = (),
|
1945
1941
|
) -> CustomJob:
|
1946
1942
|
"""
|
1947
|
-
|
1943
|
+
Get a CustomJob.
|
1948
1944
|
|
1949
1945
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1950
1946
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1967,6 +1963,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
1967
1963
|
return result
|
1968
1964
|
|
1969
1965
|
@GoogleBaseHook.fallback_to_default_project_id
|
1966
|
+
@deprecated(
|
1967
|
+
reason="Please use `PipelineJobHook.list_pipeline_jobs`",
|
1968
|
+
category=AirflowProviderDeprecationWarning,
|
1969
|
+
)
|
1970
1970
|
def list_pipeline_jobs(
|
1971
1971
|
self,
|
1972
1972
|
project_id: str,
|
@@ -1980,7 +1980,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1980
1980
|
metadata: Sequence[tuple[str, str]] = (),
|
1981
1981
|
) -> ListPipelineJobsPager:
|
1982
1982
|
"""
|
1983
|
-
|
1983
|
+
List PipelineJobs in a Location.
|
1984
1984
|
|
1985
1985
|
This method is deprecated, please use `PipelineJobHook.list_pipeline_jobs` method.
|
1986
1986
|
|
@@ -2039,11 +2039,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
2039
2039
|
:param timeout: The timeout for this request.
|
2040
2040
|
:param metadata: Strings which should be sent along with the request as metadata.
|
2041
2041
|
"""
|
2042
|
-
warnings.warn(
|
2043
|
-
"This method is deprecated, please use `PipelineJobHook.list_pipeline_jobs` method.",
|
2044
|
-
AirflowProviderDeprecationWarning,
|
2045
|
-
stacklevel=2,
|
2046
|
-
)
|
2047
2042
|
client = self.get_pipeline_service_client(region)
|
2048
2043
|
parent = client.common_location_path(project_id, region)
|
2049
2044
|
|
@@ -2075,7 +2070,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
2075
2070
|
metadata: Sequence[tuple[str, str]] = (),
|
2076
2071
|
) -> ListTrainingPipelinesPager:
|
2077
2072
|
"""
|
2078
|
-
|
2073
|
+
List TrainingPipelines in a Location.
|
2079
2074
|
|
2080
2075
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
2081
2076
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -2136,7 +2131,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
2136
2131
|
metadata: Sequence[tuple[str, str]] = (),
|
2137
2132
|
) -> ListCustomJobsPager:
|
2138
2133
|
"""
|
2139
|
-
|
2134
|
+
List CustomJobs in a Location.
|
2140
2135
|
|
2141
2136
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
2142
2137
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|