apache-airflow-providers-google 10.14.0rc1__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/ads/hooks/ads.py +1 -2
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +208 -256
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +18 -19
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +10 -10
- airflow/providers/google/cloud/hooks/dataproc.py +132 -14
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +39 -27
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +8 -9
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +11 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +444 -69
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +12 -2
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -0
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -0
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +34 -14
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/dataproc.py +25 -0
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +87 -23
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +4 -4
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +126 -124
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -55,7 +55,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
55
55
|
self._batch_prediction_job: BatchPredictionJob | None = None
|
56
56
|
|
57
57
|
def get_job_service_client(self, region: str | None = None) -> JobServiceClient:
|
58
|
-
"""
|
58
|
+
"""Return JobServiceClient object."""
|
59
59
|
if region and region != "global":
|
60
60
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
61
61
|
else:
|
@@ -66,7 +66,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
66
66
|
)
|
67
67
|
|
68
68
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
69
|
-
"""
|
69
|
+
"""Wait for long-lasting operation to complete."""
|
70
70
|
try:
|
71
71
|
return operation.result(timeout=timeout)
|
72
72
|
except Exception:
|
@@ -75,7 +75,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
75
75
|
|
76
76
|
@staticmethod
|
77
77
|
def extract_batch_prediction_job_id(obj: dict) -> str:
|
78
|
-
"""
|
78
|
+
"""Return unique id of the batch_prediction_job."""
|
79
79
|
return obj["name"].rpartition("/")[-1]
|
80
80
|
|
81
81
|
def cancel_batch_prediction_job(self) -> None:
|
@@ -252,7 +252,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
252
252
|
metadata: Sequence[tuple[str, str]] = (),
|
253
253
|
) -> Operation:
|
254
254
|
"""
|
255
|
-
|
255
|
+
Delete a BatchPredictionJob. Can only be called on jobs that already finished.
|
256
256
|
|
257
257
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
258
258
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -285,7 +285,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
285
285
|
metadata: Sequence[tuple[str, str]] = (),
|
286
286
|
) -> BatchPredictionJob:
|
287
287
|
"""
|
288
|
-
|
288
|
+
Get a BatchPredictionJob.
|
289
289
|
|
290
290
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
291
291
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -321,7 +321,7 @@ class BatchPredictionJobHook(GoogleBaseHook):
|
|
321
321
|
metadata: Sequence[tuple[str, str]] = (),
|
322
322
|
) -> ListBatchPredictionJobsPager:
|
323
323
|
"""
|
324
|
-
|
324
|
+
List BatchPredictionJobs in a Location.
|
325
325
|
|
326
326
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
327
327
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -18,9 +18,9 @@
|
|
18
18
|
"""This module contains a Google Cloud Vertex AI hook."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
import warnings
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
23
|
+
from deprecated import deprecated
|
24
24
|
from google.api_core.client_options import ClientOptions
|
25
25
|
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
26
26
|
from google.cloud.aiplatform import (
|
@@ -73,7 +73,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
73
73
|
self,
|
74
74
|
region: str | None = None,
|
75
75
|
) -> PipelineServiceClient:
|
76
|
-
"""
|
76
|
+
"""Return PipelineServiceClient object."""
|
77
77
|
if region and region != "global":
|
78
78
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
79
79
|
else:
|
@@ -86,7 +86,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
86
86
|
self,
|
87
87
|
region: str | None = None,
|
88
88
|
) -> JobServiceClient:
|
89
|
-
"""
|
89
|
+
"""Return JobServiceClient object."""
|
90
90
|
if region and region != "global":
|
91
91
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
92
92
|
else:
|
@@ -119,7 +119,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
119
119
|
model_encryption_spec_key_name: str | None = None,
|
120
120
|
staging_bucket: str | None = None,
|
121
121
|
) -> CustomContainerTrainingJob:
|
122
|
-
"""
|
122
|
+
"""Return CustomContainerTrainingJob object."""
|
123
123
|
return CustomContainerTrainingJob(
|
124
124
|
display_name=display_name,
|
125
125
|
container_uri=container_uri,
|
@@ -168,7 +168,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
168
168
|
model_encryption_spec_key_name: str | None = None,
|
169
169
|
staging_bucket: str | None = None,
|
170
170
|
):
|
171
|
-
"""
|
171
|
+
"""Return CustomPythonPackageTrainingJob object."""
|
172
172
|
return CustomPythonPackageTrainingJob(
|
173
173
|
display_name=display_name,
|
174
174
|
container_uri=container_uri,
|
@@ -218,7 +218,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
218
218
|
model_encryption_spec_key_name: str | None = None,
|
219
219
|
staging_bucket: str | None = None,
|
220
220
|
):
|
221
|
-
"""
|
221
|
+
"""Return CustomTrainingJob object."""
|
222
222
|
return CustomTrainingJob(
|
223
223
|
display_name=display_name,
|
224
224
|
script_path=script_path,
|
@@ -246,21 +246,21 @@ class CustomJobHook(GoogleBaseHook):
|
|
246
246
|
|
247
247
|
@staticmethod
|
248
248
|
def extract_model_id(obj: dict) -> str:
|
249
|
-
"""
|
249
|
+
"""Return unique id of the Model."""
|
250
250
|
return obj["name"].rpartition("/")[-1]
|
251
251
|
|
252
252
|
@staticmethod
|
253
253
|
def extract_training_id(resource_name: str) -> str:
|
254
|
-
"""
|
254
|
+
"""Return unique id of the Training pipeline."""
|
255
255
|
return resource_name.rpartition("/")[-1]
|
256
256
|
|
257
257
|
@staticmethod
|
258
258
|
def extract_custom_job_id(custom_job_name: str) -> str:
|
259
|
-
"""
|
259
|
+
"""Return unique id of the Custom Job pipeline."""
|
260
260
|
return custom_job_name.rpartition("/")[-1]
|
261
261
|
|
262
262
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
263
|
-
"""
|
263
|
+
"""Wait for long-lasting operation to complete."""
|
264
264
|
try:
|
265
265
|
return operation.result(timeout=timeout)
|
266
266
|
except Exception:
|
@@ -358,6 +358,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
358
358
|
return model, training_id, custom_job_id
|
359
359
|
|
360
360
|
@GoogleBaseHook.fallback_to_default_project_id
|
361
|
+
@deprecated(
|
362
|
+
reason="Please use `PipelineJobHook.cancel_pipeline_job`",
|
363
|
+
category=AirflowProviderDeprecationWarning,
|
364
|
+
)
|
361
365
|
def cancel_pipeline_job(
|
362
366
|
self,
|
363
367
|
project_id: str,
|
@@ -368,7 +372,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
368
372
|
metadata: Sequence[tuple[str, str]] = (),
|
369
373
|
) -> None:
|
370
374
|
"""
|
371
|
-
|
375
|
+
Cancel a PipelineJob.
|
372
376
|
|
373
377
|
Starts asynchronous cancellation on the PipelineJob. The server makes the best
|
374
378
|
effort to cancel the pipeline, but success is not guaranteed. Clients can use
|
@@ -388,11 +392,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
388
392
|
:param timeout: The timeout for this request.
|
389
393
|
:param metadata: Strings which should be sent along with the request as metadata.
|
390
394
|
"""
|
391
|
-
warnings.warn(
|
392
|
-
"This method is deprecated, please use `PipelineJobHook.cancel_pipeline_job` method.",
|
393
|
-
AirflowProviderDeprecationWarning,
|
394
|
-
stacklevel=2,
|
395
|
-
)
|
396
395
|
client = self.get_pipeline_service_client(region)
|
397
396
|
name = client.pipeline_job_path(project_id, region, pipeline_job)
|
398
397
|
|
@@ -416,7 +415,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
416
415
|
metadata: Sequence[tuple[str, str]] = (),
|
417
416
|
) -> None:
|
418
417
|
"""
|
419
|
-
|
418
|
+
Cancel a TrainingPipeline.
|
420
419
|
|
421
420
|
Starts asynchronous cancellation on the TrainingPipeline. The server makes
|
422
421
|
the best effort to cancel the pipeline, but success is not guaranteed. Clients can use
|
@@ -457,7 +456,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
457
456
|
metadata: Sequence[tuple[str, str]] = (),
|
458
457
|
) -> None:
|
459
458
|
"""
|
460
|
-
|
459
|
+
Cancel a CustomJob.
|
461
460
|
|
462
461
|
Starts asynchronous cancellation on the CustomJob. The server makes the best effort
|
463
462
|
to cancel the job, but success is not guaranteed. Clients can use
|
@@ -488,6 +487,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
488
487
|
)
|
489
488
|
|
490
489
|
@GoogleBaseHook.fallback_to_default_project_id
|
490
|
+
@deprecated(
|
491
|
+
reason="Please use `PipelineJobHook.create_pipeline_job`",
|
492
|
+
category=AirflowProviderDeprecationWarning,
|
493
|
+
)
|
491
494
|
def create_pipeline_job(
|
492
495
|
self,
|
493
496
|
project_id: str,
|
@@ -499,7 +502,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
499
502
|
metadata: Sequence[tuple[str, str]] = (),
|
500
503
|
) -> PipelineJob:
|
501
504
|
"""
|
502
|
-
|
505
|
+
Create a PipelineJob. A PipelineJob will run immediately when created.
|
503
506
|
|
504
507
|
This method is deprecated, please use `PipelineJobHook.create_pipeline_job` method.
|
505
508
|
|
@@ -514,11 +517,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
514
517
|
:param timeout: The timeout for this request.
|
515
518
|
:param metadata: Strings which should be sent along with the request as metadata.
|
516
519
|
"""
|
517
|
-
warnings.warn(
|
518
|
-
"This method is deprecated, please use `PipelineJobHook.create_pipeline_job` method.",
|
519
|
-
AirflowProviderDeprecationWarning,
|
520
|
-
stacklevel=2,
|
521
|
-
)
|
522
520
|
client = self.get_pipeline_service_client(region)
|
523
521
|
parent = client.common_location_path(project_id, region)
|
524
522
|
|
@@ -545,7 +543,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
545
543
|
metadata: Sequence[tuple[str, str]] = (),
|
546
544
|
) -> TrainingPipeline:
|
547
545
|
"""
|
548
|
-
|
546
|
+
Create a TrainingPipeline. A created TrainingPipeline right away will be attempted to be run.
|
549
547
|
|
550
548
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
551
549
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -579,7 +577,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
579
577
|
metadata: Sequence[tuple[str, str]] = (),
|
580
578
|
) -> CustomJob:
|
581
579
|
"""
|
582
|
-
|
580
|
+
Create a CustomJob. A created CustomJob right away will be attempted to be run.
|
583
581
|
|
584
582
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
585
583
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1755,6 +1753,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
1755
1753
|
return model, training_id, custom_job_id
|
1756
1754
|
|
1757
1755
|
@GoogleBaseHook.fallback_to_default_project_id
|
1756
|
+
@deprecated(
|
1757
|
+
reason="Please use `PipelineJobHook.delete_pipeline_job`",
|
1758
|
+
category=AirflowProviderDeprecationWarning,
|
1759
|
+
)
|
1758
1760
|
def delete_pipeline_job(
|
1759
1761
|
self,
|
1760
1762
|
project_id: str,
|
@@ -1765,7 +1767,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1765
1767
|
metadata: Sequence[tuple[str, str]] = (),
|
1766
1768
|
) -> Operation:
|
1767
1769
|
"""
|
1768
|
-
|
1770
|
+
Delete a PipelineJob.
|
1769
1771
|
|
1770
1772
|
This method is deprecated, please use `PipelineJobHook.delete_pipeline_job` method.
|
1771
1773
|
|
@@ -1776,11 +1778,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
1776
1778
|
:param timeout: The timeout for this request.
|
1777
1779
|
:param metadata: Strings which should be sent along with the request as metadata.
|
1778
1780
|
"""
|
1779
|
-
warnings.warn(
|
1780
|
-
"This method is deprecated, please use `PipelineJobHook.delete_pipeline_job` method.",
|
1781
|
-
AirflowProviderDeprecationWarning,
|
1782
|
-
stacklevel=2,
|
1783
|
-
)
|
1784
1781
|
client = self.get_pipeline_service_client(region)
|
1785
1782
|
name = client.pipeline_job_path(project_id, region, pipeline_job)
|
1786
1783
|
|
@@ -1805,7 +1802,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1805
1802
|
metadata: Sequence[tuple[str, str]] = (),
|
1806
1803
|
) -> Operation:
|
1807
1804
|
"""
|
1808
|
-
|
1805
|
+
Delete a TrainingPipeline.
|
1809
1806
|
|
1810
1807
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1811
1808
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1838,7 +1835,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1838
1835
|
metadata: Sequence[tuple[str, str]] = (),
|
1839
1836
|
) -> Operation:
|
1840
1837
|
"""
|
1841
|
-
|
1838
|
+
Delete a CustomJob.
|
1842
1839
|
|
1843
1840
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1844
1841
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1861,6 +1858,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
1861
1858
|
return result
|
1862
1859
|
|
1863
1860
|
@GoogleBaseHook.fallback_to_default_project_id
|
1861
|
+
@deprecated(
|
1862
|
+
reason="Please use `PipelineJobHook.get_pipeline_job`",
|
1863
|
+
category=AirflowProviderDeprecationWarning,
|
1864
|
+
)
|
1864
1865
|
def get_pipeline_job(
|
1865
1866
|
self,
|
1866
1867
|
project_id: str,
|
@@ -1871,7 +1872,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1871
1872
|
metadata: Sequence[tuple[str, str]] = (),
|
1872
1873
|
) -> PipelineJob:
|
1873
1874
|
"""
|
1874
|
-
|
1875
|
+
Get a PipelineJob.
|
1875
1876
|
|
1876
1877
|
This method is deprecated, please use `PipelineJobHook.get_pipeline_job` method.
|
1877
1878
|
|
@@ -1882,11 +1883,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
1882
1883
|
:param timeout: The timeout for this request.
|
1883
1884
|
:param metadata: Strings which should be sent along with the request as metadata.
|
1884
1885
|
"""
|
1885
|
-
warnings.warn(
|
1886
|
-
"This method is deprecated, please use `PipelineJobHook.get_pipeline_job` method.",
|
1887
|
-
AirflowProviderDeprecationWarning,
|
1888
|
-
stacklevel=2,
|
1889
|
-
)
|
1890
1886
|
client = self.get_pipeline_service_client(region)
|
1891
1887
|
name = client.pipeline_job_path(project_id, region, pipeline_job)
|
1892
1888
|
|
@@ -1911,7 +1907,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1911
1907
|
metadata: Sequence[tuple[str, str]] = (),
|
1912
1908
|
) -> TrainingPipeline:
|
1913
1909
|
"""
|
1914
|
-
|
1910
|
+
Get a TrainingPipeline.
|
1915
1911
|
|
1916
1912
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1917
1913
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1944,7 +1940,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1944
1940
|
metadata: Sequence[tuple[str, str]] = (),
|
1945
1941
|
) -> CustomJob:
|
1946
1942
|
"""
|
1947
|
-
|
1943
|
+
Get a CustomJob.
|
1948
1944
|
|
1949
1945
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
1950
1946
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -1967,6 +1963,10 @@ class CustomJobHook(GoogleBaseHook):
|
|
1967
1963
|
return result
|
1968
1964
|
|
1969
1965
|
@GoogleBaseHook.fallback_to_default_project_id
|
1966
|
+
@deprecated(
|
1967
|
+
reason="Please use `PipelineJobHook.list_pipeline_jobs`",
|
1968
|
+
category=AirflowProviderDeprecationWarning,
|
1969
|
+
)
|
1970
1970
|
def list_pipeline_jobs(
|
1971
1971
|
self,
|
1972
1972
|
project_id: str,
|
@@ -1980,7 +1980,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
1980
1980
|
metadata: Sequence[tuple[str, str]] = (),
|
1981
1981
|
) -> ListPipelineJobsPager:
|
1982
1982
|
"""
|
1983
|
-
|
1983
|
+
List PipelineJobs in a Location.
|
1984
1984
|
|
1985
1985
|
This method is deprecated, please use `PipelineJobHook.list_pipeline_jobs` method.
|
1986
1986
|
|
@@ -2039,11 +2039,6 @@ class CustomJobHook(GoogleBaseHook):
|
|
2039
2039
|
:param timeout: The timeout for this request.
|
2040
2040
|
:param metadata: Strings which should be sent along with the request as metadata.
|
2041
2041
|
"""
|
2042
|
-
warnings.warn(
|
2043
|
-
"This method is deprecated, please use `PipelineJobHook.list_pipeline_jobs` method.",
|
2044
|
-
AirflowProviderDeprecationWarning,
|
2045
|
-
stacklevel=2,
|
2046
|
-
)
|
2047
2042
|
client = self.get_pipeline_service_client(region)
|
2048
2043
|
parent = client.common_location_path(project_id, region)
|
2049
2044
|
|
@@ -2075,7 +2070,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
2075
2070
|
metadata: Sequence[tuple[str, str]] = (),
|
2076
2071
|
) -> ListTrainingPipelinesPager:
|
2077
2072
|
"""
|
2078
|
-
|
2073
|
+
List TrainingPipelines in a Location.
|
2079
2074
|
|
2080
2075
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
2081
2076
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -2136,7 +2131,7 @@ class CustomJobHook(GoogleBaseHook):
|
|
2136
2131
|
metadata: Sequence[tuple[str, str]] = (),
|
2137
2132
|
) -> ListCustomJobsPager:
|
2138
2133
|
"""
|
2139
|
-
|
2134
|
+
List CustomJobs in a Location.
|
2140
2135
|
|
2141
2136
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
2142
2137
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -52,7 +52,7 @@ class DatasetHook(GoogleBaseHook):
|
|
52
52
|
super().__init__(**kwargs)
|
53
53
|
|
54
54
|
def get_dataset_service_client(self, region: str | None = None) -> DatasetServiceClient:
|
55
|
-
"""
|
55
|
+
"""Return DatasetServiceClient."""
|
56
56
|
if region and region != "global":
|
57
57
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
58
58
|
else:
|
@@ -63,7 +63,7 @@ class DatasetHook(GoogleBaseHook):
|
|
63
63
|
)
|
64
64
|
|
65
65
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
66
|
-
"""
|
66
|
+
"""Wait for long-lasting operation to complete."""
|
67
67
|
try:
|
68
68
|
return operation.result(timeout=timeout)
|
69
69
|
except Exception:
|
@@ -72,7 +72,7 @@ class DatasetHook(GoogleBaseHook):
|
|
72
72
|
|
73
73
|
@staticmethod
|
74
74
|
def extract_dataset_id(obj: dict) -> str:
|
75
|
-
"""
|
75
|
+
"""Return unique id of the dataset."""
|
76
76
|
return obj["name"].rpartition("/")[-1]
|
77
77
|
|
78
78
|
@GoogleBaseHook.fallback_to_default_project_id
|
@@ -86,7 +86,7 @@ class DatasetHook(GoogleBaseHook):
|
|
86
86
|
metadata: Sequence[tuple[str, str]] = (),
|
87
87
|
) -> Operation:
|
88
88
|
"""
|
89
|
-
|
89
|
+
Create a Dataset.
|
90
90
|
|
91
91
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
92
92
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -120,7 +120,7 @@ class DatasetHook(GoogleBaseHook):
|
|
120
120
|
metadata: Sequence[tuple[str, str]] = (),
|
121
121
|
) -> Operation:
|
122
122
|
"""
|
123
|
-
|
123
|
+
Delete a Dataset.
|
124
124
|
|
125
125
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
126
126
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -154,7 +154,7 @@ class DatasetHook(GoogleBaseHook):
|
|
154
154
|
metadata: Sequence[tuple[str, str]] = (),
|
155
155
|
) -> Operation:
|
156
156
|
"""
|
157
|
-
|
157
|
+
Export data from a Dataset.
|
158
158
|
|
159
159
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
160
160
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -191,7 +191,7 @@ class DatasetHook(GoogleBaseHook):
|
|
191
191
|
metadata: Sequence[tuple[str, str]] = (),
|
192
192
|
) -> AnnotationSpec:
|
193
193
|
"""
|
194
|
-
|
194
|
+
Get an AnnotationSpec.
|
195
195
|
|
196
196
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
197
197
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -228,7 +228,7 @@ class DatasetHook(GoogleBaseHook):
|
|
228
228
|
metadata: Sequence[tuple[str, str]] = (),
|
229
229
|
) -> Dataset:
|
230
230
|
"""
|
231
|
-
|
231
|
+
Get a Dataset.
|
232
232
|
|
233
233
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
234
234
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -264,7 +264,7 @@ class DatasetHook(GoogleBaseHook):
|
|
264
264
|
metadata: Sequence[tuple[str, str]] = (),
|
265
265
|
) -> Operation:
|
266
266
|
"""
|
267
|
-
|
267
|
+
Import data into a Dataset.
|
268
268
|
|
269
269
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
270
270
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -306,7 +306,7 @@ class DatasetHook(GoogleBaseHook):
|
|
306
306
|
metadata: Sequence[tuple[str, str]] = (),
|
307
307
|
) -> ListAnnotationsPager:
|
308
308
|
"""
|
309
|
-
|
309
|
+
List Annotations belongs to a data item.
|
310
310
|
|
311
311
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
312
312
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -356,7 +356,7 @@ class DatasetHook(GoogleBaseHook):
|
|
356
356
|
metadata: Sequence[tuple[str, str]] = (),
|
357
357
|
) -> ListDataItemsPager:
|
358
358
|
"""
|
359
|
-
|
359
|
+
List DataItems in a Dataset.
|
360
360
|
|
361
361
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
362
362
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -404,7 +404,7 @@ class DatasetHook(GoogleBaseHook):
|
|
404
404
|
metadata: Sequence[tuple[str, str]] = (),
|
405
405
|
) -> ListDatasetsPager:
|
406
406
|
"""
|
407
|
-
|
407
|
+
List Datasets in a Location.
|
408
408
|
|
409
409
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
410
410
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -448,7 +448,7 @@ class DatasetHook(GoogleBaseHook):
|
|
448
448
|
metadata: Sequence[tuple[str, str]] = (),
|
449
449
|
) -> Dataset:
|
450
450
|
"""
|
451
|
-
|
451
|
+
Update a Dataset.
|
452
452
|
|
453
453
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
454
454
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -47,7 +47,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
47
47
|
super().__init__(**kwargs)
|
48
48
|
|
49
49
|
def get_endpoint_service_client(self, region: str | None = None) -> EndpointServiceClient:
|
50
|
-
"""
|
50
|
+
"""Return EndpointServiceClient."""
|
51
51
|
if region and region != "global":
|
52
52
|
client_options = ClientOptions(api_endpoint=f"{region}-aiplatform.googleapis.com:443")
|
53
53
|
else:
|
@@ -58,7 +58,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
58
58
|
)
|
59
59
|
|
60
60
|
def wait_for_operation(self, operation: Operation, timeout: float | None = None):
|
61
|
-
"""
|
61
|
+
"""Wait for long-lasting operation to complete."""
|
62
62
|
try:
|
63
63
|
return operation.result(timeout=timeout)
|
64
64
|
except Exception:
|
@@ -67,12 +67,12 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
67
67
|
|
68
68
|
@staticmethod
|
69
69
|
def extract_endpoint_id(obj: dict) -> str:
|
70
|
-
"""
|
70
|
+
"""Return unique id of the endpoint."""
|
71
71
|
return obj["name"].rpartition("/")[-1]
|
72
72
|
|
73
73
|
@staticmethod
|
74
74
|
def extract_deployed_model_id(obj: dict) -> str:
|
75
|
-
"""
|
75
|
+
"""Return unique id of the deploy model."""
|
76
76
|
return obj["deployed_model"]["id"]
|
77
77
|
|
78
78
|
@GoogleBaseHook.fallback_to_default_project_id
|
@@ -87,7 +87,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
87
87
|
metadata: Sequence[tuple[str, str]] = (),
|
88
88
|
) -> Operation:
|
89
89
|
"""
|
90
|
-
|
90
|
+
Create an Endpoint.
|
91
91
|
|
92
92
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
93
93
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -124,7 +124,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
124
124
|
metadata: Sequence[tuple[str, str]] = (),
|
125
125
|
) -> Operation:
|
126
126
|
"""
|
127
|
-
|
127
|
+
Delete an Endpoint.
|
128
128
|
|
129
129
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
130
130
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -210,7 +210,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
210
210
|
metadata: Sequence[tuple[str, str]] = (),
|
211
211
|
) -> Endpoint:
|
212
212
|
"""
|
213
|
-
|
213
|
+
Get an Endpoint.
|
214
214
|
|
215
215
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
216
216
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -247,7 +247,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
247
247
|
metadata: Sequence[tuple[str, str]] = (),
|
248
248
|
) -> ListEndpointsPager:
|
249
249
|
"""
|
250
|
-
|
250
|
+
List Endpoints in a Location.
|
251
251
|
|
252
252
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
253
253
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|
@@ -352,7 +352,7 @@ class EndpointServiceHook(GoogleBaseHook):
|
|
352
352
|
metadata: Sequence[tuple[str, str]] = (),
|
353
353
|
) -> Endpoint:
|
354
354
|
"""
|
355
|
-
|
355
|
+
Update an Endpoint.
|
356
356
|
|
357
357
|
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
358
358
|
:param region: Required. The ID of the Google Cloud region that the service belongs to.
|