apache-airflow-providers-google 10.14.0rc2__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +1 -1
- airflow/providers/google/cloud/hooks/automl.py +13 -13
- airflow/providers/google/cloud/hooks/bigquery.py +193 -246
- airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
- airflow/providers/google/cloud/hooks/bigtable.py +8 -8
- airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
- airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
- airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
- airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
- airflow/providers/google/cloud/hooks/cloud_sql.py +17 -17
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
- airflow/providers/google/cloud/hooks/compute.py +16 -16
- airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
- airflow/providers/google/cloud/hooks/dataflow.py +48 -49
- airflow/providers/google/cloud/hooks/dataform.py +16 -16
- airflow/providers/google/cloud/hooks/datafusion.py +15 -15
- airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
- airflow/providers/google/cloud/hooks/dataplex.py +19 -19
- airflow/providers/google/cloud/hooks/dataprep.py +8 -8
- airflow/providers/google/cloud/hooks/dataproc.py +88 -0
- airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
- airflow/providers/google/cloud/hooks/datastore.py +3 -3
- airflow/providers/google/cloud/hooks/dlp.py +25 -25
- airflow/providers/google/cloud/hooks/gcs.py +25 -23
- airflow/providers/google/cloud/hooks/gdm.py +3 -3
- airflow/providers/google/cloud/hooks/kms.py +3 -3
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
- airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
- airflow/providers/google/cloud/hooks/looker.py +7 -7
- airflow/providers/google/cloud/hooks/mlengine.py +12 -12
- airflow/providers/google/cloud/hooks/natural_language.py +2 -2
- airflow/providers/google/cloud/hooks/os_login.py +1 -1
- airflow/providers/google/cloud/hooks/pubsub.py +9 -9
- airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
- airflow/providers/google/cloud/hooks/spanner.py +11 -11
- airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
- airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
- airflow/providers/google/cloud/hooks/tasks.py +11 -11
- airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
- airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
- airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
- airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
- airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
- airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
- airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
- airflow/providers/google/cloud/hooks/vision.py +1 -1
- airflow/providers/google/cloud/hooks/workflows.py +10 -10
- airflow/providers/google/cloud/links/datafusion.py +12 -5
- airflow/providers/google/cloud/operators/bigquery.py +9 -11
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
- airflow/providers/google/cloud/operators/dataflow.py +16 -16
- airflow/providers/google/cloud/operators/datafusion.py +9 -1
- airflow/providers/google/cloud/operators/dataproc.py +298 -65
- airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
- airflow/providers/google/cloud/operators/life_sciences.py +10 -9
- airflow/providers/google/cloud/operators/mlengine.py +96 -96
- airflow/providers/google/cloud/operators/pubsub.py +2 -0
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
- airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
- airflow/providers/google/cloud/sensors/bigquery.py +20 -16
- airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
- airflow/providers/google/cloud/sensors/gcs.py +8 -7
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
- airflow/providers/google/cloud/triggers/bigquery.py +12 -12
- airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
- airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
- airflow/providers/google/cloud/triggers/dataflow.py +3 -1
- airflow/providers/google/cloud/triggers/datafusion.py +2 -2
- airflow/providers/google/cloud/triggers/dataplex.py +2 -2
- airflow/providers/google/cloud/triggers/dataproc.py +2 -2
- airflow/providers/google/cloud/triggers/gcs.py +12 -8
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/triggers/mlengine.py +2 -2
- airflow/providers/google/cloud/triggers/pubsub.py +1 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
- airflow/providers/google/cloud/utils/bigquery.py +2 -2
- airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
- airflow/providers/google/cloud/utils/dataform.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +2 -2
- airflow/providers/google/cloud/utils/helpers.py +2 -2
- airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
- airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
- airflow/providers/google/common/auth_backend/google_openid.py +2 -2
- airflow/providers/google/common/hooks/base_google.py +29 -22
- airflow/providers/google/common/hooks/discovery_api.py +2 -2
- airflow/providers/google/common/utils/id_token_credentials.py +5 -5
- airflow/providers/google/firebase/hooks/firestore.py +3 -3
- airflow/providers/google/get_provider_info.py +7 -2
- airflow/providers/google/leveldb/hooks/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
- airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
- airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
- airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
- airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
- airflow/providers/google/suite/hooks/calendar.py +2 -2
- airflow/providers/google/suite/hooks/drive.py +7 -7
- airflow/providers/google/suite/hooks/sheets.py +8 -8
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +121 -120
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.14.0rc2.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -75,7 +75,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
75
75
|
|
76
76
|
def get_conn(self):
|
77
77
|
"""
|
78
|
-
|
78
|
+
Retrieve connection to Google Compute Engine.
|
79
79
|
|
80
80
|
:return: Google Compute Engine services object
|
81
81
|
:rtype: dict
|
@@ -86,15 +86,15 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
86
86
|
return self._conn
|
87
87
|
|
88
88
|
def get_compute_instance_template_client(self):
|
89
|
-
"""
|
89
|
+
"""Return Compute Engine Instance Template Client."""
|
90
90
|
return InstanceTemplatesClient(credentials=self.get_credentials(), client_info=self.client_info)
|
91
91
|
|
92
92
|
def get_compute_instance_client(self):
|
93
|
-
"""
|
93
|
+
"""Return Compute Engine Instance Client."""
|
94
94
|
return InstancesClient(credentials=self.get_credentials(), client_info=self.client_info)
|
95
95
|
|
96
96
|
def get_compute_instance_group_managers_client(self):
|
97
|
-
"""
|
97
|
+
"""Return Compute Engine Instance Group Managers Client."""
|
98
98
|
return InstanceGroupManagersClient(credentials=self.get_credentials(), client_info=self.client_info)
|
99
99
|
|
100
100
|
@GoogleBaseHook.fallback_to_default_project_id
|
@@ -108,7 +108,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
108
108
|
metadata: Sequence[tuple[str, str]] = (),
|
109
109
|
) -> None:
|
110
110
|
"""
|
111
|
-
|
111
|
+
Create Instance Template using body specified.
|
112
112
|
|
113
113
|
Must be called with keyword arguments rather than positional.
|
114
114
|
|
@@ -160,7 +160,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
160
160
|
metadata: Sequence[tuple[str, str]] = (),
|
161
161
|
) -> None:
|
162
162
|
"""
|
163
|
-
|
163
|
+
Delete Instance Template.
|
164
164
|
|
165
165
|
Deleting an Instance Template is permanent and cannot be undone. It is not
|
166
166
|
possible to delete templates that are already in use by a managed instance
|
@@ -213,7 +213,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
213
213
|
metadata: Sequence[tuple[str, str]] = (),
|
214
214
|
) -> InstanceTemplate:
|
215
215
|
"""
|
216
|
-
|
216
|
+
Retrieve Instance Template by project_id and resource_id.
|
217
217
|
|
218
218
|
Must be called with keyword arguments rather than positional.
|
219
219
|
|
@@ -263,7 +263,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
263
263
|
metadata: Sequence[tuple[str, str]] = (),
|
264
264
|
) -> None:
|
265
265
|
"""
|
266
|
-
|
266
|
+
Create Instance using body specified.
|
267
267
|
|
268
268
|
Must be called with keyword arguments rather than positional.
|
269
269
|
|
@@ -337,7 +337,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
337
337
|
metadata: Sequence[tuple[str, str]] = (),
|
338
338
|
) -> Instance:
|
339
339
|
"""
|
340
|
-
|
340
|
+
Retrieve Instance by project_id and resource_id.
|
341
341
|
|
342
342
|
Must be called with keyword arguments rather than positional.
|
343
343
|
|
@@ -439,7 +439,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
439
439
|
@GoogleBaseHook.fallback_to_default_project_id
|
440
440
|
def start_instance(self, zone: str, resource_id: str, project_id: str) -> None:
|
441
441
|
"""
|
442
|
-
|
442
|
+
Start an existing instance defined by project_id, zone and resource_id.
|
443
443
|
|
444
444
|
Must be called with keyword arguments rather than positional.
|
445
445
|
|
@@ -465,7 +465,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
465
465
|
@GoogleBaseHook.fallback_to_default_project_id
|
466
466
|
def stop_instance(self, zone: str, resource_id: str, project_id: str) -> None:
|
467
467
|
"""
|
468
|
-
|
468
|
+
Stop an instance defined by project_id, zone and resource_id.
|
469
469
|
|
470
470
|
Must be called with keyword arguments rather than positional.
|
471
471
|
|
@@ -491,7 +491,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
491
491
|
@GoogleBaseHook.fallback_to_default_project_id
|
492
492
|
def set_machine_type(self, zone: str, resource_id: str, body: dict, project_id: str) -> None:
|
493
493
|
"""
|
494
|
-
|
494
|
+
Set machine type of an instance defined by project_id, zone and resource_id.
|
495
495
|
|
496
496
|
Must be called with keyword arguments rather than positional.
|
497
497
|
|
@@ -532,7 +532,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
532
532
|
metadata: Sequence[tuple[str, str]] = (),
|
533
533
|
) -> None:
|
534
534
|
"""
|
535
|
-
|
535
|
+
Create an Instance Group Managers using the body specified.
|
536
536
|
|
537
537
|
After the group is created, instances in the group are created using the specified Instance Template.
|
538
538
|
Must be called with keyword arguments rather than positional.
|
@@ -589,7 +589,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
589
589
|
metadata: Sequence[tuple[str, str]] = (),
|
590
590
|
) -> InstanceGroupManager:
|
591
591
|
"""
|
592
|
-
|
592
|
+
Retrieve Instance Group Manager by project_id, zone and resource_id.
|
593
593
|
|
594
594
|
Must be called with keyword arguments rather than positional.
|
595
595
|
|
@@ -737,7 +737,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
737
737
|
self, project_id: str, operation_name: str, zone: str | None = None
|
738
738
|
) -> None:
|
739
739
|
"""
|
740
|
-
|
740
|
+
Wait for the named operation to complete - checks status of the async call.
|
741
741
|
|
742
742
|
:param operation_name: name of the operation
|
743
743
|
:param zone: optional region of the request (might be None for global operations)
|
@@ -793,7 +793,7 @@ class ComputeEngineHook(GoogleBaseHook):
|
|
793
793
|
@GoogleBaseHook.fallback_to_default_project_id
|
794
794
|
def get_instance_info(self, zone: str, resource_id: str, project_id: str) -> dict[str, Any]:
|
795
795
|
"""
|
796
|
-
|
796
|
+
Get instance information.
|
797
797
|
|
798
798
|
:param zone: Google Cloud zone where the Instance Group Manager exists
|
799
799
|
:param resource_id: Name of the Instance Group Manager
|
@@ -327,7 +327,7 @@ class ComputeEngineSSHHook(SSHHook):
|
|
327
327
|
break
|
328
328
|
else:
|
329
329
|
new_dict = {"key": "ssh-keys", "value": keys}
|
330
|
-
metadata["items"] = [new_dict]
|
330
|
+
metadata["items"] = [*items, new_dict]
|
331
331
|
|
332
332
|
self._compute_hook.set_instance_metadata(
|
333
333
|
zone=self.zone, resource_id=self.instance_name, metadata=metadata, project_id=self.project_id
|
@@ -73,7 +73,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
73
73
|
self._client: DataCatalogClient | None = None
|
74
74
|
|
75
75
|
def get_conn(self) -> DataCatalogClient:
|
76
|
-
"""
|
76
|
+
"""Retrieve client library object that allow access to Cloud Data Catalog service."""
|
77
77
|
if not self._client:
|
78
78
|
self._client = DataCatalogClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
|
79
79
|
return self._client
|
@@ -91,7 +91,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
91
91
|
metadata: Sequence[tuple[str, str]] = (),
|
92
92
|
) -> Entry:
|
93
93
|
"""
|
94
|
-
|
94
|
+
Create an entry.
|
95
95
|
|
96
96
|
Currently only entries of 'FILESET' type can be created.
|
97
97
|
|
@@ -133,7 +133,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
133
133
|
metadata: Sequence[tuple[str, str]] = (),
|
134
134
|
) -> EntryGroup:
|
135
135
|
"""
|
136
|
-
|
136
|
+
Create an EntryGroup.
|
137
137
|
|
138
138
|
:param location: Required. The location of the entry group to create.
|
139
139
|
:param entry_group_id: Required. The id of the entry group to create. The id must begin with a letter
|
@@ -179,7 +179,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
179
179
|
metadata: Sequence[tuple[str, str]] = (),
|
180
180
|
) -> Tag:
|
181
181
|
"""
|
182
|
-
|
182
|
+
Create a tag on an entry.
|
183
183
|
|
184
184
|
:param location: Required. The location of the tag to create.
|
185
185
|
:param entry_group: Required. Entry group ID under which the tag is created.
|
@@ -242,7 +242,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
242
242
|
metadata: Sequence[tuple[str, str]] = (),
|
243
243
|
) -> TagTemplate:
|
244
244
|
"""
|
245
|
-
|
245
|
+
Create a tag template.
|
246
246
|
|
247
247
|
:param location: Required. The location of the tag template to create.
|
248
248
|
:param tag_template_id: Required. The id of the tag template to create.
|
@@ -301,7 +301,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
301
301
|
metadata: Sequence[tuple[str, str]] = (),
|
302
302
|
) -> TagTemplateField:
|
303
303
|
r"""
|
304
|
-
|
304
|
+
Create a field in a tag template.
|
305
305
|
|
306
306
|
:param location: Required. The location of the tag template field to create.
|
307
307
|
:param tag_template: Required. The id of the tag template to create.
|
@@ -353,7 +353,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
353
353
|
metadata: Sequence[tuple[str, str]] = (),
|
354
354
|
) -> None:
|
355
355
|
"""
|
356
|
-
|
356
|
+
Delete an existing entry.
|
357
357
|
|
358
358
|
:param location: Required. The location of the entry to delete.
|
359
359
|
:param entry_group: Required. Entry group ID for entries that is deleted.
|
@@ -383,7 +383,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
383
383
|
metadata: Sequence[tuple[str, str]] = (),
|
384
384
|
) -> None:
|
385
385
|
"""
|
386
|
-
|
386
|
+
Delete an EntryGroup.
|
387
387
|
|
388
388
|
Only entry groups that do not contain entries can be deleted.
|
389
389
|
|
@@ -419,7 +419,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
419
419
|
metadata: Sequence[tuple[str, str]] = (),
|
420
420
|
) -> None:
|
421
421
|
"""
|
422
|
-
|
422
|
+
Delete a tag.
|
423
423
|
|
424
424
|
:param location: Required. The location of the tag to delete.
|
425
425
|
:param entry_group: Entry group ID for tag that is deleted.
|
@@ -454,7 +454,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
454
454
|
metadata: Sequence[tuple[str, str]] = (),
|
455
455
|
) -> None:
|
456
456
|
"""
|
457
|
-
|
457
|
+
Delete a tag template and all tags using the template.
|
458
458
|
|
459
459
|
:param location: Required. The location of the tag template to delete.
|
460
460
|
:param tag_template: ID for tag template that is deleted.
|
@@ -491,7 +491,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
491
491
|
metadata: Sequence[tuple[str, str]] = (),
|
492
492
|
) -> None:
|
493
493
|
"""
|
494
|
-
|
494
|
+
Delete a field in a tag template and all uses of that field.
|
495
495
|
|
496
496
|
:param location: Required. The location of the tag template to delete.
|
497
497
|
:param tag_template: Tag Template ID for tag template field that is deleted.
|
@@ -526,7 +526,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
526
526
|
metadata: Sequence[tuple[str, str]] = (),
|
527
527
|
) -> Entry:
|
528
528
|
"""
|
529
|
-
|
529
|
+
Get an entry.
|
530
530
|
|
531
531
|
:param location: Required. The location of the entry to get.
|
532
532
|
:param entry_group: Required. The entry group of the entry to get.
|
@@ -562,7 +562,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
562
562
|
metadata: Sequence[tuple[str, str]] = (),
|
563
563
|
) -> EntryGroup:
|
564
564
|
"""
|
565
|
-
|
565
|
+
Get an entry group.
|
566
566
|
|
567
567
|
:param location: Required. The location of the entry group to get.
|
568
568
|
:param entry_group: The ID of the entry group to get.
|
@@ -605,7 +605,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
605
605
|
metadata: Sequence[tuple[str, str]] = (),
|
606
606
|
) -> TagTemplate:
|
607
607
|
"""
|
608
|
-
|
608
|
+
Get a tag template.
|
609
609
|
|
610
610
|
:param location: Required. The location of the tag template to get.
|
611
611
|
:param tag_template: Required. The ID of the tag template to get.
|
@@ -643,7 +643,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
643
643
|
metadata: Sequence[tuple[str, str]] = (),
|
644
644
|
):
|
645
645
|
"""
|
646
|
-
|
646
|
+
List the tags on an Entry.
|
647
647
|
|
648
648
|
:param location: Required. The location of the tags to get.
|
649
649
|
:param entry_group: Required. The entry group of the tags to get.
|
@@ -688,7 +688,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
688
688
|
metadata: Sequence[tuple[str, str]] = (),
|
689
689
|
) -> Tag:
|
690
690
|
"""
|
691
|
-
|
691
|
+
Get for a tag with a specific template for a specific entry.
|
692
692
|
|
693
693
|
:param location: Required. The location which contains the entry to search for.
|
694
694
|
:param entry_group: The entry group ID which contains the entry to search for.
|
@@ -779,7 +779,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
779
779
|
metadata: Sequence[tuple[str, str]] = (),
|
780
780
|
) -> TagTemplateField:
|
781
781
|
"""
|
782
|
-
|
782
|
+
Rename a field in a tag template.
|
783
783
|
|
784
784
|
:param location: Required. The location of the tag template field to rename.
|
785
785
|
:param tag_template: The tag template ID for field that is renamed.
|
@@ -824,7 +824,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
824
824
|
metadata: Sequence[tuple[str, str]] = (),
|
825
825
|
):
|
826
826
|
r"""
|
827
|
-
|
827
|
+
Search Data Catalog for multiple resources like entries, tags that match a query.
|
828
828
|
|
829
829
|
This does not return the complete resource, only the resource identifier and high level fields.
|
830
830
|
Clients can subsequently call ``Get`` methods.
|
@@ -898,7 +898,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
898
898
|
metadata: Sequence[tuple[str, str]] = (),
|
899
899
|
) -> Entry:
|
900
900
|
"""
|
901
|
-
|
901
|
+
Update an existing entry.
|
902
902
|
|
903
903
|
:param entry: Required. The updated entry. The "name" field must be set.
|
904
904
|
|
@@ -968,7 +968,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
968
968
|
metadata: Sequence[tuple[str, str]] = (),
|
969
969
|
) -> Tag:
|
970
970
|
"""
|
971
|
-
|
971
|
+
Update an existing tag.
|
972
972
|
|
973
973
|
:param tag: Required. The updated tag. The "name" field must be set.
|
974
974
|
|
@@ -1038,7 +1038,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
1038
1038
|
metadata: Sequence[tuple[str, str]] = (),
|
1039
1039
|
) -> TagTemplate:
|
1040
1040
|
"""
|
1041
|
-
|
1041
|
+
Update a tag template.
|
1042
1042
|
|
1043
1043
|
This method cannot be used to update the fields of a template. The tag
|
1044
1044
|
template fields are represented as separate resources and should be updated using their own
|
@@ -1112,7 +1112,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
|
|
1112
1112
|
metadata: Sequence[tuple[str, str]] = (),
|
1113
1113
|
):
|
1114
1114
|
"""
|
1115
|
-
|
1115
|
+
Update a field in a tag template. This method cannot be used to update the field type.
|
1116
1116
|
|
1117
1117
|
:param tag_template_field: Required. The template to update.
|
1118
1118
|
|
@@ -29,6 +29,7 @@ import warnings
|
|
29
29
|
from copy import deepcopy
|
30
30
|
from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
|
31
31
|
|
32
|
+
from deprecated import deprecated
|
32
33
|
from google.cloud.dataflow_v1beta3 import GetJobRequest, Job, JobState, JobsV1Beta3AsyncClient, JobView
|
33
34
|
from google.cloud.dataflow_v1beta3.types.jobs import ListJobsRequest
|
34
35
|
from googleapiclient.discovery import build
|
@@ -86,7 +87,7 @@ def process_line_and_extract_dataflow_job_id_callback(
|
|
86
87
|
def _fallback_variable_parameter(parameter_name: str, variable_key_name: str) -> Callable[[T], T]:
|
87
88
|
def _wrapper(func: T) -> T:
|
88
89
|
"""
|
89
|
-
|
90
|
+
Fallback for location from `region` key in `variables` parameters.
|
90
91
|
|
91
92
|
:param func: function to wrap
|
92
93
|
:return: result of the function call
|
@@ -224,7 +225,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
224
225
|
|
225
226
|
def is_job_running(self) -> bool:
|
226
227
|
"""
|
227
|
-
|
228
|
+
Check if job is still running in dataflow.
|
228
229
|
|
229
230
|
:return: True if job is running.
|
230
231
|
"""
|
@@ -236,7 +237,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
236
237
|
|
237
238
|
def _get_current_jobs(self) -> list[dict]:
|
238
239
|
"""
|
239
|
-
|
240
|
+
Get list of jobs that start with job name or id.
|
240
241
|
|
241
242
|
:return: list of jobs including id's
|
242
243
|
"""
|
@@ -254,7 +255,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
254
255
|
|
255
256
|
def fetch_job_by_id(self, job_id: str) -> dict:
|
256
257
|
"""
|
257
|
-
|
258
|
+
Fetch the job with the specified Job ID.
|
258
259
|
|
259
260
|
:param job_id: Job ID to get.
|
260
261
|
:return: the Job
|
@@ -273,7 +274,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
273
274
|
|
274
275
|
def fetch_job_metrics_by_id(self, job_id: str) -> dict:
|
275
276
|
"""
|
276
|
-
|
277
|
+
Fetch the job metrics with the specified Job ID.
|
277
278
|
|
278
279
|
:param job_id: Job ID to get.
|
279
280
|
:return: the JobMetrics. See:
|
@@ -292,7 +293,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
292
293
|
|
293
294
|
def _fetch_list_job_messages_responses(self, job_id: str) -> Generator[dict, None, None]:
|
294
295
|
"""
|
295
|
-
|
296
|
+
Fetch ListJobMessagesResponse with the specified Job ID.
|
296
297
|
|
297
298
|
:param job_id: Job ID to get.
|
298
299
|
:return: yields the ListJobMessagesResponse. See:
|
@@ -320,7 +321,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
320
321
|
|
321
322
|
def fetch_job_messages_by_id(self, job_id: str) -> list[dict]:
|
322
323
|
"""
|
323
|
-
|
324
|
+
Fetch the job messages with the specified Job ID.
|
324
325
|
|
325
326
|
:param job_id: Job ID to get.
|
326
327
|
:return: the list of JobMessages. See:
|
@@ -333,7 +334,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
333
334
|
|
334
335
|
def fetch_job_autoscaling_events_by_id(self, job_id: str) -> list[dict]:
|
335
336
|
"""
|
336
|
-
|
337
|
+
Fetch the job autoscaling events with the specified Job ID.
|
337
338
|
|
338
339
|
:param job_id: Job ID to get.
|
339
340
|
:return: the list of AutoscalingEvents. See:
|
@@ -374,7 +375,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
374
375
|
|
375
376
|
def _refresh_jobs(self) -> None:
|
376
377
|
"""
|
377
|
-
|
378
|
+
Get all jobs by name.
|
378
379
|
|
379
380
|
:return: jobs
|
380
381
|
"""
|
@@ -392,7 +393,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
392
393
|
|
393
394
|
def _check_dataflow_job_state(self, job) -> bool:
|
394
395
|
"""
|
395
|
-
|
396
|
+
Check the state of one job in dataflow for this task if job failed raise exception.
|
396
397
|
|
397
398
|
:return: True if job is done.
|
398
399
|
:raise: Exception
|
@@ -439,7 +440,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
439
440
|
)
|
440
441
|
|
441
442
|
def wait_for_done(self) -> None:
|
442
|
-
"""
|
443
|
+
"""Wait for result of submitted job."""
|
443
444
|
self.log.info("Start waiting for done.")
|
444
445
|
self._refresh_jobs()
|
445
446
|
while self._jobs and not all(self._check_dataflow_job_state(job) for job in self._jobs):
|
@@ -449,7 +450,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
449
450
|
|
450
451
|
def get_jobs(self, refresh: bool = False) -> list[dict]:
|
451
452
|
"""
|
452
|
-
|
453
|
+
Return Dataflow jobs.
|
453
454
|
|
454
455
|
:param refresh: Forces the latest data to be fetched.
|
455
456
|
:return: list of jobs
|
@@ -462,7 +463,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
462
463
|
return self._jobs
|
463
464
|
|
464
465
|
def _wait_for_states(self, expected_states: set[str]):
|
465
|
-
"""
|
466
|
+
"""Wait for the jobs to reach a certain state."""
|
466
467
|
if not self._jobs:
|
467
468
|
raise ValueError("The _jobs should be set")
|
468
469
|
while True:
|
@@ -485,7 +486,7 @@ class _DataflowJobsController(LoggingMixin):
|
|
485
486
|
time.sleep(self._poll_sleep)
|
486
487
|
|
487
488
|
def cancel(self) -> None:
|
488
|
-
"""
|
489
|
+
"""Cancel or drains current job."""
|
489
490
|
self._jobs = [
|
490
491
|
job for job in self.get_jobs() if job["currentState"] not in DataflowJobStatus.TERMINAL_STATES
|
491
492
|
]
|
@@ -560,13 +561,22 @@ class DataflowHook(GoogleBaseHook):
|
|
560
561
|
)
|
561
562
|
|
562
563
|
def get_conn(self) -> build:
|
563
|
-
"""
|
564
|
+
"""Return a Google Cloud Dataflow service object."""
|
564
565
|
http_authorized = self._authorize()
|
565
566
|
return build("dataflow", "v1b3", http=http_authorized, cache_discovery=False)
|
566
567
|
|
567
568
|
@_fallback_to_location_from_variables
|
568
569
|
@_fallback_to_project_id_from_variables
|
569
570
|
@GoogleBaseHook.fallback_to_default_project_id
|
571
|
+
@deprecated(
|
572
|
+
reason=(
|
573
|
+
"This method is deprecated. "
|
574
|
+
"Please use `airflow.providers.apache.beam.hooks.beam.start.start_java_pipeline` "
|
575
|
+
"to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done` "
|
576
|
+
"to wait for the required pipeline state."
|
577
|
+
),
|
578
|
+
category=AirflowProviderDeprecationWarning,
|
579
|
+
)
|
570
580
|
def start_java_dataflow(
|
571
581
|
self,
|
572
582
|
job_name: str,
|
@@ -580,7 +590,7 @@ class DataflowHook(GoogleBaseHook):
|
|
580
590
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
581
591
|
) -> None:
|
582
592
|
"""
|
583
|
-
|
593
|
+
Start Dataflow java job.
|
584
594
|
|
585
595
|
:param job_name: The name of the job.
|
586
596
|
:param variables: Variables passed to the job.
|
@@ -593,16 +603,6 @@ class DataflowHook(GoogleBaseHook):
|
|
593
603
|
:param on_new_job_id_callback: Callback called when the job ID is known.
|
594
604
|
:param location: Job location.
|
595
605
|
"""
|
596
|
-
warnings.warn(
|
597
|
-
""""This method is deprecated.
|
598
|
-
Please use `airflow.providers.apache.beam.hooks.beam.start.start_java_pipeline`
|
599
|
-
to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done`
|
600
|
-
to wait for the required pipeline state.
|
601
|
-
""",
|
602
|
-
AirflowProviderDeprecationWarning,
|
603
|
-
stacklevel=3,
|
604
|
-
)
|
605
|
-
|
606
606
|
name = self.build_dataflow_job_name(job_name, append_job_name)
|
607
607
|
|
608
608
|
variables["jobName"] = name
|
@@ -642,7 +642,7 @@ class DataflowHook(GoogleBaseHook):
|
|
642
642
|
environment: dict | None = None,
|
643
643
|
) -> dict:
|
644
644
|
"""
|
645
|
-
|
645
|
+
Start Dataflow template job.
|
646
646
|
|
647
647
|
:param job_name: The name of the job.
|
648
648
|
:param variables: Map of job runtime environment options.
|
@@ -767,7 +767,7 @@ class DataflowHook(GoogleBaseHook):
|
|
767
767
|
on_new_job_callback: Callable[[dict], None] | None = None,
|
768
768
|
) -> dict:
|
769
769
|
"""
|
770
|
-
|
770
|
+
Start flex templates with the Dataflow pipeline.
|
771
771
|
|
772
772
|
:param body: The request body. See:
|
773
773
|
https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.locations.flexTemplates/launch#request-body
|
@@ -816,6 +816,15 @@ class DataflowHook(GoogleBaseHook):
|
|
816
816
|
@_fallback_to_location_from_variables
|
817
817
|
@_fallback_to_project_id_from_variables
|
818
818
|
@GoogleBaseHook.fallback_to_default_project_id
|
819
|
+
@deprecated(
|
820
|
+
reason=(
|
821
|
+
"This method is deprecated. "
|
822
|
+
"Please use `airflow.providers.apache.beam.hooks.beam.start.start_python_pipeline` "
|
823
|
+
"to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done` "
|
824
|
+
"to wait for the required pipeline state."
|
825
|
+
),
|
826
|
+
category=AirflowProviderDeprecationWarning,
|
827
|
+
)
|
819
828
|
def start_python_dataflow(
|
820
829
|
self,
|
821
830
|
job_name: str,
|
@@ -831,7 +840,7 @@ class DataflowHook(GoogleBaseHook):
|
|
831
840
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
832
841
|
):
|
833
842
|
"""
|
834
|
-
|
843
|
+
Start Dataflow job.
|
835
844
|
|
836
845
|
:param job_name: The name of the job.
|
837
846
|
:param variables: Variables passed to the job.
|
@@ -859,16 +868,6 @@ class DataflowHook(GoogleBaseHook):
|
|
859
868
|
:param on_new_job_id_callback: Callback called when the job ID is known.
|
860
869
|
:param location: Job location.
|
861
870
|
"""
|
862
|
-
warnings.warn(
|
863
|
-
"""This method is deprecated.
|
864
|
-
Please use `airflow.providers.apache.beam.hooks.beam.start.start_python_pipeline`
|
865
|
-
to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done`
|
866
|
-
to wait for the required pipeline state.
|
867
|
-
""",
|
868
|
-
AirflowProviderDeprecationWarning,
|
869
|
-
stacklevel=3,
|
870
|
-
)
|
871
|
-
|
872
871
|
name = self.build_dataflow_job_name(job_name, append_job_name)
|
873
872
|
variables["job_name"] = name
|
874
873
|
variables["region"] = location
|
@@ -892,7 +891,7 @@ class DataflowHook(GoogleBaseHook):
|
|
892
891
|
|
893
892
|
@staticmethod
|
894
893
|
def build_dataflow_job_name(job_name: str, append_job_name: bool = True) -> str:
|
895
|
-
"""
|
894
|
+
"""Build Dataflow job name."""
|
896
895
|
base_job_name = str(job_name).replace("_", "-")
|
897
896
|
|
898
897
|
if not re.fullmatch(r"[a-z]([-a-z0-9]*[a-z0-9])?", base_job_name):
|
@@ -919,7 +918,7 @@ class DataflowHook(GoogleBaseHook):
|
|
919
918
|
variables: dict | None = None,
|
920
919
|
) -> bool:
|
921
920
|
"""
|
922
|
-
|
921
|
+
Check if jos is still running in dataflow.
|
923
922
|
|
924
923
|
:param name: The name of the job.
|
925
924
|
:param project_id: Optional, the Google Cloud project ID in which to start a job.
|
@@ -955,7 +954,7 @@ class DataflowHook(GoogleBaseHook):
|
|
955
954
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
956
955
|
) -> None:
|
957
956
|
"""
|
958
|
-
|
957
|
+
Cancel the job with the specified name prefix or Job ID.
|
959
958
|
|
960
959
|
Parameter ``name`` and ``job_id`` are mutually exclusive.
|
961
960
|
|
@@ -990,7 +989,7 @@ class DataflowHook(GoogleBaseHook):
|
|
990
989
|
on_new_job_callback: Callable[[dict], None] | None = None,
|
991
990
|
):
|
992
991
|
"""
|
993
|
-
|
992
|
+
Start Dataflow SQL query.
|
994
993
|
|
995
994
|
:param job_name: The unique name to assign to the Cloud Dataflow job.
|
996
995
|
:param query: The SQL query to execute.
|
@@ -1083,7 +1082,7 @@ class DataflowHook(GoogleBaseHook):
|
|
1083
1082
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
1084
1083
|
) -> dict:
|
1085
1084
|
"""
|
1086
|
-
|
1085
|
+
Get the job with the specified Job ID.
|
1087
1086
|
|
1088
1087
|
:param job_id: Job ID to get.
|
1089
1088
|
:param project_id: Optional, the Google Cloud project ID in which to start a job.
|
@@ -1107,7 +1106,7 @@ class DataflowHook(GoogleBaseHook):
|
|
1107
1106
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
1108
1107
|
) -> dict:
|
1109
1108
|
"""
|
1110
|
-
|
1109
|
+
Get the job metrics with the specified Job ID.
|
1111
1110
|
|
1112
1111
|
:param job_id: Job ID to get.
|
1113
1112
|
:param project_id: Optional, the Google Cloud project ID in which to start a job.
|
@@ -1132,7 +1131,7 @@ class DataflowHook(GoogleBaseHook):
|
|
1132
1131
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
1133
1132
|
) -> list[dict]:
|
1134
1133
|
"""
|
1135
|
-
|
1134
|
+
Get the job messages with the specified Job ID.
|
1136
1135
|
|
1137
1136
|
:param job_id: Job ID to get.
|
1138
1137
|
:param project_id: Optional, the Google Cloud project ID in which to start a job.
|
@@ -1156,7 +1155,7 @@ class DataflowHook(GoogleBaseHook):
|
|
1156
1155
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
1157
1156
|
) -> list[dict]:
|
1158
1157
|
"""
|
1159
|
-
|
1158
|
+
Get the job autoscaling events with the specified Job ID.
|
1160
1159
|
|
1161
1160
|
:param job_id: Job ID to get.
|
1162
1161
|
:param project_id: Optional, the Google Cloud project ID in which to start a job.
|
@@ -1266,7 +1265,7 @@ class AsyncDataflowHook(GoogleBaseAsyncHook):
|
|
1266
1265
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
1267
1266
|
) -> Job:
|
1268
1267
|
"""
|
1269
|
-
|
1268
|
+
Get the job with the specified Job ID.
|
1270
1269
|
|
1271
1270
|
:param job_id: Job ID to get.
|
1272
1271
|
:param project_id: the Google Cloud project ID in which to start a job.
|
@@ -1301,7 +1300,7 @@ class AsyncDataflowHook(GoogleBaseAsyncHook):
|
|
1301
1300
|
location: str = DEFAULT_DATAFLOW_LOCATION,
|
1302
1301
|
) -> JobState:
|
1303
1302
|
"""
|
1304
|
-
|
1303
|
+
Get the job status with the specified Job ID.
|
1305
1304
|
|
1306
1305
|
:param job_id: Job ID to get.
|
1307
1306
|
:param project_id: the Google Cloud project ID in which to start a job.
|