apache-airflow-providers-google 10.14.0rc1__py3-none-any.whl → 10.15.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +1 -2
  3. airflow/providers/google/cloud/hooks/automl.py +13 -13
  4. airflow/providers/google/cloud/hooks/bigquery.py +208 -256
  5. airflow/providers/google/cloud/hooks/bigquery_dts.py +6 -6
  6. airflow/providers/google/cloud/hooks/bigtable.py +8 -8
  7. airflow/providers/google/cloud/hooks/cloud_batch.py +1 -1
  8. airflow/providers/google/cloud/hooks/cloud_build.py +19 -20
  9. airflow/providers/google/cloud/hooks/cloud_composer.py +4 -4
  10. airflow/providers/google/cloud/hooks/cloud_memorystore.py +10 -10
  11. airflow/providers/google/cloud/hooks/cloud_run.py +1 -1
  12. airflow/providers/google/cloud/hooks/cloud_sql.py +18 -19
  13. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +3 -3
  14. airflow/providers/google/cloud/hooks/compute.py +16 -16
  15. airflow/providers/google/cloud/hooks/compute_ssh.py +1 -1
  16. airflow/providers/google/cloud/hooks/datacatalog.py +22 -22
  17. airflow/providers/google/cloud/hooks/dataflow.py +48 -49
  18. airflow/providers/google/cloud/hooks/dataform.py +16 -16
  19. airflow/providers/google/cloud/hooks/datafusion.py +15 -15
  20. airflow/providers/google/cloud/hooks/datapipeline.py +3 -3
  21. airflow/providers/google/cloud/hooks/dataplex.py +19 -19
  22. airflow/providers/google/cloud/hooks/dataprep.py +10 -10
  23. airflow/providers/google/cloud/hooks/dataproc.py +132 -14
  24. airflow/providers/google/cloud/hooks/dataproc_metastore.py +13 -13
  25. airflow/providers/google/cloud/hooks/datastore.py +3 -3
  26. airflow/providers/google/cloud/hooks/dlp.py +25 -25
  27. airflow/providers/google/cloud/hooks/gcs.py +39 -27
  28. airflow/providers/google/cloud/hooks/gdm.py +3 -3
  29. airflow/providers/google/cloud/hooks/kms.py +3 -3
  30. airflow/providers/google/cloud/hooks/kubernetes_engine.py +63 -48
  31. airflow/providers/google/cloud/hooks/life_sciences.py +13 -12
  32. airflow/providers/google/cloud/hooks/looker.py +8 -9
  33. airflow/providers/google/cloud/hooks/mlengine.py +12 -12
  34. airflow/providers/google/cloud/hooks/natural_language.py +2 -2
  35. airflow/providers/google/cloud/hooks/os_login.py +1 -1
  36. airflow/providers/google/cloud/hooks/pubsub.py +9 -9
  37. airflow/providers/google/cloud/hooks/secret_manager.py +1 -1
  38. airflow/providers/google/cloud/hooks/spanner.py +11 -11
  39. airflow/providers/google/cloud/hooks/speech_to_text.py +1 -1
  40. airflow/providers/google/cloud/hooks/stackdriver.py +7 -7
  41. airflow/providers/google/cloud/hooks/tasks.py +11 -11
  42. airflow/providers/google/cloud/hooks/text_to_speech.py +1 -1
  43. airflow/providers/google/cloud/hooks/translate.py +1 -1
  44. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +13 -13
  45. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +6 -6
  46. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +45 -50
  47. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +13 -13
  48. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +9 -9
  49. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +128 -11
  50. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +10 -10
  51. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +8 -8
  52. airflow/providers/google/cloud/hooks/video_intelligence.py +2 -2
  53. airflow/providers/google/cloud/hooks/vision.py +1 -1
  54. airflow/providers/google/cloud/hooks/workflows.py +10 -10
  55. airflow/providers/google/cloud/links/datafusion.py +12 -5
  56. airflow/providers/google/cloud/operators/bigquery.py +11 -11
  57. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +3 -1
  58. airflow/providers/google/cloud/operators/dataflow.py +16 -16
  59. airflow/providers/google/cloud/operators/datafusion.py +9 -1
  60. airflow/providers/google/cloud/operators/dataproc.py +444 -69
  61. airflow/providers/google/cloud/operators/kubernetes_engine.py +6 -6
  62. airflow/providers/google/cloud/operators/life_sciences.py +10 -9
  63. airflow/providers/google/cloud/operators/mlengine.py +96 -96
  64. airflow/providers/google/cloud/operators/pubsub.py +2 -0
  65. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +33 -3
  66. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +59 -2
  67. airflow/providers/google/cloud/secrets/secret_manager.py +8 -7
  68. airflow/providers/google/cloud/sensors/bigquery.py +20 -16
  69. airflow/providers/google/cloud/sensors/cloud_composer.py +11 -8
  70. airflow/providers/google/cloud/sensors/dataproc_metastore.py +12 -2
  71. airflow/providers/google/cloud/sensors/gcs.py +8 -7
  72. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -0
  73. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +4 -4
  74. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -0
  75. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
  76. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  77. airflow/providers/google/cloud/transfers/mysql_to_gcs.py +1 -1
  78. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +1 -1
  79. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +1 -1
  80. airflow/providers/google/cloud/transfers/presto_to_gcs.py +1 -1
  81. airflow/providers/google/cloud/transfers/s3_to_gcs.py +3 -3
  82. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
  83. airflow/providers/google/cloud/transfers/sql_to_gcs.py +3 -3
  84. airflow/providers/google/cloud/transfers/trino_to_gcs.py +1 -1
  85. airflow/providers/google/cloud/triggers/bigquery.py +12 -12
  86. airflow/providers/google/cloud/triggers/bigquery_dts.py +1 -1
  87. airflow/providers/google/cloud/triggers/cloud_batch.py +3 -1
  88. airflow/providers/google/cloud/triggers/cloud_build.py +2 -2
  89. airflow/providers/google/cloud/triggers/cloud_run.py +1 -1
  90. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +6 -6
  91. airflow/providers/google/cloud/triggers/dataflow.py +3 -1
  92. airflow/providers/google/cloud/triggers/datafusion.py +2 -2
  93. airflow/providers/google/cloud/triggers/dataplex.py +2 -2
  94. airflow/providers/google/cloud/triggers/dataproc.py +34 -14
  95. airflow/providers/google/cloud/triggers/gcs.py +12 -8
  96. airflow/providers/google/cloud/triggers/kubernetes_engine.py +2 -2
  97. airflow/providers/google/cloud/triggers/mlengine.py +2 -2
  98. airflow/providers/google/cloud/triggers/pubsub.py +1 -1
  99. airflow/providers/google/cloud/triggers/vertex_ai.py +99 -0
  100. airflow/providers/google/cloud/utils/bigquery.py +2 -2
  101. airflow/providers/google/cloud/utils/credentials_provider.py +2 -2
  102. airflow/providers/google/cloud/utils/dataform.py +1 -1
  103. airflow/providers/google/cloud/utils/dataproc.py +25 -0
  104. airflow/providers/google/cloud/utils/field_validator.py +2 -2
  105. airflow/providers/google/cloud/utils/helpers.py +2 -2
  106. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +1 -1
  107. airflow/providers/google/cloud/utils/mlengine_prediction_summary.py +1 -1
  108. airflow/providers/google/common/auth_backend/google_openid.py +2 -2
  109. airflow/providers/google/common/hooks/base_google.py +87 -23
  110. airflow/providers/google/common/hooks/discovery_api.py +2 -2
  111. airflow/providers/google/common/utils/id_token_credentials.py +5 -5
  112. airflow/providers/google/firebase/hooks/firestore.py +3 -3
  113. airflow/providers/google/get_provider_info.py +7 -2
  114. airflow/providers/google/leveldb/hooks/leveldb.py +4 -4
  115. airflow/providers/google/marketing_platform/hooks/analytics.py +11 -14
  116. airflow/providers/google/marketing_platform/hooks/campaign_manager.py +11 -11
  117. airflow/providers/google/marketing_platform/hooks/display_video.py +13 -13
  118. airflow/providers/google/marketing_platform/hooks/search_ads.py +4 -4
  119. airflow/providers/google/marketing_platform/operators/analytics.py +37 -32
  120. airflow/providers/google/suite/hooks/calendar.py +2 -2
  121. airflow/providers/google/suite/hooks/drive.py +7 -7
  122. airflow/providers/google/suite/hooks/sheets.py +8 -8
  123. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/METADATA +11 -11
  124. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/RECORD +126 -124
  125. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/WHEEL +0 -0
  126. {apache_airflow_providers_google-10.14.0rc1.dist-info → apache_airflow_providers_google-10.15.0rc1.dist-info}/entry_points.txt +0 -0
@@ -75,7 +75,7 @@ class ComputeEngineHook(GoogleBaseHook):
75
75
 
76
76
  def get_conn(self):
77
77
  """
78
- Retrieves connection to Google Compute Engine.
78
+ Retrieve connection to Google Compute Engine.
79
79
 
80
80
  :return: Google Compute Engine services object
81
81
  :rtype: dict
@@ -86,15 +86,15 @@ class ComputeEngineHook(GoogleBaseHook):
86
86
  return self._conn
87
87
 
88
88
  def get_compute_instance_template_client(self):
89
- """Returns Compute Engine Instance Template Client."""
89
+ """Return Compute Engine Instance Template Client."""
90
90
  return InstanceTemplatesClient(credentials=self.get_credentials(), client_info=self.client_info)
91
91
 
92
92
  def get_compute_instance_client(self):
93
- """Returns Compute Engine Instance Client."""
93
+ """Return Compute Engine Instance Client."""
94
94
  return InstancesClient(credentials=self.get_credentials(), client_info=self.client_info)
95
95
 
96
96
  def get_compute_instance_group_managers_client(self):
97
- """Returns Compute Engine Instance Group Managers Client."""
97
+ """Return Compute Engine Instance Group Managers Client."""
98
98
  return InstanceGroupManagersClient(credentials=self.get_credentials(), client_info=self.client_info)
99
99
 
100
100
  @GoogleBaseHook.fallback_to_default_project_id
@@ -108,7 +108,7 @@ class ComputeEngineHook(GoogleBaseHook):
108
108
  metadata: Sequence[tuple[str, str]] = (),
109
109
  ) -> None:
110
110
  """
111
- Creates Instance Template using body specified.
111
+ Create Instance Template using body specified.
112
112
 
113
113
  Must be called with keyword arguments rather than positional.
114
114
 
@@ -160,7 +160,7 @@ class ComputeEngineHook(GoogleBaseHook):
160
160
  metadata: Sequence[tuple[str, str]] = (),
161
161
  ) -> None:
162
162
  """
163
- Deletes Instance Template.
163
+ Delete Instance Template.
164
164
 
165
165
  Deleting an Instance Template is permanent and cannot be undone. It is not
166
166
  possible to delete templates that are already in use by a managed instance
@@ -213,7 +213,7 @@ class ComputeEngineHook(GoogleBaseHook):
213
213
  metadata: Sequence[tuple[str, str]] = (),
214
214
  ) -> InstanceTemplate:
215
215
  """
216
- Retrieves Instance Template by project_id and resource_id.
216
+ Retrieve Instance Template by project_id and resource_id.
217
217
 
218
218
  Must be called with keyword arguments rather than positional.
219
219
 
@@ -263,7 +263,7 @@ class ComputeEngineHook(GoogleBaseHook):
263
263
  metadata: Sequence[tuple[str, str]] = (),
264
264
  ) -> None:
265
265
  """
266
- Creates Instance using body specified.
266
+ Create Instance using body specified.
267
267
 
268
268
  Must be called with keyword arguments rather than positional.
269
269
 
@@ -337,7 +337,7 @@ class ComputeEngineHook(GoogleBaseHook):
337
337
  metadata: Sequence[tuple[str, str]] = (),
338
338
  ) -> Instance:
339
339
  """
340
- Retrieves Instance by project_id and resource_id.
340
+ Retrieve Instance by project_id and resource_id.
341
341
 
342
342
  Must be called with keyword arguments rather than positional.
343
343
 
@@ -439,7 +439,7 @@ class ComputeEngineHook(GoogleBaseHook):
439
439
  @GoogleBaseHook.fallback_to_default_project_id
440
440
  def start_instance(self, zone: str, resource_id: str, project_id: str) -> None:
441
441
  """
442
- Starts an existing instance defined by project_id, zone and resource_id.
442
+ Start an existing instance defined by project_id, zone and resource_id.
443
443
 
444
444
  Must be called with keyword arguments rather than positional.
445
445
 
@@ -465,7 +465,7 @@ class ComputeEngineHook(GoogleBaseHook):
465
465
  @GoogleBaseHook.fallback_to_default_project_id
466
466
  def stop_instance(self, zone: str, resource_id: str, project_id: str) -> None:
467
467
  """
468
- Stops an instance defined by project_id, zone and resource_id.
468
+ Stop an instance defined by project_id, zone and resource_id.
469
469
 
470
470
  Must be called with keyword arguments rather than positional.
471
471
 
@@ -491,7 +491,7 @@ class ComputeEngineHook(GoogleBaseHook):
491
491
  @GoogleBaseHook.fallback_to_default_project_id
492
492
  def set_machine_type(self, zone: str, resource_id: str, body: dict, project_id: str) -> None:
493
493
  """
494
- Sets machine type of an instance defined by project_id, zone and resource_id.
494
+ Set machine type of an instance defined by project_id, zone and resource_id.
495
495
 
496
496
  Must be called with keyword arguments rather than positional.
497
497
 
@@ -532,7 +532,7 @@ class ComputeEngineHook(GoogleBaseHook):
532
532
  metadata: Sequence[tuple[str, str]] = (),
533
533
  ) -> None:
534
534
  """
535
- Creates an Instance Group Managers using the body specified.
535
+ Create an Instance Group Managers using the body specified.
536
536
 
537
537
  After the group is created, instances in the group are created using the specified Instance Template.
538
538
  Must be called with keyword arguments rather than positional.
@@ -589,7 +589,7 @@ class ComputeEngineHook(GoogleBaseHook):
589
589
  metadata: Sequence[tuple[str, str]] = (),
590
590
  ) -> InstanceGroupManager:
591
591
  """
592
- Retrieves Instance Group Manager by project_id, zone and resource_id.
592
+ Retrieve Instance Group Manager by project_id, zone and resource_id.
593
593
 
594
594
  Must be called with keyword arguments rather than positional.
595
595
 
@@ -737,7 +737,7 @@ class ComputeEngineHook(GoogleBaseHook):
737
737
  self, project_id: str, operation_name: str, zone: str | None = None
738
738
  ) -> None:
739
739
  """
740
- Waits for the named operation to complete - checks status of the async call.
740
+ Wait for the named operation to complete - checks status of the async call.
741
741
 
742
742
  :param operation_name: name of the operation
743
743
  :param zone: optional region of the request (might be None for global operations)
@@ -793,7 +793,7 @@ class ComputeEngineHook(GoogleBaseHook):
793
793
  @GoogleBaseHook.fallback_to_default_project_id
794
794
  def get_instance_info(self, zone: str, resource_id: str, project_id: str) -> dict[str, Any]:
795
795
  """
796
- Gets instance information.
796
+ Get instance information.
797
797
 
798
798
  :param zone: Google Cloud zone where the Instance Group Manager exists
799
799
  :param resource_id: Name of the Instance Group Manager
@@ -327,7 +327,7 @@ class ComputeEngineSSHHook(SSHHook):
327
327
  break
328
328
  else:
329
329
  new_dict = {"key": "ssh-keys", "value": keys}
330
- metadata["items"] = [new_dict]
330
+ metadata["items"] = [*items, new_dict]
331
331
 
332
332
  self._compute_hook.set_instance_metadata(
333
333
  zone=self.zone, resource_id=self.instance_name, metadata=metadata, project_id=self.project_id
@@ -73,7 +73,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
73
73
  self._client: DataCatalogClient | None = None
74
74
 
75
75
  def get_conn(self) -> DataCatalogClient:
76
- """Retrieves client library object that allow access to Cloud Data Catalog service."""
76
+ """Retrieve client library object that allow access to Cloud Data Catalog service."""
77
77
  if not self._client:
78
78
  self._client = DataCatalogClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
79
79
  return self._client
@@ -91,7 +91,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
91
91
  metadata: Sequence[tuple[str, str]] = (),
92
92
  ) -> Entry:
93
93
  """
94
- Creates an entry.
94
+ Create an entry.
95
95
 
96
96
  Currently only entries of 'FILESET' type can be created.
97
97
 
@@ -133,7 +133,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
133
133
  metadata: Sequence[tuple[str, str]] = (),
134
134
  ) -> EntryGroup:
135
135
  """
136
- Creates an EntryGroup.
136
+ Create an EntryGroup.
137
137
 
138
138
  :param location: Required. The location of the entry group to create.
139
139
  :param entry_group_id: Required. The id of the entry group to create. The id must begin with a letter
@@ -179,7 +179,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
179
179
  metadata: Sequence[tuple[str, str]] = (),
180
180
  ) -> Tag:
181
181
  """
182
- Creates a tag on an entry.
182
+ Create a tag on an entry.
183
183
 
184
184
  :param location: Required. The location of the tag to create.
185
185
  :param entry_group: Required. Entry group ID under which the tag is created.
@@ -242,7 +242,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
242
242
  metadata: Sequence[tuple[str, str]] = (),
243
243
  ) -> TagTemplate:
244
244
  """
245
- Creates a tag template.
245
+ Create a tag template.
246
246
 
247
247
  :param location: Required. The location of the tag template to create.
248
248
  :param tag_template_id: Required. The id of the tag template to create.
@@ -301,7 +301,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
301
301
  metadata: Sequence[tuple[str, str]] = (),
302
302
  ) -> TagTemplateField:
303
303
  r"""
304
- Creates a field in a tag template.
304
+ Create a field in a tag template.
305
305
 
306
306
  :param location: Required. The location of the tag template field to create.
307
307
  :param tag_template: Required. The id of the tag template to create.
@@ -353,7 +353,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
353
353
  metadata: Sequence[tuple[str, str]] = (),
354
354
  ) -> None:
355
355
  """
356
- Deletes an existing entry.
356
+ Delete an existing entry.
357
357
 
358
358
  :param location: Required. The location of the entry to delete.
359
359
  :param entry_group: Required. Entry group ID for entries that is deleted.
@@ -383,7 +383,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
383
383
  metadata: Sequence[tuple[str, str]] = (),
384
384
  ) -> None:
385
385
  """
386
- Deletes an EntryGroup.
386
+ Delete an EntryGroup.
387
387
 
388
388
  Only entry groups that do not contain entries can be deleted.
389
389
 
@@ -419,7 +419,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
419
419
  metadata: Sequence[tuple[str, str]] = (),
420
420
  ) -> None:
421
421
  """
422
- Deletes a tag.
422
+ Delete a tag.
423
423
 
424
424
  :param location: Required. The location of the tag to delete.
425
425
  :param entry_group: Entry group ID for tag that is deleted.
@@ -454,7 +454,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
454
454
  metadata: Sequence[tuple[str, str]] = (),
455
455
  ) -> None:
456
456
  """
457
- Deletes a tag template and all tags using the template.
457
+ Delete a tag template and all tags using the template.
458
458
 
459
459
  :param location: Required. The location of the tag template to delete.
460
460
  :param tag_template: ID for tag template that is deleted.
@@ -491,7 +491,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
491
491
  metadata: Sequence[tuple[str, str]] = (),
492
492
  ) -> None:
493
493
  """
494
- Deletes a field in a tag template and all uses of that field.
494
+ Delete a field in a tag template and all uses of that field.
495
495
 
496
496
  :param location: Required. The location of the tag template to delete.
497
497
  :param tag_template: Tag Template ID for tag template field that is deleted.
@@ -526,7 +526,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
526
526
  metadata: Sequence[tuple[str, str]] = (),
527
527
  ) -> Entry:
528
528
  """
529
- Gets an entry.
529
+ Get an entry.
530
530
 
531
531
  :param location: Required. The location of the entry to get.
532
532
  :param entry_group: Required. The entry group of the entry to get.
@@ -562,7 +562,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
562
562
  metadata: Sequence[tuple[str, str]] = (),
563
563
  ) -> EntryGroup:
564
564
  """
565
- Gets an entry group.
565
+ Get an entry group.
566
566
 
567
567
  :param location: Required. The location of the entry group to get.
568
568
  :param entry_group: The ID of the entry group to get.
@@ -605,7 +605,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
605
605
  metadata: Sequence[tuple[str, str]] = (),
606
606
  ) -> TagTemplate:
607
607
  """
608
- Gets a tag template.
608
+ Get a tag template.
609
609
 
610
610
  :param location: Required. The location of the tag template to get.
611
611
  :param tag_template: Required. The ID of the tag template to get.
@@ -643,7 +643,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
643
643
  metadata: Sequence[tuple[str, str]] = (),
644
644
  ):
645
645
  """
646
- Lists the tags on an Entry.
646
+ List the tags on an Entry.
647
647
 
648
648
  :param location: Required. The location of the tags to get.
649
649
  :param entry_group: Required. The entry group of the tags to get.
@@ -688,7 +688,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
688
688
  metadata: Sequence[tuple[str, str]] = (),
689
689
  ) -> Tag:
690
690
  """
691
- Gets for a tag with a specific template for a specific entry.
691
+ Get for a tag with a specific template for a specific entry.
692
692
 
693
693
  :param location: Required. The location which contains the entry to search for.
694
694
  :param entry_group: The entry group ID which contains the entry to search for.
@@ -779,7 +779,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
779
779
  metadata: Sequence[tuple[str, str]] = (),
780
780
  ) -> TagTemplateField:
781
781
  """
782
- Renames a field in a tag template.
782
+ Rename a field in a tag template.
783
783
 
784
784
  :param location: Required. The location of the tag template field to rename.
785
785
  :param tag_template: The tag template ID for field that is renamed.
@@ -824,7 +824,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
824
824
  metadata: Sequence[tuple[str, str]] = (),
825
825
  ):
826
826
  r"""
827
- Searches Data Catalog for multiple resources like entries, tags that match a query.
827
+ Search Data Catalog for multiple resources like entries, tags that match a query.
828
828
 
829
829
  This does not return the complete resource, only the resource identifier and high level fields.
830
830
  Clients can subsequently call ``Get`` methods.
@@ -898,7 +898,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
898
898
  metadata: Sequence[tuple[str, str]] = (),
899
899
  ) -> Entry:
900
900
  """
901
- Updates an existing entry.
901
+ Update an existing entry.
902
902
 
903
903
  :param entry: Required. The updated entry. The "name" field must be set.
904
904
 
@@ -968,7 +968,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
968
968
  metadata: Sequence[tuple[str, str]] = (),
969
969
  ) -> Tag:
970
970
  """
971
- Updates an existing tag.
971
+ Update an existing tag.
972
972
 
973
973
  :param tag: Required. The updated tag. The "name" field must be set.
974
974
 
@@ -1038,7 +1038,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
1038
1038
  metadata: Sequence[tuple[str, str]] = (),
1039
1039
  ) -> TagTemplate:
1040
1040
  """
1041
- Updates a tag template.
1041
+ Update a tag template.
1042
1042
 
1043
1043
  This method cannot be used to update the fields of a template. The tag
1044
1044
  template fields are represented as separate resources and should be updated using their own
@@ -1112,7 +1112,7 @@ class CloudDataCatalogHook(GoogleBaseHook):
1112
1112
  metadata: Sequence[tuple[str, str]] = (),
1113
1113
  ):
1114
1114
  """
1115
- Updates a field in a tag template. This method cannot be used to update the field type.
1115
+ Update a field in a tag template. This method cannot be used to update the field type.
1116
1116
 
1117
1117
  :param tag_template_field: Required. The template to update.
1118
1118
 
@@ -29,6 +29,7 @@ import warnings
29
29
  from copy import deepcopy
30
30
  from typing import TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast
31
31
 
32
+ from deprecated import deprecated
32
33
  from google.cloud.dataflow_v1beta3 import GetJobRequest, Job, JobState, JobsV1Beta3AsyncClient, JobView
33
34
  from google.cloud.dataflow_v1beta3.types.jobs import ListJobsRequest
34
35
  from googleapiclient.discovery import build
@@ -86,7 +87,7 @@ def process_line_and_extract_dataflow_job_id_callback(
86
87
  def _fallback_variable_parameter(parameter_name: str, variable_key_name: str) -> Callable[[T], T]:
87
88
  def _wrapper(func: T) -> T:
88
89
  """
89
- Decorator that provides fallback for location from `region` key in `variables` parameters.
90
+ Fallback for location from `region` key in `variables` parameters.
90
91
 
91
92
  :param func: function to wrap
92
93
  :return: result of the function call
@@ -224,7 +225,7 @@ class _DataflowJobsController(LoggingMixin):
224
225
 
225
226
  def is_job_running(self) -> bool:
226
227
  """
227
- Helper method to check if job is still running in dataflow.
228
+ Check if job is still running in dataflow.
228
229
 
229
230
  :return: True if job is running.
230
231
  """
@@ -236,7 +237,7 @@ class _DataflowJobsController(LoggingMixin):
236
237
 
237
238
  def _get_current_jobs(self) -> list[dict]:
238
239
  """
239
- Helper method to get list of jobs that start with job name or id.
240
+ Get list of jobs that start with job name or id.
240
241
 
241
242
  :return: list of jobs including id's
242
243
  """
@@ -254,7 +255,7 @@ class _DataflowJobsController(LoggingMixin):
254
255
 
255
256
  def fetch_job_by_id(self, job_id: str) -> dict:
256
257
  """
257
- Helper method to fetch the job with the specified Job ID.
258
+ Fetch the job with the specified Job ID.
258
259
 
259
260
  :param job_id: Job ID to get.
260
261
  :return: the Job
@@ -273,7 +274,7 @@ class _DataflowJobsController(LoggingMixin):
273
274
 
274
275
  def fetch_job_metrics_by_id(self, job_id: str) -> dict:
275
276
  """
276
- Helper method to fetch the job metrics with the specified Job ID.
277
+ Fetch the job metrics with the specified Job ID.
277
278
 
278
279
  :param job_id: Job ID to get.
279
280
  :return: the JobMetrics. See:
@@ -292,7 +293,7 @@ class _DataflowJobsController(LoggingMixin):
292
293
 
293
294
  def _fetch_list_job_messages_responses(self, job_id: str) -> Generator[dict, None, None]:
294
295
  """
295
- Helper method to fetch ListJobMessagesResponse with the specified Job ID.
296
+ Fetch ListJobMessagesResponse with the specified Job ID.
296
297
 
297
298
  :param job_id: Job ID to get.
298
299
  :return: yields the ListJobMessagesResponse. See:
@@ -320,7 +321,7 @@ class _DataflowJobsController(LoggingMixin):
320
321
 
321
322
  def fetch_job_messages_by_id(self, job_id: str) -> list[dict]:
322
323
  """
323
- Helper method to fetch the job messages with the specified Job ID.
324
+ Fetch the job messages with the specified Job ID.
324
325
 
325
326
  :param job_id: Job ID to get.
326
327
  :return: the list of JobMessages. See:
@@ -333,7 +334,7 @@ class _DataflowJobsController(LoggingMixin):
333
334
 
334
335
  def fetch_job_autoscaling_events_by_id(self, job_id: str) -> list[dict]:
335
336
  """
336
- Helper method to fetch the job autoscaling events with the specified Job ID.
337
+ Fetch the job autoscaling events with the specified Job ID.
337
338
 
338
339
  :param job_id: Job ID to get.
339
340
  :return: the list of AutoscalingEvents. See:
@@ -374,7 +375,7 @@ class _DataflowJobsController(LoggingMixin):
374
375
 
375
376
  def _refresh_jobs(self) -> None:
376
377
  """
377
- Helper method to get all jobs by name.
378
+ Get all jobs by name.
378
379
 
379
380
  :return: jobs
380
381
  """
@@ -392,7 +393,7 @@ class _DataflowJobsController(LoggingMixin):
392
393
 
393
394
  def _check_dataflow_job_state(self, job) -> bool:
394
395
  """
395
- Helper method to check the state of one job in dataflow for this task if job failed raise exception.
396
+ Check the state of one job in dataflow for this task if job failed raise exception.
396
397
 
397
398
  :return: True if job is done.
398
399
  :raise: Exception
@@ -439,7 +440,7 @@ class _DataflowJobsController(LoggingMixin):
439
440
  )
440
441
 
441
442
  def wait_for_done(self) -> None:
442
- """Helper method to wait for result of submitted job."""
443
+ """Wait for result of submitted job."""
443
444
  self.log.info("Start waiting for done.")
444
445
  self._refresh_jobs()
445
446
  while self._jobs and not all(self._check_dataflow_job_state(job) for job in self._jobs):
@@ -449,7 +450,7 @@ class _DataflowJobsController(LoggingMixin):
449
450
 
450
451
  def get_jobs(self, refresh: bool = False) -> list[dict]:
451
452
  """
452
- Returns Dataflow jobs.
453
+ Return Dataflow jobs.
453
454
 
454
455
  :param refresh: Forces the latest data to be fetched.
455
456
  :return: list of jobs
@@ -462,7 +463,7 @@ class _DataflowJobsController(LoggingMixin):
462
463
  return self._jobs
463
464
 
464
465
  def _wait_for_states(self, expected_states: set[str]):
465
- """Waiting for the jobs to reach a certain state."""
466
+ """Wait for the jobs to reach a certain state."""
466
467
  if not self._jobs:
467
468
  raise ValueError("The _jobs should be set")
468
469
  while True:
@@ -485,7 +486,7 @@ class _DataflowJobsController(LoggingMixin):
485
486
  time.sleep(self._poll_sleep)
486
487
 
487
488
  def cancel(self) -> None:
488
- """Cancels or drains current job."""
489
+ """Cancel or drains current job."""
489
490
  self._jobs = [
490
491
  job for job in self.get_jobs() if job["currentState"] not in DataflowJobStatus.TERMINAL_STATES
491
492
  ]
@@ -560,13 +561,22 @@ class DataflowHook(GoogleBaseHook):
560
561
  )
561
562
 
562
563
  def get_conn(self) -> build:
563
- """Returns a Google Cloud Dataflow service object."""
564
+ """Return a Google Cloud Dataflow service object."""
564
565
  http_authorized = self._authorize()
565
566
  return build("dataflow", "v1b3", http=http_authorized, cache_discovery=False)
566
567
 
567
568
  @_fallback_to_location_from_variables
568
569
  @_fallback_to_project_id_from_variables
569
570
  @GoogleBaseHook.fallback_to_default_project_id
571
+ @deprecated(
572
+ reason=(
573
+ "This method is deprecated. "
574
+ "Please use `airflow.providers.apache.beam.hooks.beam.start.start_java_pipeline` "
575
+ "to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done` "
576
+ "to wait for the required pipeline state."
577
+ ),
578
+ category=AirflowProviderDeprecationWarning,
579
+ )
570
580
  def start_java_dataflow(
571
581
  self,
572
582
  job_name: str,
@@ -580,7 +590,7 @@ class DataflowHook(GoogleBaseHook):
580
590
  location: str = DEFAULT_DATAFLOW_LOCATION,
581
591
  ) -> None:
582
592
  """
583
- Starts Dataflow java job.
593
+ Start Dataflow java job.
584
594
 
585
595
  :param job_name: The name of the job.
586
596
  :param variables: Variables passed to the job.
@@ -593,16 +603,6 @@ class DataflowHook(GoogleBaseHook):
593
603
  :param on_new_job_id_callback: Callback called when the job ID is known.
594
604
  :param location: Job location.
595
605
  """
596
- warnings.warn(
597
- """"This method is deprecated.
598
- Please use `airflow.providers.apache.beam.hooks.beam.start.start_java_pipeline`
599
- to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done`
600
- to wait for the required pipeline state.
601
- """,
602
- AirflowProviderDeprecationWarning,
603
- stacklevel=3,
604
- )
605
-
606
606
  name = self.build_dataflow_job_name(job_name, append_job_name)
607
607
 
608
608
  variables["jobName"] = name
@@ -642,7 +642,7 @@ class DataflowHook(GoogleBaseHook):
642
642
  environment: dict | None = None,
643
643
  ) -> dict:
644
644
  """
645
- Starts Dataflow template job.
645
+ Start Dataflow template job.
646
646
 
647
647
  :param job_name: The name of the job.
648
648
  :param variables: Map of job runtime environment options.
@@ -767,7 +767,7 @@ class DataflowHook(GoogleBaseHook):
767
767
  on_new_job_callback: Callable[[dict], None] | None = None,
768
768
  ) -> dict:
769
769
  """
770
- Starts flex templates with the Dataflow pipeline.
770
+ Start flex templates with the Dataflow pipeline.
771
771
 
772
772
  :param body: The request body. See:
773
773
  https://cloud.google.com/dataflow/docs/reference/rest/v1b3/projects.locations.flexTemplates/launch#request-body
@@ -816,6 +816,15 @@ class DataflowHook(GoogleBaseHook):
816
816
  @_fallback_to_location_from_variables
817
817
  @_fallback_to_project_id_from_variables
818
818
  @GoogleBaseHook.fallback_to_default_project_id
819
+ @deprecated(
820
+ reason=(
821
+ "This method is deprecated. "
822
+ "Please use `airflow.providers.apache.beam.hooks.beam.start.start_python_pipeline` "
823
+ "to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done` "
824
+ "to wait for the required pipeline state."
825
+ ),
826
+ category=AirflowProviderDeprecationWarning,
827
+ )
819
828
  def start_python_dataflow(
820
829
  self,
821
830
  job_name: str,
@@ -831,7 +840,7 @@ class DataflowHook(GoogleBaseHook):
831
840
  location: str = DEFAULT_DATAFLOW_LOCATION,
832
841
  ):
833
842
  """
834
- Starts Dataflow job.
843
+ Start Dataflow job.
835
844
 
836
845
  :param job_name: The name of the job.
837
846
  :param variables: Variables passed to the job.
@@ -859,16 +868,6 @@ class DataflowHook(GoogleBaseHook):
859
868
  :param on_new_job_id_callback: Callback called when the job ID is known.
860
869
  :param location: Job location.
861
870
  """
862
- warnings.warn(
863
- """This method is deprecated.
864
- Please use `airflow.providers.apache.beam.hooks.beam.start.start_python_pipeline`
865
- to start pipeline and `providers.google.cloud.hooks.dataflow.DataflowHook.wait_for_done`
866
- to wait for the required pipeline state.
867
- """,
868
- AirflowProviderDeprecationWarning,
869
- stacklevel=3,
870
- )
871
-
872
871
  name = self.build_dataflow_job_name(job_name, append_job_name)
873
872
  variables["job_name"] = name
874
873
  variables["region"] = location
@@ -892,7 +891,7 @@ class DataflowHook(GoogleBaseHook):
892
891
 
893
892
  @staticmethod
894
893
  def build_dataflow_job_name(job_name: str, append_job_name: bool = True) -> str:
895
- """Builds Dataflow job name."""
894
+ """Build Dataflow job name."""
896
895
  base_job_name = str(job_name).replace("_", "-")
897
896
 
898
897
  if not re.fullmatch(r"[a-z]([-a-z0-9]*[a-z0-9])?", base_job_name):
@@ -919,7 +918,7 @@ class DataflowHook(GoogleBaseHook):
919
918
  variables: dict | None = None,
920
919
  ) -> bool:
921
920
  """
922
- Helper method to check if jos is still running in dataflow.
921
+ Check if jos is still running in dataflow.
923
922
 
924
923
  :param name: The name of the job.
925
924
  :param project_id: Optional, the Google Cloud project ID in which to start a job.
@@ -955,7 +954,7 @@ class DataflowHook(GoogleBaseHook):
955
954
  location: str = DEFAULT_DATAFLOW_LOCATION,
956
955
  ) -> None:
957
956
  """
958
- Cancels the job with the specified name prefix or Job ID.
957
+ Cancel the job with the specified name prefix or Job ID.
959
958
 
960
959
  Parameter ``name`` and ``job_id`` are mutually exclusive.
961
960
 
@@ -990,7 +989,7 @@ class DataflowHook(GoogleBaseHook):
990
989
  on_new_job_callback: Callable[[dict], None] | None = None,
991
990
  ):
992
991
  """
993
- Starts Dataflow SQL query.
992
+ Start Dataflow SQL query.
994
993
 
995
994
  :param job_name: The unique name to assign to the Cloud Dataflow job.
996
995
  :param query: The SQL query to execute.
@@ -1083,7 +1082,7 @@ class DataflowHook(GoogleBaseHook):
1083
1082
  location: str = DEFAULT_DATAFLOW_LOCATION,
1084
1083
  ) -> dict:
1085
1084
  """
1086
- Gets the job with the specified Job ID.
1085
+ Get the job with the specified Job ID.
1087
1086
 
1088
1087
  :param job_id: Job ID to get.
1089
1088
  :param project_id: Optional, the Google Cloud project ID in which to start a job.
@@ -1107,7 +1106,7 @@ class DataflowHook(GoogleBaseHook):
1107
1106
  location: str = DEFAULT_DATAFLOW_LOCATION,
1108
1107
  ) -> dict:
1109
1108
  """
1110
- Gets the job metrics with the specified Job ID.
1109
+ Get the job metrics with the specified Job ID.
1111
1110
 
1112
1111
  :param job_id: Job ID to get.
1113
1112
  :param project_id: Optional, the Google Cloud project ID in which to start a job.
@@ -1132,7 +1131,7 @@ class DataflowHook(GoogleBaseHook):
1132
1131
  location: str = DEFAULT_DATAFLOW_LOCATION,
1133
1132
  ) -> list[dict]:
1134
1133
  """
1135
- Gets the job messages with the specified Job ID.
1134
+ Get the job messages with the specified Job ID.
1136
1135
 
1137
1136
  :param job_id: Job ID to get.
1138
1137
  :param project_id: Optional, the Google Cloud project ID in which to start a job.
@@ -1156,7 +1155,7 @@ class DataflowHook(GoogleBaseHook):
1156
1155
  location: str = DEFAULT_DATAFLOW_LOCATION,
1157
1156
  ) -> list[dict]:
1158
1157
  """
1159
- Gets the job autoscaling events with the specified Job ID.
1158
+ Get the job autoscaling events with the specified Job ID.
1160
1159
 
1161
1160
  :param job_id: Job ID to get.
1162
1161
  :param project_id: Optional, the Google Cloud project ID in which to start a job.
@@ -1266,7 +1265,7 @@ class AsyncDataflowHook(GoogleBaseAsyncHook):
1266
1265
  location: str = DEFAULT_DATAFLOW_LOCATION,
1267
1266
  ) -> Job:
1268
1267
  """
1269
- Gets the job with the specified Job ID.
1268
+ Get the job with the specified Job ID.
1270
1269
 
1271
1270
  :param job_id: Job ID to get.
1272
1271
  :param project_id: the Google Cloud project ID in which to start a job.
@@ -1301,7 +1300,7 @@ class AsyncDataflowHook(GoogleBaseAsyncHook):
1301
1300
  location: str = DEFAULT_DATAFLOW_LOCATION,
1302
1301
  ) -> JobState:
1303
1302
  """
1304
- Gets the job status with the specified Job ID.
1303
+ Get the job status with the specified Job ID.
1305
1304
 
1306
1305
  :param job_id: Job ID to get.
1307
1306
  :param project_id: the Google Cloud project ID in which to start a job.