apache-airflow-providers-google 16.0.0rc1__py3-none-any.whl → 16.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +9 -5
  3. airflow/providers/google/ads/operators/ads.py +1 -1
  4. airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
  5. airflow/providers/google/cloud/hooks/bigquery.py +2 -3
  6. airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
  7. airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
  8. airflow/providers/google/cloud/hooks/dataflow.py +2 -2
  9. airflow/providers/google/cloud/hooks/dataplex.py +1 -1
  10. airflow/providers/google/cloud/hooks/dataprep.py +4 -1
  11. airflow/providers/google/cloud/hooks/gcs.py +2 -2
  12. airflow/providers/google/cloud/hooks/looker.py +5 -1
  13. airflow/providers/google/cloud/hooks/mlengine.py +2 -1
  14. airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
  15. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  16. airflow/providers/google/cloud/hooks/translate.py +1 -1
  17. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
  18. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +43 -14
  19. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
  20. airflow/providers/google/cloud/hooks/vision.py +2 -2
  21. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  22. airflow/providers/google/cloud/links/base.py +75 -11
  23. airflow/providers/google/cloud/links/bigquery.py +0 -47
  24. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  25. airflow/providers/google/cloud/links/bigtable.py +0 -48
  26. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  27. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  28. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  29. airflow/providers/google/cloud/links/cloud_run.py +1 -33
  30. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  31. airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
  32. airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
  33. airflow/providers/google/cloud/links/compute.py +0 -58
  34. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  35. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  36. airflow/providers/google/cloud/links/dataflow.py +0 -34
  37. airflow/providers/google/cloud/links/dataform.py +0 -64
  38. airflow/providers/google/cloud/links/datafusion.py +1 -96
  39. airflow/providers/google/cloud/links/dataplex.py +0 -154
  40. airflow/providers/google/cloud/links/dataprep.py +0 -24
  41. airflow/providers/google/cloud/links/dataproc.py +14 -90
  42. airflow/providers/google/cloud/links/datastore.py +0 -31
  43. airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
  44. airflow/providers/google/cloud/links/life_sciences.py +0 -19
  45. airflow/providers/google/cloud/links/managed_kafka.py +0 -70
  46. airflow/providers/google/cloud/links/mlengine.py +0 -70
  47. airflow/providers/google/cloud/links/pubsub.py +0 -32
  48. airflow/providers/google/cloud/links/spanner.py +0 -33
  49. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  50. airflow/providers/google/cloud/links/translate.py +16 -186
  51. airflow/providers/google/cloud/links/vertex_ai.py +8 -224
  52. airflow/providers/google/cloud/links/workflows.py +0 -52
  53. airflow/providers/google/cloud/operators/alloy_db.py +69 -54
  54. airflow/providers/google/cloud/operators/automl.py +16 -14
  55. airflow/providers/google/cloud/operators/bigquery.py +0 -15
  56. airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
  57. airflow/providers/google/cloud/operators/bigtable.py +35 -6
  58. airflow/providers/google/cloud/operators/cloud_base.py +21 -1
  59. airflow/providers/google/cloud/operators/cloud_build.py +74 -31
  60. airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
  61. airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
  62. airflow/providers/google/cloud/operators/cloud_run.py +0 -1
  63. airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
  64. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
  65. airflow/providers/google/cloud/operators/compute.py +7 -39
  66. airflow/providers/google/cloud/operators/datacatalog.py +156 -20
  67. airflow/providers/google/cloud/operators/dataflow.py +37 -14
  68. airflow/providers/google/cloud/operators/dataform.py +14 -4
  69. airflow/providers/google/cloud/operators/datafusion.py +4 -12
  70. airflow/providers/google/cloud/operators/dataplex.py +180 -96
  71. airflow/providers/google/cloud/operators/dataprep.py +0 -4
  72. airflow/providers/google/cloud/operators/dataproc.py +10 -16
  73. airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
  74. airflow/providers/google/cloud/operators/datastore.py +21 -5
  75. airflow/providers/google/cloud/operators/dlp.py +3 -26
  76. airflow/providers/google/cloud/operators/functions.py +15 -6
  77. airflow/providers/google/cloud/operators/gcs.py +0 -7
  78. airflow/providers/google/cloud/operators/kubernetes_engine.py +50 -7
  79. airflow/providers/google/cloud/operators/life_sciences.py +0 -1
  80. airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
  81. airflow/providers/google/cloud/operators/mlengine.py +0 -1
  82. airflow/providers/google/cloud/operators/pubsub.py +2 -4
  83. airflow/providers/google/cloud/operators/spanner.py +0 -4
  84. airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
  85. airflow/providers/google/cloud/operators/stackdriver.py +0 -8
  86. airflow/providers/google/cloud/operators/tasks.py +0 -11
  87. airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
  88. airflow/providers/google/cloud/operators/translate.py +37 -13
  89. airflow/providers/google/cloud/operators/translate_speech.py +0 -1
  90. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
  91. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
  92. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
  93. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
  94. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
  95. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
  96. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -25
  97. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
  98. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
  99. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +25 -6
  100. airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
  101. airflow/providers/google/cloud/operators/workflows.py +1 -9
  102. airflow/providers/google/cloud/sensors/bigquery.py +1 -1
  103. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
  104. airflow/providers/google/cloud/sensors/bigtable.py +15 -3
  105. airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
  106. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
  107. airflow/providers/google/cloud/sensors/dataflow.py +3 -3
  108. airflow/providers/google/cloud/sensors/dataform.py +6 -1
  109. airflow/providers/google/cloud/sensors/datafusion.py +6 -1
  110. airflow/providers/google/cloud/sensors/dataplex.py +6 -1
  111. airflow/providers/google/cloud/sensors/dataprep.py +6 -1
  112. airflow/providers/google/cloud/sensors/dataproc.py +6 -1
  113. airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
  114. airflow/providers/google/cloud/sensors/gcs.py +9 -3
  115. airflow/providers/google/cloud/sensors/looker.py +6 -1
  116. airflow/providers/google/cloud/sensors/pubsub.py +8 -3
  117. airflow/providers/google/cloud/sensors/tasks.py +6 -1
  118. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
  119. airflow/providers/google/cloud/sensors/workflows.py +6 -1
  120. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
  121. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
  122. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +1 -2
  123. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
  124. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
  125. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
  126. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  127. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
  128. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +1 -1
  129. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
  130. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
  131. airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
  132. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
  133. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
  134. airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
  135. airflow/providers/google/cloud/transfers/http_to_gcs.py +1 -1
  136. airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
  137. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
  138. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
  139. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
  140. airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
  141. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  142. airflow/providers/google/common/auth_backend/google_openid.py +2 -1
  143. airflow/providers/google/common/deprecated.py +2 -1
  144. airflow/providers/google/common/hooks/base_google.py +7 -3
  145. airflow/providers/google/common/links/storage.py +0 -22
  146. airflow/providers/google/firebase/operators/firestore.py +1 -1
  147. airflow/providers/google/get_provider_info.py +0 -11
  148. airflow/providers/google/leveldb/hooks/leveldb.py +5 -1
  149. airflow/providers/google/leveldb/operators/leveldb.py +1 -1
  150. airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
  151. airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
  152. airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
  153. airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
  154. airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
  155. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
  156. airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
  157. airflow/providers/google/suite/operators/sheets.py +3 -3
  158. airflow/providers/google/suite/sensors/drive.py +6 -1
  159. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
  160. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  161. airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
  162. airflow/providers/google/version_compat.py +28 -0
  163. {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/METADATA +19 -20
  164. {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/RECORD +166 -166
  165. {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/WHEEL +0 -0
  166. {apache_airflow_providers_google-16.0.0rc1.dist-info → apache_airflow_providers_google-16.1.0rc1.dist-info}/entry_points.txt +0 -0
@@ -21,7 +21,7 @@ from __future__ import annotations
21
21
 
22
22
  from collections.abc import Sequence
23
23
  from functools import cached_property
24
- from typing import TYPE_CHECKING
24
+ from typing import TYPE_CHECKING, Any
25
25
 
26
26
  from google.api_core.exceptions import AlreadyExists, NotFound
27
27
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
@@ -150,9 +150,17 @@ class ManagedKafkaCreateClusterOperator(ManagedKafkaBaseOperator):
150
150
  self.cluster_id = cluster_id
151
151
  self.request_id = request_id
152
152
 
153
+ @property
154
+ def extra_links_params(self) -> dict[str, Any]:
155
+ return {
156
+ "location": self.location,
157
+ "cluster_id": self.cluster_id,
158
+ "project_id": self.project_id,
159
+ }
160
+
153
161
  def execute(self, context: Context):
154
162
  self.log.info("Creating an Apache Kafka cluster.")
155
- ApacheKafkaClusterLink.persist(context=context, task_instance=self, cluster_id=self.cluster_id)
163
+ ApacheKafkaClusterLink.persist(context=context)
156
164
  try:
157
165
  operation = self.hook.create_cluster(
158
166
  project_id=self.project_id,
@@ -227,8 +235,14 @@ class ManagedKafkaListClustersOperator(ManagedKafkaBaseOperator):
227
235
  self.filter = filter
228
236
  self.order_by = order_by
229
237
 
238
+ @property
239
+ def extra_links_params(self) -> dict[str, Any]:
240
+ return {
241
+ "project_id": self.project_id,
242
+ }
243
+
230
244
  def execute(self, context: Context):
231
- ApacheKafkaClusterListLink.persist(context=context, task_instance=self)
245
+ ApacheKafkaClusterListLink.persist(context=context)
232
246
  self.log.info("Listing Clusters from location %s.", self.location)
233
247
  try:
234
248
  cluster_list_pager = self.hook.list_clusters(
@@ -242,8 +256,7 @@ class ManagedKafkaListClustersOperator(ManagedKafkaBaseOperator):
242
256
  timeout=self.timeout,
243
257
  metadata=self.metadata,
244
258
  )
245
- self.xcom_push(
246
- context=context,
259
+ context["ti"].xcom_push(
247
260
  key="cluster_page",
248
261
  value=types.ListClustersResponse.to_dict(cluster_list_pager._response),
249
262
  )
@@ -285,12 +298,16 @@ class ManagedKafkaGetClusterOperator(ManagedKafkaBaseOperator):
285
298
  super().__init__(*args, **kwargs)
286
299
  self.cluster_id = cluster_id
287
300
 
301
+ @property
302
+ def extra_links_params(self) -> dict[str, Any]:
303
+ return {
304
+ "location": self.location,
305
+ "cluster_id": self.cluster_id,
306
+ "project_id": self.project_id,
307
+ }
308
+
288
309
  def execute(self, context: Context):
289
- ApacheKafkaClusterLink.persist(
290
- context=context,
291
- task_instance=self,
292
- cluster_id=self.cluster_id,
293
- )
310
+ ApacheKafkaClusterLink.persist(context=context)
294
311
  self.log.info("Getting Cluster: %s", self.cluster_id)
295
312
  try:
296
313
  cluster = self.hook.get_cluster(
@@ -362,12 +379,16 @@ class ManagedKafkaUpdateClusterOperator(ManagedKafkaBaseOperator):
362
379
  self.update_mask = update_mask
363
380
  self.request_id = request_id
364
381
 
382
+ @property
383
+ def extra_links_params(self) -> dict[str, Any]:
384
+ return {
385
+ "location": self.location,
386
+ "cluster_id": self.cluster_id,
387
+ "project_id": self.project_id,
388
+ }
389
+
365
390
  def execute(self, context: Context):
366
- ApacheKafkaClusterLink.persist(
367
- context=context,
368
- task_instance=self,
369
- cluster_id=self.cluster_id,
370
- )
391
+ ApacheKafkaClusterLink.persist(context=context)
371
392
  self.log.info("Updating an Apache Kafka cluster.")
372
393
  try:
373
394
  operation = self.hook.update_cluster(
@@ -497,14 +518,18 @@ class ManagedKafkaCreateTopicOperator(ManagedKafkaBaseOperator):
497
518
  self.topic_id = topic_id
498
519
  self.topic = topic
499
520
 
521
+ @property
522
+ def extra_links_params(self) -> dict[str, Any]:
523
+ return {
524
+ "location": self.location,
525
+ "cluster_id": self.cluster_id,
526
+ "topic_id": self.topic_id,
527
+ "project_id": self.project_id,
528
+ }
529
+
500
530
  def execute(self, context: Context):
501
531
  self.log.info("Creating an Apache Kafka topic.")
502
- ApacheKafkaTopicLink.persist(
503
- context=context,
504
- task_instance=self,
505
- cluster_id=self.cluster_id,
506
- topic_id=self.topic_id,
507
- )
532
+ ApacheKafkaTopicLink.persist(context=context)
508
533
  try:
509
534
  topic_obj = self.hook.create_topic(
510
535
  project_id=self.project_id,
@@ -574,8 +599,16 @@ class ManagedKafkaListTopicsOperator(ManagedKafkaBaseOperator):
574
599
  self.page_size = page_size
575
600
  self.page_token = page_token
576
601
 
602
+ @property
603
+ def extra_links_params(self) -> dict[str, Any]:
604
+ return {
605
+ "location": self.location,
606
+ "cluster_id": self.cluster_id,
607
+ "project_id": self.project_id,
608
+ }
609
+
577
610
  def execute(self, context: Context):
578
- ApacheKafkaClusterLink.persist(context=context, task_instance=self, cluster_id=self.cluster_id)
611
+ ApacheKafkaClusterLink.persist(context=context)
579
612
  self.log.info("Listing Topics for cluster %s.", self.cluster_id)
580
613
  try:
581
614
  topic_list_pager = self.hook.list_topics(
@@ -588,8 +621,7 @@ class ManagedKafkaListTopicsOperator(ManagedKafkaBaseOperator):
588
621
  timeout=self.timeout,
589
622
  metadata=self.metadata,
590
623
  )
591
- self.xcom_push(
592
- context=context,
624
+ context["ti"].xcom_push(
593
625
  key="topic_page",
594
626
  value=types.ListTopicsResponse.to_dict(topic_list_pager._response),
595
627
  )
@@ -636,13 +668,17 @@ class ManagedKafkaGetTopicOperator(ManagedKafkaBaseOperator):
636
668
  self.cluster_id = cluster_id
637
669
  self.topic_id = topic_id
638
670
 
671
+ @property
672
+ def extra_links_params(self) -> dict[str, Any]:
673
+ return {
674
+ "location": self.location,
675
+ "cluster_id": self.cluster_id,
676
+ "topic_id": self.topic_id,
677
+ "project_id": self.project_id,
678
+ }
679
+
639
680
  def execute(self, context: Context):
640
- ApacheKafkaTopicLink.persist(
641
- context=context,
642
- task_instance=self,
643
- cluster_id=self.cluster_id,
644
- topic_id=self.topic_id,
645
- )
681
+ ApacheKafkaTopicLink.persist(context=context)
646
682
  self.log.info("Getting Topic: %s", self.topic_id)
647
683
  try:
648
684
  topic = self.hook.get_topic(
@@ -707,13 +743,17 @@ class ManagedKafkaUpdateTopicOperator(ManagedKafkaBaseOperator):
707
743
  self.topic = topic
708
744
  self.update_mask = update_mask
709
745
 
746
+ @property
747
+ def extra_links_params(self) -> dict[str, Any]:
748
+ return {
749
+ "location": self.location,
750
+ "cluster_id": self.cluster_id,
751
+ "topic_id": self.topic_id,
752
+ "project_id": self.project_id,
753
+ }
754
+
710
755
  def execute(self, context: Context):
711
- ApacheKafkaTopicLink.persist(
712
- context=context,
713
- task_instance=self,
714
- cluster_id=self.cluster_id,
715
- topic_id=self.topic_id,
716
- )
756
+ ApacheKafkaTopicLink.persist(context=context)
717
757
  self.log.info("Updating an Apache Kafka topic.")
718
758
  try:
719
759
  topic_obj = self.hook.update_topic(
@@ -833,8 +873,16 @@ class ManagedKafkaListConsumerGroupsOperator(ManagedKafkaBaseOperator):
833
873
  self.page_size = page_size
834
874
  self.page_token = page_token
835
875
 
876
+ @property
877
+ def extra_links_params(self) -> dict[str, Any]:
878
+ return {
879
+ "location": self.location,
880
+ "cluster_id": self.cluster_id,
881
+ "project_id": self.project_id,
882
+ }
883
+
836
884
  def execute(self, context: Context):
837
- ApacheKafkaClusterLink.persist(context=context, task_instance=self, cluster_id=self.cluster_id)
885
+ ApacheKafkaClusterLink.persist(context=context)
838
886
  self.log.info("Listing Consumer Groups for cluster %s.", self.cluster_id)
839
887
  try:
840
888
  consumer_group_list_pager = self.hook.list_consumer_groups(
@@ -847,8 +895,7 @@ class ManagedKafkaListConsumerGroupsOperator(ManagedKafkaBaseOperator):
847
895
  timeout=self.timeout,
848
896
  metadata=self.metadata,
849
897
  )
850
- self.xcom_push(
851
- context=context,
898
+ context["ti"].xcom_push(
852
899
  key="consumer_group_page",
853
900
  value=types.ListConsumerGroupsResponse.to_dict(consumer_group_list_pager._response),
854
901
  )
@@ -895,13 +942,17 @@ class ManagedKafkaGetConsumerGroupOperator(ManagedKafkaBaseOperator):
895
942
  self.cluster_id = cluster_id
896
943
  self.consumer_group_id = consumer_group_id
897
944
 
945
+ @property
946
+ def extra_links_params(self) -> dict[str, Any]:
947
+ return {
948
+ "location": self.location,
949
+ "cluster_id": self.cluster_id,
950
+ "consumer_group_id": self.consumer_group_id,
951
+ "project_id": self.project_id,
952
+ }
953
+
898
954
  def execute(self, context: Context):
899
- ApacheKafkaConsumerGroupLink.persist(
900
- context=context,
901
- task_instance=self,
902
- cluster_id=self.cluster_id,
903
- consumer_group_id=self.consumer_group_id,
904
- )
955
+ ApacheKafkaConsumerGroupLink.persist(context=context)
905
956
  self.log.info("Getting Consumer Group: %s", self.consumer_group_id)
906
957
  try:
907
958
  consumer_group = self.hook.get_consumer_group(
@@ -971,13 +1022,17 @@ class ManagedKafkaUpdateConsumerGroupOperator(ManagedKafkaBaseOperator):
971
1022
  self.consumer_group = consumer_group
972
1023
  self.update_mask = update_mask
973
1024
 
1025
+ @property
1026
+ def extra_links_params(self) -> dict[str, Any]:
1027
+ return {
1028
+ "location": self.location,
1029
+ "cluster_id": self.cluster_id,
1030
+ "consumer_group_id": self.consumer_group_id,
1031
+ "project_id": self.project_id,
1032
+ }
1033
+
974
1034
  def execute(self, context: Context):
975
- ApacheKafkaConsumerGroupLink.persist(
976
- context=context,
977
- task_instance=self,
978
- cluster_id=self.cluster_id,
979
- consumer_group_id=self.consumer_group_id,
980
- )
1035
+ ApacheKafkaConsumerGroupLink.persist(context=context)
981
1036
  self.log.info("Updating an Apache Kafka consumer group.")
982
1037
  try:
983
1038
  consumer_group_obj = self.hook.update_consumer_group(
@@ -104,7 +104,6 @@ class MLEngineCreateModelOperator(GoogleCloudBaseOperator):
104
104
  if project_id:
105
105
  MLEngineModelLink.persist(
106
106
  context=context,
107
- task_instance=self,
108
107
  project_id=project_id,
109
108
  model_id=self.model["name"],
110
109
  )
@@ -25,8 +25,8 @@ This module contains Google PubSub operators.
25
25
 
26
26
  from __future__ import annotations
27
27
 
28
- from collections.abc import Sequence
29
- from typing import TYPE_CHECKING, Any, Callable
28
+ from collections.abc import Callable, Sequence
29
+ from typing import TYPE_CHECKING, Any
30
30
 
31
31
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
32
32
  from google.cloud.pubsub_v1.types import (
@@ -183,7 +183,6 @@ class PubSubCreateTopicOperator(GoogleCloudBaseOperator):
183
183
  self.log.info("Created topic %s", self.topic)
184
184
  PubSubTopicLink.persist(
185
185
  context=context,
186
- task_instance=self,
187
186
  topic_id=self.topic,
188
187
  project_id=self.project_id or hook.project_id,
189
188
  )
@@ -392,7 +391,6 @@ class PubSubCreateSubscriptionOperator(GoogleCloudBaseOperator):
392
391
  self.log.info("Created subscription for topic %s", self.topic)
393
392
  PubSubSubscriptionLink.persist(
394
393
  context=context,
395
- task_instance=self,
396
394
  subscription_id=self.subscription or result, # result returns subscription name
397
395
  project_id=self.project_id or hook.project_id,
398
396
  )
@@ -122,7 +122,6 @@ class SpannerDeployInstanceOperator(GoogleCloudBaseOperator):
122
122
  )
123
123
  SpannerInstanceLink.persist(
124
124
  context=context,
125
- task_instance=self,
126
125
  instance_id=self.instance_id,
127
126
  project_id=self.project_id or hook.project_id,
128
127
  )
@@ -290,7 +289,6 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
290
289
  )
291
290
  SpannerDatabaseLink.persist(
292
291
  context=context,
293
- task_instance=self,
294
292
  instance_id=self.instance_id,
295
293
  database_id=self.database_id,
296
294
  project_id=self.project_id or hook.project_id,
@@ -380,7 +378,6 @@ class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
380
378
  )
381
379
  SpannerDatabaseLink.persist(
382
380
  context=context,
383
- task_instance=self,
384
381
  instance_id=self.instance_id,
385
382
  database_id=self.database_id,
386
383
  project_id=self.project_id or hook.project_id,
@@ -496,7 +493,6 @@ class SpannerUpdateDatabaseInstanceOperator(GoogleCloudBaseOperator):
496
493
  )
497
494
  SpannerDatabaseLink.persist(
498
495
  context=context,
499
- task_instance=self,
500
496
  instance_id=self.instance_id,
501
497
  database_id=self.database_id,
502
498
  project_id=self.project_id or hook.project_id,
@@ -117,7 +117,6 @@ class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
117
117
  if self.audio.uri:
118
118
  FileDetailsLink.persist(
119
119
  context=context,
120
- task_instance=self,
121
120
  # Slice from: "gs://{BUCKET_NAME}/{FILE_NAME}" to: "{BUCKET_NAME}/{FILE_NAME}"
122
121
  uri=self.audio.uri[5:],
123
122
  project_id=self.project_id or hook.project_id,
@@ -145,7 +145,6 @@ class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
145
145
  )
146
146
  StackdriverPoliciesLink.persist(
147
147
  context=context,
148
- operator_instance=self,
149
148
  project_id=self.project_id or self.hook.project_id,
150
149
  )
151
150
  return [AlertPolicy.to_dict(policy) for policy in result]
@@ -228,7 +227,6 @@ class StackdriverEnableAlertPoliciesOperator(GoogleCloudBaseOperator):
228
227
  )
229
228
  StackdriverPoliciesLink.persist(
230
229
  context=context,
231
- operator_instance=self,
232
230
  project_id=self.project_id or self.hook.project_id,
233
231
  )
234
232
 
@@ -311,7 +309,6 @@ class StackdriverDisableAlertPoliciesOperator(GoogleCloudBaseOperator):
311
309
  )
312
310
  StackdriverPoliciesLink.persist(
313
311
  context=context,
314
- operator_instance=self,
315
312
  project_id=self.project_id or self.hook.project_id,
316
313
  )
317
314
 
@@ -394,7 +391,6 @@ class StackdriverUpsertAlertOperator(GoogleCloudBaseOperator):
394
391
  )
395
392
  StackdriverPoliciesLink.persist(
396
393
  context=context,
397
- operator_instance=self,
398
394
  project_id=self.project_id or self.hook.project_id,
399
395
  )
400
396
 
@@ -580,7 +576,6 @@ class StackdriverListNotificationChannelsOperator(GoogleCloudBaseOperator):
580
576
  )
581
577
  StackdriverNotificationsLink.persist(
582
578
  context=context,
583
- operator_instance=self,
584
579
  project_id=self.project_id or self.hook.project_id,
585
580
  )
586
581
  return [NotificationChannel.to_dict(channel) for channel in channels]
@@ -666,7 +661,6 @@ class StackdriverEnableNotificationChannelsOperator(GoogleCloudBaseOperator):
666
661
  )
667
662
  StackdriverNotificationsLink.persist(
668
663
  context=context,
669
- operator_instance=self,
670
664
  project_id=self.project_id or self.hook.project_id,
671
665
  )
672
666
 
@@ -751,7 +745,6 @@ class StackdriverDisableNotificationChannelsOperator(GoogleCloudBaseOperator):
751
745
  )
752
746
  StackdriverNotificationsLink.persist(
753
747
  context=context,
754
- operator_instance=self,
755
748
  project_id=self.project_id or self.hook.project_id,
756
749
  )
757
750
 
@@ -838,7 +831,6 @@ class StackdriverUpsertNotificationChannelOperator(GoogleCloudBaseOperator):
838
831
  )
839
832
  StackdriverNotificationsLink.persist(
840
833
  context=context,
841
- operator_instance=self,
842
834
  project_id=self.project_id or self.hook.project_id,
843
835
  )
844
836
 
@@ -137,7 +137,6 @@ class CloudTasksQueueCreateOperator(GoogleCloudBaseOperator):
137
137
  metadata=self.metadata,
138
138
  )
139
139
  CloudTasksQueueLink.persist(
140
- operator_instance=self,
141
140
  context=context,
142
141
  queue_name=queue.name,
143
142
  )
@@ -236,7 +235,6 @@ class CloudTasksQueueUpdateOperator(GoogleCloudBaseOperator):
236
235
  metadata=self.metadata,
237
236
  )
238
237
  CloudTasksQueueLink.persist(
239
- operator_instance=self,
240
238
  context=context,
241
239
  queue_name=queue.name,
242
240
  )
@@ -319,7 +317,6 @@ class CloudTasksQueueGetOperator(GoogleCloudBaseOperator):
319
317
  metadata=self.metadata,
320
318
  )
321
319
  CloudTasksQueueLink.persist(
322
- operator_instance=self,
323
320
  context=context,
324
321
  queue_name=queue.name,
325
322
  )
@@ -406,7 +403,6 @@ class CloudTasksQueuesListOperator(GoogleCloudBaseOperator):
406
403
  metadata=self.metadata,
407
404
  )
408
405
  CloudTasksLink.persist(
409
- operator_instance=self,
410
406
  context=context,
411
407
  project_id=self.project_id or hook.project_id,
412
408
  )
@@ -564,7 +560,6 @@ class CloudTasksQueuePurgeOperator(GoogleCloudBaseOperator):
564
560
  metadata=self.metadata,
565
561
  )
566
562
  CloudTasksQueueLink.persist(
567
- operator_instance=self,
568
563
  context=context,
569
564
  queue_name=queue.name,
570
565
  )
@@ -647,7 +642,6 @@ class CloudTasksQueuePauseOperator(GoogleCloudBaseOperator):
647
642
  metadata=self.metadata,
648
643
  )
649
644
  CloudTasksQueueLink.persist(
650
- operator_instance=self,
651
645
  context=context,
652
646
  queue_name=queue.name,
653
647
  )
@@ -730,7 +724,6 @@ class CloudTasksQueueResumeOperator(GoogleCloudBaseOperator):
730
724
  metadata=self.metadata,
731
725
  )
732
726
  CloudTasksQueueLink.persist(
733
- operator_instance=self,
734
727
  context=context,
735
728
  queue_name=queue.name,
736
729
  )
@@ -830,7 +823,6 @@ class CloudTasksTaskCreateOperator(GoogleCloudBaseOperator):
830
823
  metadata=self.metadata,
831
824
  )
832
825
  CloudTasksQueueLink.persist(
833
- operator_instance=self,
834
826
  context=context,
835
827
  queue_name=task.name,
836
828
  )
@@ -923,7 +915,6 @@ class CloudTasksTaskGetOperator(GoogleCloudBaseOperator):
923
915
  metadata=self.metadata,
924
916
  )
925
917
  CloudTasksQueueLink.persist(
926
- operator_instance=self,
927
918
  context=context,
928
919
  queue_name=task.name,
929
920
  )
@@ -1016,7 +1007,6 @@ class CloudTasksTasksListOperator(GoogleCloudBaseOperator):
1016
1007
  metadata=self.metadata,
1017
1008
  )
1018
1009
  CloudTasksQueueLink.persist(
1019
- operator_instance=self,
1020
1010
  context=context,
1021
1011
  queue_name=f"projects/{self.project_id or hook.project_id}/"
1022
1012
  f"locations/{self.location}/queues/{self.queue_name}",
@@ -1190,7 +1180,6 @@ class CloudTasksTaskRunOperator(GoogleCloudBaseOperator):
1190
1180
  metadata=self.metadata,
1191
1181
  )
1192
1182
  CloudTasksQueueLink.persist(
1193
- operator_instance=self,
1194
1183
  context=context,
1195
1184
  queue_name=task.name,
1196
1185
  )
@@ -150,7 +150,6 @@ class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
150
150
  )
151
151
  FileDetailsLink.persist(
152
152
  context=context,
153
- task_instance=self,
154
153
  uri=f"{self.target_bucket_name}/{self.target_filename}",
155
154
  project_id=cloud_storage_hook.project_id,
156
155
  )
@@ -37,6 +37,7 @@ from airflow.providers.google.cloud.links.translate import (
37
37
  TranslationNativeDatasetLink,
38
38
  )
39
39
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
40
+ from airflow.providers.google.cloud.operators.vertex_ai.dataset import DatasetImportDataResultsCheckHelper
40
41
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
41
42
 
42
43
  if TYPE_CHECKING:
@@ -394,7 +395,6 @@ class TranslateTextBatchOperator(GoogleCloudBaseOperator):
394
395
  self.log.info("Translate text batch job started.")
395
396
  TranslateTextBatchLink.persist(
396
397
  context=context,
397
- task_instance=self,
398
398
  project_id=self.project_id or hook.project_id,
399
399
  output_config=self.output_config,
400
400
  )
@@ -480,15 +480,15 @@ class TranslateCreateDatasetOperator(GoogleCloudBaseOperator):
480
480
  result = hook.wait_for_operation_result(result_operation)
481
481
  result = type(result).to_dict(result)
482
482
  dataset_id = hook.extract_object_id(result)
483
- self.xcom_push(context, key="dataset_id", value=dataset_id)
483
+ context["ti"].xcom_push(key="dataset_id", value=dataset_id)
484
484
  self.log.info("Dataset creation complete. The dataset_id: %s.", dataset_id)
485
485
 
486
486
  project_id = self.project_id or hook.project_id
487
487
  TranslationNativeDatasetLink.persist(
488
488
  context=context,
489
- task_instance=self,
490
489
  dataset_id=dataset_id,
491
490
  project_id=project_id,
491
+ location=self.location,
492
492
  )
493
493
  return result
494
494
 
@@ -556,7 +556,6 @@ class TranslateDatasetsListOperator(GoogleCloudBaseOperator):
556
556
  project_id = self.project_id or hook.project_id
557
557
  TranslationDatasetsListLink.persist(
558
558
  context=context,
559
- task_instance=self,
560
559
  project_id=project_id,
561
560
  )
562
561
  self.log.info("Requesting datasets list")
@@ -577,7 +576,7 @@ class TranslateDatasetsListOperator(GoogleCloudBaseOperator):
577
576
  return result_ids
578
577
 
579
578
 
580
- class TranslateImportDataOperator(GoogleCloudBaseOperator):
579
+ class TranslateImportDataOperator(GoogleCloudBaseOperator, DatasetImportDataResultsCheckHelper):
581
580
  """
582
581
  Import data to the translation dataset.
583
582
 
@@ -604,6 +603,7 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
604
603
  If set as a sequence, the identities from the list must grant
605
604
  Service Account Token Creator IAM role to the directly preceding identity, with first
606
605
  account from the list granting this role to the originating account (templated).
606
+ :param raise_for_empty_result: Raise an error if no additional data has been populated after the import.
607
607
  """
608
608
 
609
609
  template_fields: Sequence[str] = (
@@ -629,6 +629,7 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
629
629
  retry: Retry | _MethodDefault = DEFAULT,
630
630
  gcp_conn_id: str = "google_cloud_default",
631
631
  impersonation_chain: str | Sequence[str] | None = None,
632
+ raise_for_empty_result: bool = False,
632
633
  **kwargs,
633
634
  ) -> None:
634
635
  super().__init__(**kwargs)
@@ -641,9 +642,21 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
641
642
  self.retry = retry
642
643
  self.gcp_conn_id = gcp_conn_id
643
644
  self.impersonation_chain = impersonation_chain
645
+ self.raise_for_empty_result = raise_for_empty_result
644
646
 
645
647
  def execute(self, context: Context):
646
648
  hook = TranslateHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
649
+ initial_dataset_size = self._get_number_of_ds_items(
650
+ dataset=hook.get_dataset(
651
+ dataset_id=self.dataset_id,
652
+ project_id=self.project_id,
653
+ location=self.location,
654
+ retry=self.retry,
655
+ timeout=self.timeout,
656
+ metadata=self.metadata,
657
+ ),
658
+ total_key_name="example_count",
659
+ )
647
660
  self.log.info("Importing data to dataset...")
648
661
  operation = hook.import_dataset_data(
649
662
  dataset_id=self.dataset_id,
@@ -657,12 +670,27 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
657
670
  project_id = self.project_id or hook.project_id
658
671
  TranslationNativeDatasetLink.persist(
659
672
  context=context,
660
- task_instance=self,
661
673
  dataset_id=self.dataset_id,
662
674
  project_id=project_id,
675
+ location=self.location,
663
676
  )
664
677
  hook.wait_for_operation_done(operation=operation, timeout=self.timeout)
678
+
679
+ result_dataset_size = self._get_number_of_ds_items(
680
+ dataset=hook.get_dataset(
681
+ dataset_id=self.dataset_id,
682
+ project_id=self.project_id,
683
+ location=self.location,
684
+ retry=self.retry,
685
+ timeout=self.timeout,
686
+ metadata=self.metadata,
687
+ ),
688
+ total_key_name="example_count",
689
+ )
690
+ if self.raise_for_empty_result:
691
+ self._raise_for_empty_import_result(self.dataset_id, initial_dataset_size, result_dataset_size)
665
692
  self.log.info("Importing data finished!")
693
+ return {"total_imported": int(result_dataset_size) - int(initial_dataset_size)}
666
694
 
667
695
 
668
696
  class TranslateDeleteDatasetOperator(GoogleCloudBaseOperator):
@@ -821,16 +849,16 @@ class TranslateCreateModelOperator(GoogleCloudBaseOperator):
821
849
  result = hook.wait_for_operation_result(operation=result_operation)
822
850
  result = type(result).to_dict(result)
823
851
  model_id = hook.extract_object_id(result)
824
- self.xcom_push(context, key="model_id", value=model_id)
852
+ context["ti"].xcom_push(key="model_id", value=model_id)
825
853
  self.log.info("Model creation complete. The model_id: %s.", model_id)
826
854
 
827
855
  project_id = self.project_id or hook.project_id
828
856
  TranslationModelLink.persist(
829
857
  context=context,
830
- task_instance=self,
831
858
  dataset_id=self.dataset_id,
832
859
  model_id=model_id,
833
860
  project_id=project_id,
861
+ location=self.location,
834
862
  )
835
863
  return result
836
864
 
@@ -898,7 +926,6 @@ class TranslateModelsListOperator(GoogleCloudBaseOperator):
898
926
  project_id = self.project_id or hook.project_id
899
927
  TranslationModelsListLink.persist(
900
928
  context=context,
901
- task_instance=self,
902
929
  project_id=project_id,
903
930
  )
904
931
  self.log.info("Requesting models list")
@@ -1141,7 +1168,6 @@ class TranslateDocumentOperator(GoogleCloudBaseOperator):
1141
1168
  if self.document_output_config:
1142
1169
  TranslateResultByOutputConfigLink.persist(
1143
1170
  context=context,
1144
- task_instance=self,
1145
1171
  project_id=self.project_id or hook.project_id,
1146
1172
  output_config=self.document_output_config,
1147
1173
  )
@@ -1304,7 +1330,6 @@ class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
1304
1330
  self.log.info("Batch document translation job started.")
1305
1331
  TranslateResultByOutputConfigLink.persist(
1306
1332
  context=context,
1307
- task_instance=self,
1308
1333
  project_id=self.project_id or hook.project_id,
1309
1334
  output_config=self.output_config,
1310
1335
  )
@@ -1411,7 +1436,7 @@ class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
1411
1436
  result = type(result).to_dict(result)
1412
1437
 
1413
1438
  glossary_id = hook.extract_object_id(result)
1414
- self.xcom_push(context, key="glossary_id", value=glossary_id)
1439
+ context["ti"].xcom_push(key="glossary_id", value=glossary_id)
1415
1440
  self.log.info("Glossary creation complete. The glossary_id: %s.", glossary_id)
1416
1441
  return result
1417
1442
 
@@ -1610,7 +1635,6 @@ class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
1610
1635
  project_id = self.project_id or hook.project_id
1611
1636
  TranslationGlossariesListLink.persist(
1612
1637
  context=context,
1613
- task_instance=self,
1614
1638
  project_id=project_id,
1615
1639
  )
1616
1640
  self.log.info("Requesting glossaries list")
@@ -173,7 +173,6 @@ class CloudTranslateSpeechOperator(GoogleCloudBaseOperator):
173
173
  if self.audio.uri:
174
174
  FileDetailsLink.persist(
175
175
  context=context,
176
- task_instance=self,
177
176
  # Slice from: "gs://{BUCKET_NAME}/{FILE_NAME}" to: "{BUCKET_NAME}/{FILE_NAME}"
178
177
  uri=self.audio.uri[5:],
179
178
  project_id=self.project_id or translate_hook.project_id,