apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
  3. airflow/providers/google/__init__.py +3 -3
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +52 -43
  7. airflow/providers/google/ads/operators/ads.py +2 -2
  8. airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
  9. airflow/providers/google/assets/gcs.py +1 -11
  10. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  11. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  12. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  13. airflow/providers/google/cloud/hooks/bigquery.py +195 -318
  14. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  15. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  16. airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
  17. airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
  18. airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
  19. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  20. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  21. airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
  22. airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
  23. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
  24. airflow/providers/google/cloud/hooks/compute.py +7 -6
  25. airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
  26. airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
  27. airflow/providers/google/cloud/hooks/dataflow.py +87 -242
  28. airflow/providers/google/cloud/hooks/dataform.py +9 -14
  29. airflow/providers/google/cloud/hooks/datafusion.py +7 -9
  30. airflow/providers/google/cloud/hooks/dataplex.py +13 -12
  31. airflow/providers/google/cloud/hooks/dataprep.py +2 -2
  32. airflow/providers/google/cloud/hooks/dataproc.py +76 -74
  33. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  34. airflow/providers/google/cloud/hooks/dlp.py +5 -4
  35. airflow/providers/google/cloud/hooks/gcs.py +144 -33
  36. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  37. airflow/providers/google/cloud/hooks/kms.py +3 -2
  38. airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
  39. airflow/providers/google/cloud/hooks/looker.py +6 -1
  40. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  41. airflow/providers/google/cloud/hooks/mlengine.py +7 -8
  42. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  43. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  44. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  45. airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
  46. airflow/providers/google/cloud/hooks/spanner.py +75 -10
  47. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  48. airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
  49. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  50. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  51. airflow/providers/google/cloud/hooks/translate.py +8 -17
  52. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
  53. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
  54. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
  55. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  56. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
  57. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  58. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
  59. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  60. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
  61. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
  62. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  63. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  64. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  65. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  66. airflow/providers/google/cloud/hooks/vision.py +7 -7
  67. airflow/providers/google/cloud/hooks/workflows.py +4 -3
  68. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  69. airflow/providers/google/cloud/links/base.py +77 -7
  70. airflow/providers/google/cloud/links/bigquery.py +0 -47
  71. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  72. airflow/providers/google/cloud/links/bigtable.py +0 -48
  73. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  74. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  75. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  76. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  77. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  78. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
  79. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  80. airflow/providers/google/cloud/links/compute.py +0 -58
  81. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  82. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  83. airflow/providers/google/cloud/links/dataflow.py +0 -34
  84. airflow/providers/google/cloud/links/dataform.py +0 -64
  85. airflow/providers/google/cloud/links/datafusion.py +1 -90
  86. airflow/providers/google/cloud/links/dataplex.py +0 -154
  87. airflow/providers/google/cloud/links/dataprep.py +0 -24
  88. airflow/providers/google/cloud/links/dataproc.py +11 -89
  89. airflow/providers/google/cloud/links/datastore.py +0 -31
  90. airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
  91. airflow/providers/google/cloud/links/managed_kafka.py +11 -51
  92. airflow/providers/google/cloud/links/mlengine.py +0 -70
  93. airflow/providers/google/cloud/links/pubsub.py +0 -32
  94. airflow/providers/google/cloud/links/spanner.py +0 -33
  95. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  96. airflow/providers/google/cloud/links/translate.py +17 -187
  97. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  98. airflow/providers/google/cloud/links/workflows.py +0 -52
  99. airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
  100. airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
  101. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  102. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  103. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  104. airflow/providers/google/cloud/openlineage/facets.py +141 -40
  105. airflow/providers/google/cloud/openlineage/mixins.py +14 -13
  106. airflow/providers/google/cloud/openlineage/utils.py +19 -3
  107. airflow/providers/google/cloud/operators/alloy_db.py +76 -61
  108. airflow/providers/google/cloud/operators/bigquery.py +104 -667
  109. airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
  110. airflow/providers/google/cloud/operators/bigtable.py +38 -7
  111. airflow/providers/google/cloud/operators/cloud_base.py +22 -1
  112. airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
  113. airflow/providers/google/cloud/operators/cloud_build.py +80 -36
  114. airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
  115. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  116. airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
  117. airflow/providers/google/cloud/operators/cloud_run.py +39 -20
  118. airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
  119. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
  120. airflow/providers/google/cloud/operators/compute.py +18 -50
  121. airflow/providers/google/cloud/operators/datacatalog.py +167 -29
  122. airflow/providers/google/cloud/operators/dataflow.py +38 -15
  123. airflow/providers/google/cloud/operators/dataform.py +19 -7
  124. airflow/providers/google/cloud/operators/datafusion.py +43 -43
  125. airflow/providers/google/cloud/operators/dataplex.py +212 -126
  126. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  127. airflow/providers/google/cloud/operators/dataproc.py +134 -207
  128. airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
  129. airflow/providers/google/cloud/operators/datastore.py +22 -6
  130. airflow/providers/google/cloud/operators/dlp.py +24 -45
  131. airflow/providers/google/cloud/operators/functions.py +21 -14
  132. airflow/providers/google/cloud/operators/gcs.py +15 -12
  133. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  134. airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
  135. airflow/providers/google/cloud/operators/looker.py +1 -1
  136. airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
  137. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  138. airflow/providers/google/cloud/operators/pubsub.py +69 -21
  139. airflow/providers/google/cloud/operators/spanner.py +53 -45
  140. airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
  141. airflow/providers/google/cloud/operators/stackdriver.py +5 -11
  142. airflow/providers/google/cloud/operators/tasks.py +6 -15
  143. airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
  144. airflow/providers/google/cloud/operators/translate.py +46 -20
  145. airflow/providers/google/cloud/operators/translate_speech.py +4 -3
  146. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
  147. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
  148. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
  149. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
  150. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
  151. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  152. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  153. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
  154. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
  155. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
  156. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
  157. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  158. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  159. airflow/providers/google/cloud/operators/vision.py +7 -5
  160. airflow/providers/google/cloud/operators/workflows.py +24 -19
  161. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  162. airflow/providers/google/cloud/sensors/bigquery.py +2 -2
  163. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
  164. airflow/providers/google/cloud/sensors/bigtable.py +14 -6
  165. airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
  166. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
  167. airflow/providers/google/cloud/sensors/dataflow.py +27 -10
  168. airflow/providers/google/cloud/sensors/dataform.py +2 -2
  169. airflow/providers/google/cloud/sensors/datafusion.py +4 -4
  170. airflow/providers/google/cloud/sensors/dataplex.py +7 -5
  171. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  172. airflow/providers/google/cloud/sensors/dataproc.py +10 -9
  173. airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
  174. airflow/providers/google/cloud/sensors/gcs.py +22 -21
  175. airflow/providers/google/cloud/sensors/looker.py +5 -5
  176. airflow/providers/google/cloud/sensors/pubsub.py +20 -20
  177. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  178. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
  179. airflow/providers/google/cloud/sensors/workflows.py +6 -4
  180. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  181. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  182. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
  183. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  184. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
  185. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  186. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  187. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  188. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  189. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  190. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
  191. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
  192. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
  193. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
  194. airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
  195. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
  196. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  197. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  198. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  199. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  200. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  201. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  202. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
  203. airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
  204. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  205. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
  206. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  207. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  208. airflow/providers/google/cloud/triggers/bigquery.py +75 -34
  209. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  210. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  211. airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
  212. airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
  213. airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
  214. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
  215. airflow/providers/google/cloud/triggers/dataflow.py +125 -2
  216. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  217. airflow/providers/google/cloud/triggers/dataplex.py +16 -3
  218. airflow/providers/google/cloud/triggers/dataproc.py +124 -53
  219. airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
  220. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  221. airflow/providers/google/cloud/triggers/pubsub.py +17 -20
  222. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  223. airflow/providers/google/cloud/utils/bigquery.py +5 -7
  224. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  225. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  226. airflow/providers/google/cloud/utils/dataform.py +1 -1
  227. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  228. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  229. airflow/providers/google/cloud/utils/validators.py +43 -0
  230. airflow/providers/google/common/auth_backend/google_openid.py +26 -9
  231. airflow/providers/google/common/consts.py +2 -1
  232. airflow/providers/google/common/deprecated.py +2 -1
  233. airflow/providers/google/common/hooks/base_google.py +40 -43
  234. airflow/providers/google/common/hooks/operation_helpers.py +78 -0
  235. airflow/providers/google/common/links/storage.py +0 -22
  236. airflow/providers/google/common/utils/get_secret.py +31 -0
  237. airflow/providers/google/common/utils/id_token_credentials.py +4 -5
  238. airflow/providers/google/firebase/operators/firestore.py +2 -2
  239. airflow/providers/google/get_provider_info.py +61 -216
  240. airflow/providers/google/go_module_utils.py +35 -3
  241. airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
  242. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  243. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  244. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  245. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  246. airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
  247. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
  248. airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
  249. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  250. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  251. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  252. airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
  253. airflow/providers/google/suite/hooks/calendar.py +1 -1
  254. airflow/providers/google/suite/hooks/drive.py +2 -2
  255. airflow/providers/google/suite/hooks/sheets.py +15 -1
  256. airflow/providers/google/suite/operators/sheets.py +8 -3
  257. airflow/providers/google/suite/sensors/drive.py +2 -2
  258. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
  259. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  260. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  261. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  262. airflow/providers/google/version_compat.py +15 -1
  263. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
  264. apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
  265. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
  266. apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
  267. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  268. airflow/providers/google/cloud/hooks/automl.py +0 -679
  269. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  270. airflow/providers/google/cloud/links/automl.py +0 -193
  271. airflow/providers/google/cloud/operators/automl.py +0 -1360
  272. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  273. airflow/providers/google/cloud/operators/mlengine.py +0 -1515
  274. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
  275. apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
  276. /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
  277. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
  278. {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -23,34 +23,19 @@ from collections.abc import MutableSequence, Sequence
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING, Any
25
25
 
26
+ from google.protobuf.json_format import MessageToDict
27
+
26
28
  from airflow.exceptions import AirflowException
27
29
  from airflow.providers.google.cloud.triggers.dataplex import (
28
30
  DataplexDataProfileJobTrigger,
29
31
  DataplexDataQualityJobTrigger,
30
32
  )
31
- from google.protobuf.json_format import MessageToDict
32
33
 
33
34
  if TYPE_CHECKING:
34
- from airflow.utils.context import Context
35
35
  from google.protobuf.field_mask_pb2 import FieldMask
36
36
 
37
- from googleapiclient.errors import HttpError
37
+ from airflow.providers.common.compat.sdk import Context
38
38
 
39
- from airflow.configuration import conf
40
- from airflow.providers.google.cloud.hooks.dataplex import AirflowDataQualityScanException, DataplexHook
41
- from airflow.providers.google.cloud.links.dataplex import (
42
- DataplexCatalogAspectTypeLink,
43
- DataplexCatalogAspectTypesLink,
44
- DataplexCatalogEntryGroupLink,
45
- DataplexCatalogEntryGroupsLink,
46
- DataplexCatalogEntryLink,
47
- DataplexCatalogEntryTypeLink,
48
- DataplexCatalogEntryTypesLink,
49
- DataplexLakeLink,
50
- DataplexTaskLink,
51
- DataplexTasksLink,
52
- )
53
- from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
54
39
  from google.api_core.exceptions import AlreadyExists, GoogleAPICallError, NotFound
55
40
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
56
41
  from google.api_core.retry import Retry, exponential_sleep_generator
@@ -72,6 +57,23 @@ from google.cloud.dataplex_v1.types import (
72
57
  Task,
73
58
  Zone,
74
59
  )
60
+ from googleapiclient.errors import HttpError
61
+
62
+ from airflow.configuration import conf
63
+ from airflow.providers.google.cloud.hooks.dataplex import AirflowDataQualityScanException, DataplexHook
64
+ from airflow.providers.google.cloud.links.dataplex import (
65
+ DataplexCatalogAspectTypeLink,
66
+ DataplexCatalogAspectTypesLink,
67
+ DataplexCatalogEntryGroupLink,
68
+ DataplexCatalogEntryGroupsLink,
69
+ DataplexCatalogEntryLink,
70
+ DataplexCatalogEntryTypeLink,
71
+ DataplexCatalogEntryTypesLink,
72
+ DataplexLakeLink,
73
+ DataplexTaskLink,
74
+ DataplexTasksLink,
75
+ )
76
+ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
75
77
 
76
78
 
77
79
  class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
@@ -148,6 +150,15 @@ class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
148
150
  self.impersonation_chain = impersonation_chain
149
151
  self.asynchronous = asynchronous
150
152
 
153
+ @property
154
+ def extra_links_params(self) -> dict[str, Any]:
155
+ return {
156
+ "lake_id": self.lake_id,
157
+ "task_id": self.dataplex_task_id,
158
+ "region": self.region,
159
+ "project_id": self.project_id,
160
+ }
161
+
151
162
  def execute(self, context: Context) -> dict:
152
163
  hook = DataplexHook(
153
164
  gcp_conn_id=self.gcp_conn_id,
@@ -155,7 +166,7 @@ class DataplexCreateTaskOperator(GoogleCloudBaseOperator):
155
166
  impersonation_chain=self.impersonation_chain,
156
167
  )
157
168
  self.log.info("Creating Dataplex task %s", self.dataplex_task_id)
158
- DataplexTaskLink.persist(context=context, task_instance=self)
169
+ DataplexTaskLink.persist(context=context)
159
170
 
160
171
  try:
161
172
  operation = hook.create_task(
@@ -349,6 +360,14 @@ class DataplexListTasksOperator(GoogleCloudBaseOperator):
349
360
  self.gcp_conn_id = gcp_conn_id
350
361
  self.impersonation_chain = impersonation_chain
351
362
 
363
+ @property
364
+ def extra_links_params(self) -> dict[str, Any]:
365
+ return {
366
+ "project_id": self.project_id,
367
+ "lake_id": self.lake_id,
368
+ "region": self.region,
369
+ }
370
+
352
371
  def execute(self, context: Context) -> list[dict]:
353
372
  hook = DataplexHook(
354
373
  gcp_conn_id=self.gcp_conn_id,
@@ -356,7 +375,7 @@ class DataplexListTasksOperator(GoogleCloudBaseOperator):
356
375
  impersonation_chain=self.impersonation_chain,
357
376
  )
358
377
  self.log.info("Listing Dataplex tasks from lake %s", self.lake_id)
359
- DataplexTasksLink.persist(context=context, task_instance=self)
378
+ DataplexTasksLink.persist(context=context)
360
379
 
361
380
  tasks = hook.list_tasks(
362
381
  project_id=self.project_id,
@@ -428,6 +447,15 @@ class DataplexGetTaskOperator(GoogleCloudBaseOperator):
428
447
  self.gcp_conn_id = gcp_conn_id
429
448
  self.impersonation_chain = impersonation_chain
430
449
 
450
+ @property
451
+ def extra_links_params(self) -> dict[str, Any]:
452
+ return {
453
+ "lake_id": self.lake_id,
454
+ "task_id": self.dataplex_task_id,
455
+ "region": self.region,
456
+ "project_id": self.project_id,
457
+ }
458
+
431
459
  def execute(self, context: Context) -> dict:
432
460
  hook = DataplexHook(
433
461
  gcp_conn_id=self.gcp_conn_id,
@@ -435,7 +463,7 @@ class DataplexGetTaskOperator(GoogleCloudBaseOperator):
435
463
  impersonation_chain=self.impersonation_chain,
436
464
  )
437
465
  self.log.info("Retrieving Dataplex task %s", self.dataplex_task_id)
438
- DataplexTaskLink.persist(context=context, task_instance=self)
466
+ DataplexTaskLink.persist(context=context)
439
467
 
440
468
  task = hook.get_task(
441
469
  project_id=self.project_id,
@@ -446,7 +474,7 @@ class DataplexGetTaskOperator(GoogleCloudBaseOperator):
446
474
  timeout=self.timeout,
447
475
  metadata=self.metadata,
448
476
  )
449
- DataplexTasksLink.persist(context=context, task_instance=self)
477
+ DataplexTasksLink.persist(context=context)
450
478
  return Task.to_dict(task)
451
479
 
452
480
 
@@ -520,6 +548,14 @@ class DataplexCreateLakeOperator(GoogleCloudBaseOperator):
520
548
  self.impersonation_chain = impersonation_chain
521
549
  self.asynchronous = asynchronous
522
550
 
551
+ @property
552
+ def extra_links_params(self) -> dict[str, Any]:
553
+ return {
554
+ "lake_id": self.lake_id,
555
+ "region": self.region,
556
+ "project_id": self.project_id,
557
+ }
558
+
523
559
  def execute(self, context: Context) -> dict:
524
560
  hook = DataplexHook(
525
561
  gcp_conn_id=self.gcp_conn_id,
@@ -564,10 +600,7 @@ class DataplexCreateLakeOperator(GoogleCloudBaseOperator):
564
600
  if lake["state"] != "CREATING":
565
601
  break
566
602
  time.sleep(time_to_wait)
567
- DataplexLakeLink.persist(
568
- context=context,
569
- task_instance=self,
570
- )
603
+ DataplexLakeLink.persist(context=context)
571
604
  return Lake.to_dict(lake)
572
605
 
573
606
 
@@ -623,6 +656,14 @@ class DataplexDeleteLakeOperator(GoogleCloudBaseOperator):
623
656
  self.gcp_conn_id = gcp_conn_id
624
657
  self.impersonation_chain = impersonation_chain
625
658
 
659
+ @property
660
+ def extra_links_params(self) -> dict[str, Any]:
661
+ return {
662
+ "lake_id": self.lake_id,
663
+ "region": self.region,
664
+ "project_id": self.project_id,
665
+ }
666
+
626
667
  def execute(self, context: Context) -> None:
627
668
  hook = DataplexHook(
628
669
  gcp_conn_id=self.gcp_conn_id,
@@ -640,7 +681,7 @@ class DataplexDeleteLakeOperator(GoogleCloudBaseOperator):
640
681
  timeout=self.timeout,
641
682
  metadata=self.metadata,
642
683
  )
643
- DataplexLakeLink.persist(context=context, task_instance=self)
684
+ DataplexLakeLink.persist(context=context)
644
685
  hook.wait_for_operation(timeout=self.timeout, operation=operation)
645
686
  self.log.info("Dataplex lake %s deleted successfully!", self.lake_id)
646
687
 
@@ -1041,11 +1082,11 @@ class DataplexRunDataQualityScanOperator(GoogleCloudBaseOperator):
1041
1082
  """
1042
1083
  job_state = event["job_state"]
1043
1084
  job_id = event["job_id"]
1044
- if job_state == DataScanJob.State.FAILED:
1085
+ if job_state == DataScanJob.State.FAILED.name: # type: ignore
1045
1086
  raise AirflowException(f"Job failed:\n{job_id}")
1046
- if job_state == DataScanJob.State.CANCELLED:
1087
+ if job_state == DataScanJob.State.CANCELLED.name: # type: ignore
1047
1088
  raise AirflowException(f"Job was cancelled:\n{job_id}")
1048
- if job_state == DataScanJob.State.SUCCEEDED:
1089
+ if job_state == DataScanJob.State.SUCCEEDED.name: # type: ignore
1049
1090
  job = event["job"]
1050
1091
  if not job["data_quality_result"]["passed"]:
1051
1092
  if self.fail_on_dq_failure:
@@ -1219,11 +1260,11 @@ class DataplexGetDataQualityScanResultOperator(GoogleCloudBaseOperator):
1219
1260
  job_state = event["job_state"]
1220
1261
  job_id = event["job_id"]
1221
1262
  job = event["job"]
1222
- if job_state == DataScanJob.State.FAILED:
1263
+ if job_state == DataScanJob.State.FAILED.name: # type: ignore
1223
1264
  raise AirflowException(f"Job failed:\n{job_id}")
1224
- if job_state == DataScanJob.State.CANCELLED:
1265
+ if job_state == DataScanJob.State.CANCELLED.name: # type: ignore
1225
1266
  raise AirflowException(f"Job was cancelled:\n{job_id}")
1226
- if job_state == DataScanJob.State.SUCCEEDED:
1267
+ if job_state == DataScanJob.State.SUCCEEDED.name: # type: ignore
1227
1268
  if not job["data_quality_result"]["passed"]:
1228
1269
  if self.fail_on_dq_failure:
1229
1270
  raise AirflowDataQualityScanException(
@@ -1598,12 +1639,12 @@ class DataplexRunDataProfileScanOperator(GoogleCloudBaseOperator):
1598
1639
  result_timeout=self.result_timeout,
1599
1640
  )
1600
1641
 
1601
- if job.state == DataScanJob.State.FAILED:
1642
+ if job.state == DataScanJob.State.FAILED.name: # type: ignore
1602
1643
  raise AirflowException(f"Data Profile job failed: {job_id}")
1603
- if job.state == DataScanJob.State.SUCCEEDED:
1644
+ if job.state == DataScanJob.State.SUCCEEDED.name: # type: ignore
1604
1645
  self.log.info("Data Profile job executed successfully.")
1605
1646
  else:
1606
- self.log.info("Data Profile job execution returned status: %s", job.status)
1647
+ self.log.info("Data Profile job execution returned status: %s", job.state)
1607
1648
 
1608
1649
  return job_id
1609
1650
 
@@ -1616,11 +1657,11 @@ class DataplexRunDataProfileScanOperator(GoogleCloudBaseOperator):
1616
1657
  """
1617
1658
  job_state = event["job_state"]
1618
1659
  job_id = event["job_id"]
1619
- if job_state == DataScanJob.State.FAILED:
1660
+ if job_state == DataScanJob.State.FAILED.name: # type: ignore
1620
1661
  raise AirflowException(f"Job failed:\n{job_id}")
1621
- if job_state == DataScanJob.State.CANCELLED:
1662
+ if job_state == DataScanJob.State.CANCELLED.name: # type: ignore
1622
1663
  raise AirflowException(f"Job was cancelled:\n{job_id}")
1623
- if job_state == DataScanJob.State.SUCCEEDED:
1664
+ if job_state == DataScanJob.State.SUCCEEDED.name: # type: ignore
1624
1665
  self.log.info("Data Profile job executed successfully.")
1625
1666
  return job_id
1626
1667
 
@@ -2177,6 +2218,13 @@ class DataplexCatalogBaseOperator(GoogleCloudBaseOperator):
2177
2218
  impersonation_chain=self.impersonation_chain,
2178
2219
  )
2179
2220
 
2221
+ @property
2222
+ def extra_links_params(self) -> dict[str, Any]:
2223
+ return {
2224
+ "location": self.location,
2225
+ "project_id": self.project_id,
2226
+ }
2227
+
2180
2228
 
2181
2229
  class DataplexCatalogCreateEntryGroupOperator(DataplexCatalogBaseOperator):
2182
2230
  """
@@ -2228,12 +2276,15 @@ class DataplexCatalogCreateEntryGroupOperator(DataplexCatalogBaseOperator):
2228
2276
  self.entry_group_configuration = entry_group_configuration
2229
2277
  self.validate_request = validate_request
2230
2278
 
2231
- def execute(self, context: Context):
2232
- DataplexCatalogEntryGroupLink.persist(
2233
- context=context,
2234
- task_instance=self,
2235
- )
2279
+ @property
2280
+ def extra_links_params(self) -> dict[str, Any]:
2281
+ return {
2282
+ **super().extra_links_params,
2283
+ "entry_group_id": self.entry_group_id,
2284
+ }
2236
2285
 
2286
+ def execute(self, context: Context):
2287
+ DataplexCatalogEntryGroupLink.persist(context=context)
2237
2288
  if self.validate_request:
2238
2289
  self.log.info("Validating a Create Dataplex Catalog EntryGroup request.")
2239
2290
  else:
@@ -2314,11 +2365,15 @@ class DataplexCatalogGetEntryGroupOperator(DataplexCatalogBaseOperator):
2314
2365
  super().__init__(*args, **kwargs)
2315
2366
  self.entry_group_id = entry_group_id
2316
2367
 
2368
+ @property
2369
+ def extra_links_params(self) -> dict[str, Any]:
2370
+ return {
2371
+ **super().extra_links_params,
2372
+ "entry_group_id": self.entry_group_id,
2373
+ }
2374
+
2317
2375
  def execute(self, context: Context):
2318
- DataplexCatalogEntryGroupLink.persist(
2319
- context=context,
2320
- task_instance=self,
2321
- )
2376
+ DataplexCatalogEntryGroupLink.persist(context=context)
2322
2377
  self.log.info(
2323
2378
  "Retrieving Dataplex Catalog EntryGroup %s.",
2324
2379
  self.entry_group_id,
@@ -2460,10 +2515,7 @@ class DataplexCatalogListEntryGroupsOperator(DataplexCatalogBaseOperator):
2460
2515
  self.order_by = order_by
2461
2516
 
2462
2517
  def execute(self, context: Context):
2463
- DataplexCatalogEntryGroupsLink.persist(
2464
- context=context,
2465
- task_instance=self,
2466
- )
2518
+ DataplexCatalogEntryGroupsLink.persist(context=context)
2467
2519
  self.log.info(
2468
2520
  "Listing Dataplex Catalog EntryGroup from location %s.",
2469
2521
  self.location,
@@ -2481,8 +2533,7 @@ class DataplexCatalogListEntryGroupsOperator(DataplexCatalogBaseOperator):
2481
2533
  metadata=self.metadata,
2482
2534
  )
2483
2535
  self.log.info("EntryGroup on page: %s", entry_group_on_page)
2484
- self.xcom_push(
2485
- context=context,
2536
+ context["ti"].xcom_push(
2486
2537
  key="entry_group_page",
2487
2538
  value=ListEntryGroupsResponse.to_dict(entry_group_on_page._response),
2488
2539
  )
@@ -2552,12 +2603,15 @@ class DataplexCatalogUpdateEntryGroupOperator(DataplexCatalogBaseOperator):
2552
2603
  self.update_mask = update_mask
2553
2604
  self.validate_request = validate_request
2554
2605
 
2555
- def execute(self, context: Context):
2556
- DataplexCatalogEntryGroupLink.persist(
2557
- context=context,
2558
- task_instance=self,
2559
- )
2606
+ @property
2607
+ def extra_links_params(self) -> dict[str, Any]:
2608
+ return {
2609
+ **super().extra_links_params,
2610
+ "entry_group_id": self.entry_group_id,
2611
+ }
2560
2612
 
2613
+ def execute(self, context: Context):
2614
+ DataplexCatalogEntryGroupLink.persist(context=context)
2561
2615
  if self.validate_request:
2562
2616
  self.log.info("Validating an Update Dataplex Catalog EntryGroup request.")
2563
2617
  else:
@@ -2642,12 +2696,15 @@ class DataplexCatalogCreateEntryTypeOperator(DataplexCatalogBaseOperator):
2642
2696
  self.entry_type_configuration = entry_type_configuration
2643
2697
  self.validate_request = validate_request
2644
2698
 
2645
- def execute(self, context: Context):
2646
- DataplexCatalogEntryTypeLink.persist(
2647
- context=context,
2648
- task_instance=self,
2649
- )
2699
+ @property
2700
+ def extra_links_params(self) -> dict[str, Any]:
2701
+ return {
2702
+ **super().extra_links_params,
2703
+ "entry_type_id": self.entry_type_id,
2704
+ }
2650
2705
 
2706
+ def execute(self, context: Context):
2707
+ DataplexCatalogEntryTypeLink.persist(context=context)
2651
2708
  if self.validate_request:
2652
2709
  self.log.info("Validating a Create Dataplex Catalog EntryType request.")
2653
2710
  else:
@@ -2728,11 +2785,15 @@ class DataplexCatalogGetEntryTypeOperator(DataplexCatalogBaseOperator):
2728
2785
  super().__init__(*args, **kwargs)
2729
2786
  self.entry_type_id = entry_type_id
2730
2787
 
2788
+ @property
2789
+ def extra_links_params(self) -> dict[str, Any]:
2790
+ return {
2791
+ **super().extra_links_params,
2792
+ "entry_type_id": self.entry_type_id,
2793
+ }
2794
+
2731
2795
  def execute(self, context: Context):
2732
- DataplexCatalogEntryTypeLink.persist(
2733
- context=context,
2734
- task_instance=self,
2735
- )
2796
+ DataplexCatalogEntryTypeLink.persist(context=context)
2736
2797
  self.log.info(
2737
2798
  "Retrieving Dataplex Catalog EntryType %s.",
2738
2799
  self.entry_type_id,
@@ -2874,10 +2935,7 @@ class DataplexCatalogListEntryTypesOperator(DataplexCatalogBaseOperator):
2874
2935
  self.order_by = order_by
2875
2936
 
2876
2937
  def execute(self, context: Context):
2877
- DataplexCatalogEntryTypesLink.persist(
2878
- context=context,
2879
- task_instance=self,
2880
- )
2938
+ DataplexCatalogEntryTypesLink.persist(context=context)
2881
2939
  self.log.info(
2882
2940
  "Listing Dataplex Catalog EntryType from location %s.",
2883
2941
  self.location,
@@ -2895,8 +2953,7 @@ class DataplexCatalogListEntryTypesOperator(DataplexCatalogBaseOperator):
2895
2953
  metadata=self.metadata,
2896
2954
  )
2897
2955
  self.log.info("EntryType on page: %s", entry_type_on_page)
2898
- self.xcom_push(
2899
- context=context,
2956
+ context["ti"].xcom_push(
2900
2957
  key="entry_type_page",
2901
2958
  value=ListEntryTypesResponse.to_dict(entry_type_on_page._response),
2902
2959
  )
@@ -2966,12 +3023,15 @@ class DataplexCatalogUpdateEntryTypeOperator(DataplexCatalogBaseOperator):
2966
3023
  self.update_mask = update_mask
2967
3024
  self.validate_request = validate_request
2968
3025
 
2969
- def execute(self, context: Context):
2970
- DataplexCatalogEntryTypeLink.persist(
2971
- context=context,
2972
- task_instance=self,
2973
- )
3026
+ @property
3027
+ def extra_links_params(self) -> dict[str, Any]:
3028
+ return {
3029
+ **super().extra_links_params,
3030
+ "entry_type_id": self.entry_type_id,
3031
+ }
2974
3032
 
3033
+ def execute(self, context: Context):
3034
+ DataplexCatalogEntryTypeLink.persist(context=context)
2975
3035
  if self.validate_request:
2976
3036
  self.log.info("Validating an Update Dataplex Catalog EntryType request.")
2977
3037
  else:
@@ -3056,12 +3116,15 @@ class DataplexCatalogCreateAspectTypeOperator(DataplexCatalogBaseOperator):
3056
3116
  self.aspect_type_configuration = aspect_type_configuration
3057
3117
  self.validate_request = validate_request
3058
3118
 
3059
- def execute(self, context: Context):
3060
- DataplexCatalogAspectTypeLink.persist(
3061
- context=context,
3062
- task_instance=self,
3063
- )
3119
+ @property
3120
+ def extra_links_params(self) -> dict[str, Any]:
3121
+ return {
3122
+ **super().extra_links_params,
3123
+ "aspect_type_id": self.aspect_type_id,
3124
+ }
3064
3125
 
3126
+ def execute(self, context: Context):
3127
+ DataplexCatalogAspectTypeLink.persist(context=context)
3065
3128
  if self.validate_request:
3066
3129
  self.log.info("Validating a Create Dataplex Catalog AspectType request.")
3067
3130
  else:
@@ -3142,11 +3205,15 @@ class DataplexCatalogGetAspectTypeOperator(DataplexCatalogBaseOperator):
3142
3205
  super().__init__(*args, **kwargs)
3143
3206
  self.aspect_type_id = aspect_type_id
3144
3207
 
3208
+ @property
3209
+ def extra_links_params(self) -> dict[str, Any]:
3210
+ return {
3211
+ **super().extra_links_params,
3212
+ "aspect_type_id": self.aspect_type_id,
3213
+ }
3214
+
3145
3215
  def execute(self, context: Context):
3146
- DataplexCatalogAspectTypeLink.persist(
3147
- context=context,
3148
- task_instance=self,
3149
- )
3216
+ DataplexCatalogAspectTypeLink.persist(context=context)
3150
3217
  self.log.info(
3151
3218
  "Retrieving Dataplex Catalog AspectType %s.",
3152
3219
  self.aspect_type_id,
@@ -3221,10 +3288,7 @@ class DataplexCatalogListAspectTypesOperator(DataplexCatalogBaseOperator):
3221
3288
  self.order_by = order_by
3222
3289
 
3223
3290
  def execute(self, context: Context):
3224
- DataplexCatalogAspectTypesLink.persist(
3225
- context=context,
3226
- task_instance=self,
3227
- )
3291
+ DataplexCatalogAspectTypesLink.persist(context=context)
3228
3292
  self.log.info(
3229
3293
  "Listing Dataplex Catalog AspectType from location %s.",
3230
3294
  self.location,
@@ -3242,8 +3306,7 @@ class DataplexCatalogListAspectTypesOperator(DataplexCatalogBaseOperator):
3242
3306
  metadata=self.metadata,
3243
3307
  )
3244
3308
  self.log.info("AspectType on page: %s", aspect_type_on_page)
3245
- self.xcom_push(
3246
- context=context,
3309
+ context["ti"].xcom_push(
3247
3310
  key="aspect_type_page",
3248
3311
  value=ListAspectTypesResponse.to_dict(aspect_type_on_page._response),
3249
3312
  )
@@ -3313,12 +3376,15 @@ class DataplexCatalogUpdateAspectTypeOperator(DataplexCatalogBaseOperator):
3313
3376
  self.update_mask = update_mask
3314
3377
  self.validate_request = validate_request
3315
3378
 
3316
- def execute(self, context: Context):
3317
- DataplexCatalogAspectTypeLink.persist(
3318
- context=context,
3319
- task_instance=self,
3320
- )
3379
+ @property
3380
+ def extra_links_params(self) -> dict[str, Any]:
3381
+ return {
3382
+ **super().extra_links_params,
3383
+ "aspect_type_id": self.aspect_type_id,
3384
+ }
3321
3385
 
3386
+ def execute(self, context: Context):
3387
+ DataplexCatalogAspectTypeLink.persist(context=context)
3322
3388
  if self.validate_request:
3323
3389
  self.log.info("Validating an Update Dataplex Catalog AspectType request.")
3324
3390
  else:
@@ -3491,12 +3557,16 @@ class DataplexCatalogCreateEntryOperator(DataplexCatalogBaseOperator):
3491
3557
  f"Missing required fields in Entry configuration: {', '.join(missing_fields)}. "
3492
3558
  )
3493
3559
 
3494
- def execute(self, context: Context):
3495
- DataplexCatalogEntryLink.persist(
3496
- context=context,
3497
- task_instance=self,
3498
- )
3560
+ @property
3561
+ def extra_links_params(self) -> dict[str, Any]:
3562
+ return {
3563
+ **super().extra_links_params,
3564
+ "entry_id": self.entry_id,
3565
+ "entry_group_id": self.entry_group_id,
3566
+ }
3499
3567
 
3568
+ def execute(self, context: Context):
3569
+ DataplexCatalogEntryLink.persist(context=context)
3500
3570
  self._validate_fields(self.entry_configuration)
3501
3571
  try:
3502
3572
  entry = self.hook.create_entry(
@@ -3597,11 +3667,16 @@ class DataplexCatalogGetEntryOperator(DataplexCatalogBaseOperator):
3597
3667
  self.aspect_types = aspect_types
3598
3668
  self.paths = paths
3599
3669
 
3670
+ @property
3671
+ def extra_links_params(self) -> dict[str, Any]:
3672
+ return {
3673
+ **super().extra_links_params,
3674
+ "entry_id": self.entry_id,
3675
+ "entry_group_id": self.entry_group_id,
3676
+ }
3677
+
3600
3678
  def execute(self, context: Context):
3601
- DataplexCatalogEntryLink.persist(
3602
- context=context,
3603
- task_instance=self,
3604
- )
3679
+ DataplexCatalogEntryLink.persist(context=context)
3605
3680
  self.log.info(
3606
3681
  "Retrieving Dataplex Catalog Entry %s.",
3607
3682
  self.entry_id,
@@ -3699,11 +3774,15 @@ class DataplexCatalogListEntriesOperator(DataplexCatalogBaseOperator):
3699
3774
  self.page_token = page_token
3700
3775
  self.filter_by = filter_by
3701
3776
 
3777
+ @property
3778
+ def extra_links_params(self) -> dict[str, Any]:
3779
+ return {
3780
+ **super().extra_links_params,
3781
+ "entry_group_id": self.entry_group_id,
3782
+ }
3783
+
3702
3784
  def execute(self, context: Context):
3703
- DataplexCatalogEntryGroupLink.persist(
3704
- context=context,
3705
- task_instance=self,
3706
- )
3785
+ DataplexCatalogEntryGroupLink.persist(context=context)
3707
3786
  self.log.info(
3708
3787
  "Listing Dataplex Catalog Entry from location %s.",
3709
3788
  self.location,
@@ -3721,8 +3800,7 @@ class DataplexCatalogListEntriesOperator(DataplexCatalogBaseOperator):
3721
3800
  metadata=self.metadata,
3722
3801
  )
3723
3802
  self.log.info("Entries on page: %s", entries_on_page)
3724
- self.xcom_push(
3725
- context=context,
3803
+ context["ti"].xcom_push(
3726
3804
  key="entry_page",
3727
3805
  value=ListEntriesResponse.to_dict(entries_on_page._response),
3728
3806
  )
@@ -3819,8 +3897,7 @@ class DataplexCatalogSearchEntriesOperator(DataplexCatalogBaseOperator):
3819
3897
  metadata=self.metadata,
3820
3898
  )
3821
3899
  self.log.info("Entries on page: %s", entries_on_page)
3822
- self.xcom_push(
3823
- context=context,
3900
+ context["ti"].xcom_push(
3824
3901
  key="entry_page",
3825
3902
  value=SearchEntriesResponse.to_dict(entries_on_page._response),
3826
3903
  )
@@ -3901,11 +3978,16 @@ class DataplexCatalogLookupEntryOperator(DataplexCatalogBaseOperator):
3901
3978
  self.aspect_types = aspect_types
3902
3979
  self.paths = paths
3903
3980
 
3981
+ @property
3982
+ def extra_links_params(self) -> dict[str, Any]:
3983
+ return {
3984
+ **super().extra_links_params,
3985
+ "entry_id": self.entry_id,
3986
+ "entry_group_id": self.entry_group_id,
3987
+ }
3988
+
3904
3989
  def execute(self, context: Context):
3905
- DataplexCatalogEntryLink.persist(
3906
- context=context,
3907
- task_instance=self,
3908
- )
3990
+ DataplexCatalogEntryLink.persist(context=context)
3909
3991
  self.log.info(
3910
3992
  "Looking for Dataplex Catalog Entry %s.",
3911
3993
  self.entry_id,
@@ -3961,7 +4043,7 @@ class DataplexCatalogUpdateEntryOperator(DataplexCatalogBaseOperator):
3961
4043
  :param entry_configuration: Required. The updated configuration body of the Entry.
3962
4044
  :param allow_missing: Optional. If set to true and entry doesn't exist, the service will create it.
3963
4045
  :param delete_missing_aspects: Optional. If set to true and the aspect_keys specify aspect
3964
- ranges, the service deletes any existing aspects from that range that weren't provided
4046
+ ranges, the service deletes any existing aspects from that range that were not provided
3965
4047
  in the request.
3966
4048
  :param aspect_keys: Optional. The map keys of the Aspects which the service should modify.
3967
4049
  It supports the following syntax:
@@ -4020,12 +4102,16 @@ class DataplexCatalogUpdateEntryOperator(DataplexCatalogBaseOperator):
4020
4102
  self.delete_missing_aspects = delete_missing_aspects
4021
4103
  self.aspect_keys = aspect_keys
4022
4104
 
4023
- def execute(self, context: Context):
4024
- DataplexCatalogEntryLink.persist(
4025
- context=context,
4026
- task_instance=self,
4027
- )
4105
+ @property
4106
+ def extra_links_params(self) -> dict[str, Any]:
4107
+ return {
4108
+ **super().extra_links_params,
4109
+ "entry_id": self.entry_id,
4110
+ "entry_group_id": self.entry_group_id,
4111
+ }
4028
4112
 
4113
+ def execute(self, context: Context):
4114
+ DataplexCatalogEntryLink.persist(context=context)
4029
4115
  try:
4030
4116
  entry = self.hook.update_entry(
4031
4117
  location=self.location,
@@ -28,7 +28,7 @@ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseO
28
28
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
29
 
30
30
  if TYPE_CHECKING:
31
- from airflow.utils.context import Context
31
+ from airflow.providers.common.compat.sdk import Context
32
32
 
33
33
 
34
34
  class DataprepGetJobsForJobGroupOperator(GoogleCloudBaseOperator):
@@ -113,7 +113,6 @@ class DataprepGetJobGroupOperator(GoogleCloudBaseOperator):
113
113
  if self.project_id:
114
114
  DataprepJobGroupLink.persist(
115
115
  context=context,
116
- task_instance=self,
117
116
  project_id=self.project_id,
118
117
  job_group_id=int(self.job_group_id),
119
118
  )
@@ -170,7 +169,6 @@ class DataprepRunJobGroupOperator(GoogleCloudBaseOperator):
170
169
  if self.project_id and job_group_id:
171
170
  DataprepJobGroupLink.persist(
172
171
  context=context,
173
- task_instance=self,
174
172
  project_id=self.project_id,
175
173
  job_group_id=int(job_group_id),
176
174
  )
@@ -230,7 +228,6 @@ class DataprepCopyFlowOperator(GoogleCloudBaseOperator):
230
228
  if self.project_id and copied_flow_id:
231
229
  DataprepFlowLink.persist(
232
230
  context=context,
233
- task_instance=self,
234
231
  project_id=self.project_id,
235
232
  flow_id=int(copied_flow_id),
236
233
  )
@@ -303,7 +300,6 @@ class DataprepRunFlowOperator(GoogleCloudBaseOperator):
303
300
  job_group_id = response["data"][0]["id"]
304
301
  DataprepJobGroupLink.persist(
305
302
  context=context,
306
- task_instance=self,
307
303
  project_id=self.project_id,
308
304
  job_group_id=int(job_group_id),
309
305
  )