apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
  3. airflow/providers/google/__init__.py +3 -3
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +52 -43
  7. airflow/providers/google/ads/operators/ads.py +2 -2
  8. airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
  9. airflow/providers/google/assets/gcs.py +1 -11
  10. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  11. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  12. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  13. airflow/providers/google/cloud/hooks/bigquery.py +195 -318
  14. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  15. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  16. airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
  17. airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
  18. airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
  19. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  20. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  21. airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
  22. airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
  23. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
  24. airflow/providers/google/cloud/hooks/compute.py +7 -6
  25. airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
  26. airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
  27. airflow/providers/google/cloud/hooks/dataflow.py +87 -242
  28. airflow/providers/google/cloud/hooks/dataform.py +9 -14
  29. airflow/providers/google/cloud/hooks/datafusion.py +7 -9
  30. airflow/providers/google/cloud/hooks/dataplex.py +13 -12
  31. airflow/providers/google/cloud/hooks/dataprep.py +2 -2
  32. airflow/providers/google/cloud/hooks/dataproc.py +76 -74
  33. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  34. airflow/providers/google/cloud/hooks/dlp.py +5 -4
  35. airflow/providers/google/cloud/hooks/gcs.py +144 -33
  36. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  37. airflow/providers/google/cloud/hooks/kms.py +3 -2
  38. airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
  39. airflow/providers/google/cloud/hooks/looker.py +6 -1
  40. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  41. airflow/providers/google/cloud/hooks/mlengine.py +7 -8
  42. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  43. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  44. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  45. airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
  46. airflow/providers/google/cloud/hooks/spanner.py +75 -10
  47. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  48. airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
  49. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  50. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  51. airflow/providers/google/cloud/hooks/translate.py +8 -17
  52. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
  53. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
  54. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
  55. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  56. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
  57. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  58. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
  59. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  60. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
  61. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
  62. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  63. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  64. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  65. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  66. airflow/providers/google/cloud/hooks/vision.py +7 -7
  67. airflow/providers/google/cloud/hooks/workflows.py +4 -3
  68. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  69. airflow/providers/google/cloud/links/base.py +77 -7
  70. airflow/providers/google/cloud/links/bigquery.py +0 -47
  71. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  72. airflow/providers/google/cloud/links/bigtable.py +0 -48
  73. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  74. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  75. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  76. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  77. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  78. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
  79. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  80. airflow/providers/google/cloud/links/compute.py +0 -58
  81. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  82. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  83. airflow/providers/google/cloud/links/dataflow.py +0 -34
  84. airflow/providers/google/cloud/links/dataform.py +0 -64
  85. airflow/providers/google/cloud/links/datafusion.py +1 -90
  86. airflow/providers/google/cloud/links/dataplex.py +0 -154
  87. airflow/providers/google/cloud/links/dataprep.py +0 -24
  88. airflow/providers/google/cloud/links/dataproc.py +11 -89
  89. airflow/providers/google/cloud/links/datastore.py +0 -31
  90. airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
  91. airflow/providers/google/cloud/links/managed_kafka.py +11 -51
  92. airflow/providers/google/cloud/links/mlengine.py +0 -70
  93. airflow/providers/google/cloud/links/pubsub.py +0 -32
  94. airflow/providers/google/cloud/links/spanner.py +0 -33
  95. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  96. airflow/providers/google/cloud/links/translate.py +17 -187
  97. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  98. airflow/providers/google/cloud/links/workflows.py +0 -52
  99. airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
  100. airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
  101. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  102. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  103. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  104. airflow/providers/google/cloud/openlineage/facets.py +141 -40
  105. airflow/providers/google/cloud/openlineage/mixins.py +14 -13
  106. airflow/providers/google/cloud/openlineage/utils.py +19 -3
  107. airflow/providers/google/cloud/operators/alloy_db.py +76 -61
  108. airflow/providers/google/cloud/operators/bigquery.py +104 -667
  109. airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
  110. airflow/providers/google/cloud/operators/bigtable.py +38 -7
  111. airflow/providers/google/cloud/operators/cloud_base.py +22 -1
  112. airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
  113. airflow/providers/google/cloud/operators/cloud_build.py +80 -36
  114. airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
  115. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  116. airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
  117. airflow/providers/google/cloud/operators/cloud_run.py +39 -20
  118. airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
  119. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
  120. airflow/providers/google/cloud/operators/compute.py +18 -50
  121. airflow/providers/google/cloud/operators/datacatalog.py +167 -29
  122. airflow/providers/google/cloud/operators/dataflow.py +38 -15
  123. airflow/providers/google/cloud/operators/dataform.py +19 -7
  124. airflow/providers/google/cloud/operators/datafusion.py +43 -43
  125. airflow/providers/google/cloud/operators/dataplex.py +212 -126
  126. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  127. airflow/providers/google/cloud/operators/dataproc.py +134 -207
  128. airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
  129. airflow/providers/google/cloud/operators/datastore.py +22 -6
  130. airflow/providers/google/cloud/operators/dlp.py +24 -45
  131. airflow/providers/google/cloud/operators/functions.py +21 -14
  132. airflow/providers/google/cloud/operators/gcs.py +15 -12
  133. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  134. airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
  135. airflow/providers/google/cloud/operators/looker.py +1 -1
  136. airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
  137. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  138. airflow/providers/google/cloud/operators/pubsub.py +69 -21
  139. airflow/providers/google/cloud/operators/spanner.py +53 -45
  140. airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
  141. airflow/providers/google/cloud/operators/stackdriver.py +5 -11
  142. airflow/providers/google/cloud/operators/tasks.py +6 -15
  143. airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
  144. airflow/providers/google/cloud/operators/translate.py +46 -20
  145. airflow/providers/google/cloud/operators/translate_speech.py +4 -3
  146. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
  147. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
  148. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
  149. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
  150. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
  151. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  152. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  153. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
  154. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
  155. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
  156. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
  157. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  158. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  159. airflow/providers/google/cloud/operators/vision.py +7 -5
  160. airflow/providers/google/cloud/operators/workflows.py +24 -19
  161. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  162. airflow/providers/google/cloud/sensors/bigquery.py +2 -2
  163. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
  164. airflow/providers/google/cloud/sensors/bigtable.py +14 -6
  165. airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
  166. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
  167. airflow/providers/google/cloud/sensors/dataflow.py +27 -10
  168. airflow/providers/google/cloud/sensors/dataform.py +2 -2
  169. airflow/providers/google/cloud/sensors/datafusion.py +4 -4
  170. airflow/providers/google/cloud/sensors/dataplex.py +7 -5
  171. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  172. airflow/providers/google/cloud/sensors/dataproc.py +10 -9
  173. airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
  174. airflow/providers/google/cloud/sensors/gcs.py +22 -21
  175. airflow/providers/google/cloud/sensors/looker.py +5 -5
  176. airflow/providers/google/cloud/sensors/pubsub.py +20 -20
  177. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  178. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
  179. airflow/providers/google/cloud/sensors/workflows.py +6 -4
  180. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  181. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  182. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
  183. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  184. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
  185. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  186. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  187. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  188. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  189. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  190. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
  191. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
  192. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
  193. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
  194. airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
  195. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
  196. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  197. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  198. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  199. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  200. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  201. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  202. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
  203. airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
  204. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  205. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
  206. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  207. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  208. airflow/providers/google/cloud/triggers/bigquery.py +75 -34
  209. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  210. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  211. airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
  212. airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
  213. airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
  214. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
  215. airflow/providers/google/cloud/triggers/dataflow.py +125 -2
  216. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  217. airflow/providers/google/cloud/triggers/dataplex.py +16 -3
  218. airflow/providers/google/cloud/triggers/dataproc.py +124 -53
  219. airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
  220. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  221. airflow/providers/google/cloud/triggers/pubsub.py +17 -20
  222. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  223. airflow/providers/google/cloud/utils/bigquery.py +5 -7
  224. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  225. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  226. airflow/providers/google/cloud/utils/dataform.py +1 -1
  227. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  228. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  229. airflow/providers/google/cloud/utils/validators.py +43 -0
  230. airflow/providers/google/common/auth_backend/google_openid.py +26 -9
  231. airflow/providers/google/common/consts.py +2 -1
  232. airflow/providers/google/common/deprecated.py +2 -1
  233. airflow/providers/google/common/hooks/base_google.py +40 -43
  234. airflow/providers/google/common/hooks/operation_helpers.py +78 -0
  235. airflow/providers/google/common/links/storage.py +0 -22
  236. airflow/providers/google/common/utils/get_secret.py +31 -0
  237. airflow/providers/google/common/utils/id_token_credentials.py +4 -5
  238. airflow/providers/google/firebase/operators/firestore.py +2 -2
  239. airflow/providers/google/get_provider_info.py +61 -216
  240. airflow/providers/google/go_module_utils.py +35 -3
  241. airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
  242. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  243. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  244. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  245. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  246. airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
  247. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
  248. airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
  249. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  250. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  251. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  252. airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
  253. airflow/providers/google/suite/hooks/calendar.py +1 -1
  254. airflow/providers/google/suite/hooks/drive.py +2 -2
  255. airflow/providers/google/suite/hooks/sheets.py +15 -1
  256. airflow/providers/google/suite/operators/sheets.py +8 -3
  257. airflow/providers/google/suite/sensors/drive.py +2 -2
  258. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
  259. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  260. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  261. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  262. airflow/providers/google/version_compat.py +15 -1
  263. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
  264. apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
  265. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
  266. apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
  267. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  268. airflow/providers/google/cloud/hooks/automl.py +0 -679
  269. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  270. airflow/providers/google/cloud/links/automl.py +0 -193
  271. airflow/providers/google/cloud/operators/automl.py +0 -1360
  272. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  273. airflow/providers/google/cloud/operators/mlengine.py +0 -1515
  274. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
  275. apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
  276. /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
  277. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
  278. {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -20,6 +20,7 @@
20
20
  from __future__ import annotations
21
21
 
22
22
  from collections.abc import Sequence
23
+ from functools import cached_property
23
24
  from typing import TYPE_CHECKING
24
25
 
25
26
  from airflow.exceptions import AirflowException
@@ -29,7 +30,8 @@ from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseO
29
30
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
30
31
 
31
32
  if TYPE_CHECKING:
32
- from airflow.utils.context import Context
33
+ from airflow.providers.common.compat.sdk import Context
34
+ from airflow.providers.openlineage.extractors import OperatorLineage
33
35
 
34
36
 
35
37
  class SpannerDeployInstanceOperator(GoogleCloudBaseOperator):
@@ -122,7 +124,6 @@ class SpannerDeployInstanceOperator(GoogleCloudBaseOperator):
122
124
  )
123
125
  SpannerInstanceLink.persist(
124
126
  context=context,
125
- task_instance=self,
126
127
  instance_id=self.instance_id,
127
128
  project_id=self.project_id or hook.project_id,
128
129
  )
@@ -188,13 +189,12 @@ class SpannerDeleteInstanceOperator(GoogleCloudBaseOperator):
188
189
  )
189
190
  if hook.get_instance(project_id=self.project_id, instance_id=self.instance_id):
190
191
  return hook.delete_instance(project_id=self.project_id, instance_id=self.instance_id)
191
- else:
192
- self.log.info(
193
- "Instance '%s' does not exist in project '%s'. Aborting delete.",
194
- self.instance_id,
195
- self.project_id,
196
- )
197
- return True
192
+ self.log.info(
193
+ "Instance '%s' does not exist in project '%s'. Aborting delete.",
194
+ self.instance_id,
195
+ self.project_id,
196
+ )
197
+ return True
198
198
 
199
199
 
200
200
  class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
@@ -256,6 +256,13 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
256
256
  self.impersonation_chain = impersonation_chain
257
257
  super().__init__(**kwargs)
258
258
 
259
+ @cached_property
260
+ def hook(self) -> SpannerHook:
261
+ return SpannerHook(
262
+ gcp_conn_id=self.gcp_conn_id,
263
+ impersonation_chain=self.impersonation_chain,
264
+ )
265
+
259
266
  def _validate_inputs(self) -> None:
260
267
  if self.project_id == "":
261
268
  raise AirflowException("The required parameter 'project_id' is empty")
@@ -267,10 +274,6 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
267
274
  raise AirflowException("The required parameter 'query' is empty")
268
275
 
269
276
  def execute(self, context: Context):
270
- hook = SpannerHook(
271
- gcp_conn_id=self.gcp_conn_id,
272
- impersonation_chain=self.impersonation_chain,
273
- )
274
277
  if isinstance(self.query, str):
275
278
  queries = [x.strip() for x in self.query.split(";")]
276
279
  self.sanitize_queries(queries)
@@ -282,8 +285,8 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
282
285
  self.instance_id,
283
286
  self.database_id,
284
287
  )
285
- self.log.info(queries)
286
- hook.execute_dml(
288
+ self.log.info("Executing queries: %s", queries)
289
+ result_rows_count_per_query = self.hook.execute_dml(
287
290
  project_id=self.project_id,
288
291
  instance_id=self.instance_id,
289
292
  database_id=self.database_id,
@@ -291,11 +294,11 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
291
294
  )
292
295
  SpannerDatabaseLink.persist(
293
296
  context=context,
294
- task_instance=self,
295
297
  instance_id=self.instance_id,
296
298
  database_id=self.database_id,
297
- project_id=self.project_id or hook.project_id,
299
+ project_id=self.project_id or self.hook.project_id,
298
300
  )
301
+ return result_rows_count_per_query
299
302
 
300
303
  @staticmethod
301
304
  def sanitize_queries(queries: list[str]) -> None:
@@ -307,6 +310,17 @@ class SpannerQueryDatabaseInstanceOperator(GoogleCloudBaseOperator):
307
310
  if queries and queries[-1] == "":
308
311
  queries.pop()
309
312
 
313
+ def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage | None:
314
+ """Build a generic OpenLineage facet, aligned with SQL-based operators."""
315
+ from airflow.providers.common.compat.openlineage.utils.sql import get_openlineage_facets_with_sql
316
+
317
+ return get_openlineage_facets_with_sql(
318
+ hook=self.hook,
319
+ sql=self.query,
320
+ conn_id=self.gcp_conn_id,
321
+ database=self.database_id,
322
+ )
323
+
310
324
 
311
325
  class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
312
326
  """
@@ -381,7 +395,6 @@ class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
381
395
  )
382
396
  SpannerDatabaseLink.persist(
383
397
  context=context,
384
- task_instance=self,
385
398
  instance_id=self.instance_id,
386
399
  database_id=self.database_id,
387
400
  project_id=self.project_id or hook.project_id,
@@ -401,14 +414,12 @@ class SpannerDeployDatabaseInstanceOperator(GoogleCloudBaseOperator):
401
414
  database_id=self.database_id,
402
415
  ddl_statements=self.ddl_statements,
403
416
  )
404
- else:
405
- self.log.info(
406
- "The database '%s' in project '%s' and instance '%s'"
407
- " already exists. Nothing to do. Exiting.",
408
- self.database_id,
409
- self.project_id,
410
- self.instance_id,
411
- )
417
+ self.log.info(
418
+ "The database '%s' in project '%s' and instance '%s' already exists. Nothing to do. Exiting.",
419
+ self.database_id,
420
+ self.project_id,
421
+ self.instance_id,
422
+ )
412
423
  return True
413
424
 
414
425
 
@@ -497,21 +508,19 @@ class SpannerUpdateDatabaseInstanceOperator(GoogleCloudBaseOperator):
497
508
  f"and instance '{self.instance_id}' is missing. "
498
509
  f"Create the database first before you can update it."
499
510
  )
500
- else:
501
- SpannerDatabaseLink.persist(
502
- context=context,
503
- task_instance=self,
504
- instance_id=self.instance_id,
505
- database_id=self.database_id,
506
- project_id=self.project_id or hook.project_id,
507
- )
508
- return hook.update_database(
509
- project_id=self.project_id,
510
- instance_id=self.instance_id,
511
- database_id=self.database_id,
512
- ddl_statements=self.ddl_statements,
513
- operation_id=self.operation_id,
514
- )
511
+ SpannerDatabaseLink.persist(
512
+ context=context,
513
+ instance_id=self.instance_id,
514
+ database_id=self.database_id,
515
+ project_id=self.project_id or hook.project_id,
516
+ )
517
+ return hook.update_database(
518
+ project_id=self.project_id,
519
+ instance_id=self.instance_id,
520
+ database_id=self.database_id,
521
+ ddl_statements=self.ddl_statements,
522
+ operation_id=self.operation_id,
523
+ )
515
524
 
516
525
 
517
526
  class SpannerDeleteDatabaseInstanceOperator(GoogleCloudBaseOperator):
@@ -590,7 +599,6 @@ class SpannerDeleteDatabaseInstanceOperator(GoogleCloudBaseOperator):
590
599
  self.instance_id,
591
600
  )
592
601
  return True
593
- else:
594
- return hook.delete_database(
595
- project_id=self.project_id, instance_id=self.instance_id, database_id=self.database_id
596
- )
602
+ return hook.delete_database(
603
+ project_id=self.project_id, instance_id=self.instance_id, database_id=self.database_id
604
+ )
@@ -22,19 +22,21 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
26
+ from google.protobuf.json_format import MessageToDict
27
+
25
28
  from airflow.exceptions import AirflowException
26
29
  from airflow.providers.google.cloud.hooks.speech_to_text import CloudSpeechToTextHook, RecognitionAudio
27
30
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
28
31
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
32
  from airflow.providers.google.common.links.storage import FileDetailsLink
30
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
- from google.protobuf.json_format import MessageToDict
32
33
 
33
34
  if TYPE_CHECKING:
34
- from airflow.utils.context import Context
35
35
  from google.api_core.retry import Retry
36
36
  from google.cloud.speech_v1.types import RecognitionConfig
37
37
 
38
+ from airflow.providers.common.compat.sdk import Context
39
+
38
40
 
39
41
  class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
40
42
  """
@@ -115,7 +117,6 @@ class CloudSpeechToTextRecognizeSpeechOperator(GoogleCloudBaseOperator):
115
117
  if self.audio.uri:
116
118
  FileDetailsLink.persist(
117
119
  context=context,
118
- task_instance=self,
119
120
  # Slice from: "gs://{BUCKET_NAME}/{FILE_NAME}" to: "{BUCKET_NAME}/{FILE_NAME}"
120
121
  uri=self.audio.uri[5:],
121
122
  project_id=self.project_id or hook.project_id,
@@ -20,6 +20,9 @@ from __future__ import annotations
20
20
  from collections.abc import Sequence
21
21
  from typing import TYPE_CHECKING
22
22
 
23
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
24
+ from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
25
+
23
26
  from airflow.providers.google.cloud.hooks.stackdriver import StackdriverHook
24
27
  from airflow.providers.google.cloud.links.stackdriver import (
25
28
  StackdriverNotificationsLink,
@@ -27,13 +30,12 @@ from airflow.providers.google.cloud.links.stackdriver import (
27
30
  )
28
31
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
29
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
30
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
- from google.cloud.monitoring_v3 import AlertPolicy, NotificationChannel
32
33
 
33
34
  if TYPE_CHECKING:
34
- from airflow.utils.context import Context
35
35
  from google.api_core.retry import Retry
36
36
 
37
+ from airflow.providers.common.compat.sdk import Context
38
+
37
39
 
38
40
  class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
39
41
  """
@@ -143,7 +145,6 @@ class StackdriverListAlertPoliciesOperator(GoogleCloudBaseOperator):
143
145
  )
144
146
  StackdriverPoliciesLink.persist(
145
147
  context=context,
146
- operator_instance=self,
147
148
  project_id=self.project_id or self.hook.project_id,
148
149
  )
149
150
  return [AlertPolicy.to_dict(policy) for policy in result]
@@ -226,7 +227,6 @@ class StackdriverEnableAlertPoliciesOperator(GoogleCloudBaseOperator):
226
227
  )
227
228
  StackdriverPoliciesLink.persist(
228
229
  context=context,
229
- operator_instance=self,
230
230
  project_id=self.project_id or self.hook.project_id,
231
231
  )
232
232
 
@@ -309,7 +309,6 @@ class StackdriverDisableAlertPoliciesOperator(GoogleCloudBaseOperator):
309
309
  )
310
310
  StackdriverPoliciesLink.persist(
311
311
  context=context,
312
- operator_instance=self,
313
312
  project_id=self.project_id or self.hook.project_id,
314
313
  )
315
314
 
@@ -392,7 +391,6 @@ class StackdriverUpsertAlertOperator(GoogleCloudBaseOperator):
392
391
  )
393
392
  StackdriverPoliciesLink.persist(
394
393
  context=context,
395
- operator_instance=self,
396
394
  project_id=self.project_id or self.hook.project_id,
397
395
  )
398
396
 
@@ -578,7 +576,6 @@ class StackdriverListNotificationChannelsOperator(GoogleCloudBaseOperator):
578
576
  )
579
577
  StackdriverNotificationsLink.persist(
580
578
  context=context,
581
- operator_instance=self,
582
579
  project_id=self.project_id or self.hook.project_id,
583
580
  )
584
581
  return [NotificationChannel.to_dict(channel) for channel in channels]
@@ -664,7 +661,6 @@ class StackdriverEnableNotificationChannelsOperator(GoogleCloudBaseOperator):
664
661
  )
665
662
  StackdriverNotificationsLink.persist(
666
663
  context=context,
667
- operator_instance=self,
668
664
  project_id=self.project_id or self.hook.project_id,
669
665
  )
670
666
 
@@ -749,7 +745,6 @@ class StackdriverDisableNotificationChannelsOperator(GoogleCloudBaseOperator):
749
745
  )
750
746
  StackdriverNotificationsLink.persist(
751
747
  context=context,
752
- operator_instance=self,
753
748
  project_id=self.project_id or self.hook.project_id,
754
749
  )
755
750
 
@@ -836,7 +831,6 @@ class StackdriverUpsertNotificationChannelOperator(GoogleCloudBaseOperator):
836
831
  )
837
832
  StackdriverNotificationsLink.persist(
838
833
  context=context,
839
- operator_instance=self,
840
834
  project_id=self.project_id or self.hook.project_id,
841
835
  )
842
836
 
@@ -22,19 +22,21 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
+ from google.api_core.exceptions import AlreadyExists
26
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
+ from google.cloud.tasks_v2.types import Queue, Task
28
+
25
29
  from airflow.providers.google.cloud.hooks.tasks import CloudTasksHook
26
30
  from airflow.providers.google.cloud.links.cloud_tasks import CloudTasksLink, CloudTasksQueueLink
27
31
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
28
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
29
- from google.api_core.exceptions import AlreadyExists
30
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
- from google.cloud.tasks_v2.types import Queue, Task
32
33
 
33
34
  if TYPE_CHECKING:
34
- from airflow.utils.context import Context
35
35
  from google.api_core.retry import Retry
36
36
  from google.protobuf.field_mask_pb2 import FieldMask
37
37
 
38
+ from airflow.providers.common.compat.sdk import Context
39
+
38
40
 
39
41
  MetaData = Sequence[tuple[str, str]]
40
42
 
@@ -135,7 +137,6 @@ class CloudTasksQueueCreateOperator(GoogleCloudBaseOperator):
135
137
  metadata=self.metadata,
136
138
  )
137
139
  CloudTasksQueueLink.persist(
138
- operator_instance=self,
139
140
  context=context,
140
141
  queue_name=queue.name,
141
142
  )
@@ -234,7 +235,6 @@ class CloudTasksQueueUpdateOperator(GoogleCloudBaseOperator):
234
235
  metadata=self.metadata,
235
236
  )
236
237
  CloudTasksQueueLink.persist(
237
- operator_instance=self,
238
238
  context=context,
239
239
  queue_name=queue.name,
240
240
  )
@@ -317,7 +317,6 @@ class CloudTasksQueueGetOperator(GoogleCloudBaseOperator):
317
317
  metadata=self.metadata,
318
318
  )
319
319
  CloudTasksQueueLink.persist(
320
- operator_instance=self,
321
320
  context=context,
322
321
  queue_name=queue.name,
323
322
  )
@@ -404,7 +403,6 @@ class CloudTasksQueuesListOperator(GoogleCloudBaseOperator):
404
403
  metadata=self.metadata,
405
404
  )
406
405
  CloudTasksLink.persist(
407
- operator_instance=self,
408
406
  context=context,
409
407
  project_id=self.project_id or hook.project_id,
410
408
  )
@@ -562,7 +560,6 @@ class CloudTasksQueuePurgeOperator(GoogleCloudBaseOperator):
562
560
  metadata=self.metadata,
563
561
  )
564
562
  CloudTasksQueueLink.persist(
565
- operator_instance=self,
566
563
  context=context,
567
564
  queue_name=queue.name,
568
565
  )
@@ -645,7 +642,6 @@ class CloudTasksQueuePauseOperator(GoogleCloudBaseOperator):
645
642
  metadata=self.metadata,
646
643
  )
647
644
  CloudTasksQueueLink.persist(
648
- operator_instance=self,
649
645
  context=context,
650
646
  queue_name=queue.name,
651
647
  )
@@ -728,7 +724,6 @@ class CloudTasksQueueResumeOperator(GoogleCloudBaseOperator):
728
724
  metadata=self.metadata,
729
725
  )
730
726
  CloudTasksQueueLink.persist(
731
- operator_instance=self,
732
727
  context=context,
733
728
  queue_name=queue.name,
734
729
  )
@@ -828,7 +823,6 @@ class CloudTasksTaskCreateOperator(GoogleCloudBaseOperator):
828
823
  metadata=self.metadata,
829
824
  )
830
825
  CloudTasksQueueLink.persist(
831
- operator_instance=self,
832
826
  context=context,
833
827
  queue_name=task.name,
834
828
  )
@@ -921,7 +915,6 @@ class CloudTasksTaskGetOperator(GoogleCloudBaseOperator):
921
915
  metadata=self.metadata,
922
916
  )
923
917
  CloudTasksQueueLink.persist(
924
- operator_instance=self,
925
918
  context=context,
926
919
  queue_name=task.name,
927
920
  )
@@ -1014,7 +1007,6 @@ class CloudTasksTasksListOperator(GoogleCloudBaseOperator):
1014
1007
  metadata=self.metadata,
1015
1008
  )
1016
1009
  CloudTasksQueueLink.persist(
1017
- operator_instance=self,
1018
1010
  context=context,
1019
1011
  queue_name=f"projects/{self.project_id or hook.project_id}/"
1020
1012
  f"locations/{self.location}/queues/{self.queue_name}",
@@ -1188,7 +1180,6 @@ class CloudTasksTaskRunOperator(GoogleCloudBaseOperator):
1188
1180
  metadata=self.metadata,
1189
1181
  )
1190
1182
  CloudTasksQueueLink.persist(
1191
- operator_instance=self,
1192
1183
  context=context,
1193
1184
  queue_name=task.name,
1194
1185
  )
@@ -23,19 +23,21 @@ from collections.abc import Sequence
23
23
  from tempfile import NamedTemporaryFile
24
24
  from typing import TYPE_CHECKING
25
25
 
26
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
+
26
28
  from airflow.exceptions import AirflowException
27
29
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
28
30
  from airflow.providers.google.cloud.hooks.text_to_speech import CloudTextToSpeechHook
29
31
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
30
32
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
31
33
  from airflow.providers.google.common.links.storage import FileDetailsLink
32
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
34
 
34
35
  if TYPE_CHECKING:
35
- from airflow.utils.context import Context
36
36
  from google.api_core.retry import Retry
37
37
  from google.cloud.texttospeech_v1.types import AudioConfig, SynthesisInput, VoiceSelectionParams
38
38
 
39
+ from airflow.providers.common.compat.sdk import Context
40
+
39
41
 
40
42
  class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
41
43
  """
@@ -148,7 +150,6 @@ class CloudTextToSpeechSynthesizeOperator(GoogleCloudBaseOperator):
148
150
  )
149
151
  FileDetailsLink.persist(
150
152
  context=context,
151
- task_instance=self,
152
153
  uri=f"{self.target_bucket_name}/{self.target_filename}",
153
154
  project_id=cloud_storage_hook.project_id,
154
155
  )
@@ -22,6 +22,9 @@ from __future__ import annotations
22
22
  from collections.abc import MutableMapping, MutableSequence, Sequence
23
23
  from typing import TYPE_CHECKING, cast
24
24
 
25
+ from google.api_core.exceptions import GoogleAPICallError
26
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
27
+
25
28
  from airflow.exceptions import AirflowException
26
29
  from airflow.providers.google.cloud.hooks.translate import CloudTranslateHook, TranslateHook
27
30
  from airflow.providers.google.cloud.links.translate import (
@@ -34,12 +37,10 @@ from airflow.providers.google.cloud.links.translate import (
34
37
  TranslationNativeDatasetLink,
35
38
  )
36
39
  from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
40
+ from airflow.providers.google.cloud.operators.vertex_ai.dataset import DatasetImportDataResultsCheckHelper
37
41
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
38
- from google.api_core.exceptions import GoogleAPICallError
39
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
40
42
 
41
43
  if TYPE_CHECKING:
42
- from airflow.utils.context import Context
43
44
  from google.api_core.retry import Retry
44
45
  from google.cloud.translate_v3.types import (
45
46
  BatchDocumentInputConfig,
@@ -55,6 +56,8 @@ if TYPE_CHECKING:
55
56
  )
56
57
  from google.cloud.translate_v3.types.translation_service import Glossary, GlossaryInputConfig
57
58
 
59
+ from airflow.providers.common.compat.sdk import Context
60
+
58
61
 
59
62
  class CloudTranslateTextOperator(GoogleCloudBaseOperator):
60
63
  """
@@ -392,7 +395,6 @@ class TranslateTextBatchOperator(GoogleCloudBaseOperator):
392
395
  self.log.info("Translate text batch job started.")
393
396
  TranslateTextBatchLink.persist(
394
397
  context=context,
395
- task_instance=self,
396
398
  project_id=self.project_id or hook.project_id,
397
399
  output_config=self.output_config,
398
400
  )
@@ -478,15 +480,15 @@ class TranslateCreateDatasetOperator(GoogleCloudBaseOperator):
478
480
  result = hook.wait_for_operation_result(result_operation)
479
481
  result = type(result).to_dict(result)
480
482
  dataset_id = hook.extract_object_id(result)
481
- self.xcom_push(context, key="dataset_id", value=dataset_id)
483
+ context["ti"].xcom_push(key="dataset_id", value=dataset_id)
482
484
  self.log.info("Dataset creation complete. The dataset_id: %s.", dataset_id)
483
485
 
484
486
  project_id = self.project_id or hook.project_id
485
487
  TranslationNativeDatasetLink.persist(
486
488
  context=context,
487
- task_instance=self,
488
489
  dataset_id=dataset_id,
489
490
  project_id=project_id,
491
+ location=self.location,
490
492
  )
491
493
  return result
492
494
 
@@ -554,7 +556,6 @@ class TranslateDatasetsListOperator(GoogleCloudBaseOperator):
554
556
  project_id = self.project_id or hook.project_id
555
557
  TranslationDatasetsListLink.persist(
556
558
  context=context,
557
- task_instance=self,
558
559
  project_id=project_id,
559
560
  )
560
561
  self.log.info("Requesting datasets list")
@@ -575,7 +576,7 @@ class TranslateDatasetsListOperator(GoogleCloudBaseOperator):
575
576
  return result_ids
576
577
 
577
578
 
578
- class TranslateImportDataOperator(GoogleCloudBaseOperator):
579
+ class TranslateImportDataOperator(GoogleCloudBaseOperator, DatasetImportDataResultsCheckHelper):
579
580
  """
580
581
  Import data to the translation dataset.
581
582
 
@@ -602,6 +603,7 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
602
603
  If set as a sequence, the identities from the list must grant
603
604
  Service Account Token Creator IAM role to the directly preceding identity, with first
604
605
  account from the list granting this role to the originating account (templated).
606
+ :param raise_for_empty_result: Raise an error if no additional data has been populated after the import.
605
607
  """
606
608
 
607
609
  template_fields: Sequence[str] = (
@@ -627,6 +629,7 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
627
629
  retry: Retry | _MethodDefault = DEFAULT,
628
630
  gcp_conn_id: str = "google_cloud_default",
629
631
  impersonation_chain: str | Sequence[str] | None = None,
632
+ raise_for_empty_result: bool = False,
630
633
  **kwargs,
631
634
  ) -> None:
632
635
  super().__init__(**kwargs)
@@ -639,9 +642,21 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
639
642
  self.retry = retry
640
643
  self.gcp_conn_id = gcp_conn_id
641
644
  self.impersonation_chain = impersonation_chain
645
+ self.raise_for_empty_result = raise_for_empty_result
642
646
 
643
647
  def execute(self, context: Context):
644
648
  hook = TranslateHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
649
+ initial_dataset_size = self._get_number_of_ds_items(
650
+ dataset=hook.get_dataset(
651
+ dataset_id=self.dataset_id,
652
+ project_id=self.project_id,
653
+ location=self.location,
654
+ retry=self.retry,
655
+ timeout=self.timeout,
656
+ metadata=self.metadata,
657
+ ),
658
+ total_key_name="example_count",
659
+ )
645
660
  self.log.info("Importing data to dataset...")
646
661
  operation = hook.import_dataset_data(
647
662
  dataset_id=self.dataset_id,
@@ -655,12 +670,27 @@ class TranslateImportDataOperator(GoogleCloudBaseOperator):
655
670
  project_id = self.project_id or hook.project_id
656
671
  TranslationNativeDatasetLink.persist(
657
672
  context=context,
658
- task_instance=self,
659
673
  dataset_id=self.dataset_id,
660
674
  project_id=project_id,
675
+ location=self.location,
661
676
  )
662
677
  hook.wait_for_operation_done(operation=operation, timeout=self.timeout)
678
+
679
+ result_dataset_size = self._get_number_of_ds_items(
680
+ dataset=hook.get_dataset(
681
+ dataset_id=self.dataset_id,
682
+ project_id=self.project_id,
683
+ location=self.location,
684
+ retry=self.retry,
685
+ timeout=self.timeout,
686
+ metadata=self.metadata,
687
+ ),
688
+ total_key_name="example_count",
689
+ )
690
+ if self.raise_for_empty_result:
691
+ self._raise_for_empty_import_result(self.dataset_id, initial_dataset_size, result_dataset_size)
663
692
  self.log.info("Importing data finished!")
693
+ return {"total_imported": int(result_dataset_size) - int(initial_dataset_size)}
664
694
 
665
695
 
666
696
  class TranslateDeleteDatasetOperator(GoogleCloudBaseOperator):
@@ -819,16 +849,16 @@ class TranslateCreateModelOperator(GoogleCloudBaseOperator):
819
849
  result = hook.wait_for_operation_result(operation=result_operation)
820
850
  result = type(result).to_dict(result)
821
851
  model_id = hook.extract_object_id(result)
822
- self.xcom_push(context, key="model_id", value=model_id)
852
+ context["ti"].xcom_push(key="model_id", value=model_id)
823
853
  self.log.info("Model creation complete. The model_id: %s.", model_id)
824
854
 
825
855
  project_id = self.project_id or hook.project_id
826
856
  TranslationModelLink.persist(
827
857
  context=context,
828
- task_instance=self,
829
858
  dataset_id=self.dataset_id,
830
859
  model_id=model_id,
831
860
  project_id=project_id,
861
+ location=self.location,
832
862
  )
833
863
  return result
834
864
 
@@ -896,7 +926,6 @@ class TranslateModelsListOperator(GoogleCloudBaseOperator):
896
926
  project_id = self.project_id or hook.project_id
897
927
  TranslationModelsListLink.persist(
898
928
  context=context,
899
- task_instance=self,
900
929
  project_id=project_id,
901
930
  )
902
931
  self.log.info("Requesting models list")
@@ -1139,11 +1168,10 @@ class TranslateDocumentOperator(GoogleCloudBaseOperator):
1139
1168
  if self.document_output_config:
1140
1169
  TranslateResultByOutputConfigLink.persist(
1141
1170
  context=context,
1142
- task_instance=self,
1143
1171
  project_id=self.project_id or hook.project_id,
1144
1172
  output_config=self.document_output_config,
1145
1173
  )
1146
- return cast(dict, type(doc_translation_result).to_dict(doc_translation_result))
1174
+ return cast("dict", type(doc_translation_result).to_dict(doc_translation_result))
1147
1175
 
1148
1176
 
1149
1177
  class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
@@ -1302,13 +1330,12 @@ class TranslateDocumentBatchOperator(GoogleCloudBaseOperator):
1302
1330
  self.log.info("Batch document translation job started.")
1303
1331
  TranslateResultByOutputConfigLink.persist(
1304
1332
  context=context,
1305
- task_instance=self,
1306
1333
  project_id=self.project_id or hook.project_id,
1307
1334
  output_config=self.output_config,
1308
1335
  )
1309
1336
  result = hook.wait_for_operation_result(batch_document_translate_operation)
1310
1337
  self.log.info("Batch document translation job finished")
1311
- return cast(dict, type(result).to_dict(result))
1338
+ return cast("dict", type(result).to_dict(result))
1312
1339
 
1313
1340
 
1314
1341
  class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
@@ -1409,7 +1436,7 @@ class TranslateCreateGlossaryOperator(GoogleCloudBaseOperator):
1409
1436
  result = type(result).to_dict(result)
1410
1437
 
1411
1438
  glossary_id = hook.extract_object_id(result)
1412
- self.xcom_push(context, key="glossary_id", value=glossary_id)
1439
+ context["ti"].xcom_push(key="glossary_id", value=glossary_id)
1413
1440
  self.log.info("Glossary creation complete. The glossary_id: %s.", glossary_id)
1414
1441
  return result
1415
1442
 
@@ -1608,7 +1635,6 @@ class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
1608
1635
  project_id = self.project_id or hook.project_id
1609
1636
  TranslationGlossariesListLink.persist(
1610
1637
  context=context,
1611
- task_instance=self,
1612
1638
  project_id=project_id,
1613
1639
  )
1614
1640
  self.log.info("Requesting glossaries list")
@@ -1628,8 +1654,8 @@ class TranslateListGlossariesOperator(GoogleCloudBaseOperator):
1628
1654
  raise AirflowException(e)
1629
1655
 
1630
1656
  result_ids = []
1631
- for glossary_item in results_pager:
1632
- glossary_item = type(glossary_item).to_dict(glossary_item)
1657
+ for glossary_item_raw in results_pager:
1658
+ glossary_item = type(glossary_item_raw).to_dict(glossary_item_raw)
1633
1659
  glossary_id = hook.extract_object_id(glossary_item)
1634
1660
  result_ids.append(glossary_id)
1635
1661
  self.log.info("Fetching the glossaries list complete. Glossary id-s: %s", result_ids)