apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
  3. airflow/providers/google/__init__.py +3 -3
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +52 -43
  7. airflow/providers/google/ads/operators/ads.py +2 -2
  8. airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
  9. airflow/providers/google/assets/gcs.py +1 -11
  10. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  11. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  12. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  13. airflow/providers/google/cloud/hooks/bigquery.py +195 -318
  14. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  15. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  16. airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
  17. airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
  18. airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
  19. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  20. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  21. airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
  22. airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
  23. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
  24. airflow/providers/google/cloud/hooks/compute.py +7 -6
  25. airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
  26. airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
  27. airflow/providers/google/cloud/hooks/dataflow.py +87 -242
  28. airflow/providers/google/cloud/hooks/dataform.py +9 -14
  29. airflow/providers/google/cloud/hooks/datafusion.py +7 -9
  30. airflow/providers/google/cloud/hooks/dataplex.py +13 -12
  31. airflow/providers/google/cloud/hooks/dataprep.py +2 -2
  32. airflow/providers/google/cloud/hooks/dataproc.py +76 -74
  33. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  34. airflow/providers/google/cloud/hooks/dlp.py +5 -4
  35. airflow/providers/google/cloud/hooks/gcs.py +144 -33
  36. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  37. airflow/providers/google/cloud/hooks/kms.py +3 -2
  38. airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
  39. airflow/providers/google/cloud/hooks/looker.py +6 -1
  40. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  41. airflow/providers/google/cloud/hooks/mlengine.py +7 -8
  42. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  43. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  44. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  45. airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
  46. airflow/providers/google/cloud/hooks/spanner.py +75 -10
  47. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  48. airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
  49. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  50. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  51. airflow/providers/google/cloud/hooks/translate.py +8 -17
  52. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
  53. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
  54. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
  55. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  56. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
  57. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  58. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
  59. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  60. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
  61. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
  62. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  63. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  64. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  65. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  66. airflow/providers/google/cloud/hooks/vision.py +7 -7
  67. airflow/providers/google/cloud/hooks/workflows.py +4 -3
  68. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  69. airflow/providers/google/cloud/links/base.py +77 -7
  70. airflow/providers/google/cloud/links/bigquery.py +0 -47
  71. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  72. airflow/providers/google/cloud/links/bigtable.py +0 -48
  73. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  74. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  75. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  76. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  77. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  78. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
  79. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  80. airflow/providers/google/cloud/links/compute.py +0 -58
  81. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  82. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  83. airflow/providers/google/cloud/links/dataflow.py +0 -34
  84. airflow/providers/google/cloud/links/dataform.py +0 -64
  85. airflow/providers/google/cloud/links/datafusion.py +1 -90
  86. airflow/providers/google/cloud/links/dataplex.py +0 -154
  87. airflow/providers/google/cloud/links/dataprep.py +0 -24
  88. airflow/providers/google/cloud/links/dataproc.py +11 -89
  89. airflow/providers/google/cloud/links/datastore.py +0 -31
  90. airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
  91. airflow/providers/google/cloud/links/managed_kafka.py +11 -51
  92. airflow/providers/google/cloud/links/mlengine.py +0 -70
  93. airflow/providers/google/cloud/links/pubsub.py +0 -32
  94. airflow/providers/google/cloud/links/spanner.py +0 -33
  95. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  96. airflow/providers/google/cloud/links/translate.py +17 -187
  97. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  98. airflow/providers/google/cloud/links/workflows.py +0 -52
  99. airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
  100. airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
  101. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  102. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  103. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  104. airflow/providers/google/cloud/openlineage/facets.py +141 -40
  105. airflow/providers/google/cloud/openlineage/mixins.py +14 -13
  106. airflow/providers/google/cloud/openlineage/utils.py +19 -3
  107. airflow/providers/google/cloud/operators/alloy_db.py +76 -61
  108. airflow/providers/google/cloud/operators/bigquery.py +104 -667
  109. airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
  110. airflow/providers/google/cloud/operators/bigtable.py +38 -7
  111. airflow/providers/google/cloud/operators/cloud_base.py +22 -1
  112. airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
  113. airflow/providers/google/cloud/operators/cloud_build.py +80 -36
  114. airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
  115. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  116. airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
  117. airflow/providers/google/cloud/operators/cloud_run.py +39 -20
  118. airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
  119. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
  120. airflow/providers/google/cloud/operators/compute.py +18 -50
  121. airflow/providers/google/cloud/operators/datacatalog.py +167 -29
  122. airflow/providers/google/cloud/operators/dataflow.py +38 -15
  123. airflow/providers/google/cloud/operators/dataform.py +19 -7
  124. airflow/providers/google/cloud/operators/datafusion.py +43 -43
  125. airflow/providers/google/cloud/operators/dataplex.py +212 -126
  126. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  127. airflow/providers/google/cloud/operators/dataproc.py +134 -207
  128. airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
  129. airflow/providers/google/cloud/operators/datastore.py +22 -6
  130. airflow/providers/google/cloud/operators/dlp.py +24 -45
  131. airflow/providers/google/cloud/operators/functions.py +21 -14
  132. airflow/providers/google/cloud/operators/gcs.py +15 -12
  133. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  134. airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
  135. airflow/providers/google/cloud/operators/looker.py +1 -1
  136. airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
  137. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  138. airflow/providers/google/cloud/operators/pubsub.py +69 -21
  139. airflow/providers/google/cloud/operators/spanner.py +53 -45
  140. airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
  141. airflow/providers/google/cloud/operators/stackdriver.py +5 -11
  142. airflow/providers/google/cloud/operators/tasks.py +6 -15
  143. airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
  144. airflow/providers/google/cloud/operators/translate.py +46 -20
  145. airflow/providers/google/cloud/operators/translate_speech.py +4 -3
  146. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
  147. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
  148. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
  149. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
  150. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
  151. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  152. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  153. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
  154. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
  155. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
  156. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
  157. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  158. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  159. airflow/providers/google/cloud/operators/vision.py +7 -5
  160. airflow/providers/google/cloud/operators/workflows.py +24 -19
  161. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  162. airflow/providers/google/cloud/sensors/bigquery.py +2 -2
  163. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
  164. airflow/providers/google/cloud/sensors/bigtable.py +14 -6
  165. airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
  166. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
  167. airflow/providers/google/cloud/sensors/dataflow.py +27 -10
  168. airflow/providers/google/cloud/sensors/dataform.py +2 -2
  169. airflow/providers/google/cloud/sensors/datafusion.py +4 -4
  170. airflow/providers/google/cloud/sensors/dataplex.py +7 -5
  171. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  172. airflow/providers/google/cloud/sensors/dataproc.py +10 -9
  173. airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
  174. airflow/providers/google/cloud/sensors/gcs.py +22 -21
  175. airflow/providers/google/cloud/sensors/looker.py +5 -5
  176. airflow/providers/google/cloud/sensors/pubsub.py +20 -20
  177. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  178. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
  179. airflow/providers/google/cloud/sensors/workflows.py +6 -4
  180. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  181. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  182. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
  183. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  184. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
  185. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  186. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  187. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  188. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  189. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  190. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
  191. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
  192. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
  193. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
  194. airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
  195. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
  196. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  197. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  198. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  199. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  200. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  201. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  202. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
  203. airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
  204. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  205. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
  206. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  207. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  208. airflow/providers/google/cloud/triggers/bigquery.py +75 -34
  209. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  210. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  211. airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
  212. airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
  213. airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
  214. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
  215. airflow/providers/google/cloud/triggers/dataflow.py +125 -2
  216. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  217. airflow/providers/google/cloud/triggers/dataplex.py +16 -3
  218. airflow/providers/google/cloud/triggers/dataproc.py +124 -53
  219. airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
  220. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  221. airflow/providers/google/cloud/triggers/pubsub.py +17 -20
  222. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  223. airflow/providers/google/cloud/utils/bigquery.py +5 -7
  224. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  225. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  226. airflow/providers/google/cloud/utils/dataform.py +1 -1
  227. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  228. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  229. airflow/providers/google/cloud/utils/validators.py +43 -0
  230. airflow/providers/google/common/auth_backend/google_openid.py +26 -9
  231. airflow/providers/google/common/consts.py +2 -1
  232. airflow/providers/google/common/deprecated.py +2 -1
  233. airflow/providers/google/common/hooks/base_google.py +40 -43
  234. airflow/providers/google/common/hooks/operation_helpers.py +78 -0
  235. airflow/providers/google/common/links/storage.py +0 -22
  236. airflow/providers/google/common/utils/get_secret.py +31 -0
  237. airflow/providers/google/common/utils/id_token_credentials.py +4 -5
  238. airflow/providers/google/firebase/operators/firestore.py +2 -2
  239. airflow/providers/google/get_provider_info.py +61 -216
  240. airflow/providers/google/go_module_utils.py +35 -3
  241. airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
  242. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  243. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  244. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  245. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  246. airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
  247. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
  248. airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
  249. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  250. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  251. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  252. airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
  253. airflow/providers/google/suite/hooks/calendar.py +1 -1
  254. airflow/providers/google/suite/hooks/drive.py +2 -2
  255. airflow/providers/google/suite/hooks/sheets.py +15 -1
  256. airflow/providers/google/suite/operators/sheets.py +8 -3
  257. airflow/providers/google/suite/sensors/drive.py +2 -2
  258. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
  259. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  260. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  261. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  262. airflow/providers/google/version_compat.py +15 -1
  263. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
  264. apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
  265. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
  266. apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
  267. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  268. airflow/providers/google/cloud/hooks/automl.py +0 -679
  269. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  270. airflow/providers/google/cloud/links/automl.py +0 -193
  271. airflow/providers/google/cloud/operators/automl.py +0 -1360
  272. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  273. airflow/providers/google/cloud/operators/mlengine.py +0 -1515
  274. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
  275. apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
  276. /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
  277. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
  278. {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -31,7 +31,7 @@ import pendulum
31
31
  from slugify import slugify
32
32
 
33
33
  from airflow.providers.google.cloud.transfers.sql_to_gcs import BaseSQLToGCSOperator
34
- from airflow.providers.postgres.hooks.postgres import PostgresHook
34
+ from airflow.providers.postgres.hooks.postgres import USE_PSYCOPG3, PostgresHook
35
35
 
36
36
  if TYPE_CHECKING:
37
37
  from airflow.providers.openlineage.extractors import OperatorLineage
@@ -52,14 +52,24 @@ class _PostgresServerSideCursorDecorator:
52
52
  self.initialized = False
53
53
 
54
54
  def __iter__(self):
55
+ """Make the cursor iterable."""
55
56
  return self
56
57
 
57
58
  def __next__(self):
59
+ """Fetch next row from the cursor."""
60
+ if USE_PSYCOPG3:
61
+ if self.rows:
62
+ return self.rows.pop()
63
+ self.initialized = True
64
+ row = self.cursor.fetchone()
65
+ if row is None:
66
+ raise StopIteration
67
+ return row
68
+ # psycopg2
58
69
  if self.rows:
59
70
  return self.rows.pop()
60
- else:
61
- self.initialized = True
62
- return next(self.cursor)
71
+ self.initialized = True
72
+ return next(self.cursor)
63
73
 
64
74
  @property
65
75
  def description(self):
@@ -142,13 +152,29 @@ class PostgresToGCSOperator(BaseSQLToGCSOperator):
142
152
  return PostgresHook(postgres_conn_id=self.postgres_conn_id)
143
153
 
144
154
  def query(self):
145
- """Query Postgres and returns a cursor to the results."""
155
+ """Execute the query and return a cursor."""
146
156
  conn = self.db_hook.get_conn()
147
- cursor = conn.cursor(name=self._unique_name())
148
- cursor.execute(self.sql, self.parameters)
149
- if self.use_server_side_cursor:
150
- cursor.itersize = self.cursor_itersize
151
- return _PostgresServerSideCursorDecorator(cursor)
157
+
158
+ if USE_PSYCOPG3:
159
+ from psycopg.types.json import register_default_adapters
160
+
161
+ # Register JSON handlers for this connection if not already done
162
+ register_default_adapters(conn)
163
+
164
+ if self.use_server_side_cursor:
165
+ cursor_name = f"airflow_{self.task_id.replace('-', '_')}_{uuid.uuid4().hex}"[:63]
166
+ cursor = conn.cursor(name=cursor_name)
167
+ cursor.itersize = self.cursor_itersize
168
+ cursor.execute(self.sql, self.parameters)
169
+ return _PostgresServerSideCursorDecorator(cursor)
170
+ cursor = conn.cursor()
171
+ cursor.execute(self.sql, self.parameters)
172
+ else:
173
+ cursor = conn.cursor(name=self._unique_name())
174
+ cursor.execute(self.sql, self.parameters)
175
+ if self.use_server_side_cursor:
176
+ cursor.itersize = self.cursor_itersize
177
+ return _PostgresServerSideCursorDecorator(cursor)
152
178
  return cursor
153
179
 
154
180
  def field_to_bigquery(self, field) -> dict[str, str]:
@@ -183,8 +209,14 @@ class PostgresToGCSOperator(BaseSQLToGCSOperator):
183
209
  hours=formatted_time.tm_hour, minutes=formatted_time.tm_min, seconds=formatted_time.tm_sec
184
210
  )
185
211
  return str(time_delta)
186
- if stringify_dict and isinstance(value, dict):
187
- return json.dumps(value)
212
+ if stringify_dict:
213
+ if USE_PSYCOPG3:
214
+ from psycopg.types.json import Json
215
+
216
+ if isinstance(value, (dict, Json)):
217
+ return json.dumps(value)
218
+ elif isinstance(value, dict):
219
+ return json.dumps(value)
188
220
  if isinstance(value, Decimal):
189
221
  return float(value)
190
222
  return value
@@ -57,7 +57,7 @@ except ImportError:
57
57
  from airflow.providers.amazon.aws.operators.s3_list import S3ListOperator # type: ignore[no-redef]
58
58
 
59
59
  if TYPE_CHECKING:
60
- from airflow.utils.context import Context
60
+ from airflow.providers.common.compat.sdk import Context
61
61
 
62
62
 
63
63
  class S3ToGCSOperator(S3ListOperator):
@@ -181,21 +181,27 @@ class S3ToGCSOperator(S3ListOperator):
181
181
  'The destination Google Cloud Storage path must end with a slash "/" or be empty.'
182
182
  )
183
183
 
184
- def execute(self, context: Context):
185
- self._check_inputs()
184
+ def _get_files(self, context: Context, gcs_hook: GCSHook) -> list[str]:
186
185
  # use the super method to list all the files in an S3 bucket/key
187
186
  s3_objects = super().execute(context)
188
187
 
188
+ if not self.replace:
189
+ s3_objects = self.exclude_existing_objects(s3_objects=s3_objects, gcs_hook=gcs_hook)
190
+
191
+ return s3_objects
192
+
193
+ def execute(self, context: Context):
194
+ self._check_inputs()
189
195
  gcs_hook = GCSHook(
190
196
  gcp_conn_id=self.gcp_conn_id,
191
197
  impersonation_chain=self.google_impersonation_chain,
192
198
  )
193
- if not self.replace:
194
- s3_objects = self.exclude_existing_objects(s3_objects=s3_objects, gcs_hook=gcs_hook)
195
-
196
199
  s3_hook = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
200
+
201
+ s3_objects = self._get_files(context, gcs_hook)
197
202
  if not s3_objects:
198
203
  self.log.info("In sync, no files needed to be uploaded to Google Cloud Storage")
204
+
199
205
  elif self.deferrable:
200
206
  self.transfer_files_async(s3_objects, gcs_hook, s3_hook)
201
207
  else:
@@ -21,12 +21,12 @@ import tempfile
21
21
  from collections.abc import Sequence
22
22
  from typing import TYPE_CHECKING
23
23
 
24
- from airflow.models import BaseOperator
25
24
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
25
+ from airflow.providers.google.version_compat import BaseOperator
26
26
  from airflow.providers.salesforce.hooks.salesforce import SalesforceHook
27
27
 
28
28
  if TYPE_CHECKING:
29
- from airflow.utils.context import Context
29
+ from airflow.providers.common.compat.sdk import Context
30
30
 
31
31
 
32
32
  class SalesforceToGcsOperator(BaseOperator):
@@ -26,12 +26,12 @@ from tempfile import NamedTemporaryFile
26
26
  from typing import TYPE_CHECKING
27
27
 
28
28
  from airflow.exceptions import AirflowException
29
- from airflow.models import BaseOperator
30
29
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
30
+ from airflow.providers.google.version_compat import BaseOperator
31
31
  from airflow.providers.sftp.hooks.sftp import SFTPHook
32
32
 
33
33
  if TYPE_CHECKING:
34
- from airflow.utils.context import Context
34
+ from airflow.providers.common.compat.sdk import Context
35
35
 
36
36
 
37
37
  WILDCARD = "*"
@@ -73,6 +73,13 @@ class SFTPToGCSOperator(BaseOperator):
73
73
  Service Account Token Creator IAM role to the directly preceding identity, with first
74
74
  account from the list granting this role to the originating account (templated).
75
75
  :param sftp_prefetch: Whether to enable SFTP prefetch, the default is True.
76
+ :param use_stream: Determines the transfer method from SFTP to GCS.
77
+ When ``False`` (default), the file downloads locally
78
+ then uploads (may require significant disk space).
79
+ When ``True``, the file streams directly without using local disk.
80
+ Defaults to ``False``.
81
+ :param fail_on_file_not_exist: If True, operator fails when file does not exist,
82
+ if False, operator will not fail and skips transfer. Default is True.
76
83
  """
77
84
 
78
85
  template_fields: Sequence[str] = (
@@ -95,6 +102,8 @@ class SFTPToGCSOperator(BaseOperator):
95
102
  move_object: bool = False,
96
103
  impersonation_chain: str | Sequence[str] | None = None,
97
104
  sftp_prefetch: bool = True,
105
+ use_stream: bool = False,
106
+ fail_on_file_not_exist: bool = True,
98
107
  **kwargs,
99
108
  ) -> None:
100
109
  super().__init__(**kwargs)
@@ -109,6 +118,8 @@ class SFTPToGCSOperator(BaseOperator):
109
118
  self.move_object = move_object
110
119
  self.impersonation_chain = impersonation_chain
111
120
  self.sftp_prefetch = sftp_prefetch
121
+ self.use_stream = use_stream
122
+ self.fail_on_file_not_exist = fail_on_file_not_exist
112
123
 
113
124
  @cached_property
114
125
  def sftp_hook(self):
@@ -149,7 +160,13 @@ class SFTPToGCSOperator(BaseOperator):
149
160
  destination_object = (
150
161
  self.destination_path if self.destination_path else self.source_path.rsplit("/", 1)[1]
151
162
  )
152
- self._copy_single_object(gcs_hook, self.sftp_hook, self.source_path, destination_object)
163
+ try:
164
+ self._copy_single_object(gcs_hook, self.sftp_hook, self.source_path, destination_object)
165
+ except FileNotFoundError as e:
166
+ if self.fail_on_file_not_exist:
167
+ raise e
168
+ self.log.info("File %s not found on SFTP server. Skipping transfer.", self.source_path)
169
+ return
153
170
 
154
171
  def _copy_single_object(
155
172
  self,
@@ -165,17 +182,22 @@ class SFTPToGCSOperator(BaseOperator):
165
182
  self.destination_bucket,
166
183
  destination_object,
167
184
  )
168
-
169
- with NamedTemporaryFile("w") as tmp:
170
- sftp_hook.retrieve_file(source_path, tmp.name, prefetch=self.sftp_prefetch)
171
-
172
- gcs_hook.upload(
173
- bucket_name=self.destination_bucket,
174
- object_name=destination_object,
175
- filename=tmp.name,
176
- mime_type=self.mime_type,
177
- gzip=self.gzip,
178
- )
185
+ if self.use_stream:
186
+ dest_bucket = gcs_hook.get_bucket(self.destination_bucket)
187
+ dest_blob = dest_bucket.blob(destination_object)
188
+ with dest_blob.open("wb") as write_stream:
189
+ sftp_hook.retrieve_file(source_path, write_stream, prefetch=self.sftp_prefetch)
190
+ else:
191
+ with NamedTemporaryFile("w") as tmp:
192
+ sftp_hook.retrieve_file(source_path, tmp.name, prefetch=self.sftp_prefetch)
193
+
194
+ gcs_hook.upload(
195
+ bucket_name=self.destination_bucket,
196
+ object_name=destination_object,
197
+ filename=tmp.name,
198
+ mime_type=self.mime_type,
199
+ gzip=self.gzip,
200
+ )
179
201
 
180
202
  if self.move_object:
181
203
  self.log.info("Executing delete of %s", source_path)
@@ -21,12 +21,12 @@ from collections.abc import Sequence
21
21
  from tempfile import NamedTemporaryFile
22
22
  from typing import TYPE_CHECKING, Any
23
23
 
24
- from airflow.models import BaseOperator
25
24
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
26
25
  from airflow.providers.google.suite.hooks.sheets import GSheetsHook
26
+ from airflow.providers.google.version_compat import BaseOperator
27
27
 
28
28
  if TYPE_CHECKING:
29
- from airflow.utils.context import Context
29
+ from airflow.providers.common.compat.sdk import Context
30
30
 
31
31
 
32
32
  class GoogleSheetsToGCSOperator(BaseOperator):
@@ -130,5 +130,5 @@ class GoogleSheetsToGCSOperator(BaseOperator):
130
130
  gcs_path_to_file = self._upload_data(gcs_hook, sheet_hook, sheet_range, data)
131
131
  destination_array.append(gcs_path_to_file)
132
132
 
133
- self.xcom_push(context, "destination_objects", destination_array)
133
+ context["ti"].xcom_push(key="destination_objects", value=destination_array)
134
134
  return destination_array
@@ -30,12 +30,12 @@ from typing import TYPE_CHECKING, Any
30
30
  import pyarrow as pa
31
31
  import pyarrow.parquet as pq
32
32
 
33
- from airflow.models import BaseOperator
34
33
  from airflow.providers.google.cloud.hooks.gcs import GCSHook
34
+ from airflow.providers.google.version_compat import BaseOperator
35
35
 
36
36
  if TYPE_CHECKING:
37
37
  from airflow.providers.common.compat.openlineage.facet import OutputDataset
38
- from airflow.utils.context import Context
38
+ from airflow.providers.common.compat.sdk import Context
39
39
 
40
40
 
41
41
  class BaseSQLToGCSOperator(BaseOperator):
@@ -295,21 +295,21 @@ class BaseSQLToGCSOperator(BaseOperator):
295
295
 
296
296
  # Proceed to write the row to the localfile
297
297
  if self.export_format == "csv":
298
- row = self.convert_types(schema, col_type_dict, row)
298
+ row2 = self.convert_types(schema, col_type_dict, row)
299
299
  if self.null_marker is not None:
300
- row = [value or self.null_marker for value in row]
301
- csv_writer.writerow(row)
300
+ row2 = [value or self.null_marker for value in row2]
301
+ csv_writer.writerow(row2)
302
302
  elif self.export_format == "parquet":
303
- row = self.convert_types(schema, col_type_dict, row)
303
+ row2 = self.convert_types(schema, col_type_dict, row)
304
304
  if self.null_marker is not None:
305
- row = [value or self.null_marker for value in row]
306
- rows_buffer.append(row)
305
+ row2 = [value or self.null_marker for value in row2]
306
+ rows_buffer.append(row2)
307
307
  if len(rows_buffer) >= self.parquet_row_group_size:
308
308
  self._write_rows_to_parquet(parquet_writer, rows_buffer)
309
309
  rows_buffer = []
310
310
  else:
311
- row = self.convert_types(schema, col_type_dict, row)
312
- row_dict = dict(zip(schema, row))
311
+ row2 = self.convert_types(schema, col_type_dict, row)
312
+ row_dict = dict(zip(schema, row2))
313
313
 
314
314
  json.dump(row_dict, tmp_file_handle, sort_keys=True, ensure_ascii=False)
315
315
 
@@ -22,17 +22,21 @@ from typing import TYPE_CHECKING, Any, SupportsAbs
22
22
 
23
23
  from aiohttp import ClientSession
24
24
  from aiohttp.client_exceptions import ClientResponseError
25
+ from asgiref.sync import sync_to_async
25
26
 
26
27
  from airflow.exceptions import AirflowException
27
- from airflow.models.taskinstance import TaskInstance
28
28
  from airflow.providers.google.cloud.hooks.bigquery import BigQueryAsyncHook, BigQueryTableAsyncHook
29
+ from airflow.providers.google.version_compat import AIRFLOW_V_3_0_PLUS
29
30
  from airflow.triggers.base import BaseTrigger, TriggerEvent
30
- from airflow.utils.session import provide_session
31
31
  from airflow.utils.state import TaskInstanceState
32
32
 
33
33
  if TYPE_CHECKING:
34
34
  from sqlalchemy.orm.session import Session
35
35
 
36
+ if not AIRFLOW_V_3_0_PLUS:
37
+ from airflow.models.taskinstance import TaskInstance
38
+ from airflow.utils.session import provide_session
39
+
36
40
 
37
41
  class BigQueryInsertJobTrigger(BaseTrigger):
38
42
  """
@@ -97,16 +101,39 @@ class BigQueryInsertJobTrigger(BaseTrigger):
97
101
  },
98
102
  )
99
103
 
100
- @provide_session
101
- def get_task_instance(self, session: Session) -> TaskInstance:
102
- query = session.query(TaskInstance).filter(
103
- TaskInstance.dag_id == self.task_instance.dag_id,
104
- TaskInstance.task_id == self.task_instance.task_id,
105
- TaskInstance.run_id == self.task_instance.run_id,
106
- TaskInstance.map_index == self.task_instance.map_index,
104
+ if not AIRFLOW_V_3_0_PLUS:
105
+
106
+ @provide_session
107
+ def get_task_instance(self, session: Session) -> TaskInstance:
108
+ query = session.query(TaskInstance).filter(
109
+ TaskInstance.dag_id == self.task_instance.dag_id,
110
+ TaskInstance.task_id == self.task_instance.task_id,
111
+ TaskInstance.run_id == self.task_instance.run_id,
112
+ TaskInstance.map_index == self.task_instance.map_index,
113
+ )
114
+ task_instance = query.one_or_none()
115
+ if task_instance is None:
116
+ raise AirflowException(
117
+ "TaskInstance with dag_id: %s, task_id: %s, run_id: %s and map_index: %s is not found",
118
+ self.task_instance.dag_id,
119
+ self.task_instance.task_id,
120
+ self.task_instance.run_id,
121
+ self.task_instance.map_index,
122
+ )
123
+ return task_instance
124
+
125
+ async def get_task_state(self):
126
+ from airflow.sdk.execution_time.task_runner import RuntimeTaskInstance
127
+
128
+ task_states_response = await sync_to_async(RuntimeTaskInstance.get_task_states)(
129
+ dag_id=self.task_instance.dag_id,
130
+ task_ids=[self.task_instance.task_id],
131
+ run_ids=[self.task_instance.run_id],
132
+ map_index=self.task_instance.map_index,
107
133
  )
108
- task_instance = query.one_or_none()
109
- if task_instance is None:
134
+ try:
135
+ task_state = task_states_response[self.task_instance.run_id][self.task_instance.task_id]
136
+ except Exception:
110
137
  raise AirflowException(
111
138
  "TaskInstance with dag_id: %s, task_id: %s, run_id: %s and map_index: %s is not found",
112
139
  self.task_instance.dag_id,
@@ -114,20 +141,24 @@ class BigQueryInsertJobTrigger(BaseTrigger):
114
141
  self.task_instance.run_id,
115
142
  self.task_instance.map_index,
116
143
  )
117
- return task_instance
144
+ return task_state
118
145
 
119
- def safe_to_cancel(self) -> bool:
146
+ async def safe_to_cancel(self) -> bool:
120
147
  """
121
148
  Whether it is safe to cancel the external job which is being executed by this trigger.
122
149
 
123
150
  This is to avoid the case that `asyncio.CancelledError` is called because the trigger itself is stopped.
124
151
  Because in those cases, we should NOT cancel the external job.
125
152
  """
126
- # Database query is needed to get the latest state of the task instance.
127
- task_instance = self.get_task_instance() # type: ignore[call-arg]
128
- return task_instance.state != TaskInstanceState.DEFERRED
129
-
130
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
153
+ if AIRFLOW_V_3_0_PLUS:
154
+ task_state = await self.get_task_state()
155
+ else:
156
+ # Database query is needed to get the latest state of the task instance.
157
+ task_instance = self.get_task_instance() # type: ignore[call-arg]
158
+ task_state = task_instance.state
159
+ return task_state != TaskInstanceState.DEFERRED
160
+
161
+ async def run(self) -> AsyncIterator[TriggerEvent]:
131
162
  """Get current job execution status and yields a TriggerEvent."""
132
163
  hook = self._get_async_hook()
133
164
  try:
@@ -136,6 +167,7 @@ class BigQueryInsertJobTrigger(BaseTrigger):
136
167
  job_id=self.job_id, project_id=self.project_id, location=self.location
137
168
  )
138
169
  if job_status["status"] == "success":
170
+ self.log.info("BigQuery Job succeeded")
139
171
  yield TriggerEvent(
140
172
  {
141
173
  "job_id": self.job_id,
@@ -145,7 +177,13 @@ class BigQueryInsertJobTrigger(BaseTrigger):
145
177
  )
146
178
  return
147
179
  elif job_status["status"] == "error":
148
- yield TriggerEvent(job_status)
180
+ self.log.info("BigQuery Job failed: %s", job_status)
181
+ yield TriggerEvent(
182
+ {
183
+ "status": job_status["status"],
184
+ "message": job_status["message"],
185
+ }
186
+ )
149
187
  return
150
188
  else:
151
189
  self.log.info(
@@ -155,7 +193,7 @@ class BigQueryInsertJobTrigger(BaseTrigger):
155
193
  )
156
194
  await asyncio.sleep(self.poll_interval)
157
195
  except asyncio.CancelledError:
158
- if self.job_id and self.cancel_on_kill and self.safe_to_cancel():
196
+ if self.job_id and self.cancel_on_kill and await self.safe_to_cancel():
159
197
  self.log.info(
160
198
  "The job is safe to cancel the as airflow TaskInstance is not in deferred state."
161
199
  )
@@ -165,9 +203,7 @@ class BigQueryInsertJobTrigger(BaseTrigger):
165
203
  self.location,
166
204
  self.job_id,
167
205
  )
168
- await hook.cancel_job( # type: ignore[union-attr]
169
- job_id=self.job_id, project_id=self.project_id, location=self.location
170
- )
206
+ await hook.cancel_job(job_id=self.job_id, project_id=self.project_id, location=self.location)
171
207
  else:
172
208
  self.log.info(
173
209
  "Trigger may have shutdown. Skipping to cancel job because the airflow "
@@ -204,7 +240,7 @@ class BigQueryCheckTrigger(BigQueryInsertJobTrigger):
204
240
  },
205
241
  )
206
242
 
207
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
243
+ async def run(self) -> AsyncIterator[TriggerEvent]:
208
244
  """Get current job execution status and yields a TriggerEvent."""
209
245
  hook = self._get_async_hook()
210
246
  try:
@@ -281,7 +317,7 @@ class BigQueryGetDataTrigger(BigQueryInsertJobTrigger):
281
317
  },
282
318
  )
283
319
 
284
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
320
+ async def run(self) -> AsyncIterator[TriggerEvent]:
285
321
  """Get current job execution status and yields a TriggerEvent with response data."""
286
322
  hook = self._get_async_hook()
287
323
  try:
@@ -305,7 +341,12 @@ class BigQueryGetDataTrigger(BigQueryInsertJobTrigger):
305
341
  )
306
342
  return
307
343
  elif job_status["status"] == "error":
308
- yield TriggerEvent(job_status)
344
+ yield TriggerEvent(
345
+ {
346
+ "status": job_status["status"],
347
+ "message": job_status["message"],
348
+ }
349
+ )
309
350
  return
310
351
  else:
311
352
  self.log.info(
@@ -406,7 +447,7 @@ class BigQueryIntervalCheckTrigger(BigQueryInsertJobTrigger):
406
447
  },
407
448
  )
408
449
 
409
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
450
+ async def run(self) -> AsyncIterator[TriggerEvent]:
410
451
  """Get current job execution status and yields a TriggerEvent."""
411
452
  hook = self._get_async_hook()
412
453
  try:
@@ -554,7 +595,7 @@ class BigQueryValueCheckTrigger(BigQueryInsertJobTrigger):
554
595
  },
555
596
  )
556
597
 
557
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
598
+ async def run(self) -> AsyncIterator[TriggerEvent]:
558
599
  """Get current job execution status and yields a TriggerEvent."""
559
600
  hook = self._get_async_hook()
560
601
  try:
@@ -564,9 +605,9 @@ class BigQueryValueCheckTrigger(BigQueryInsertJobTrigger):
564
605
  if response_from_hook["status"] == "success":
565
606
  query_results = await hook.get_job_output(job_id=self.job_id, project_id=self.project_id)
566
607
  records = hook.get_records(query_results)
567
- records = records.pop(0) if records else None
568
- hook.value_check(self.sql, self.pass_value, records, self.tolerance)
569
- yield TriggerEvent({"status": "success", "message": "Job completed", "records": records})
608
+ _records = records.pop(0) if records else None
609
+ hook.value_check(self.sql, self.pass_value, _records, self.tolerance)
610
+ yield TriggerEvent({"status": "success", "message": "Job completed", "records": _records})
570
611
  return
571
612
  elif response_from_hook["status"] == "pending":
572
613
  self.log.info("Query is still running...")
@@ -640,7 +681,7 @@ class BigQueryTableExistenceTrigger(BaseTrigger):
640
681
  gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
641
682
  )
642
683
 
643
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
684
+ async def run(self) -> AsyncIterator[TriggerEvent]:
644
685
  """Will run until the table exists in the Google Big Query."""
645
686
  try:
646
687
  while True:
@@ -723,7 +764,7 @@ class BigQueryTablePartitionExistenceTrigger(BigQueryTableExistenceTrigger):
723
764
  },
724
765
  )
725
766
 
726
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
767
+ async def run(self) -> AsyncIterator[TriggerEvent]:
727
768
  """Will run until the table exists in the Google Big Query."""
728
769
  hook = BigQueryAsyncHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain)
729
770
  job_id = None
@@ -744,7 +785,7 @@ class BigQueryTablePartitionExistenceTrigger(BigQueryTableExistenceTrigger):
744
785
  return
745
786
  job_id = None
746
787
  elif job_status["status"] == "error":
747
- yield TriggerEvent(job_status)
788
+ yield TriggerEvent({"status": job_status["status"]})
748
789
  return
749
790
  self.log.info("Sleeping for %s seconds.", self.poll_interval)
750
791
  await asyncio.sleep(self.poll_interval)
@@ -21,9 +21,10 @@ import asyncio
21
21
  from collections.abc import AsyncIterator, Sequence
22
22
  from typing import Any
23
23
 
24
+ from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
25
+
24
26
  from airflow.providers.google.cloud.hooks.bigquery_dts import AsyncBiqQueryDataTransferServiceHook
25
27
  from airflow.triggers.base import BaseTrigger, TriggerEvent
26
- from google.cloud.bigquery_datatransfer_v1 import TransferRun, TransferState
27
28
 
28
29
 
29
30
  class BigQueryDataTransferRunTrigger(BaseTrigger):
@@ -20,9 +20,10 @@ import asyncio
20
20
  from collections.abc import AsyncIterator, Sequence
21
21
  from typing import Any
22
22
 
23
+ from google.cloud.batch_v1 import Job, JobStatus
24
+
23
25
  from airflow.providers.google.cloud.hooks.cloud_batch import CloudBatchAsyncHook
24
26
  from airflow.triggers.base import BaseTrigger, TriggerEvent
25
- from google.cloud.batch_v1 import Job, JobStatus
26
27
 
27
28
  DEFAULT_BATCH_LOCATION = "us-central1"
28
29
 
@@ -20,9 +20,10 @@ import asyncio
20
20
  from collections.abc import AsyncIterator, Sequence
21
21
  from typing import Any
22
22
 
23
+ from google.cloud.devtools.cloudbuild_v1.types import Build
24
+
23
25
  from airflow.providers.google.cloud.hooks.cloud_build import CloudBuildAsyncHook
24
26
  from airflow.triggers.base import BaseTrigger, TriggerEvent
25
- from google.cloud.devtools.cloudbuild_v1.types import Build
26
27
 
27
28
 
28
29
  class CloudBuildCreateBuildTrigger(BaseTrigger):
@@ -75,7 +76,7 @@ class CloudBuildCreateBuildTrigger(BaseTrigger):
75
76
  },
76
77
  )
77
78
 
78
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
79
+ async def run(self) -> AsyncIterator[TriggerEvent]:
79
80
  """Get current build execution status and yields a TriggerEvent."""
80
81
  hook = self._get_async_hook()
81
82
  try: