apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. airflow/providers/google/3rd-party-licenses/LICENSES.txt +14 -0
  2. airflow/providers/google/3rd-party-licenses/NOTICE +5 -0
  3. airflow/providers/google/__init__.py +3 -3
  4. airflow/providers/google/_vendor/__init__.py +0 -0
  5. airflow/providers/google/_vendor/json_merge_patch.py +91 -0
  6. airflow/providers/google/ads/hooks/ads.py +52 -43
  7. airflow/providers/google/ads/operators/ads.py +2 -2
  8. airflow/providers/google/ads/transfers/ads_to_gcs.py +3 -19
  9. airflow/providers/google/assets/gcs.py +1 -11
  10. airflow/providers/google/cloud/_internal_client/secret_manager_client.py +3 -2
  11. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  12. airflow/providers/google/cloud/hooks/alloy_db.py +2 -3
  13. airflow/providers/google/cloud/hooks/bigquery.py +195 -318
  14. airflow/providers/google/cloud/hooks/bigquery_dts.py +8 -8
  15. airflow/providers/google/cloud/hooks/bigtable.py +3 -2
  16. airflow/providers/google/cloud/hooks/cloud_batch.py +8 -9
  17. airflow/providers/google/cloud/hooks/cloud_build.py +6 -65
  18. airflow/providers/google/cloud/hooks/cloud_composer.py +292 -24
  19. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  20. airflow/providers/google/cloud/hooks/cloud_memorystore.py +4 -3
  21. airflow/providers/google/cloud/hooks/cloud_run.py +20 -11
  22. airflow/providers/google/cloud/hooks/cloud_sql.py +136 -64
  23. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +35 -15
  24. airflow/providers/google/cloud/hooks/compute.py +7 -6
  25. airflow/providers/google/cloud/hooks/compute_ssh.py +7 -4
  26. airflow/providers/google/cloud/hooks/datacatalog.py +12 -3
  27. airflow/providers/google/cloud/hooks/dataflow.py +87 -242
  28. airflow/providers/google/cloud/hooks/dataform.py +9 -14
  29. airflow/providers/google/cloud/hooks/datafusion.py +7 -9
  30. airflow/providers/google/cloud/hooks/dataplex.py +13 -12
  31. airflow/providers/google/cloud/hooks/dataprep.py +2 -2
  32. airflow/providers/google/cloud/hooks/dataproc.py +76 -74
  33. airflow/providers/google/cloud/hooks/dataproc_metastore.py +4 -3
  34. airflow/providers/google/cloud/hooks/dlp.py +5 -4
  35. airflow/providers/google/cloud/hooks/gcs.py +144 -33
  36. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  37. airflow/providers/google/cloud/hooks/kms.py +3 -2
  38. airflow/providers/google/cloud/hooks/kubernetes_engine.py +22 -17
  39. airflow/providers/google/cloud/hooks/looker.py +6 -1
  40. airflow/providers/google/cloud/hooks/managed_kafka.py +227 -3
  41. airflow/providers/google/cloud/hooks/mlengine.py +7 -8
  42. airflow/providers/google/cloud/hooks/natural_language.py +3 -2
  43. airflow/providers/google/cloud/hooks/os_login.py +3 -2
  44. airflow/providers/google/cloud/hooks/pubsub.py +6 -6
  45. airflow/providers/google/cloud/hooks/secret_manager.py +105 -12
  46. airflow/providers/google/cloud/hooks/spanner.py +75 -10
  47. airflow/providers/google/cloud/hooks/speech_to_text.py +3 -2
  48. airflow/providers/google/cloud/hooks/stackdriver.py +18 -18
  49. airflow/providers/google/cloud/hooks/tasks.py +4 -3
  50. airflow/providers/google/cloud/hooks/text_to_speech.py +3 -2
  51. airflow/providers/google/cloud/hooks/translate.py +8 -17
  52. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +8 -222
  53. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +9 -15
  54. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +33 -283
  55. airflow/providers/google/cloud/hooks/vertex_ai/dataset.py +5 -12
  56. airflow/providers/google/cloud/hooks/vertex_ai/endpoint_service.py +6 -12
  57. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  58. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +311 -10
  59. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  60. airflow/providers/google/cloud/hooks/vertex_ai/hyperparameter_tuning_job.py +7 -13
  61. airflow/providers/google/cloud/hooks/vertex_ai/model_service.py +8 -12
  62. airflow/providers/google/cloud/hooks/vertex_ai/pipeline_job.py +6 -12
  63. airflow/providers/google/cloud/hooks/vertex_ai/prediction_service.py +3 -2
  64. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  65. airflow/providers/google/cloud/hooks/video_intelligence.py +3 -2
  66. airflow/providers/google/cloud/hooks/vision.py +7 -7
  67. airflow/providers/google/cloud/hooks/workflows.py +4 -3
  68. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  69. airflow/providers/google/cloud/links/base.py +77 -7
  70. airflow/providers/google/cloud/links/bigquery.py +0 -47
  71. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  72. airflow/providers/google/cloud/links/bigtable.py +0 -48
  73. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  74. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  75. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  76. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  77. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  78. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -46
  79. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  80. airflow/providers/google/cloud/links/compute.py +0 -58
  81. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  82. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  83. airflow/providers/google/cloud/links/dataflow.py +0 -34
  84. airflow/providers/google/cloud/links/dataform.py +0 -64
  85. airflow/providers/google/cloud/links/datafusion.py +1 -90
  86. airflow/providers/google/cloud/links/dataplex.py +0 -154
  87. airflow/providers/google/cloud/links/dataprep.py +0 -24
  88. airflow/providers/google/cloud/links/dataproc.py +11 -89
  89. airflow/providers/google/cloud/links/datastore.py +0 -31
  90. airflow/providers/google/cloud/links/kubernetes_engine.py +11 -61
  91. airflow/providers/google/cloud/links/managed_kafka.py +11 -51
  92. airflow/providers/google/cloud/links/mlengine.py +0 -70
  93. airflow/providers/google/cloud/links/pubsub.py +0 -32
  94. airflow/providers/google/cloud/links/spanner.py +0 -33
  95. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  96. airflow/providers/google/cloud/links/translate.py +17 -187
  97. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  98. airflow/providers/google/cloud/links/workflows.py +0 -52
  99. airflow/providers/google/cloud/log/gcs_task_handler.py +166 -118
  100. airflow/providers/google/cloud/log/stackdriver_task_handler.py +14 -9
  101. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  102. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  103. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  104. airflow/providers/google/cloud/openlineage/facets.py +141 -40
  105. airflow/providers/google/cloud/openlineage/mixins.py +14 -13
  106. airflow/providers/google/cloud/openlineage/utils.py +19 -3
  107. airflow/providers/google/cloud/operators/alloy_db.py +76 -61
  108. airflow/providers/google/cloud/operators/bigquery.py +104 -667
  109. airflow/providers/google/cloud/operators/bigquery_dts.py +12 -12
  110. airflow/providers/google/cloud/operators/bigtable.py +38 -7
  111. airflow/providers/google/cloud/operators/cloud_base.py +22 -1
  112. airflow/providers/google/cloud/operators/cloud_batch.py +18 -18
  113. airflow/providers/google/cloud/operators/cloud_build.py +80 -36
  114. airflow/providers/google/cloud/operators/cloud_composer.py +157 -71
  115. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  116. airflow/providers/google/cloud/operators/cloud_memorystore.py +74 -46
  117. airflow/providers/google/cloud/operators/cloud_run.py +39 -20
  118. airflow/providers/google/cloud/operators/cloud_sql.py +46 -61
  119. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -14
  120. airflow/providers/google/cloud/operators/compute.py +18 -50
  121. airflow/providers/google/cloud/operators/datacatalog.py +167 -29
  122. airflow/providers/google/cloud/operators/dataflow.py +38 -15
  123. airflow/providers/google/cloud/operators/dataform.py +19 -7
  124. airflow/providers/google/cloud/operators/datafusion.py +43 -43
  125. airflow/providers/google/cloud/operators/dataplex.py +212 -126
  126. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  127. airflow/providers/google/cloud/operators/dataproc.py +134 -207
  128. airflow/providers/google/cloud/operators/dataproc_metastore.py +102 -84
  129. airflow/providers/google/cloud/operators/datastore.py +22 -6
  130. airflow/providers/google/cloud/operators/dlp.py +24 -45
  131. airflow/providers/google/cloud/operators/functions.py +21 -14
  132. airflow/providers/google/cloud/operators/gcs.py +15 -12
  133. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  134. airflow/providers/google/cloud/operators/kubernetes_engine.py +115 -106
  135. airflow/providers/google/cloud/operators/looker.py +1 -1
  136. airflow/providers/google/cloud/operators/managed_kafka.py +362 -40
  137. airflow/providers/google/cloud/operators/natural_language.py +5 -3
  138. airflow/providers/google/cloud/operators/pubsub.py +69 -21
  139. airflow/providers/google/cloud/operators/spanner.py +53 -45
  140. airflow/providers/google/cloud/operators/speech_to_text.py +5 -4
  141. airflow/providers/google/cloud/operators/stackdriver.py +5 -11
  142. airflow/providers/google/cloud/operators/tasks.py +6 -15
  143. airflow/providers/google/cloud/operators/text_to_speech.py +4 -3
  144. airflow/providers/google/cloud/operators/translate.py +46 -20
  145. airflow/providers/google/cloud/operators/translate_speech.py +4 -3
  146. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +44 -34
  147. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +34 -12
  148. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +62 -53
  149. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +75 -11
  150. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +48 -12
  151. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  152. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  153. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
  154. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +16 -12
  155. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +62 -14
  156. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +35 -10
  157. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  158. airflow/providers/google/cloud/operators/video_intelligence.py +5 -3
  159. airflow/providers/google/cloud/operators/vision.py +7 -5
  160. airflow/providers/google/cloud/operators/workflows.py +24 -19
  161. airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
  162. airflow/providers/google/cloud/sensors/bigquery.py +2 -2
  163. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -4
  164. airflow/providers/google/cloud/sensors/bigtable.py +14 -6
  165. airflow/providers/google/cloud/sensors/cloud_composer.py +535 -33
  166. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -5
  167. airflow/providers/google/cloud/sensors/dataflow.py +27 -10
  168. airflow/providers/google/cloud/sensors/dataform.py +2 -2
  169. airflow/providers/google/cloud/sensors/datafusion.py +4 -4
  170. airflow/providers/google/cloud/sensors/dataplex.py +7 -5
  171. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  172. airflow/providers/google/cloud/sensors/dataproc.py +10 -9
  173. airflow/providers/google/cloud/sensors/dataproc_metastore.py +4 -3
  174. airflow/providers/google/cloud/sensors/gcs.py +22 -21
  175. airflow/providers/google/cloud/sensors/looker.py +5 -5
  176. airflow/providers/google/cloud/sensors/pubsub.py +20 -20
  177. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  178. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
  179. airflow/providers/google/cloud/sensors/workflows.py +6 -4
  180. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  181. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  182. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
  183. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  184. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +14 -13
  185. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  186. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  187. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  188. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  189. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  190. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +18 -22
  191. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +4 -5
  192. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +45 -38
  193. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
  194. airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
  195. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
  196. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  197. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  198. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  199. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  200. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  201. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  202. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +44 -12
  203. airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
  204. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  205. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +36 -14
  206. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  207. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  208. airflow/providers/google/cloud/triggers/bigquery.py +75 -34
  209. airflow/providers/google/cloud/triggers/bigquery_dts.py +2 -1
  210. airflow/providers/google/cloud/triggers/cloud_batch.py +2 -1
  211. airflow/providers/google/cloud/triggers/cloud_build.py +3 -2
  212. airflow/providers/google/cloud/triggers/cloud_composer.py +303 -47
  213. airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
  214. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +96 -5
  215. airflow/providers/google/cloud/triggers/dataflow.py +125 -2
  216. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  217. airflow/providers/google/cloud/triggers/dataplex.py +16 -3
  218. airflow/providers/google/cloud/triggers/dataproc.py +124 -53
  219. airflow/providers/google/cloud/triggers/kubernetes_engine.py +46 -28
  220. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  221. airflow/providers/google/cloud/triggers/pubsub.py +17 -20
  222. airflow/providers/google/cloud/triggers/vertex_ai.py +8 -7
  223. airflow/providers/google/cloud/utils/bigquery.py +5 -7
  224. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  225. airflow/providers/google/cloud/utils/credentials_provider.py +4 -3
  226. airflow/providers/google/cloud/utils/dataform.py +1 -1
  227. airflow/providers/google/cloud/utils/external_token_supplier.py +0 -1
  228. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  229. airflow/providers/google/cloud/utils/validators.py +43 -0
  230. airflow/providers/google/common/auth_backend/google_openid.py +26 -9
  231. airflow/providers/google/common/consts.py +2 -1
  232. airflow/providers/google/common/deprecated.py +2 -1
  233. airflow/providers/google/common/hooks/base_google.py +40 -43
  234. airflow/providers/google/common/hooks/operation_helpers.py +78 -0
  235. airflow/providers/google/common/links/storage.py +0 -22
  236. airflow/providers/google/common/utils/get_secret.py +31 -0
  237. airflow/providers/google/common/utils/id_token_credentials.py +4 -5
  238. airflow/providers/google/firebase/operators/firestore.py +2 -2
  239. airflow/providers/google/get_provider_info.py +61 -216
  240. airflow/providers/google/go_module_utils.py +35 -3
  241. airflow/providers/google/leveldb/hooks/leveldb.py +30 -6
  242. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  243. airflow/providers/google/marketing_platform/hooks/analytics_admin.py +3 -2
  244. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  245. airflow/providers/google/marketing_platform/hooks/search_ads.py +1 -1
  246. airflow/providers/google/marketing_platform/links/analytics_admin.py +4 -5
  247. airflow/providers/google/marketing_platform/operators/analytics_admin.py +7 -6
  248. airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
  249. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  250. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  251. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  252. airflow/providers/google/marketing_platform/sensors/display_video.py +4 -64
  253. airflow/providers/google/suite/hooks/calendar.py +1 -1
  254. airflow/providers/google/suite/hooks/drive.py +2 -2
  255. airflow/providers/google/suite/hooks/sheets.py +15 -1
  256. airflow/providers/google/suite/operators/sheets.py +8 -3
  257. airflow/providers/google/suite/sensors/drive.py +2 -2
  258. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
  259. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  260. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  261. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  262. airflow/providers/google/version_compat.py +15 -1
  263. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +117 -72
  264. apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
  265. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +1 -1
  266. apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
  267. airflow/providers/google/cloud/example_dags/example_cloud_task.py +0 -54
  268. airflow/providers/google/cloud/hooks/automl.py +0 -679
  269. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  270. airflow/providers/google/cloud/links/automl.py +0 -193
  271. airflow/providers/google/cloud/operators/automl.py +0 -1360
  272. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  273. airflow/providers/google/cloud/operators/mlengine.py +0 -1515
  274. airflow/providers/google/cloud/utils/mlengine_operator_utils.py +0 -273
  275. apache_airflow_providers_google-14.0.0.dist-info/RECORD +0 -318
  276. /airflow/providers/google/cloud/{example_dags → bundles}/__init__.py +0 -0
  277. {apache_airflow_providers_google-14.0.0.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
  278. {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -19,23 +19,81 @@
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
+ import base64
23
+ import datetime
24
+ import json
25
+ import time
22
26
  from collections.abc import Sequence
23
27
  from copy import deepcopy
24
28
  from typing import TYPE_CHECKING
25
29
 
30
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
31
+ from google.auth.transport import requests as google_requests
32
+ from google.cloud.managedkafka_v1 import Cluster, ConsumerGroup, ManagedKafkaClient, Topic, types
33
+
26
34
  from airflow.exceptions import AirflowException
27
35
  from airflow.providers.google.common.consts import CLIENT_INFO
28
36
  from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
29
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
30
- from google.cloud.managedkafka_v1 import Cluster, ManagedKafkaClient, Topic, types
31
37
 
32
38
  if TYPE_CHECKING:
33
39
  from google.api_core.operation import Operation
34
40
  from google.api_core.retry import Retry
35
- from google.cloud.managedkafka_v1.services.managed_kafka.pagers import ListClustersPager, ListTopicsPager
41
+ from google.auth.credentials import Credentials
42
+ from google.cloud.managedkafka_v1.services.managed_kafka.pagers import (
43
+ ListClustersPager,
44
+ ListConsumerGroupsPager,
45
+ ListTopicsPager,
46
+ )
36
47
  from google.protobuf.field_mask_pb2 import FieldMask
37
48
 
38
49
 
50
+ class ManagedKafkaTokenProvider:
51
+ """Helper for providing authentication token for establishing connection via confluent to Apache Kafka cluster managed by Google Cloud."""
52
+
53
+ def __init__(
54
+ self,
55
+ credentials: Credentials,
56
+ ):
57
+ self._credentials = credentials
58
+ self._header = json.dumps(dict(typ="JWT", alg="GOOG_OAUTH2_TOKEN"))
59
+
60
+ def _valid_credentials(self):
61
+ if not self._credentials.valid:
62
+ self._credentials.refresh(google_requests.Request())
63
+ return self._credentials
64
+
65
+ def _get_jwt(self, credentials):
66
+ return json.dumps(
67
+ dict(
68
+ exp=credentials.expiry.timestamp(),
69
+ iss="Google",
70
+ iat=datetime.datetime.now(datetime.timezone.utc).timestamp(),
71
+ scope="kafka",
72
+ sub=credentials.service_account_email,
73
+ )
74
+ )
75
+
76
+ def _b64_encode(self, source):
77
+ return base64.urlsafe_b64encode(source.encode("utf-8")).decode("utf-8").rstrip("=")
78
+
79
+ def _get_kafka_access_token(self, credentials):
80
+ return ".".join(
81
+ [
82
+ self._b64_encode(self._header),
83
+ self._b64_encode(self._get_jwt(credentials)),
84
+ self._b64_encode(credentials.token),
85
+ ]
86
+ )
87
+
88
+ def confluent_token(self):
89
+ credentials = self._valid_credentials()
90
+
91
+ utc_expiry = credentials.expiry.replace(tzinfo=datetime.timezone.utc)
92
+ expiry_seconds = (utc_expiry - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
93
+
94
+ return self._get_kafka_access_token(credentials), time.time() + expiry_seconds
95
+
96
+
39
97
  class ManagedKafkaHook(GoogleBaseHook):
40
98
  """Hook for Managed Service for Apache Kafka APIs."""
41
99
 
@@ -62,6 +120,12 @@ class ManagedKafkaHook(GoogleBaseHook):
62
120
  error = operation.exception(timeout=timeout)
63
121
  raise AirflowException(error)
64
122
 
123
+ def get_confluent_token(self, config_str: str):
124
+ """Get the authentication token for confluent client."""
125
+ token_provider = ManagedKafkaTokenProvider(credentials=self.get_credentials())
126
+ token = token_provider.confluent_token()
127
+ return token
128
+
65
129
  @GoogleBaseHook.fallback_to_default_project_id
66
130
  def create_cluster(
67
131
  self,
@@ -480,3 +544,163 @@ class ManagedKafkaHook(GoogleBaseHook):
480
544
  timeout=timeout,
481
545
  metadata=metadata,
482
546
  )
547
+
548
+ @GoogleBaseHook.fallback_to_default_project_id
549
+ def list_consumer_groups(
550
+ self,
551
+ project_id: str,
552
+ location: str,
553
+ cluster_id: str,
554
+ page_size: int | None = None,
555
+ page_token: str | None = None,
556
+ retry: Retry | _MethodDefault = DEFAULT,
557
+ timeout: float | None = None,
558
+ metadata: Sequence[tuple[str, str]] = (),
559
+ ) -> ListConsumerGroupsPager:
560
+ """
561
+ List the consumer groups in a given cluster.
562
+
563
+ :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
564
+ :param location: Required. The ID of the Google Cloud region that the service belongs to.
565
+ :param cluster_id: Required. The ID of the cluster whose consumer groups are to be listed.
566
+ :param page_size: Optional. The maximum number of consumer groups to return. The service may return
567
+ fewer than this value. If unset or zero, all consumer groups for the parent is returned.
568
+ :param page_token: Optional. A page token, received from a previous ``ListConsumerGroups`` call.
569
+ Provide this to retrieve the subsequent page. When paginating, all other parameters provided to
570
+ ``ListConsumerGroups`` must match the call that provided the page token.
571
+ :param retry: Designation of what errors, if any, should be retried.
572
+ :param timeout: The timeout for this request.
573
+ :param metadata: Strings which should be sent along with the request as metadata.
574
+ """
575
+ client = self.get_managed_kafka_client()
576
+ parent = client.cluster_path(project_id, location, cluster_id)
577
+
578
+ result = client.list_consumer_groups(
579
+ request={
580
+ "parent": parent,
581
+ "page_size": page_size,
582
+ "page_token": page_token,
583
+ },
584
+ retry=retry,
585
+ timeout=timeout,
586
+ metadata=metadata,
587
+ )
588
+ return result
589
+
590
+ @GoogleBaseHook.fallback_to_default_project_id
591
+ def get_consumer_group(
592
+ self,
593
+ project_id: str,
594
+ location: str,
595
+ cluster_id: str,
596
+ consumer_group_id: str,
597
+ retry: Retry | _MethodDefault = DEFAULT,
598
+ timeout: float | None = None,
599
+ metadata: Sequence[tuple[str, str]] = (),
600
+ ) -> types.ConsumerGroup:
601
+ """
602
+ Return the properties of a single consumer group.
603
+
604
+ :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
605
+ :param location: Required. The ID of the Google Cloud region that the service belongs to.
606
+ :param cluster_id: Required. The ID of the cluster whose consumer group is to be returned.
607
+ :param consumer_group_id: Required. The ID of the consumer group whose configuration to return.
608
+ :param retry: Designation of what errors, if any, should be retried.
609
+ :param timeout: The timeout for this request.
610
+ :param metadata: Strings which should be sent along with the request as metadata.
611
+ """
612
+ client = self.get_managed_kafka_client()
613
+ name = client.consumer_group_path(project_id, location, cluster_id, consumer_group_id)
614
+
615
+ result = client.get_consumer_group(
616
+ request={
617
+ "name": name,
618
+ },
619
+ retry=retry,
620
+ timeout=timeout,
621
+ metadata=metadata,
622
+ )
623
+ return result
624
+
625
+ @GoogleBaseHook.fallback_to_default_project_id
626
+ def update_consumer_group(
627
+ self,
628
+ project_id: str,
629
+ location: str,
630
+ cluster_id: str,
631
+ consumer_group_id: str,
632
+ consumer_group: types.ConsumerGroup | dict,
633
+ update_mask: FieldMask | dict,
634
+ retry: Retry | _MethodDefault = DEFAULT,
635
+ timeout: float | None = None,
636
+ metadata: Sequence[tuple[str, str]] = (),
637
+ ) -> types.ConsumerGroup:
638
+ """
639
+ Update the properties of a single consumer group.
640
+
641
+ :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
642
+ :param location: Required. The ID of the Google Cloud region that the service belongs to.
643
+ :param cluster_id: Required. The ID of the cluster whose topic is to be updated.
644
+ :param consumer_group_id: Required. The ID of the consumer group whose configuration to update.
645
+ :param consumer_group: Required. The consumer_group to update. Its ``name`` field must be populated.
646
+ :param update_mask: Required. Field mask is used to specify the fields to be overwritten in the
647
+ ConsumerGroup resource by the update. The fields specified in the update_mask are relative to the
648
+ resource, not the full request. A field will be overwritten if it is in the mask.
649
+ :param retry: Designation of what errors, if any, should be retried.
650
+ :param timeout: The timeout for this request.
651
+ :param metadata: Strings which should be sent along with the request as metadata.
652
+ """
653
+ client = self.get_managed_kafka_client()
654
+ _consumer_group = (
655
+ deepcopy(consumer_group)
656
+ if isinstance(consumer_group, dict)
657
+ else ConsumerGroup.to_dict(consumer_group)
658
+ )
659
+ _consumer_group["name"] = client.consumer_group_path(
660
+ project_id, location, cluster_id, consumer_group_id
661
+ )
662
+
663
+ result = client.update_consumer_group(
664
+ request={
665
+ "update_mask": update_mask,
666
+ "consumer_group": _consumer_group,
667
+ },
668
+ retry=retry,
669
+ timeout=timeout,
670
+ metadata=metadata,
671
+ )
672
+ return result
673
+
674
+ @GoogleBaseHook.fallback_to_default_project_id
675
+ def delete_consumer_group(
676
+ self,
677
+ project_id: str,
678
+ location: str,
679
+ cluster_id: str,
680
+ consumer_group_id: str,
681
+ retry: Retry | _MethodDefault = DEFAULT,
682
+ timeout: float | None = None,
683
+ metadata: Sequence[tuple[str, str]] = (),
684
+ ) -> None:
685
+ """
686
+ Delete a single consumer group.
687
+
688
+ :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
689
+ :param location: Required. The ID of the Google Cloud region that the service belongs to.
690
+ :param cluster_id: Required. The ID of the cluster whose consumer group is to be deleted.
691
+ :param consumer_group_id: Required. The ID of the consumer group to delete.
692
+ :param retry: Designation of what errors, if any, should be retried.
693
+ :param timeout: The timeout for this request.
694
+ :param metadata: Strings which should be sent along with the request as metadata.
695
+ """
696
+ client = self.get_managed_kafka_client()
697
+ name = client.consumer_group_path(project_id, location, cluster_id, consumer_group_id)
698
+
699
+ client.delete_consumer_group(
700
+ request={
701
+ "name": name,
702
+ },
703
+ retry=retry,
704
+ timeout=timeout,
705
+ metadata=metadata,
706
+ )
@@ -23,7 +23,8 @@ import contextlib
23
23
  import logging
24
24
  import random
25
25
  import time
26
- from typing import TYPE_CHECKING, Callable
26
+ from collections.abc import Callable
27
+ from typing import TYPE_CHECKING
27
28
 
28
29
  from aiohttp import ClientSession
29
30
  from gcloud.aio.auth import AioSession, Token
@@ -78,8 +79,7 @@ def _poll_with_exponential_delay(
78
79
  if e.resp.status != 429:
79
80
  log.info("Something went wrong. Not retrying: %s", format(e))
80
81
  raise
81
- else:
82
- time.sleep((2**i) + random.random())
82
+ time.sleep((2**i) + random.random())
83
83
 
84
84
  raise ValueError(f"Connection could not be established after {max_n} retries.")
85
85
 
@@ -219,12 +219,11 @@ class MLEngineHook(GoogleBaseHook):
219
219
  if e.resp.status == 404:
220
220
  self.log.error("Job with job_id %s does not exist. ", job_id)
221
221
  raise
222
- elif e.resp.status == 400:
222
+ if e.resp.status == 400:
223
223
  self.log.info("Job with job_id %s is already complete, cancellation aborted.", job_id)
224
224
  return {}
225
- else:
226
- self.log.error("Failed to cancel MLEngine job: %s", e)
227
- raise
225
+ self.log.error("Failed to cancel MLEngine job: %s", e)
226
+ raise
228
227
 
229
228
  def get_job(self, project_id: str, job_id: str) -> dict:
230
229
  """
@@ -589,7 +588,7 @@ class MLEngineAsyncHook(GoogleBaseAsyncHook):
589
588
  job = await self.get_job(
590
589
  project_id=project_id,
591
590
  job_id=job_id,
592
- session=session, # type: ignore
591
+ session=session, # type: ignore
593
592
  )
594
593
  job = await job.json(content_type=None)
595
594
  self.log.info("Retrieving json_response: %s", job)
@@ -22,8 +22,6 @@ from __future__ import annotations
22
22
  from collections.abc import Sequence
23
23
  from typing import TYPE_CHECKING
24
24
 
25
- from airflow.providers.google.common.consts import CLIENT_INFO
26
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
27
25
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
28
26
  from google.cloud.language_v1 import EncodingType, LanguageServiceClient
29
27
  from google.cloud.language_v1.types import (
@@ -37,6 +35,9 @@ from google.cloud.language_v1.types import (
37
35
  Document,
38
36
  )
39
37
 
38
+ from airflow.providers.google.common.consts import CLIENT_INFO
39
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
40
+
40
41
  if TYPE_CHECKING:
41
42
  from google.api_core.retry import Retry
42
43
 
@@ -27,11 +27,12 @@ from __future__ import annotations
27
27
  from collections.abc import Sequence
28
28
  from typing import TYPE_CHECKING
29
29
 
30
- from airflow.providers.google.common.consts import CLIENT_INFO
31
- from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
32
30
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
33
31
  from google.cloud.oslogin_v1 import ImportSshPublicKeyResponse, OsLoginServiceClient
34
32
 
33
+ from airflow.providers.google.common.consts import CLIENT_INFO
34
+ from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
35
+
35
36
  if TYPE_CHECKING:
36
37
  from google.api_core.retry import Retry
37
38
 
@@ -33,6 +33,12 @@ from functools import cached_property
33
33
  from typing import TYPE_CHECKING, Any
34
34
  from uuid import uuid4
35
35
 
36
+ from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
37
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
38
+ from google.cloud.exceptions import NotFound
39
+ from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
40
+ from google.cloud.pubsub_v1.types import PublisherOptions
41
+ from google.pubsub_v1.services.subscriber.async_client import SubscriberAsyncClient
36
42
  from googleapiclient.errors import HttpError
37
43
 
38
44
  from airflow.exceptions import AirflowProviderDeprecationWarning
@@ -43,12 +49,6 @@ from airflow.providers.google.common.hooks.base_google import (
43
49
  GoogleBaseHook,
44
50
  )
45
51
  from airflow.version import version
46
- from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
47
- from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
48
- from google.cloud.exceptions import NotFound
49
- from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
50
- from google.cloud.pubsub_v1.types import PublisherOptions
51
- from google.pubsub_v1.services.subscriber.async_client import SubscriberAsyncClient
52
52
 
53
53
  if TYPE_CHECKING:
54
54
  from google.api_core.retry import Retry
@@ -23,8 +23,6 @@ from collections.abc import Sequence
23
23
  from functools import cached_property
24
24
  from typing import TYPE_CHECKING
25
25
 
26
- from airflow.providers.google.common.consts import CLIENT_INFO
27
- from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
28
26
  from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
29
27
  from google.cloud.secretmanager_v1 import (
30
28
  AccessSecretVersionResponse,
@@ -34,6 +32,9 @@ from google.cloud.secretmanager_v1 import (
34
32
  SecretVersion,
35
33
  )
36
34
 
35
+ from airflow.providers.google.common.consts import CLIENT_INFO
36
+ from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
37
+
37
38
  if TYPE_CHECKING:
38
39
  from google.api_core.retry import Retry
39
40
  from google.cloud.secretmanager_v1.services.secret_manager_service.pagers import ListSecretsPager
@@ -46,6 +47,10 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
46
47
  See https://cloud.google.com/secret-manager
47
48
  """
48
49
 
50
+ def __init__(self, location: str | None = None, **kwargs) -> None:
51
+ super().__init__(**kwargs)
52
+ self.location = location
53
+
49
54
  @cached_property
50
55
  def client(self):
51
56
  """
@@ -53,7 +58,16 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
53
58
 
54
59
  :return: Secret Manager client.
55
60
  """
56
- return SecretManagerServiceClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
61
+ if self.location is not None:
62
+ return SecretManagerServiceClient(
63
+ credentials=self.get_credentials(),
64
+ client_info=CLIENT_INFO,
65
+ client_options={"api_endpoint": f"secretmanager.{self.location}.rep.googleapis.com"},
66
+ )
67
+ return SecretManagerServiceClient(
68
+ credentials=self.get_credentials(),
69
+ client_info=CLIENT_INFO,
70
+ )
57
71
 
58
72
  def get_conn(self) -> SecretManagerServiceClient:
59
73
  """
@@ -63,6 +77,60 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
63
77
  """
64
78
  return self.client
65
79
 
80
+ def _get_parent(self, project_id: str, location: str | None = None) -> str:
81
+ """
82
+ Return parent path.
83
+
84
+ :param project_id: Required. ID of the GCP project that owns the job.
85
+ :param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
86
+ For more details : https://cloud.google.com/secret-manager/docs/locations
87
+
88
+ :return: Parent path.
89
+ """
90
+ _location = location or self.location
91
+ if _location is not None:
92
+ return self.client.common_location_path(project_id, _location)
93
+ return self.client.common_project_path(project_id)
94
+
95
+ def _get_secret_path(self, project_id: str, secret_id: str, location: str | None = None) -> str:
96
+ """
97
+ Return secret path.
98
+
99
+ :param project_id: Required. ID of the GCP project that owns the job.
100
+ :param secret_id: Required. Secret ID for which path is required.
101
+ :param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
102
+ For more details : https://cloud.google.com/secret-manager/docs/locations
103
+
104
+ :return: Parent path.
105
+ """
106
+ _location = location or self.location
107
+ if _location is not None:
108
+ # Google's client library does not provide a method to construct regional secret paths, so constructing manually.
109
+ return f"projects/{project_id}/locations/{_location}/secrets/{secret_id}"
110
+ return self.client.secret_path(project_id, secret_id)
111
+
112
+ def _get_secret_version_path(
113
+ self, project_id: str, secret_id: str, secret_version: str, location: str | None = None
114
+ ) -> str:
115
+ """
116
+ Return secret version path.
117
+
118
+ :param project_id: Required. ID of the GCP project that owns the job.
119
+ :param secret_id: Required. Secret ID for which path is required.
120
+ :param secret_version: Required. Secret version for which path is required.
121
+ :param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
122
+ For more details : https://cloud.google.com/secret-manager/docs/locations
123
+
124
+ :return: Parent path.
125
+ """
126
+ _location = location or self.location
127
+ if _location is not None:
128
+ # Google's client library does not provide a method to construct regional secret version paths, so constructing manually.
129
+ return (
130
+ f"projects/{project_id}/locations/{_location}/secrets/{secret_id}/versions/{secret_version}"
131
+ )
132
+ return self.client.secret_version_path(project_id, secret_id, secret_version)
133
+
66
134
  @GoogleBaseHook.fallback_to_default_project_id
67
135
  def create_secret(
68
136
  self,
@@ -72,6 +140,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
72
140
  retry: Retry | _MethodDefault = DEFAULT,
73
141
  timeout: float | None = None,
74
142
  metadata: Sequence[tuple[str, str]] = (),
143
+ location: str | None = None,
75
144
  ) -> Secret:
76
145
  """
77
146
  Create a secret.
@@ -87,12 +156,20 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
87
156
  :param retry: Optional. Designation of what errors, if any, should be retried.
88
157
  :param timeout: Optional. The timeout for this request.
89
158
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
159
+ :param location: Optional. Location where secret should be created. Used for creating regional secret. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
160
+ For more details : https://cloud.google.com/secret-manager/docs/locations
90
161
  :return: Secret object.
91
162
  """
92
- _secret = secret or {"replication": {"automatic": {}}}
163
+ if not secret:
164
+ _secret: dict | Secret = {}
165
+ if (location or self.location) is None:
166
+ _secret["replication"] = {"automatic": {}}
167
+ else:
168
+ _secret = secret
169
+
93
170
  response = self.client.create_secret(
94
171
  request={
95
- "parent": f"projects/{project_id}",
172
+ "parent": self._get_parent(project_id, location),
96
173
  "secret_id": secret_id,
97
174
  "secret": _secret,
98
175
  },
@@ -112,6 +189,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
112
189
  retry: Retry | _MethodDefault = DEFAULT,
113
190
  timeout: float | None = None,
114
191
  metadata: Sequence[tuple[str, str]] = (),
192
+ location: str | None = None,
115
193
  ) -> SecretVersion:
116
194
  """
117
195
  Add a version to the secret.
@@ -127,11 +205,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
127
205
  :param retry: Optional. Designation of what errors, if any, should be retried.
128
206
  :param timeout: Optional. The timeout for this request.
129
207
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
208
+ :param location: Optional. Location where secret is located. Used for adding version to regional secret. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
209
+ For more details : https://cloud.google.com/secret-manager/docs/locations
130
210
  :return: Secret version object.
131
211
  """
132
212
  response = self.client.add_secret_version(
133
213
  request={
134
- "parent": f"projects/{project_id}/secrets/{secret_id}",
214
+ "parent": self._get_secret_path(project_id, secret_id, location),
135
215
  "payload": secret_payload,
136
216
  },
137
217
  retry=retry,
@@ -151,6 +231,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
151
231
  retry: Retry | _MethodDefault = DEFAULT,
152
232
  timeout: float | None = None,
153
233
  metadata: Sequence[tuple[str, str]] = (),
234
+ location: str | None = None,
154
235
  ) -> ListSecretsPager:
155
236
  """
156
237
  List secrets.
@@ -167,11 +248,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
167
248
  :param retry: Optional. Designation of what errors, if any, should be retried.
168
249
  :param timeout: Optional. The timeout for this request.
169
250
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
251
+ :param location: Optional. The regional secrets stored in the provided location will be listed. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
252
+ For more details : https://cloud.google.com/secret-manager/docs/locations
170
253
  :return: Secret List object.
171
254
  """
172
255
  response = self.client.list_secrets(
173
256
  request={
174
- "parent": f"projects/{project_id}",
257
+ "parent": self._get_parent(project_id, location),
175
258
  "page_size": page_size,
176
259
  "page_token": page_token,
177
260
  "filter": secret_filter,
@@ -184,18 +267,22 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
184
267
  return response
185
268
 
186
269
  @GoogleBaseHook.fallback_to_default_project_id
187
- def secret_exists(self, project_id: str, secret_id: str) -> bool:
270
+ def secret_exists(self, project_id: str, secret_id: str, location: str | None = None) -> bool:
188
271
  """
189
272
  Check whether secret exists.
190
273
 
191
274
  :param project_id: Required. ID of the GCP project that owns the job.
192
275
  If set to ``None`` or missing, the default project_id from the GCP connection is used.
193
276
  :param secret_id: Required. ID of the secret to find.
277
+ :param location: Optional. Location where secret is expected to be stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
278
+ For more details : https://cloud.google.com/secret-manager/docs/locations
194
279
  :return: True if the secret exists, False otherwise.
195
280
  """
196
281
  secret_filter = f"name:{secret_id}"
197
- secret_name = self.client.secret_path(project_id, secret_id)
198
- for secret in self.list_secrets(project_id=project_id, page_size=100, secret_filter=secret_filter):
282
+ secret_name = self._get_secret_path(project_id, secret_id, location)
283
+ for secret in self.list_secrets(
284
+ project_id=project_id, page_size=100, secret_filter=secret_filter, location=location
285
+ ):
199
286
  if secret.name.split("/")[-1] == secret_id:
200
287
  self.log.info("Secret %s exists.", secret_name)
201
288
  return True
@@ -211,6 +298,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
211
298
  retry: Retry | _MethodDefault = DEFAULT,
212
299
  timeout: float | None = None,
213
300
  metadata: Sequence[tuple[str, str]] = (),
301
+ location: str | None = None,
214
302
  ) -> AccessSecretVersionResponse:
215
303
  """
216
304
  Access a secret version.
@@ -226,11 +314,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
226
314
  :param retry: Optional. Designation of what errors, if any, should be retried.
227
315
  :param timeout: Optional. The timeout for this request.
228
316
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
317
+ :param location: Optional. Location where secret is stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
318
+ For more details : https://cloud.google.com/secret-manager/docs/locations
229
319
  :return: Access secret version response object.
230
320
  """
231
321
  response = self.client.access_secret_version(
232
322
  request={
233
- "name": self.client.secret_version_path(project_id, secret_id, secret_version),
323
+ "name": self._get_secret_version_path(project_id, secret_id, secret_version, location),
234
324
  },
235
325
  retry=retry,
236
326
  timeout=timeout,
@@ -247,6 +337,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
247
337
  retry: Retry | _MethodDefault = DEFAULT,
248
338
  timeout: float | None = None,
249
339
  metadata: Sequence[tuple[str, str]] = (),
340
+ location: str | None = None,
250
341
  ) -> None:
251
342
  """
252
343
  Delete a secret.
@@ -261,9 +352,11 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
261
352
  :param retry: Optional. Designation of what errors, if any, should be retried.
262
353
  :param timeout: Optional. The timeout for this request.
263
354
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
355
+ :param location: Optional. Location where secret is stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used.
356
+ For more details : https://cloud.google.com/secret-manager/docs/locations
264
357
  :return: Access secret version response object.
265
358
  """
266
- name = self.client.secret_path(project_id, secret_id)
359
+ name = self._get_secret_path(project_id, secret_id, location)
267
360
  self.client.delete_secret(
268
361
  request={"name": name},
269
362
  retry=retry,