apache-airflow-providers-google 16.0.0a1__py3-none-any.whl → 16.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. airflow/providers/google/__init__.py +1 -1
  2. airflow/providers/google/ads/hooks/ads.py +43 -5
  3. airflow/providers/google/ads/operators/ads.py +1 -1
  4. airflow/providers/google/ads/transfers/ads_to_gcs.py +1 -1
  5. airflow/providers/google/cloud/hooks/bigquery.py +63 -77
  6. airflow/providers/google/cloud/hooks/cloud_sql.py +8 -4
  7. airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
  8. airflow/providers/google/cloud/hooks/dataflow.py +2 -2
  9. airflow/providers/google/cloud/hooks/dataplex.py +1 -1
  10. airflow/providers/google/cloud/hooks/dataprep.py +4 -1
  11. airflow/providers/google/cloud/hooks/gcs.py +5 -5
  12. airflow/providers/google/cloud/hooks/looker.py +10 -1
  13. airflow/providers/google/cloud/hooks/mlengine.py +2 -1
  14. airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
  15. airflow/providers/google/cloud/hooks/spanner.py +2 -2
  16. airflow/providers/google/cloud/hooks/translate.py +1 -1
  17. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -36
  18. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
  19. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +44 -80
  20. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +11 -2
  21. airflow/providers/google/cloud/hooks/vision.py +2 -2
  22. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  23. airflow/providers/google/cloud/links/base.py +75 -11
  24. airflow/providers/google/cloud/links/bigquery.py +0 -47
  25. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  26. airflow/providers/google/cloud/links/bigtable.py +0 -48
  27. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  28. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  29. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  30. airflow/providers/google/cloud/links/cloud_run.py +27 -0
  31. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  32. airflow/providers/google/cloud/links/cloud_storage_transfer.py +16 -43
  33. airflow/providers/google/cloud/links/cloud_tasks.py +6 -25
  34. airflow/providers/google/cloud/links/compute.py +0 -58
  35. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  36. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  37. airflow/providers/google/cloud/links/dataflow.py +0 -34
  38. airflow/providers/google/cloud/links/dataform.py +0 -64
  39. airflow/providers/google/cloud/links/datafusion.py +1 -96
  40. airflow/providers/google/cloud/links/dataplex.py +0 -154
  41. airflow/providers/google/cloud/links/dataprep.py +0 -24
  42. airflow/providers/google/cloud/links/dataproc.py +14 -90
  43. airflow/providers/google/cloud/links/datastore.py +0 -31
  44. airflow/providers/google/cloud/links/kubernetes_engine.py +5 -59
  45. airflow/providers/google/cloud/links/life_sciences.py +0 -19
  46. airflow/providers/google/cloud/links/managed_kafka.py +0 -70
  47. airflow/providers/google/cloud/links/mlengine.py +0 -70
  48. airflow/providers/google/cloud/links/pubsub.py +0 -32
  49. airflow/providers/google/cloud/links/spanner.py +0 -33
  50. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  51. airflow/providers/google/cloud/links/translate.py +16 -186
  52. airflow/providers/google/cloud/links/vertex_ai.py +8 -224
  53. airflow/providers/google/cloud/links/workflows.py +0 -52
  54. airflow/providers/google/cloud/log/gcs_task_handler.py +4 -4
  55. airflow/providers/google/cloud/operators/alloy_db.py +69 -54
  56. airflow/providers/google/cloud/operators/automl.py +16 -14
  57. airflow/providers/google/cloud/operators/bigquery.py +49 -25
  58. airflow/providers/google/cloud/operators/bigquery_dts.py +2 -4
  59. airflow/providers/google/cloud/operators/bigtable.py +35 -6
  60. airflow/providers/google/cloud/operators/cloud_base.py +21 -1
  61. airflow/providers/google/cloud/operators/cloud_build.py +74 -31
  62. airflow/providers/google/cloud/operators/cloud_composer.py +34 -35
  63. airflow/providers/google/cloud/operators/cloud_memorystore.py +68 -42
  64. airflow/providers/google/cloud/operators/cloud_run.py +9 -1
  65. airflow/providers/google/cloud/operators/cloud_sql.py +11 -15
  66. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +0 -2
  67. airflow/providers/google/cloud/operators/compute.py +7 -39
  68. airflow/providers/google/cloud/operators/datacatalog.py +156 -20
  69. airflow/providers/google/cloud/operators/dataflow.py +37 -14
  70. airflow/providers/google/cloud/operators/dataform.py +14 -4
  71. airflow/providers/google/cloud/operators/datafusion.py +4 -12
  72. airflow/providers/google/cloud/operators/dataplex.py +180 -96
  73. airflow/providers/google/cloud/operators/dataprep.py +0 -4
  74. airflow/providers/google/cloud/operators/dataproc.py +10 -16
  75. airflow/providers/google/cloud/operators/dataproc_metastore.py +95 -87
  76. airflow/providers/google/cloud/operators/datastore.py +21 -5
  77. airflow/providers/google/cloud/operators/dlp.py +3 -26
  78. airflow/providers/google/cloud/operators/functions.py +15 -6
  79. airflow/providers/google/cloud/operators/gcs.py +1 -7
  80. airflow/providers/google/cloud/operators/kubernetes_engine.py +53 -92
  81. airflow/providers/google/cloud/operators/life_sciences.py +0 -1
  82. airflow/providers/google/cloud/operators/managed_kafka.py +106 -51
  83. airflow/providers/google/cloud/operators/mlengine.py +0 -1
  84. airflow/providers/google/cloud/operators/pubsub.py +4 -5
  85. airflow/providers/google/cloud/operators/spanner.py +0 -4
  86. airflow/providers/google/cloud/operators/speech_to_text.py +0 -1
  87. airflow/providers/google/cloud/operators/stackdriver.py +0 -8
  88. airflow/providers/google/cloud/operators/tasks.py +0 -11
  89. airflow/providers/google/cloud/operators/text_to_speech.py +0 -1
  90. airflow/providers/google/cloud/operators/translate.py +37 -13
  91. airflow/providers/google/cloud/operators/translate_speech.py +0 -1
  92. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +31 -18
  93. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +28 -8
  94. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +38 -25
  95. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +69 -7
  96. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +42 -8
  97. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +531 -0
  98. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +93 -117
  99. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +10 -8
  100. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +56 -10
  101. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +29 -6
  102. airflow/providers/google/cloud/operators/vertex_ai/ray.py +9 -6
  103. airflow/providers/google/cloud/operators/workflows.py +1 -9
  104. airflow/providers/google/cloud/sensors/bigquery.py +1 -1
  105. airflow/providers/google/cloud/sensors/bigquery_dts.py +6 -1
  106. airflow/providers/google/cloud/sensors/bigtable.py +15 -3
  107. airflow/providers/google/cloud/sensors/cloud_composer.py +6 -1
  108. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +6 -1
  109. airflow/providers/google/cloud/sensors/dataflow.py +3 -3
  110. airflow/providers/google/cloud/sensors/dataform.py +6 -1
  111. airflow/providers/google/cloud/sensors/datafusion.py +6 -1
  112. airflow/providers/google/cloud/sensors/dataplex.py +6 -1
  113. airflow/providers/google/cloud/sensors/dataprep.py +6 -1
  114. airflow/providers/google/cloud/sensors/dataproc.py +6 -1
  115. airflow/providers/google/cloud/sensors/dataproc_metastore.py +6 -1
  116. airflow/providers/google/cloud/sensors/gcs.py +9 -3
  117. airflow/providers/google/cloud/sensors/looker.py +6 -1
  118. airflow/providers/google/cloud/sensors/pubsub.py +8 -3
  119. airflow/providers/google/cloud/sensors/tasks.py +6 -1
  120. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +6 -1
  121. airflow/providers/google/cloud/sensors/workflows.py +6 -1
  122. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +1 -1
  123. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +1 -1
  124. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +10 -7
  125. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +1 -2
  126. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +0 -1
  127. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -1
  128. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  129. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +1 -1
  130. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +2 -2
  131. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +1 -2
  132. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +1 -1
  133. airflow/providers/google/cloud/transfers/gcs_to_local.py +1 -1
  134. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +1 -1
  135. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +5 -1
  136. airflow/providers/google/cloud/transfers/gdrive_to_local.py +1 -1
  137. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  138. airflow/providers/google/cloud/transfers/local_to_gcs.py +1 -1
  139. airflow/providers/google/cloud/transfers/s3_to_gcs.py +11 -5
  140. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +1 -1
  141. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +1 -1
  142. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +2 -2
  143. airflow/providers/google/cloud/transfers/sql_to_gcs.py +1 -1
  144. airflow/providers/google/cloud/triggers/bigquery.py +32 -5
  145. airflow/providers/google/cloud/triggers/dataproc.py +62 -10
  146. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  147. airflow/providers/google/common/auth_backend/google_openid.py +2 -1
  148. airflow/providers/google/common/deprecated.py +2 -1
  149. airflow/providers/google/common/hooks/base_google.py +7 -3
  150. airflow/providers/google/common/links/storage.py +0 -22
  151. airflow/providers/google/firebase/operators/firestore.py +1 -1
  152. airflow/providers/google/get_provider_info.py +14 -16
  153. airflow/providers/google/leveldb/hooks/leveldb.py +30 -1
  154. airflow/providers/google/leveldb/operators/leveldb.py +1 -1
  155. airflow/providers/google/marketing_platform/links/analytics_admin.py +3 -6
  156. airflow/providers/google/marketing_platform/operators/analytics_admin.py +0 -1
  157. airflow/providers/google/marketing_platform/operators/campaign_manager.py +4 -4
  158. airflow/providers/google/marketing_platform/operators/display_video.py +6 -6
  159. airflow/providers/google/marketing_platform/operators/search_ads.py +1 -1
  160. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +6 -1
  161. airflow/providers/google/marketing_platform/sensors/display_video.py +6 -1
  162. airflow/providers/google/suite/operators/sheets.py +3 -3
  163. airflow/providers/google/suite/sensors/drive.py +6 -1
  164. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +1 -1
  165. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  166. airflow/providers/google/suite/transfers/local_to_drive.py +1 -1
  167. airflow/providers/google/version_compat.py +28 -0
  168. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/METADATA +35 -35
  169. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/RECORD +171 -170
  170. airflow/providers/google/cloud/links/automl.py +0 -193
  171. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/WHEEL +0 -0
  172. {apache_airflow_providers_google-16.0.0a1.dist-info → apache_airflow_providers_google-16.1.0.dist-info}/entry_points.txt +0 -0
@@ -47,6 +47,10 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
47
47
  See https://cloud.google.com/secret-manager
48
48
  """
49
49
 
50
+ def __init__(self, location: str | None = None, **kwargs) -> None:
51
+ super().__init__(**kwargs)
52
+ self.location = location
53
+
50
54
  @cached_property
51
55
  def client(self):
52
56
  """
@@ -54,7 +58,16 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
54
58
 
55
59
  :return: Secret Manager client.
56
60
  """
57
- return SecretManagerServiceClient(credentials=self.get_credentials(), client_info=CLIENT_INFO)
61
+ if self.location is not None:
62
+ return SecretManagerServiceClient(
63
+ credentials=self.get_credentials(),
64
+ client_info=CLIENT_INFO,
65
+ client_options={"api_endpoint": f"secretmanager.{self.location}.rep.googleapis.com"},
66
+ )
67
+ return SecretManagerServiceClient(
68
+ credentials=self.get_credentials(),
69
+ client_info=CLIENT_INFO,
70
+ )
58
71
 
59
72
  def get_conn(self) -> SecretManagerServiceClient:
60
73
  """
@@ -64,6 +77,60 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
64
77
  """
65
78
  return self.client
66
79
 
80
+ def _get_parent(self, project_id: str, location: str | None = None) -> str:
81
+ """
82
+ Return parent path.
83
+
84
+ :param project_id: Required. ID of the GCP project that owns the job.
85
+ :param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
86
+ For more details : https://cloud.google.com/secret-manager/docs/locations
87
+
88
+ :return: Parent path.
89
+ """
90
+ _location = location or self.location
91
+ if _location is not None:
92
+ return self.client.common_location_path(project_id, _location)
93
+ return self.client.common_project_path(project_id)
94
+
95
+ def _get_secret_path(self, project_id: str, secret_id: str, location: str | None = None) -> str:
96
+ """
97
+ Return secret path.
98
+
99
+ :param project_id: Required. ID of the GCP project that owns the job.
100
+ :param secret_id: Required. Secret ID for which path is required.
101
+ :param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
102
+ For more details : https://cloud.google.com/secret-manager/docs/locations
103
+
104
+ :return: Parent path.
105
+ """
106
+ _location = location or self.location
107
+ if _location is not None:
108
+ # Google's client library does not provide a method to construct regional secret paths, so constructing manually.
109
+ return f"projects/{project_id}/locations/{_location}/secrets/{secret_id}"
110
+ return self.client.secret_path(project_id, secret_id)
111
+
112
+ def _get_secret_version_path(
113
+ self, project_id: str, secret_id: str, secret_version: str, location: str | None = None
114
+ ) -> str:
115
+ """
116
+ Return secret version path.
117
+
118
+ :param project_id: Required. ID of the GCP project that owns the job.
119
+ :param secret_id: Required. Secret ID for which path is required.
120
+ :param secret_version: Required. Secret version for which path is required.
121
+ :param location: Optional. Target location. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
122
+ For more details : https://cloud.google.com/secret-manager/docs/locations
123
+
124
+ :return: Parent path.
125
+ """
126
+ _location = location or self.location
127
+ if _location is not None:
128
+ # Google's client library does not provide a method to construct regional secret version paths, so constructing manually.
129
+ return (
130
+ f"projects/{project_id}/locations/{_location}/secrets/{secret_id}/versions/{secret_version}"
131
+ )
132
+ return self.client.secret_version_path(project_id, secret_id, secret_version)
133
+
67
134
  @GoogleBaseHook.fallback_to_default_project_id
68
135
  def create_secret(
69
136
  self,
@@ -73,6 +140,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
73
140
  retry: Retry | _MethodDefault = DEFAULT,
74
141
  timeout: float | None = None,
75
142
  metadata: Sequence[tuple[str, str]] = (),
143
+ location: str | None = None,
76
144
  ) -> Secret:
77
145
  """
78
146
  Create a secret.
@@ -88,12 +156,20 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
88
156
  :param retry: Optional. Designation of what errors, if any, should be retried.
89
157
  :param timeout: Optional. The timeout for this request.
90
158
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
159
+ :param location: Optional. Location where secret should be created. Used for creating regional secret. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
160
+ For more details : https://cloud.google.com/secret-manager/docs/locations
91
161
  :return: Secret object.
92
162
  """
93
- _secret = secret or {"replication": {"automatic": {}}}
163
+ if not secret:
164
+ _secret: dict | Secret = {}
165
+ if (location or self.location) is None:
166
+ _secret["replication"] = {"automatic": {}}
167
+ else:
168
+ _secret = secret
169
+
94
170
  response = self.client.create_secret(
95
171
  request={
96
- "parent": f"projects/{project_id}",
172
+ "parent": self._get_parent(project_id, location),
97
173
  "secret_id": secret_id,
98
174
  "secret": _secret,
99
175
  },
@@ -113,6 +189,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
113
189
  retry: Retry | _MethodDefault = DEFAULT,
114
190
  timeout: float | None = None,
115
191
  metadata: Sequence[tuple[str, str]] = (),
192
+ location: str | None = None,
116
193
  ) -> SecretVersion:
117
194
  """
118
195
  Add a version to the secret.
@@ -128,11 +205,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
128
205
  :param retry: Optional. Designation of what errors, if any, should be retried.
129
206
  :param timeout: Optional. The timeout for this request.
130
207
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
208
+ :param location: Optional. Location where secret is located. Used for adding version to regional secret. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
209
+ For more details : https://cloud.google.com/secret-manager/docs/locations
131
210
  :return: Secret version object.
132
211
  """
133
212
  response = self.client.add_secret_version(
134
213
  request={
135
- "parent": f"projects/{project_id}/secrets/{secret_id}",
214
+ "parent": self._get_secret_path(project_id, secret_id, location),
136
215
  "payload": secret_payload,
137
216
  },
138
217
  retry=retry,
@@ -152,6 +231,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
152
231
  retry: Retry | _MethodDefault = DEFAULT,
153
232
  timeout: float | None = None,
154
233
  metadata: Sequence[tuple[str, str]] = (),
234
+ location: str | None = None,
155
235
  ) -> ListSecretsPager:
156
236
  """
157
237
  List secrets.
@@ -168,11 +248,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
168
248
  :param retry: Optional. Designation of what errors, if any, should be retried.
169
249
  :param timeout: Optional. The timeout for this request.
170
250
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
251
+ :param location: Optional. The regional secrets stored in the provided location will be listed. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
252
+ For more details : https://cloud.google.com/secret-manager/docs/locations
171
253
  :return: Secret List object.
172
254
  """
173
255
  response = self.client.list_secrets(
174
256
  request={
175
- "parent": f"projects/{project_id}",
257
+ "parent": self._get_parent(project_id, location),
176
258
  "page_size": page_size,
177
259
  "page_token": page_token,
178
260
  "filter": secret_filter,
@@ -185,18 +267,22 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
185
267
  return response
186
268
 
187
269
  @GoogleBaseHook.fallback_to_default_project_id
188
- def secret_exists(self, project_id: str, secret_id: str) -> bool:
270
+ def secret_exists(self, project_id: str, secret_id: str, location: str | None = None) -> bool:
189
271
  """
190
272
  Check whether secret exists.
191
273
 
192
274
  :param project_id: Required. ID of the GCP project that owns the job.
193
275
  If set to ``None`` or missing, the default project_id from the GCP connection is used.
194
276
  :param secret_id: Required. ID of the secret to find.
277
+ :param location: Optional. Location where secret is expected to be stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
278
+ For more details : https://cloud.google.com/secret-manager/docs/locations
195
279
  :return: True if the secret exists, False otherwise.
196
280
  """
197
281
  secret_filter = f"name:{secret_id}"
198
- secret_name = self.client.secret_path(project_id, secret_id)
199
- for secret in self.list_secrets(project_id=project_id, page_size=100, secret_filter=secret_filter):
282
+ secret_name = self._get_secret_path(project_id, secret_id, location)
283
+ for secret in self.list_secrets(
284
+ project_id=project_id, page_size=100, secret_filter=secret_filter, location=location
285
+ ):
200
286
  if secret.name.split("/")[-1] == secret_id:
201
287
  self.log.info("Secret %s exists.", secret_name)
202
288
  return True
@@ -212,6 +298,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
212
298
  retry: Retry | _MethodDefault = DEFAULT,
213
299
  timeout: float | None = None,
214
300
  metadata: Sequence[tuple[str, str]] = (),
301
+ location: str | None = None,
215
302
  ) -> AccessSecretVersionResponse:
216
303
  """
217
304
  Access a secret version.
@@ -227,11 +314,13 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
227
314
  :param retry: Optional. Designation of what errors, if any, should be retried.
228
315
  :param timeout: Optional. The timeout for this request.
229
316
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
317
+ :param location: Optional. Location where secret is stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used
318
+ For more details : https://cloud.google.com/secret-manager/docs/locations
230
319
  :return: Access secret version response object.
231
320
  """
232
321
  response = self.client.access_secret_version(
233
322
  request={
234
- "name": self.client.secret_version_path(project_id, secret_id, secret_version),
323
+ "name": self._get_secret_version_path(project_id, secret_id, secret_version, location),
235
324
  },
236
325
  retry=retry,
237
326
  timeout=timeout,
@@ -248,6 +337,7 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
248
337
  retry: Retry | _MethodDefault = DEFAULT,
249
338
  timeout: float | None = None,
250
339
  metadata: Sequence[tuple[str, str]] = (),
340
+ location: str | None = None,
251
341
  ) -> None:
252
342
  """
253
343
  Delete a secret.
@@ -262,9 +352,11 @@ class GoogleCloudSecretManagerHook(GoogleBaseHook):
262
352
  :param retry: Optional. Designation of what errors, if any, should be retried.
263
353
  :param timeout: Optional. The timeout for this request.
264
354
  :param metadata: Optional. Strings which should be sent along with the request as metadata.
355
+ :param location: Optional. Location where secret is stored regionally. If set to ``None`` or missing, the location provided for GoogleCloudSecretHook instantiation is used.
356
+ For more details : https://cloud.google.com/secret-manager/docs/locations
265
357
  :return: Access secret version response object.
266
358
  """
267
- name = self.client.secret_path(project_id, secret_id)
359
+ name = self._get_secret_path(project_id, secret_id, location)
268
360
  self.client.delete_secret(
269
361
  request={"name": name},
270
362
  retry=retry,
@@ -19,8 +19,8 @@
19
19
 
20
20
  from __future__ import annotations
21
21
 
22
- from collections.abc import Sequence
23
- from typing import TYPE_CHECKING, Callable, NamedTuple
22
+ from collections.abc import Callable, Sequence
23
+ from typing import TYPE_CHECKING, NamedTuple
24
24
 
25
25
  from google.api_core.exceptions import AlreadyExists, GoogleAPICallError
26
26
  from google.cloud.spanner_v1.client import Client
@@ -429,7 +429,7 @@ class TranslateHook(GoogleBaseHook, OperationHelper):
429
429
  project_id: str,
430
430
  location: str,
431
431
  retry: Retry | _MethodDefault = DEFAULT,
432
- timeout: float | _MethodDefault = DEFAULT,
432
+ timeout: float | None | _MethodDefault = DEFAULT,
433
433
  metadata: Sequence[tuple[str, str]] = (),
434
434
  ) -> automl_translation.Dataset:
435
435
  """
@@ -185,42 +185,6 @@ class AutoMLHook(GoogleBaseHook, OperationHelper):
185
185
  model_encryption_spec_key_name=model_encryption_spec_key_name,
186
186
  )
187
187
 
188
- @deprecated(
189
- planned_removal_date="June 15, 2025",
190
- category=AirflowProviderDeprecationWarning,
191
- reason="Deprecation of AutoMLText API",
192
- )
193
- def get_auto_ml_text_training_job(
194
- self,
195
- display_name: str,
196
- prediction_type: str,
197
- multi_label: bool = False,
198
- sentiment_max: int = 10,
199
- project: str | None = None,
200
- location: str | None = None,
201
- labels: dict[str, str] | None = None,
202
- training_encryption_spec_key_name: str | None = None,
203
- model_encryption_spec_key_name: str | None = None,
204
- ) -> AutoMLTextTrainingJob:
205
- """
206
- Return AutoMLTextTrainingJob object.
207
-
208
- WARNING: Text creation API is deprecated since September 15, 2024
209
- (https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/overview).
210
- """
211
- return AutoMLTextTrainingJob(
212
- display_name=display_name,
213
- prediction_type=prediction_type,
214
- multi_label=multi_label,
215
- sentiment_max=sentiment_max,
216
- project=project,
217
- location=location,
218
- credentials=self.get_credentials(),
219
- labels=labels,
220
- training_encryption_spec_key_name=training_encryption_spec_key_name,
221
- model_encryption_spec_key_name=model_encryption_spec_key_name,
222
- )
223
-
224
188
  def get_auto_ml_video_training_job(
225
189
  self,
226
190
  display_name: str,
@@ -18,15 +18,32 @@
18
18
 
19
19
  from __future__ import annotations
20
20
 
21
+ from collections.abc import Sequence
22
+ from typing import (
23
+ TYPE_CHECKING,
24
+ )
25
+
21
26
  from google.api_core.client_options import ClientOptions
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
22
28
  from google.cloud.aiplatform_v1beta1 import (
23
29
  FeatureOnlineStoreAdminServiceClient,
30
+ FeatureOnlineStoreServiceClient,
24
31
  )
25
32
 
26
33
  from airflow.exceptions import AirflowException
27
34
  from airflow.providers.google.common.consts import CLIENT_INFO
28
35
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
29
36
 
37
+ if TYPE_CHECKING:
38
+ from google.api_core.operation import Operation
39
+ from google.api_core.retry import Retry
40
+ from google.cloud.aiplatform_v1beta1.types import (
41
+ FeatureOnlineStore,
42
+ FeatureView,
43
+ FeatureViewDataKey,
44
+ FetchFeatureValuesResponse,
45
+ )
46
+
30
47
 
31
48
  class FeatureStoreHook(GoogleBaseHook):
32
49
  """
@@ -48,6 +65,19 @@ class FeatureStoreHook(GoogleBaseHook):
48
65
  originating account.
49
66
  """
50
67
 
68
+ @staticmethod
69
+ def _get_client_options(
70
+ location: str | None = None,
71
+ custom_endpoint: str | None = None,
72
+ ) -> ClientOptions:
73
+ if custom_endpoint:
74
+ client_options = ClientOptions(api_endpoint=custom_endpoint)
75
+ elif location and location != "global":
76
+ client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
77
+ else:
78
+ client_options = ClientOptions()
79
+ return client_options
80
+
51
81
  def get_feature_online_store_admin_service_client(
52
82
  self,
53
83
  location: str | None = None,
@@ -62,12 +92,153 @@ class FeatureStoreHook(GoogleBaseHook):
62
92
  If provided and not 'global', the client will be configured to use the
63
93
  region-specific API endpoint.
64
94
  """
65
- if location and location != "global":
66
- client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
67
- else:
68
- client_options = ClientOptions()
69
95
  return FeatureOnlineStoreAdminServiceClient(
70
- credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options
96
+ credentials=self.get_credentials(),
97
+ client_info=CLIENT_INFO,
98
+ client_options=self._get_client_options(location),
99
+ )
100
+
101
+ def get_feature_online_store_service_client(
102
+ self,
103
+ location: str | None = None,
104
+ custom_endpoint: str | None = None,
105
+ ) -> FeatureOnlineStoreServiceClient:
106
+ return FeatureOnlineStoreServiceClient(
107
+ credentials=self.get_credentials(),
108
+ client_info=CLIENT_INFO,
109
+ client_options=self._get_client_options(location, custom_endpoint),
110
+ )
111
+
112
+ @GoogleBaseHook.fallback_to_default_project_id
113
+ def create_feature_online_store(
114
+ self,
115
+ feature_online_store_id: str,
116
+ feature_online_store: FeatureOnlineStore,
117
+ project_id: str = PROVIDE_PROJECT_ID,
118
+ location: str | None = None,
119
+ timeout: float | _MethodDefault = DEFAULT,
120
+ retry: Retry | _MethodDefault | None = DEFAULT,
121
+ metadata: Sequence[tuple[str, str]] = (),
122
+ ) -> Operation:
123
+ """
124
+ Create and sends request for Feature Online store.
125
+
126
+ This method initiates VertexAI Feature Online Store creation request.
127
+ Feature Online Store aims to serve and manage features data as a part of VertexAI MLOps.
128
+
129
+ :param feature_online_store_id: The ID of the online feature store.
130
+ :param feature_online_store: The configuration of the online repository.
131
+ :param project_id: The ID of the Google Cloud project that contains the
132
+ feature store. If not provided, will attempt to determine from the environment.
133
+ :param location: The Google Cloud region where the feature store is located
134
+ (e.g., 'us-central1', 'us-east1').
135
+ :param retry: Designation of what errors, if any, should be retried.
136
+ :param timeout: The timeout for this request.
137
+ :param metadata: Strings which should be sent along with the request as metadata.
138
+ """
139
+ client = self.get_feature_online_store_admin_service_client(location)
140
+ return client.create_feature_online_store(
141
+ request={
142
+ "parent": f"projects/{project_id}/locations/{location}",
143
+ "feature_online_store_id": feature_online_store_id,
144
+ "feature_online_store": feature_online_store,
145
+ },
146
+ timeout=timeout,
147
+ retry=retry,
148
+ metadata=metadata,
149
+ )
150
+
151
+ @GoogleBaseHook.fallback_to_default_project_id
152
+ def get_feature_online_store(
153
+ self,
154
+ feature_online_store_id: str,
155
+ project_id: str = PROVIDE_PROJECT_ID,
156
+ location: str | None = None,
157
+ timeout: float | _MethodDefault = DEFAULT,
158
+ retry: Retry | _MethodDefault | None = DEFAULT,
159
+ metadata: Sequence[tuple[str, str]] = (),
160
+ ) -> FeatureOnlineStore:
161
+ """
162
+ Get Feature Online store data.
163
+
164
+ Get the FeatureOnlineStore details.
165
+ Vertex AI Feature Online Store provides a centralized repository for serving ML features
166
+ and embedding indexes at low latency.
167
+
168
+ :param feature_online_store_id: The ID of the online feature store.
169
+ :param project_id: The ID of the Google Cloud project that contains the
170
+ feature store. If not provided, will attempt to determine from the environment.
171
+ :param location: The Google Cloud region where the feature store is located
172
+ (e.g., 'us-central1', 'us-east1').
173
+ :param retry: Designation of what errors, if any, should be retried.
174
+ :param timeout: The timeout for this request.
175
+ :param metadata: Strings which should be sent along with the request as metadata.
176
+ """
177
+ client = self.get_feature_online_store_admin_service_client(location)
178
+ return client.get_feature_online_store(
179
+ name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
180
+ timeout=timeout,
181
+ retry=retry,
182
+ metadata=metadata,
183
+ )
184
+
185
+ @staticmethod
186
+ def _get_featurestore_public_endpoint(feature_online_store: FeatureOnlineStore):
187
+ public_endpoint = None
188
+ featurestore_data = type(feature_online_store).to_dict(feature_online_store)
189
+ if "dedicated_serving_endpoint" in featurestore_data:
190
+ public_endpoint = featurestore_data["dedicated_serving_endpoint"].get(
191
+ "public_endpoint_domain_name"
192
+ )
193
+ return public_endpoint
194
+
195
+ @GoogleBaseHook.fallback_to_default_project_id
196
+ def create_feature_view(
197
+ self,
198
+ feature_view_id: str,
199
+ feature_view: FeatureView,
200
+ feature_online_store_id: str,
201
+ project_id: str = PROVIDE_PROJECT_ID,
202
+ run_sync_immediately: bool = False,
203
+ location: str | None = None,
204
+ timeout: float | _MethodDefault = DEFAULT,
205
+ retry: Retry | _MethodDefault | None = DEFAULT,
206
+ metadata: Sequence[tuple[str, str]] = (),
207
+ ) -> Operation:
208
+ """
209
+ Create request for Feature View creation.
210
+
211
+ This method initiates VertexAI Feature View request for the existing Feature Online Store.
212
+ Feature View represents features and data according to the source provided.
213
+
214
+ :param feature_view_id: The ID to use for the FeatureView, which will
215
+ become the final component of the FeatureView's resource name.
216
+ This value may be up to 60 characters, and valid characters are ``[a-z0-9_]``.
217
+ The first character cannot be a number. The value must be unique within a FeatureOnlineStore.
218
+ :param feature_view: The configuration of the FeatureView to create.
219
+ :param feature_online_store_id: The ID of the online feature store.
220
+ :param run_sync_immediately: If set to true, one on demand sync will be run
221
+ immediately, regardless the FeatureView.sync_config.
222
+ :param project_id: The ID of the Google Cloud project that contains the
223
+ feature store. If not provided, will attempt to determine from the environment.
224
+ :param location: The Google Cloud region where the feature store is located
225
+ (e.g., 'us-central1', 'us-east1').
226
+ :param retry: Designation of what errors, if any, should be retried.
227
+ :param timeout: The timeout for this request.
228
+ :param metadata: Strings which should be sent along with the request as metadata.
229
+ """
230
+ client = self.get_feature_online_store_admin_service_client(location)
231
+ return client.create_feature_view(
232
+ request={
233
+ "parent": f"projects/{project_id}/locations/"
234
+ f"{location}/featureOnlineStores/{feature_online_store_id}",
235
+ "feature_view_id": feature_view_id,
236
+ "feature_view": feature_view,
237
+ "run_sync_immediately": run_sync_immediately,
238
+ },
239
+ timeout=timeout,
240
+ retry=retry,
241
+ metadata=metadata,
71
242
  )
72
243
 
73
244
  def get_feature_view_sync(
@@ -135,13 +306,142 @@ class FeatureStoreHook(GoogleBaseHook):
135
306
  environment.
136
307
  """
137
308
  client = self.get_feature_online_store_admin_service_client(location)
138
- feature_view = f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}/featureViews/{feature_view_id}"
309
+ feature_view = (
310
+ f"projects/{project_id}/locations/{location}/featureOnlineStores/"
311
+ f"{feature_online_store_id}/featureViews/{feature_view_id}"
312
+ )
139
313
 
140
314
  try:
141
315
  response = client.sync_feature_view(feature_view=feature_view)
142
-
143
316
  return str(response.feature_view_sync)
144
317
 
145
318
  except Exception as e:
146
319
  self.log.error("Failed to sync feature view: %s", str(e))
147
320
  raise AirflowException(str(e))
321
+
322
+ @GoogleBaseHook.fallback_to_default_project_id
323
+ def fetch_feature_values(
324
+ self,
325
+ feature_view_id: str,
326
+ feature_online_store_id: str,
327
+ entity_id: str | None = None,
328
+ project_id: str = PROVIDE_PROJECT_ID,
329
+ endpoint_domain_name: str | None = None,
330
+ data_key: FeatureViewDataKey | None = None,
331
+ data_format: int | None = None,
332
+ location: str | None = None,
333
+ timeout: float | _MethodDefault = DEFAULT,
334
+ retry: Retry | _MethodDefault | None = DEFAULT,
335
+ metadata: Sequence[tuple[str, str]] = (),
336
+ ) -> FetchFeatureValuesResponse:
337
+ """
338
+ Fetch data from the Feature View provided.
339
+
340
+ This method fetches data from existing Feature view, filtered by provided (or default) data_key.
341
+ Helps to retrieve actual features data hosted in the VertexAI Feature Store.
342
+
343
+ :param entity_id: Simple ID to identify Entity to fetch feature values for.
344
+ :param endpoint_domain_name: Optional. Public domain name, hosting the content of Optimized
345
+ Feature Online store. Should be omitted, if bigtable configuration provided for the FeatureStore,
346
+ and default feature store endpoint will be used, based on location provided.
347
+ :param feature_view_id: The FeatureView ID to fetch data from.
348
+ :param feature_online_store_id: The ID of the online feature store.
349
+ :param data_key: Optional. The request key to fetch feature values for.
350
+ :param data_format: Optional. Response data format. If not set, FeatureViewDataFormat.KEY_VALUE
351
+ will be used.
352
+ :param project_id: The ID of the Google Cloud project that contains the
353
+ feature store. If not provided, will attempt to determine from the
354
+ environment.
355
+ :param location: The Google Cloud region where the feature store is located
356
+ (e.g., 'us-central1', 'us-east1').
357
+ :param retry: Designation of what errors, if any, should be retried.
358
+ :param timeout: The timeout for this request.
359
+ :param metadata: Strings which should be sent along with the request as metadata.
360
+ """
361
+ data_client = self.get_feature_online_store_service_client(location, endpoint_domain_name)
362
+ return data_client.fetch_feature_values(
363
+ request={
364
+ "id": entity_id,
365
+ "feature_view": f"projects/{project_id}/locations/{location}/featureOnlineStores/"
366
+ f"{feature_online_store_id}/featureViews/{feature_view_id}",
367
+ "data_key": data_key,
368
+ "data_format": data_format,
369
+ },
370
+ timeout=timeout,
371
+ retry=retry,
372
+ metadata=metadata,
373
+ )
374
+
375
+ @GoogleBaseHook.fallback_to_default_project_id
376
+ def delete_feature_view(
377
+ self,
378
+ feature_view_id: str,
379
+ feature_online_store_id: str,
380
+ project_id: str = PROVIDE_PROJECT_ID,
381
+ location: str | None = None,
382
+ timeout: float | _MethodDefault = DEFAULT,
383
+ retry: Retry | _MethodDefault | None = DEFAULT,
384
+ metadata: Sequence[tuple[str, str]] = (),
385
+ ) -> Operation:
386
+ """
387
+ Delete the Feature View.
388
+
389
+ This method deletes the Feature View from the Feature Online Store.
390
+
391
+ :param feature_view_id: The ID to use for the FeatureView, to be deleted.
392
+ :param feature_online_store_id: The ID of the online feature store.
393
+ :param project_id: The ID of the Google Cloud project that contains the
394
+ feature store. If not provided, will attempt to determine from the
395
+ environment.
396
+ :param location: The Google Cloud region where the feature store is located
397
+ (e.g., 'us-central1', 'us-east1').
398
+ :param retry: Designation of what errors, if any, should be retried.
399
+ :param timeout: The timeout for this request.
400
+ :param metadata: Strings which should be sent along with the request as metadata.
401
+ """
402
+ client = self.get_feature_online_store_admin_service_client(location)
403
+ return client.delete_feature_view(
404
+ name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}"
405
+ f"/featureViews/{feature_view_id}",
406
+ timeout=timeout,
407
+ retry=retry,
408
+ metadata=metadata,
409
+ )
410
+
411
+ @GoogleBaseHook.fallback_to_default_project_id
412
+ def delete_feature_online_store(
413
+ self,
414
+ feature_online_store_id: str,
415
+ force: bool = False,
416
+ project_id: str = PROVIDE_PROJECT_ID,
417
+ location: str | None = None,
418
+ timeout: float | _MethodDefault = DEFAULT,
419
+ retry: Retry | _MethodDefault | None = DEFAULT,
420
+ metadata: Sequence[tuple[str, str]] = (),
421
+ ) -> Operation:
422
+ """
423
+ Delete the FeatureOnlineStore.
424
+
425
+ This method deletes the Feature Online Store and all features data.
426
+ The FeatureOnlineStore must not contain any FeatureViews.
427
+
428
+ :param feature_online_store_id: The ID of the online feature store.
429
+ :param force: If set to true, any FeatureViews and Features for this FeatureOnlineStore
430
+ will also be deleted.
431
+ :param project_id: The ID of the Google Cloud project that contains the
432
+ feature store. If not provided, will attempt to determine from the
433
+ environment.
434
+ :param location: The Google Cloud region where the feature store is located
435
+ (e.g., 'us-central1', 'us-east1').
436
+ :param retry: Designation of what errors, if any, should be retried.
437
+ :param timeout: The timeout for this request.
438
+ :param metadata: Strings which should be sent along with the request as metadata.
439
+ """
440
+ client = self.get_feature_online_store_admin_service_client(location)
441
+ return client.delete_feature_online_store(
442
+ name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
443
+ force=force,
444
+ timeout=timeout,
445
+ retry=retry,
446
+ metadata=metadata,
447
+ )