apache-airflow-providers-google 15.1.0rc1__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. airflow/providers/google/3rd-party-licenses/NOTICE +2 -12
  2. airflow/providers/google/__init__.py +3 -3
  3. airflow/providers/google/ads/hooks/ads.py +39 -5
  4. airflow/providers/google/ads/operators/ads.py +2 -2
  5. airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -2
  6. airflow/providers/google/assets/gcs.py +1 -11
  7. airflow/providers/google/cloud/bundles/__init__.py +16 -0
  8. airflow/providers/google/cloud/bundles/gcs.py +161 -0
  9. airflow/providers/google/cloud/hooks/bigquery.py +166 -281
  10. airflow/providers/google/cloud/hooks/cloud_composer.py +287 -14
  11. airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
  12. airflow/providers/google/cloud/hooks/cloud_run.py +17 -9
  13. airflow/providers/google/cloud/hooks/cloud_sql.py +101 -22
  14. airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +27 -6
  15. airflow/providers/google/cloud/hooks/compute_ssh.py +5 -1
  16. airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
  17. airflow/providers/google/cloud/hooks/dataflow.py +71 -94
  18. airflow/providers/google/cloud/hooks/datafusion.py +1 -1
  19. airflow/providers/google/cloud/hooks/dataplex.py +1 -1
  20. airflow/providers/google/cloud/hooks/dataprep.py +1 -1
  21. airflow/providers/google/cloud/hooks/dataproc.py +72 -71
  22. airflow/providers/google/cloud/hooks/gcs.py +111 -14
  23. airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
  24. airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
  25. airflow/providers/google/cloud/hooks/looker.py +6 -1
  26. airflow/providers/google/cloud/hooks/mlengine.py +3 -2
  27. airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
  28. airflow/providers/google/cloud/hooks/spanner.py +73 -8
  29. airflow/providers/google/cloud/hooks/stackdriver.py +10 -8
  30. airflow/providers/google/cloud/hooks/translate.py +1 -1
  31. airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -209
  32. airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +2 -2
  33. airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +27 -1
  34. airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
  35. airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
  36. airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
  37. airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
  38. airflow/providers/google/cloud/hooks/vision.py +2 -2
  39. airflow/providers/google/cloud/hooks/workflows.py +1 -1
  40. airflow/providers/google/cloud/links/alloy_db.py +0 -46
  41. airflow/providers/google/cloud/links/base.py +77 -13
  42. airflow/providers/google/cloud/links/bigquery.py +0 -47
  43. airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
  44. airflow/providers/google/cloud/links/bigtable.py +0 -48
  45. airflow/providers/google/cloud/links/cloud_build.py +0 -73
  46. airflow/providers/google/cloud/links/cloud_functions.py +0 -33
  47. airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
  48. airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
  49. airflow/providers/google/cloud/links/cloud_sql.py +0 -33
  50. airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -44
  51. airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
  52. airflow/providers/google/cloud/links/compute.py +0 -58
  53. airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
  54. airflow/providers/google/cloud/links/datacatalog.py +23 -54
  55. airflow/providers/google/cloud/links/dataflow.py +0 -34
  56. airflow/providers/google/cloud/links/dataform.py +0 -64
  57. airflow/providers/google/cloud/links/datafusion.py +1 -96
  58. airflow/providers/google/cloud/links/dataplex.py +0 -154
  59. airflow/providers/google/cloud/links/dataprep.py +0 -24
  60. airflow/providers/google/cloud/links/dataproc.py +11 -95
  61. airflow/providers/google/cloud/links/datastore.py +0 -31
  62. airflow/providers/google/cloud/links/kubernetes_engine.py +9 -60
  63. airflow/providers/google/cloud/links/managed_kafka.py +0 -70
  64. airflow/providers/google/cloud/links/mlengine.py +0 -70
  65. airflow/providers/google/cloud/links/pubsub.py +0 -32
  66. airflow/providers/google/cloud/links/spanner.py +0 -33
  67. airflow/providers/google/cloud/links/stackdriver.py +0 -30
  68. airflow/providers/google/cloud/links/translate.py +17 -187
  69. airflow/providers/google/cloud/links/vertex_ai.py +28 -195
  70. airflow/providers/google/cloud/links/workflows.py +0 -52
  71. airflow/providers/google/cloud/log/gcs_task_handler.py +17 -9
  72. airflow/providers/google/cloud/log/stackdriver_task_handler.py +9 -6
  73. airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
  74. airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
  75. airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
  76. airflow/providers/google/cloud/openlineage/facets.py +102 -1
  77. airflow/providers/google/cloud/openlineage/mixins.py +10 -8
  78. airflow/providers/google/cloud/openlineage/utils.py +15 -1
  79. airflow/providers/google/cloud/operators/alloy_db.py +70 -55
  80. airflow/providers/google/cloud/operators/bigquery.py +73 -636
  81. airflow/providers/google/cloud/operators/bigquery_dts.py +3 -5
  82. airflow/providers/google/cloud/operators/bigtable.py +36 -7
  83. airflow/providers/google/cloud/operators/cloud_base.py +21 -1
  84. airflow/providers/google/cloud/operators/cloud_batch.py +2 -2
  85. airflow/providers/google/cloud/operators/cloud_build.py +75 -32
  86. airflow/providers/google/cloud/operators/cloud_composer.py +128 -40
  87. airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
  88. airflow/providers/google/cloud/operators/cloud_memorystore.py +69 -43
  89. airflow/providers/google/cloud/operators/cloud_run.py +23 -5
  90. airflow/providers/google/cloud/operators/cloud_sql.py +8 -16
  91. airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -11
  92. airflow/providers/google/cloud/operators/compute.py +8 -40
  93. airflow/providers/google/cloud/operators/datacatalog.py +157 -21
  94. airflow/providers/google/cloud/operators/dataflow.py +38 -15
  95. airflow/providers/google/cloud/operators/dataform.py +15 -5
  96. airflow/providers/google/cloud/operators/datafusion.py +41 -20
  97. airflow/providers/google/cloud/operators/dataplex.py +193 -109
  98. airflow/providers/google/cloud/operators/dataprep.py +1 -5
  99. airflow/providers/google/cloud/operators/dataproc.py +78 -35
  100. airflow/providers/google/cloud/operators/dataproc_metastore.py +96 -88
  101. airflow/providers/google/cloud/operators/datastore.py +22 -6
  102. airflow/providers/google/cloud/operators/dlp.py +6 -29
  103. airflow/providers/google/cloud/operators/functions.py +16 -7
  104. airflow/providers/google/cloud/operators/gcs.py +10 -8
  105. airflow/providers/google/cloud/operators/gen_ai.py +389 -0
  106. airflow/providers/google/cloud/operators/kubernetes_engine.py +60 -99
  107. airflow/providers/google/cloud/operators/looker.py +1 -1
  108. airflow/providers/google/cloud/operators/managed_kafka.py +107 -52
  109. airflow/providers/google/cloud/operators/natural_language.py +1 -1
  110. airflow/providers/google/cloud/operators/pubsub.py +60 -14
  111. airflow/providers/google/cloud/operators/spanner.py +25 -12
  112. airflow/providers/google/cloud/operators/speech_to_text.py +1 -2
  113. airflow/providers/google/cloud/operators/stackdriver.py +1 -9
  114. airflow/providers/google/cloud/operators/tasks.py +1 -12
  115. airflow/providers/google/cloud/operators/text_to_speech.py +1 -2
  116. airflow/providers/google/cloud/operators/translate.py +40 -16
  117. airflow/providers/google/cloud/operators/translate_speech.py +1 -2
  118. airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +39 -19
  119. airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +29 -9
  120. airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +54 -26
  121. airflow/providers/google/cloud/operators/vertex_ai/dataset.py +70 -8
  122. airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +43 -9
  123. airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
  124. airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
  125. airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
  126. airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +11 -9
  127. airflow/providers/google/cloud/operators/vertex_ai/model_service.py +57 -11
  128. airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +30 -7
  129. airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
  130. airflow/providers/google/cloud/operators/video_intelligence.py +1 -1
  131. airflow/providers/google/cloud/operators/vision.py +2 -2
  132. airflow/providers/google/cloud/operators/workflows.py +18 -15
  133. airflow/providers/google/cloud/sensors/bigquery.py +2 -2
  134. airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -2
  135. airflow/providers/google/cloud/sensors/bigtable.py +11 -4
  136. airflow/providers/google/cloud/sensors/cloud_composer.py +533 -29
  137. airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -2
  138. airflow/providers/google/cloud/sensors/dataflow.py +26 -9
  139. airflow/providers/google/cloud/sensors/dataform.py +2 -2
  140. airflow/providers/google/cloud/sensors/datafusion.py +4 -4
  141. airflow/providers/google/cloud/sensors/dataplex.py +2 -2
  142. airflow/providers/google/cloud/sensors/dataprep.py +2 -2
  143. airflow/providers/google/cloud/sensors/dataproc.py +2 -2
  144. airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -2
  145. airflow/providers/google/cloud/sensors/gcs.py +4 -4
  146. airflow/providers/google/cloud/sensors/looker.py +2 -2
  147. airflow/providers/google/cloud/sensors/pubsub.py +4 -4
  148. airflow/providers/google/cloud/sensors/tasks.py +2 -2
  149. airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
  150. airflow/providers/google/cloud/sensors/workflows.py +2 -2
  151. airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
  152. airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
  153. airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
  154. airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
  155. airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +4 -4
  156. airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
  157. airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
  158. airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
  159. airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
  160. airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
  161. airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +2 -2
  162. airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +3 -3
  163. airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +20 -12
  164. airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
  165. airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
  166. airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
  167. airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
  168. airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
  169. airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
  170. airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
  171. airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
  172. airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
  173. airflow/providers/google/cloud/transfers/postgres_to_gcs.py +42 -9
  174. airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
  175. airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
  176. airflow/providers/google/cloud/transfers/sftp_to_gcs.py +13 -4
  177. airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
  178. airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
  179. airflow/providers/google/cloud/triggers/bigquery.py +75 -34
  180. airflow/providers/google/cloud/triggers/cloud_build.py +1 -1
  181. airflow/providers/google/cloud/triggers/cloud_composer.py +302 -46
  182. airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
  183. airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +91 -1
  184. airflow/providers/google/cloud/triggers/dataflow.py +122 -0
  185. airflow/providers/google/cloud/triggers/datafusion.py +1 -1
  186. airflow/providers/google/cloud/triggers/dataplex.py +14 -2
  187. airflow/providers/google/cloud/triggers/dataproc.py +122 -52
  188. airflow/providers/google/cloud/triggers/kubernetes_engine.py +45 -27
  189. airflow/providers/google/cloud/triggers/mlengine.py +1 -1
  190. airflow/providers/google/cloud/triggers/pubsub.py +15 -19
  191. airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
  192. airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
  193. airflow/providers/google/cloud/utils/field_validator.py +1 -2
  194. airflow/providers/google/common/auth_backend/google_openid.py +4 -4
  195. airflow/providers/google/common/deprecated.py +2 -1
  196. airflow/providers/google/common/hooks/base_google.py +27 -8
  197. airflow/providers/google/common/links/storage.py +0 -22
  198. airflow/providers/google/common/utils/get_secret.py +31 -0
  199. airflow/providers/google/common/utils/id_token_credentials.py +3 -4
  200. airflow/providers/google/firebase/operators/firestore.py +2 -2
  201. airflow/providers/google/get_provider_info.py +56 -52
  202. airflow/providers/google/go_module_utils.py +35 -3
  203. airflow/providers/google/leveldb/hooks/leveldb.py +26 -1
  204. airflow/providers/google/leveldb/operators/leveldb.py +2 -2
  205. airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
  206. airflow/providers/google/marketing_platform/links/analytics_admin.py +5 -14
  207. airflow/providers/google/marketing_platform/operators/analytics_admin.py +1 -2
  208. airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
  209. airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
  210. airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
  211. airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
  212. airflow/providers/google/marketing_platform/sensors/display_video.py +3 -63
  213. airflow/providers/google/suite/hooks/calendar.py +1 -1
  214. airflow/providers/google/suite/hooks/sheets.py +15 -1
  215. airflow/providers/google/suite/operators/sheets.py +8 -3
  216. airflow/providers/google/suite/sensors/drive.py +2 -2
  217. airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
  218. airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
  219. airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
  220. airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
  221. airflow/providers/google/version_compat.py +15 -1
  222. {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +92 -48
  223. apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
  224. apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
  225. airflow/providers/google/cloud/hooks/automl.py +0 -673
  226. airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
  227. airflow/providers/google/cloud/links/automl.py +0 -193
  228. airflow/providers/google/cloud/operators/automl.py +0 -1362
  229. airflow/providers/google/cloud/operators/life_sciences.py +0 -119
  230. airflow/providers/google/cloud/operators/mlengine.py +0 -112
  231. apache_airflow_providers_google-15.1.0rc1.dist-info/RECORD +0 -321
  232. {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +0 -0
  233. {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
  234. {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -18,15 +18,32 @@
18
18
 
19
19
  from __future__ import annotations
20
20
 
21
+ from collections.abc import Sequence
22
+ from typing import (
23
+ TYPE_CHECKING,
24
+ )
25
+
21
26
  from google.api_core.client_options import ClientOptions
27
+ from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
22
28
  from google.cloud.aiplatform_v1beta1 import (
23
29
  FeatureOnlineStoreAdminServiceClient,
30
+ FeatureOnlineStoreServiceClient,
24
31
  )
25
32
 
26
33
  from airflow.exceptions import AirflowException
27
34
  from airflow.providers.google.common.consts import CLIENT_INFO
28
35
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
29
36
 
37
+ if TYPE_CHECKING:
38
+ from google.api_core.operation import Operation
39
+ from google.api_core.retry import Retry
40
+ from google.cloud.aiplatform_v1beta1.types import (
41
+ FeatureOnlineStore,
42
+ FeatureView,
43
+ FeatureViewDataKey,
44
+ FetchFeatureValuesResponse,
45
+ )
46
+
30
47
 
31
48
  class FeatureStoreHook(GoogleBaseHook):
32
49
  """
@@ -48,6 +65,19 @@ class FeatureStoreHook(GoogleBaseHook):
48
65
  originating account.
49
66
  """
50
67
 
68
+ @staticmethod
69
+ def _get_client_options(
70
+ location: str | None = None,
71
+ custom_endpoint: str | None = None,
72
+ ) -> ClientOptions:
73
+ if custom_endpoint:
74
+ client_options = ClientOptions(api_endpoint=custom_endpoint)
75
+ elif location and location != "global":
76
+ client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
77
+ else:
78
+ client_options = ClientOptions()
79
+ return client_options
80
+
51
81
  def get_feature_online_store_admin_service_client(
52
82
  self,
53
83
  location: str | None = None,
@@ -62,12 +92,153 @@ class FeatureStoreHook(GoogleBaseHook):
62
92
  If provided and not 'global', the client will be configured to use the
63
93
  region-specific API endpoint.
64
94
  """
65
- if location and location != "global":
66
- client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
67
- else:
68
- client_options = ClientOptions()
69
95
  return FeatureOnlineStoreAdminServiceClient(
70
- credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options
96
+ credentials=self.get_credentials(),
97
+ client_info=CLIENT_INFO,
98
+ client_options=self._get_client_options(location),
99
+ )
100
+
101
+ def get_feature_online_store_service_client(
102
+ self,
103
+ location: str | None = None,
104
+ custom_endpoint: str | None = None,
105
+ ) -> FeatureOnlineStoreServiceClient:
106
+ return FeatureOnlineStoreServiceClient(
107
+ credentials=self.get_credentials(),
108
+ client_info=CLIENT_INFO,
109
+ client_options=self._get_client_options(location, custom_endpoint),
110
+ )
111
+
112
+ @GoogleBaseHook.fallback_to_default_project_id
113
+ def create_feature_online_store(
114
+ self,
115
+ feature_online_store_id: str,
116
+ feature_online_store: FeatureOnlineStore,
117
+ project_id: str = PROVIDE_PROJECT_ID,
118
+ location: str | None = None,
119
+ timeout: float | _MethodDefault = DEFAULT,
120
+ retry: Retry | _MethodDefault | None = DEFAULT,
121
+ metadata: Sequence[tuple[str, str]] = (),
122
+ ) -> Operation:
123
+ """
124
+ Create and sends request for Feature Online store.
125
+
126
+ This method initiates VertexAI Feature Online Store creation request.
127
+ Feature Online Store aims to serve and manage features data as a part of VertexAI MLOps.
128
+
129
+ :param feature_online_store_id: The ID of the online feature store.
130
+ :param feature_online_store: The configuration of the online repository.
131
+ :param project_id: The ID of the Google Cloud project that contains the
132
+ feature store. If not provided, will attempt to determine from the environment.
133
+ :param location: The Google Cloud region where the feature store is located
134
+ (e.g., 'us-central1', 'us-east1').
135
+ :param retry: Designation of what errors, if any, should be retried.
136
+ :param timeout: The timeout for this request.
137
+ :param metadata: Strings which should be sent along with the request as metadata.
138
+ """
139
+ client = self.get_feature_online_store_admin_service_client(location)
140
+ return client.create_feature_online_store(
141
+ request={
142
+ "parent": f"projects/{project_id}/locations/{location}",
143
+ "feature_online_store_id": feature_online_store_id,
144
+ "feature_online_store": feature_online_store,
145
+ },
146
+ timeout=timeout,
147
+ retry=retry,
148
+ metadata=metadata,
149
+ )
150
+
151
+ @GoogleBaseHook.fallback_to_default_project_id
152
+ def get_feature_online_store(
153
+ self,
154
+ feature_online_store_id: str,
155
+ project_id: str = PROVIDE_PROJECT_ID,
156
+ location: str | None = None,
157
+ timeout: float | _MethodDefault = DEFAULT,
158
+ retry: Retry | _MethodDefault | None = DEFAULT,
159
+ metadata: Sequence[tuple[str, str]] = (),
160
+ ) -> FeatureOnlineStore:
161
+ """
162
+ Get Feature Online store data.
163
+
164
+ Get the FeatureOnlineStore details.
165
+ Vertex AI Feature Online Store provides a centralized repository for serving ML features
166
+ and embedding indexes at low latency.
167
+
168
+ :param feature_online_store_id: The ID of the online feature store.
169
+ :param project_id: The ID of the Google Cloud project that contains the
170
+ feature store. If not provided, will attempt to determine from the environment.
171
+ :param location: The Google Cloud region where the feature store is located
172
+ (e.g., 'us-central1', 'us-east1').
173
+ :param retry: Designation of what errors, if any, should be retried.
174
+ :param timeout: The timeout for this request.
175
+ :param metadata: Strings which should be sent along with the request as metadata.
176
+ """
177
+ client = self.get_feature_online_store_admin_service_client(location)
178
+ return client.get_feature_online_store(
179
+ name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
180
+ timeout=timeout,
181
+ retry=retry,
182
+ metadata=metadata,
183
+ )
184
+
185
+ @staticmethod
186
+ def _get_featurestore_public_endpoint(feature_online_store: FeatureOnlineStore):
187
+ public_endpoint = None
188
+ featurestore_data = type(feature_online_store).to_dict(feature_online_store)
189
+ if "dedicated_serving_endpoint" in featurestore_data:
190
+ public_endpoint = featurestore_data["dedicated_serving_endpoint"].get(
191
+ "public_endpoint_domain_name"
192
+ )
193
+ return public_endpoint
194
+
195
+ @GoogleBaseHook.fallback_to_default_project_id
196
+ def create_feature_view(
197
+ self,
198
+ feature_view_id: str,
199
+ feature_view: FeatureView,
200
+ feature_online_store_id: str,
201
+ project_id: str = PROVIDE_PROJECT_ID,
202
+ run_sync_immediately: bool = False,
203
+ location: str | None = None,
204
+ timeout: float | _MethodDefault = DEFAULT,
205
+ retry: Retry | _MethodDefault | None = DEFAULT,
206
+ metadata: Sequence[tuple[str, str]] = (),
207
+ ) -> Operation:
208
+ """
209
+ Create request for Feature View creation.
210
+
211
+ This method initiates VertexAI Feature View request for the existing Feature Online Store.
212
+ Feature View represents features and data according to the source provided.
213
+
214
+ :param feature_view_id: The ID to use for the FeatureView, which will
215
+ become the final component of the FeatureView's resource name.
216
+ This value may be up to 60 characters, and valid characters are ``[a-z0-9_]``.
217
+ The first character cannot be a number. The value must be unique within a FeatureOnlineStore.
218
+ :param feature_view: The configuration of the FeatureView to create.
219
+ :param feature_online_store_id: The ID of the online feature store.
220
+ :param run_sync_immediately: If set to true, one on demand sync will be run
221
+ immediately, regardless the FeatureView.sync_config.
222
+ :param project_id: The ID of the Google Cloud project that contains the
223
+ feature store. If not provided, will attempt to determine from the environment.
224
+ :param location: The Google Cloud region where the feature store is located
225
+ (e.g., 'us-central1', 'us-east1').
226
+ :param retry: Designation of what errors, if any, should be retried.
227
+ :param timeout: The timeout for this request.
228
+ :param metadata: Strings which should be sent along with the request as metadata.
229
+ """
230
+ client = self.get_feature_online_store_admin_service_client(location)
231
+ return client.create_feature_view(
232
+ request={
233
+ "parent": f"projects/{project_id}/locations/"
234
+ f"{location}/featureOnlineStores/{feature_online_store_id}",
235
+ "feature_view_id": feature_view_id,
236
+ "feature_view": feature_view,
237
+ "run_sync_immediately": run_sync_immediately,
238
+ },
239
+ timeout=timeout,
240
+ retry=retry,
241
+ metadata=metadata,
71
242
  )
72
243
 
73
244
  def get_feature_view_sync(
@@ -135,13 +306,142 @@ class FeatureStoreHook(GoogleBaseHook):
135
306
  environment.
136
307
  """
137
308
  client = self.get_feature_online_store_admin_service_client(location)
138
- feature_view = f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}/featureViews/{feature_view_id}"
309
+ feature_view = (
310
+ f"projects/{project_id}/locations/{location}/featureOnlineStores/"
311
+ f"{feature_online_store_id}/featureViews/{feature_view_id}"
312
+ )
139
313
 
140
314
  try:
141
315
  response = client.sync_feature_view(feature_view=feature_view)
142
-
143
316
  return str(response.feature_view_sync)
144
317
 
145
318
  except Exception as e:
146
319
  self.log.error("Failed to sync feature view: %s", str(e))
147
320
  raise AirflowException(str(e))
321
+
322
+ @GoogleBaseHook.fallback_to_default_project_id
323
+ def fetch_feature_values(
324
+ self,
325
+ feature_view_id: str,
326
+ feature_online_store_id: str,
327
+ entity_id: str | None = None,
328
+ project_id: str = PROVIDE_PROJECT_ID,
329
+ endpoint_domain_name: str | None = None,
330
+ data_key: FeatureViewDataKey | None = None,
331
+ data_format: int | None = None,
332
+ location: str | None = None,
333
+ timeout: float | _MethodDefault = DEFAULT,
334
+ retry: Retry | _MethodDefault | None = DEFAULT,
335
+ metadata: Sequence[tuple[str, str]] = (),
336
+ ) -> FetchFeatureValuesResponse:
337
+ """
338
+ Fetch data from the Feature View provided.
339
+
340
+ This method fetches data from existing Feature view, filtered by provided (or default) data_key.
341
+ Helps to retrieve actual features data hosted in the VertexAI Feature Store.
342
+
343
+ :param entity_id: Simple ID to identify Entity to fetch feature values for.
344
+ :param endpoint_domain_name: Optional. Public domain name, hosting the content of Optimized
345
+ Feature Online store. Should be omitted, if bigtable configuration provided for the FeatureStore,
346
+ and default feature store endpoint will be used, based on location provided.
347
+ :param feature_view_id: The FeatureView ID to fetch data from.
348
+ :param feature_online_store_id: The ID of the online feature store.
349
+ :param data_key: Optional. The request key to fetch feature values for.
350
+ :param data_format: Optional. Response data format. If not set, FeatureViewDataFormat.KEY_VALUE
351
+ will be used.
352
+ :param project_id: The ID of the Google Cloud project that contains the
353
+ feature store. If not provided, will attempt to determine from the
354
+ environment.
355
+ :param location: The Google Cloud region where the feature store is located
356
+ (e.g., 'us-central1', 'us-east1').
357
+ :param retry: Designation of what errors, if any, should be retried.
358
+ :param timeout: The timeout for this request.
359
+ :param metadata: Strings which should be sent along with the request as metadata.
360
+ """
361
+ data_client = self.get_feature_online_store_service_client(location, endpoint_domain_name)
362
+ return data_client.fetch_feature_values(
363
+ request={
364
+ "id": entity_id,
365
+ "feature_view": f"projects/{project_id}/locations/{location}/featureOnlineStores/"
366
+ f"{feature_online_store_id}/featureViews/{feature_view_id}",
367
+ "data_key": data_key,
368
+ "data_format": data_format,
369
+ },
370
+ timeout=timeout,
371
+ retry=retry,
372
+ metadata=metadata,
373
+ )
374
+
375
+ @GoogleBaseHook.fallback_to_default_project_id
376
+ def delete_feature_view(
377
+ self,
378
+ feature_view_id: str,
379
+ feature_online_store_id: str,
380
+ project_id: str = PROVIDE_PROJECT_ID,
381
+ location: str | None = None,
382
+ timeout: float | _MethodDefault = DEFAULT,
383
+ retry: Retry | _MethodDefault | None = DEFAULT,
384
+ metadata: Sequence[tuple[str, str]] = (),
385
+ ) -> Operation:
386
+ """
387
+ Delete the Feature View.
388
+
389
+ This method deletes the Feature View from the Feature Online Store.
390
+
391
+ :param feature_view_id: The ID to use for the FeatureView, to be deleted.
392
+ :param feature_online_store_id: The ID of the online feature store.
393
+ :param project_id: The ID of the Google Cloud project that contains the
394
+ feature store. If not provided, will attempt to determine from the
395
+ environment.
396
+ :param location: The Google Cloud region where the feature store is located
397
+ (e.g., 'us-central1', 'us-east1').
398
+ :param retry: Designation of what errors, if any, should be retried.
399
+ :param timeout: The timeout for this request.
400
+ :param metadata: Strings which should be sent along with the request as metadata.
401
+ """
402
+ client = self.get_feature_online_store_admin_service_client(location)
403
+ return client.delete_feature_view(
404
+ name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}"
405
+ f"/featureViews/{feature_view_id}",
406
+ timeout=timeout,
407
+ retry=retry,
408
+ metadata=metadata,
409
+ )
410
+
411
+ @GoogleBaseHook.fallback_to_default_project_id
412
+ def delete_feature_online_store(
413
+ self,
414
+ feature_online_store_id: str,
415
+ force: bool = False,
416
+ project_id: str = PROVIDE_PROJECT_ID,
417
+ location: str | None = None,
418
+ timeout: float | _MethodDefault = DEFAULT,
419
+ retry: Retry | _MethodDefault | None = DEFAULT,
420
+ metadata: Sequence[tuple[str, str]] = (),
421
+ ) -> Operation:
422
+ """
423
+ Delete the FeatureOnlineStore.
424
+
425
+ This method deletes the Feature Online Store and all features data.
426
+ The FeatureOnlineStore must not contain any FeatureViews.
427
+
428
+ :param feature_online_store_id: The ID of the online feature store.
429
+ :param force: If set to true, any FeatureViews and Features for this FeatureOnlineStore
430
+ will also be deleted.
431
+ :param project_id: The ID of the Google Cloud project that contains the
432
+ feature store. If not provided, will attempt to determine from the
433
+ environment.
434
+ :param location: The Google Cloud region where the feature store is located
435
+ (e.g., 'us-central1', 'us-east1').
436
+ :param retry: Designation of what errors, if any, should be retried.
437
+ :param timeout: The timeout for this request.
438
+ :param metadata: Strings which should be sent along with the request as metadata.
439
+ """
440
+ client = self.get_feature_online_store_admin_service_client(location)
441
+ return client.delete_feature_online_store(
442
+ name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
443
+ force=force,
444
+ timeout=timeout,
445
+ retry=retry,
446
+ metadata=metadata,
447
+ )
@@ -21,14 +21,15 @@ from __future__ import annotations
21
21
 
22
22
  import time
23
23
  from datetime import timedelta
24
- from typing import TYPE_CHECKING
24
+ from typing import TYPE_CHECKING, Any, Literal
25
25
 
26
26
  import vertexai
27
+ from google.cloud import aiplatform
27
28
  from vertexai.generative_models import GenerativeModel
28
- from vertexai.language_models import TextEmbeddingModel, TextGenerationModel
29
+ from vertexai.language_models import TextEmbeddingModel
30
+ from vertexai.preview import generative_models as preview_generative_model
29
31
  from vertexai.preview.caching import CachedContent
30
32
  from vertexai.preview.evaluation import EvalResult, EvalTask
31
- from vertexai.preview.generative_models import GenerativeModel as preview_generative_model
32
33
  from vertexai.preview.tuning import sft
33
34
 
34
35
  from airflow.exceptions import AirflowProviderDeprecationWarning
@@ -36,23 +37,12 @@ from airflow.providers.google.common.deprecated import deprecated
36
37
  from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
37
38
 
38
39
  if TYPE_CHECKING:
39
- from google.cloud.aiplatform_v1 import types as types_v1
40
40
  from google.cloud.aiplatform_v1beta1 import types as types_v1beta1
41
41
 
42
42
 
43
43
  class GenerativeModelHook(GoogleBaseHook):
44
44
  """Hook for Google Cloud Vertex AI Generative Model APIs."""
45
45
 
46
- @deprecated(
47
- planned_removal_date="April 09, 2025",
48
- use_instead="GenerativeModelHook.get_generative_model",
49
- category=AirflowProviderDeprecationWarning,
50
- )
51
- def get_text_generation_model(self, pretrained_model: str):
52
- """Return a Model Garden Model object based on Text Generation."""
53
- model = TextGenerationModel.from_pretrained(pretrained_model)
54
- return model
55
-
56
46
  def get_text_embedding_model(self, pretrained_model: str):
57
47
  """Return a Model Garden Model object based on Text Embedding."""
58
48
  model = TextEmbeddingModel.from_pretrained(pretrained_model)
@@ -61,7 +51,7 @@ class GenerativeModelHook(GoogleBaseHook):
61
51
  def get_generative_model(
62
52
  self,
63
53
  pretrained_model: str,
64
- system_instruction: str | None = None,
54
+ system_instruction: Any | None = None,
65
55
  generation_config: dict | None = None,
66
56
  safety_settings: dict | None = None,
67
57
  tools: list | None = None,
@@ -93,66 +83,18 @@ class GenerativeModelHook(GoogleBaseHook):
93
83
  def get_cached_context_model(
94
84
  self,
95
85
  cached_content_name: str,
96
- ) -> preview_generative_model:
86
+ ) -> Any:
97
87
  """Return a Generative Model with Cached Context."""
98
88
  cached_content = CachedContent(cached_content_name=cached_content_name)
99
89
 
100
- cached_context_model = preview_generative_model.from_cached_content(cached_content)
90
+ cached_context_model = preview_generative_model.GenerativeModel.from_cached_content(cached_content)
101
91
  return cached_context_model
102
92
 
103
93
  @deprecated(
104
- planned_removal_date="April 09, 2025",
105
- use_instead="GenerativeModelHook.generative_model_generate_content",
94
+ planned_removal_date="January 3, 2026",
95
+ use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.embed_content",
106
96
  category=AirflowProviderDeprecationWarning,
107
97
  )
108
- @GoogleBaseHook.fallback_to_default_project_id
109
- def text_generation_model_predict(
110
- self,
111
- prompt: str,
112
- pretrained_model: str,
113
- temperature: float,
114
- max_output_tokens: int,
115
- top_p: float,
116
- top_k: int,
117
- location: str,
118
- project_id: str = PROVIDE_PROJECT_ID,
119
- ) -> str:
120
- """
121
- Use the Vertex AI PaLM API to generate natural language text.
122
-
123
- :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
124
- :param location: Required. The ID of the Google Cloud location that the service belongs to.
125
- :param prompt: Required. Inputs or queries that a user or a program gives
126
- to the Vertex AI PaLM API, in order to elicit a specific response.
127
- :param pretrained_model: A pre-trained model optimized for performing natural
128
- language tasks such as classification, summarization, extraction, content
129
- creation, and ideation.
130
- :param temperature: Temperature controls the degree of randomness in token
131
- selection.
132
- :param max_output_tokens: Token limit determines the maximum amount of text
133
- output.
134
- :param top_p: Tokens are selected from most probable to least until the sum
135
- of their probabilities equals the top_p value. Defaults to 0.8.
136
- :param top_k: A top_k of 1 means the selected token is the most probable
137
- among all tokens.
138
- """
139
- vertexai.init(project=project_id, location=location, credentials=self.get_credentials())
140
-
141
- parameters = {
142
- "temperature": temperature,
143
- "max_output_tokens": max_output_tokens,
144
- "top_p": top_p,
145
- "top_k": top_k,
146
- }
147
-
148
- model = self.get_text_generation_model(pretrained_model)
149
-
150
- response = model.predict(
151
- prompt=prompt,
152
- **parameters,
153
- )
154
- return response.text
155
-
156
98
  @GoogleBaseHook.fallback_to_default_project_id
157
99
  def text_embedding_model_get_embeddings(
158
100
  self,
@@ -177,16 +119,21 @@ class GenerativeModelHook(GoogleBaseHook):
177
119
 
178
120
  return response.values
179
121
 
122
+ @deprecated(
123
+ planned_removal_date="January 3, 2026",
124
+ use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.generate_content",
125
+ category=AirflowProviderDeprecationWarning,
126
+ )
180
127
  @GoogleBaseHook.fallback_to_default_project_id
181
128
  def generative_model_generate_content(
182
129
  self,
183
130
  contents: list,
184
131
  location: str,
132
+ pretrained_model: str,
185
133
  tools: list | None = None,
186
134
  generation_config: dict | None = None,
187
135
  safety_settings: dict | None = None,
188
136
  system_instruction: str | None = None,
189
- pretrained_model: str = "gemini-pro",
190
137
  project_id: str = PROVIDE_PROJECT_ID,
191
138
  ) -> str:
192
139
  """
@@ -200,7 +147,7 @@ class GenerativeModelHook(GoogleBaseHook):
200
147
  :param safety_settings: Optional. Per request settings for blocking unsafe content.
201
148
  :param tools: Optional. A list of tools available to the model during evaluation, such as a data store.
202
149
  :param system_instruction: Optional. An instruction given to the model to guide its behavior.
203
- :param pretrained_model: By default uses the pre-trained model `gemini-pro`,
150
+ :param pretrained_model: Required. Model,
204
151
  supporting prompts with text-only input, including natural language
205
152
  tasks, multi-turn text and code chat, and code generation. It can
206
153
  output text and code.
@@ -219,6 +166,11 @@ class GenerativeModelHook(GoogleBaseHook):
219
166
 
220
167
  return response.text
221
168
 
169
+ @deprecated(
170
+ planned_removal_date="January 3, 2026",
171
+ use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.supervised_fine_tuning_train",
172
+ category=AirflowProviderDeprecationWarning,
173
+ )
222
174
  @GoogleBaseHook.fallback_to_default_project_id
223
175
  def supervised_fine_tuning_train(
224
176
  self,
@@ -228,10 +180,10 @@ class GenerativeModelHook(GoogleBaseHook):
228
180
  tuned_model_display_name: str | None = None,
229
181
  validation_dataset: str | None = None,
230
182
  epochs: int | None = None,
231
- adapter_size: int | None = None,
183
+ adapter_size: Literal[1, 4, 8, 16] | None = None,
232
184
  learning_rate_multiplier: float | None = None,
233
185
  project_id: str = PROVIDE_PROJECT_ID,
234
- ) -> types_v1.TuningJob:
186
+ ) -> Any:
235
187
  """
236
188
  Use the Supervised Fine Tuning API to create a tuning job.
237
189
 
@@ -272,12 +224,17 @@ class GenerativeModelHook(GoogleBaseHook):
272
224
 
273
225
  return sft_tuning_job
274
226
 
227
+ @deprecated(
228
+ planned_removal_date="January 3, 2026",
229
+ use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.count_tokens",
230
+ category=AirflowProviderDeprecationWarning,
231
+ )
275
232
  @GoogleBaseHook.fallback_to_default_project_id
276
233
  def count_tokens(
277
234
  self,
278
235
  contents: list,
279
236
  location: str,
280
- pretrained_model: str = "gemini-pro",
237
+ pretrained_model: str,
281
238
  project_id: str = PROVIDE_PROJECT_ID,
282
239
  ) -> types_v1beta1.CountTokensResponse:
283
240
  """
@@ -287,7 +244,7 @@ class GenerativeModelHook(GoogleBaseHook):
287
244
  :param location: Required. The ID of the Google Cloud location that the service belongs to.
288
245
  :param contents: Required. The multi-part content of a message that a user or a program
289
246
  gives to the generative model, in order to elicit a specific response.
290
- :param pretrained_model: By default uses the pre-trained model `gemini-pro`,
247
+ :param pretrained_model: Required. Model,
291
248
  supporting prompts with text-only input, including natural language
292
249
  tasks, multi-turn text and code chat, and code generation. It can
293
250
  output text and code.
@@ -359,13 +316,18 @@ class GenerativeModelHook(GoogleBaseHook):
359
316
 
360
317
  return eval_result
361
318
 
319
+ @deprecated(
320
+ planned_removal_date="January 3, 2026",
321
+ use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.create_cached_content",
322
+ category=AirflowProviderDeprecationWarning,
323
+ )
362
324
  def create_cached_content(
363
325
  self,
364
326
  model_name: str,
365
327
  location: str,
366
328
  ttl_hours: float = 1,
367
- system_instruction: str | None = None,
368
- contents: list | None = None,
329
+ system_instruction: Any | None = None,
330
+ contents: list[Any] | None = None,
369
331
  display_name: str | None = None,
370
332
  project_id: str = PROVIDE_PROJECT_ID,
371
333
  ) -> str:
@@ -393,6 +355,11 @@ class GenerativeModelHook(GoogleBaseHook):
393
355
 
394
356
  return response.name
395
357
 
358
+ @deprecated(
359
+ planned_removal_date="January 3, 2026",
360
+ use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.generate_content",
361
+ category=AirflowProviderDeprecationWarning,
362
+ )
396
363
  def generate_from_cached_content(
397
364
  self,
398
365
  location: str,
@@ -413,6 +380,9 @@ class GenerativeModelHook(GoogleBaseHook):
413
380
  :param generation_config: Optional. Generation configuration settings.
414
381
  :param safety_settings: Optional. Per request settings for blocking unsafe content.
415
382
  """
383
+ # During run of the system test it was found out that names from xcom, e.g. 3402922389 can be
384
+ # treated as int and throw an error TypeError: expected string or bytes-like object, got 'int'
385
+ cached_content_name = str(cached_content_name)
416
386
  vertexai.init(project=project_id, location=location, credentials=self.get_credentials())
417
387
 
418
388
  cached_context_model = self.get_cached_context_model(cached_content_name=cached_content_name)
@@ -424,3 +394,37 @@ class GenerativeModelHook(GoogleBaseHook):
424
394
  )
425
395
 
426
396
  return response.text
397
+
398
+
399
+ @deprecated(
400
+ planned_removal_date="January 3, 2026",
401
+ use_instead="airflow.providers.google.cloud.hooks.vertex_ai.experiment_service.ExperimentRunHook",
402
+ category=AirflowProviderDeprecationWarning,
403
+ )
404
+ class ExperimentRunHook(GoogleBaseHook):
405
+ """Use the Vertex AI SDK for Python to create and manage your experiment runs."""
406
+
407
+ @GoogleBaseHook.fallback_to_default_project_id
408
+ def delete_experiment_run(
409
+ self,
410
+ experiment_run_name: str,
411
+ experiment_name: str,
412
+ location: str,
413
+ project_id: str = PROVIDE_PROJECT_ID,
414
+ delete_backing_tensorboard_run: bool = False,
415
+ ) -> None:
416
+ """
417
+ Delete experiment run from the experiment.
418
+
419
+ :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
420
+ :param location: Required. The ID of the Google Cloud location that the service belongs to.
421
+ :param experiment_name: Required. The name of the evaluation experiment.
422
+ :param experiment_run_name: Required. The specific run name or ID for this experiment.
423
+ :param delete_backing_tensorboard_run: Whether to delete the backing Vertex AI TensorBoard run
424
+ that stores time series metrics for this run.
425
+ """
426
+ self.log.info("Next experiment run will be deleted: %s", experiment_run_name)
427
+ experiment_run = aiplatform.ExperimentRun(
428
+ run_name=experiment_run_name, experiment=experiment_name, project=project_id, location=location
429
+ )
430
+ experiment_run.delete(delete_backing_tensorboard_run=delete_backing_tensorboard_run)