apache-airflow-providers-google 15.1.0rc1__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/3rd-party-licenses/NOTICE +2 -12
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/ads/hooks/ads.py +39 -5
- airflow/providers/google/ads/operators/ads.py +2 -2
- airflow/providers/google/ads/transfers/ads_to_gcs.py +2 -2
- airflow/providers/google/assets/gcs.py +1 -11
- airflow/providers/google/cloud/bundles/__init__.py +16 -0
- airflow/providers/google/cloud/bundles/gcs.py +161 -0
- airflow/providers/google/cloud/hooks/bigquery.py +166 -281
- airflow/providers/google/cloud/hooks/cloud_composer.py +287 -14
- airflow/providers/google/cloud/hooks/cloud_logging.py +109 -0
- airflow/providers/google/cloud/hooks/cloud_run.py +17 -9
- airflow/providers/google/cloud/hooks/cloud_sql.py +101 -22
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +27 -6
- airflow/providers/google/cloud/hooks/compute_ssh.py +5 -1
- airflow/providers/google/cloud/hooks/datacatalog.py +9 -1
- airflow/providers/google/cloud/hooks/dataflow.py +71 -94
- airflow/providers/google/cloud/hooks/datafusion.py +1 -1
- airflow/providers/google/cloud/hooks/dataplex.py +1 -1
- airflow/providers/google/cloud/hooks/dataprep.py +1 -1
- airflow/providers/google/cloud/hooks/dataproc.py +72 -71
- airflow/providers/google/cloud/hooks/gcs.py +111 -14
- airflow/providers/google/cloud/hooks/gen_ai.py +196 -0
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/hooks/looker.py +6 -1
- airflow/providers/google/cloud/hooks/mlengine.py +3 -2
- airflow/providers/google/cloud/hooks/secret_manager.py +102 -10
- airflow/providers/google/cloud/hooks/spanner.py +73 -8
- airflow/providers/google/cloud/hooks/stackdriver.py +10 -8
- airflow/providers/google/cloud/hooks/translate.py +1 -1
- airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py +0 -209
- airflow/providers/google/cloud/hooks/vertex_ai/batch_prediction_job.py +2 -2
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +27 -1
- airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py +202 -0
- airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py +307 -7
- airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py +79 -75
- airflow/providers/google/cloud/hooks/vertex_ai/ray.py +223 -0
- airflow/providers/google/cloud/hooks/vision.py +2 -2
- airflow/providers/google/cloud/hooks/workflows.py +1 -1
- airflow/providers/google/cloud/links/alloy_db.py +0 -46
- airflow/providers/google/cloud/links/base.py +77 -13
- airflow/providers/google/cloud/links/bigquery.py +0 -47
- airflow/providers/google/cloud/links/bigquery_dts.py +0 -20
- airflow/providers/google/cloud/links/bigtable.py +0 -48
- airflow/providers/google/cloud/links/cloud_build.py +0 -73
- airflow/providers/google/cloud/links/cloud_functions.py +0 -33
- airflow/providers/google/cloud/links/cloud_memorystore.py +0 -58
- airflow/providers/google/cloud/links/{life_sciences.py → cloud_run.py} +5 -27
- airflow/providers/google/cloud/links/cloud_sql.py +0 -33
- airflow/providers/google/cloud/links/cloud_storage_transfer.py +17 -44
- airflow/providers/google/cloud/links/cloud_tasks.py +7 -26
- airflow/providers/google/cloud/links/compute.py +0 -58
- airflow/providers/google/cloud/links/data_loss_prevention.py +0 -169
- airflow/providers/google/cloud/links/datacatalog.py +23 -54
- airflow/providers/google/cloud/links/dataflow.py +0 -34
- airflow/providers/google/cloud/links/dataform.py +0 -64
- airflow/providers/google/cloud/links/datafusion.py +1 -96
- airflow/providers/google/cloud/links/dataplex.py +0 -154
- airflow/providers/google/cloud/links/dataprep.py +0 -24
- airflow/providers/google/cloud/links/dataproc.py +11 -95
- airflow/providers/google/cloud/links/datastore.py +0 -31
- airflow/providers/google/cloud/links/kubernetes_engine.py +9 -60
- airflow/providers/google/cloud/links/managed_kafka.py +0 -70
- airflow/providers/google/cloud/links/mlengine.py +0 -70
- airflow/providers/google/cloud/links/pubsub.py +0 -32
- airflow/providers/google/cloud/links/spanner.py +0 -33
- airflow/providers/google/cloud/links/stackdriver.py +0 -30
- airflow/providers/google/cloud/links/translate.py +17 -187
- airflow/providers/google/cloud/links/vertex_ai.py +28 -195
- airflow/providers/google/cloud/links/workflows.py +0 -52
- airflow/providers/google/cloud/log/gcs_task_handler.py +17 -9
- airflow/providers/google/cloud/log/stackdriver_task_handler.py +9 -6
- airflow/providers/google/cloud/openlineage/CloudStorageTransferJobFacet.json +68 -0
- airflow/providers/google/cloud/openlineage/CloudStorageTransferRunFacet.json +60 -0
- airflow/providers/google/cloud/openlineage/DataFusionRunFacet.json +32 -0
- airflow/providers/google/cloud/openlineage/facets.py +102 -1
- airflow/providers/google/cloud/openlineage/mixins.py +10 -8
- airflow/providers/google/cloud/openlineage/utils.py +15 -1
- airflow/providers/google/cloud/operators/alloy_db.py +70 -55
- airflow/providers/google/cloud/operators/bigquery.py +73 -636
- airflow/providers/google/cloud/operators/bigquery_dts.py +3 -5
- airflow/providers/google/cloud/operators/bigtable.py +36 -7
- airflow/providers/google/cloud/operators/cloud_base.py +21 -1
- airflow/providers/google/cloud/operators/cloud_batch.py +2 -2
- airflow/providers/google/cloud/operators/cloud_build.py +75 -32
- airflow/providers/google/cloud/operators/cloud_composer.py +128 -40
- airflow/providers/google/cloud/operators/cloud_logging_sink.py +341 -0
- airflow/providers/google/cloud/operators/cloud_memorystore.py +69 -43
- airflow/providers/google/cloud/operators/cloud_run.py +23 -5
- airflow/providers/google/cloud/operators/cloud_sql.py +8 -16
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +92 -11
- airflow/providers/google/cloud/operators/compute.py +8 -40
- airflow/providers/google/cloud/operators/datacatalog.py +157 -21
- airflow/providers/google/cloud/operators/dataflow.py +38 -15
- airflow/providers/google/cloud/operators/dataform.py +15 -5
- airflow/providers/google/cloud/operators/datafusion.py +41 -20
- airflow/providers/google/cloud/operators/dataplex.py +193 -109
- airflow/providers/google/cloud/operators/dataprep.py +1 -5
- airflow/providers/google/cloud/operators/dataproc.py +78 -35
- airflow/providers/google/cloud/operators/dataproc_metastore.py +96 -88
- airflow/providers/google/cloud/operators/datastore.py +22 -6
- airflow/providers/google/cloud/operators/dlp.py +6 -29
- airflow/providers/google/cloud/operators/functions.py +16 -7
- airflow/providers/google/cloud/operators/gcs.py +10 -8
- airflow/providers/google/cloud/operators/gen_ai.py +389 -0
- airflow/providers/google/cloud/operators/kubernetes_engine.py +60 -99
- airflow/providers/google/cloud/operators/looker.py +1 -1
- airflow/providers/google/cloud/operators/managed_kafka.py +107 -52
- airflow/providers/google/cloud/operators/natural_language.py +1 -1
- airflow/providers/google/cloud/operators/pubsub.py +60 -14
- airflow/providers/google/cloud/operators/spanner.py +25 -12
- airflow/providers/google/cloud/operators/speech_to_text.py +1 -2
- airflow/providers/google/cloud/operators/stackdriver.py +1 -9
- airflow/providers/google/cloud/operators/tasks.py +1 -12
- airflow/providers/google/cloud/operators/text_to_speech.py +1 -2
- airflow/providers/google/cloud/operators/translate.py +40 -16
- airflow/providers/google/cloud/operators/translate_speech.py +1 -2
- airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py +39 -19
- airflow/providers/google/cloud/operators/vertex_ai/batch_prediction_job.py +29 -9
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +54 -26
- airflow/providers/google/cloud/operators/vertex_ai/dataset.py +70 -8
- airflow/providers/google/cloud/operators/vertex_ai/endpoint_service.py +43 -9
- airflow/providers/google/cloud/operators/vertex_ai/experiment_service.py +435 -0
- airflow/providers/google/cloud/operators/vertex_ai/feature_store.py +532 -1
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +135 -116
- airflow/providers/google/cloud/operators/vertex_ai/hyperparameter_tuning_job.py +11 -9
- airflow/providers/google/cloud/operators/vertex_ai/model_service.py +57 -11
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +30 -7
- airflow/providers/google/cloud/operators/vertex_ai/ray.py +393 -0
- airflow/providers/google/cloud/operators/video_intelligence.py +1 -1
- airflow/providers/google/cloud/operators/vision.py +2 -2
- airflow/providers/google/cloud/operators/workflows.py +18 -15
- airflow/providers/google/cloud/sensors/bigquery.py +2 -2
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -2
- airflow/providers/google/cloud/sensors/bigtable.py +11 -4
- airflow/providers/google/cloud/sensors/cloud_composer.py +533 -29
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -2
- airflow/providers/google/cloud/sensors/dataflow.py +26 -9
- airflow/providers/google/cloud/sensors/dataform.py +2 -2
- airflow/providers/google/cloud/sensors/datafusion.py +4 -4
- airflow/providers/google/cloud/sensors/dataplex.py +2 -2
- airflow/providers/google/cloud/sensors/dataprep.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc.py +2 -2
- airflow/providers/google/cloud/sensors/dataproc_metastore.py +2 -2
- airflow/providers/google/cloud/sensors/gcs.py +4 -4
- airflow/providers/google/cloud/sensors/looker.py +2 -2
- airflow/providers/google/cloud/sensors/pubsub.py +4 -4
- airflow/providers/google/cloud/sensors/tasks.py +2 -2
- airflow/providers/google/cloud/sensors/vertex_ai/feature_store.py +2 -2
- airflow/providers/google/cloud/sensors/workflows.py +2 -2
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/bigquery_to_bigquery.py +11 -8
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +4 -4
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +7 -3
- airflow/providers/google/cloud/transfers/bigquery_to_mysql.py +12 -1
- airflow/providers/google/cloud/transfers/bigquery_to_postgres.py +24 -10
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +104 -5
- airflow/providers/google/cloud/transfers/calendar_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/cassandra_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/facebook_ads_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +20 -12
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/gcs_to_local.py +5 -3
- airflow/providers/google/cloud/transfers/gcs_to_sftp.py +10 -4
- airflow/providers/google/cloud/transfers/gdrive_to_gcs.py +6 -2
- airflow/providers/google/cloud/transfers/gdrive_to_local.py +2 -2
- airflow/providers/google/cloud/transfers/http_to_gcs.py +193 -0
- airflow/providers/google/cloud/transfers/local_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/mssql_to_gcs.py +1 -1
- airflow/providers/google/cloud/transfers/oracle_to_gcs.py +36 -11
- airflow/providers/google/cloud/transfers/postgres_to_gcs.py +42 -9
- airflow/providers/google/cloud/transfers/s3_to_gcs.py +12 -6
- airflow/providers/google/cloud/transfers/salesforce_to_gcs.py +2 -2
- airflow/providers/google/cloud/transfers/sftp_to_gcs.py +13 -4
- airflow/providers/google/cloud/transfers/sheets_to_gcs.py +3 -3
- airflow/providers/google/cloud/transfers/sql_to_gcs.py +10 -10
- airflow/providers/google/cloud/triggers/bigquery.py +75 -34
- airflow/providers/google/cloud/triggers/cloud_build.py +1 -1
- airflow/providers/google/cloud/triggers/cloud_composer.py +302 -46
- airflow/providers/google/cloud/triggers/cloud_run.py +2 -2
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +91 -1
- airflow/providers/google/cloud/triggers/dataflow.py +122 -0
- airflow/providers/google/cloud/triggers/datafusion.py +1 -1
- airflow/providers/google/cloud/triggers/dataplex.py +14 -2
- airflow/providers/google/cloud/triggers/dataproc.py +122 -52
- airflow/providers/google/cloud/triggers/kubernetes_engine.py +45 -27
- airflow/providers/google/cloud/triggers/mlengine.py +1 -1
- airflow/providers/google/cloud/triggers/pubsub.py +15 -19
- airflow/providers/google/cloud/utils/bigquery_get_data.py +1 -1
- airflow/providers/google/cloud/utils/credentials_provider.py +1 -1
- airflow/providers/google/cloud/utils/field_validator.py +1 -2
- airflow/providers/google/common/auth_backend/google_openid.py +4 -4
- airflow/providers/google/common/deprecated.py +2 -1
- airflow/providers/google/common/hooks/base_google.py +27 -8
- airflow/providers/google/common/links/storage.py +0 -22
- airflow/providers/google/common/utils/get_secret.py +31 -0
- airflow/providers/google/common/utils/id_token_credentials.py +3 -4
- airflow/providers/google/firebase/operators/firestore.py +2 -2
- airflow/providers/google/get_provider_info.py +56 -52
- airflow/providers/google/go_module_utils.py +35 -3
- airflow/providers/google/leveldb/hooks/leveldb.py +26 -1
- airflow/providers/google/leveldb/operators/leveldb.py +2 -2
- airflow/providers/google/marketing_platform/hooks/display_video.py +3 -109
- airflow/providers/google/marketing_platform/links/analytics_admin.py +5 -14
- airflow/providers/google/marketing_platform/operators/analytics_admin.py +1 -2
- airflow/providers/google/marketing_platform/operators/campaign_manager.py +5 -5
- airflow/providers/google/marketing_platform/operators/display_video.py +28 -489
- airflow/providers/google/marketing_platform/operators/search_ads.py +2 -2
- airflow/providers/google/marketing_platform/sensors/campaign_manager.py +2 -2
- airflow/providers/google/marketing_platform/sensors/display_video.py +3 -63
- airflow/providers/google/suite/hooks/calendar.py +1 -1
- airflow/providers/google/suite/hooks/sheets.py +15 -1
- airflow/providers/google/suite/operators/sheets.py +8 -3
- airflow/providers/google/suite/sensors/drive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_gdrive.py +2 -2
- airflow/providers/google/suite/transfers/gcs_to_sheets.py +1 -1
- airflow/providers/google/suite/transfers/local_to_drive.py +3 -3
- airflow/providers/google/suite/transfers/sql_to_sheets.py +5 -4
- airflow/providers/google/version_compat.py +15 -1
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/METADATA +92 -48
- apache_airflow_providers_google-19.1.0rc1.dist-info/RECORD +331 -0
- apache_airflow_providers_google-19.1.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/google/cloud/hooks/automl.py +0 -673
- airflow/providers/google/cloud/hooks/life_sciences.py +0 -159
- airflow/providers/google/cloud/links/automl.py +0 -193
- airflow/providers/google/cloud/operators/automl.py +0 -1362
- airflow/providers/google/cloud/operators/life_sciences.py +0 -119
- airflow/providers/google/cloud/operators/mlengine.py +0 -112
- apache_airflow_providers_google-15.1.0rc1.dist-info/RECORD +0 -321
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-15.1.0rc1.dist-info → apache_airflow_providers_google-19.1.0rc1.dist-info}/entry_points.txt +0 -0
- {airflow/providers/google → apache_airflow_providers_google-19.1.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -18,15 +18,32 @@
|
|
|
18
18
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
+
from collections.abc import Sequence
|
|
22
|
+
from typing import (
|
|
23
|
+
TYPE_CHECKING,
|
|
24
|
+
)
|
|
25
|
+
|
|
21
26
|
from google.api_core.client_options import ClientOptions
|
|
27
|
+
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
|
|
22
28
|
from google.cloud.aiplatform_v1beta1 import (
|
|
23
29
|
FeatureOnlineStoreAdminServiceClient,
|
|
30
|
+
FeatureOnlineStoreServiceClient,
|
|
24
31
|
)
|
|
25
32
|
|
|
26
33
|
from airflow.exceptions import AirflowException
|
|
27
34
|
from airflow.providers.google.common.consts import CLIENT_INFO
|
|
28
35
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
|
29
36
|
|
|
37
|
+
if TYPE_CHECKING:
|
|
38
|
+
from google.api_core.operation import Operation
|
|
39
|
+
from google.api_core.retry import Retry
|
|
40
|
+
from google.cloud.aiplatform_v1beta1.types import (
|
|
41
|
+
FeatureOnlineStore,
|
|
42
|
+
FeatureView,
|
|
43
|
+
FeatureViewDataKey,
|
|
44
|
+
FetchFeatureValuesResponse,
|
|
45
|
+
)
|
|
46
|
+
|
|
30
47
|
|
|
31
48
|
class FeatureStoreHook(GoogleBaseHook):
|
|
32
49
|
"""
|
|
@@ -48,6 +65,19 @@ class FeatureStoreHook(GoogleBaseHook):
|
|
|
48
65
|
originating account.
|
|
49
66
|
"""
|
|
50
67
|
|
|
68
|
+
@staticmethod
|
|
69
|
+
def _get_client_options(
|
|
70
|
+
location: str | None = None,
|
|
71
|
+
custom_endpoint: str | None = None,
|
|
72
|
+
) -> ClientOptions:
|
|
73
|
+
if custom_endpoint:
|
|
74
|
+
client_options = ClientOptions(api_endpoint=custom_endpoint)
|
|
75
|
+
elif location and location != "global":
|
|
76
|
+
client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
|
|
77
|
+
else:
|
|
78
|
+
client_options = ClientOptions()
|
|
79
|
+
return client_options
|
|
80
|
+
|
|
51
81
|
def get_feature_online_store_admin_service_client(
|
|
52
82
|
self,
|
|
53
83
|
location: str | None = None,
|
|
@@ -62,12 +92,153 @@ class FeatureStoreHook(GoogleBaseHook):
|
|
|
62
92
|
If provided and not 'global', the client will be configured to use the
|
|
63
93
|
region-specific API endpoint.
|
|
64
94
|
"""
|
|
65
|
-
if location and location != "global":
|
|
66
|
-
client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
|
|
67
|
-
else:
|
|
68
|
-
client_options = ClientOptions()
|
|
69
95
|
return FeatureOnlineStoreAdminServiceClient(
|
|
70
|
-
credentials=self.get_credentials(),
|
|
96
|
+
credentials=self.get_credentials(),
|
|
97
|
+
client_info=CLIENT_INFO,
|
|
98
|
+
client_options=self._get_client_options(location),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def get_feature_online_store_service_client(
|
|
102
|
+
self,
|
|
103
|
+
location: str | None = None,
|
|
104
|
+
custom_endpoint: str | None = None,
|
|
105
|
+
) -> FeatureOnlineStoreServiceClient:
|
|
106
|
+
return FeatureOnlineStoreServiceClient(
|
|
107
|
+
credentials=self.get_credentials(),
|
|
108
|
+
client_info=CLIENT_INFO,
|
|
109
|
+
client_options=self._get_client_options(location, custom_endpoint),
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
113
|
+
def create_feature_online_store(
|
|
114
|
+
self,
|
|
115
|
+
feature_online_store_id: str,
|
|
116
|
+
feature_online_store: FeatureOnlineStore,
|
|
117
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
118
|
+
location: str | None = None,
|
|
119
|
+
timeout: float | _MethodDefault = DEFAULT,
|
|
120
|
+
retry: Retry | _MethodDefault | None = DEFAULT,
|
|
121
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
122
|
+
) -> Operation:
|
|
123
|
+
"""
|
|
124
|
+
Create and sends request for Feature Online store.
|
|
125
|
+
|
|
126
|
+
This method initiates VertexAI Feature Online Store creation request.
|
|
127
|
+
Feature Online Store aims to serve and manage features data as a part of VertexAI MLOps.
|
|
128
|
+
|
|
129
|
+
:param feature_online_store_id: The ID of the online feature store.
|
|
130
|
+
:param feature_online_store: The configuration of the online repository.
|
|
131
|
+
:param project_id: The ID of the Google Cloud project that contains the
|
|
132
|
+
feature store. If not provided, will attempt to determine from the environment.
|
|
133
|
+
:param location: The Google Cloud region where the feature store is located
|
|
134
|
+
(e.g., 'us-central1', 'us-east1').
|
|
135
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
136
|
+
:param timeout: The timeout for this request.
|
|
137
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
138
|
+
"""
|
|
139
|
+
client = self.get_feature_online_store_admin_service_client(location)
|
|
140
|
+
return client.create_feature_online_store(
|
|
141
|
+
request={
|
|
142
|
+
"parent": f"projects/{project_id}/locations/{location}",
|
|
143
|
+
"feature_online_store_id": feature_online_store_id,
|
|
144
|
+
"feature_online_store": feature_online_store,
|
|
145
|
+
},
|
|
146
|
+
timeout=timeout,
|
|
147
|
+
retry=retry,
|
|
148
|
+
metadata=metadata,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
152
|
+
def get_feature_online_store(
|
|
153
|
+
self,
|
|
154
|
+
feature_online_store_id: str,
|
|
155
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
156
|
+
location: str | None = None,
|
|
157
|
+
timeout: float | _MethodDefault = DEFAULT,
|
|
158
|
+
retry: Retry | _MethodDefault | None = DEFAULT,
|
|
159
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
160
|
+
) -> FeatureOnlineStore:
|
|
161
|
+
"""
|
|
162
|
+
Get Feature Online store data.
|
|
163
|
+
|
|
164
|
+
Get the FeatureOnlineStore details.
|
|
165
|
+
Vertex AI Feature Online Store provides a centralized repository for serving ML features
|
|
166
|
+
and embedding indexes at low latency.
|
|
167
|
+
|
|
168
|
+
:param feature_online_store_id: The ID of the online feature store.
|
|
169
|
+
:param project_id: The ID of the Google Cloud project that contains the
|
|
170
|
+
feature store. If not provided, will attempt to determine from the environment.
|
|
171
|
+
:param location: The Google Cloud region where the feature store is located
|
|
172
|
+
(e.g., 'us-central1', 'us-east1').
|
|
173
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
174
|
+
:param timeout: The timeout for this request.
|
|
175
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
176
|
+
"""
|
|
177
|
+
client = self.get_feature_online_store_admin_service_client(location)
|
|
178
|
+
return client.get_feature_online_store(
|
|
179
|
+
name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
|
|
180
|
+
timeout=timeout,
|
|
181
|
+
retry=retry,
|
|
182
|
+
metadata=metadata,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
@staticmethod
|
|
186
|
+
def _get_featurestore_public_endpoint(feature_online_store: FeatureOnlineStore):
|
|
187
|
+
public_endpoint = None
|
|
188
|
+
featurestore_data = type(feature_online_store).to_dict(feature_online_store)
|
|
189
|
+
if "dedicated_serving_endpoint" in featurestore_data:
|
|
190
|
+
public_endpoint = featurestore_data["dedicated_serving_endpoint"].get(
|
|
191
|
+
"public_endpoint_domain_name"
|
|
192
|
+
)
|
|
193
|
+
return public_endpoint
|
|
194
|
+
|
|
195
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
196
|
+
def create_feature_view(
|
|
197
|
+
self,
|
|
198
|
+
feature_view_id: str,
|
|
199
|
+
feature_view: FeatureView,
|
|
200
|
+
feature_online_store_id: str,
|
|
201
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
202
|
+
run_sync_immediately: bool = False,
|
|
203
|
+
location: str | None = None,
|
|
204
|
+
timeout: float | _MethodDefault = DEFAULT,
|
|
205
|
+
retry: Retry | _MethodDefault | None = DEFAULT,
|
|
206
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
207
|
+
) -> Operation:
|
|
208
|
+
"""
|
|
209
|
+
Create request for Feature View creation.
|
|
210
|
+
|
|
211
|
+
This method initiates VertexAI Feature View request for the existing Feature Online Store.
|
|
212
|
+
Feature View represents features and data according to the source provided.
|
|
213
|
+
|
|
214
|
+
:param feature_view_id: The ID to use for the FeatureView, which will
|
|
215
|
+
become the final component of the FeatureView's resource name.
|
|
216
|
+
This value may be up to 60 characters, and valid characters are ``[a-z0-9_]``.
|
|
217
|
+
The first character cannot be a number. The value must be unique within a FeatureOnlineStore.
|
|
218
|
+
:param feature_view: The configuration of the FeatureView to create.
|
|
219
|
+
:param feature_online_store_id: The ID of the online feature store.
|
|
220
|
+
:param run_sync_immediately: If set to true, one on demand sync will be run
|
|
221
|
+
immediately, regardless the FeatureView.sync_config.
|
|
222
|
+
:param project_id: The ID of the Google Cloud project that contains the
|
|
223
|
+
feature store. If not provided, will attempt to determine from the environment.
|
|
224
|
+
:param location: The Google Cloud region where the feature store is located
|
|
225
|
+
(e.g., 'us-central1', 'us-east1').
|
|
226
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
227
|
+
:param timeout: The timeout for this request.
|
|
228
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
229
|
+
"""
|
|
230
|
+
client = self.get_feature_online_store_admin_service_client(location)
|
|
231
|
+
return client.create_feature_view(
|
|
232
|
+
request={
|
|
233
|
+
"parent": f"projects/{project_id}/locations/"
|
|
234
|
+
f"{location}/featureOnlineStores/{feature_online_store_id}",
|
|
235
|
+
"feature_view_id": feature_view_id,
|
|
236
|
+
"feature_view": feature_view,
|
|
237
|
+
"run_sync_immediately": run_sync_immediately,
|
|
238
|
+
},
|
|
239
|
+
timeout=timeout,
|
|
240
|
+
retry=retry,
|
|
241
|
+
metadata=metadata,
|
|
71
242
|
)
|
|
72
243
|
|
|
73
244
|
def get_feature_view_sync(
|
|
@@ -135,13 +306,142 @@ class FeatureStoreHook(GoogleBaseHook):
|
|
|
135
306
|
environment.
|
|
136
307
|
"""
|
|
137
308
|
client = self.get_feature_online_store_admin_service_client(location)
|
|
138
|
-
feature_view =
|
|
309
|
+
feature_view = (
|
|
310
|
+
f"projects/{project_id}/locations/{location}/featureOnlineStores/"
|
|
311
|
+
f"{feature_online_store_id}/featureViews/{feature_view_id}"
|
|
312
|
+
)
|
|
139
313
|
|
|
140
314
|
try:
|
|
141
315
|
response = client.sync_feature_view(feature_view=feature_view)
|
|
142
|
-
|
|
143
316
|
return str(response.feature_view_sync)
|
|
144
317
|
|
|
145
318
|
except Exception as e:
|
|
146
319
|
self.log.error("Failed to sync feature view: %s", str(e))
|
|
147
320
|
raise AirflowException(str(e))
|
|
321
|
+
|
|
322
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
323
|
+
def fetch_feature_values(
|
|
324
|
+
self,
|
|
325
|
+
feature_view_id: str,
|
|
326
|
+
feature_online_store_id: str,
|
|
327
|
+
entity_id: str | None = None,
|
|
328
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
329
|
+
endpoint_domain_name: str | None = None,
|
|
330
|
+
data_key: FeatureViewDataKey | None = None,
|
|
331
|
+
data_format: int | None = None,
|
|
332
|
+
location: str | None = None,
|
|
333
|
+
timeout: float | _MethodDefault = DEFAULT,
|
|
334
|
+
retry: Retry | _MethodDefault | None = DEFAULT,
|
|
335
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
336
|
+
) -> FetchFeatureValuesResponse:
|
|
337
|
+
"""
|
|
338
|
+
Fetch data from the Feature View provided.
|
|
339
|
+
|
|
340
|
+
This method fetches data from existing Feature view, filtered by provided (or default) data_key.
|
|
341
|
+
Helps to retrieve actual features data hosted in the VertexAI Feature Store.
|
|
342
|
+
|
|
343
|
+
:param entity_id: Simple ID to identify Entity to fetch feature values for.
|
|
344
|
+
:param endpoint_domain_name: Optional. Public domain name, hosting the content of Optimized
|
|
345
|
+
Feature Online store. Should be omitted, if bigtable configuration provided for the FeatureStore,
|
|
346
|
+
and default feature store endpoint will be used, based on location provided.
|
|
347
|
+
:param feature_view_id: The FeatureView ID to fetch data from.
|
|
348
|
+
:param feature_online_store_id: The ID of the online feature store.
|
|
349
|
+
:param data_key: Optional. The request key to fetch feature values for.
|
|
350
|
+
:param data_format: Optional. Response data format. If not set, FeatureViewDataFormat.KEY_VALUE
|
|
351
|
+
will be used.
|
|
352
|
+
:param project_id: The ID of the Google Cloud project that contains the
|
|
353
|
+
feature store. If not provided, will attempt to determine from the
|
|
354
|
+
environment.
|
|
355
|
+
:param location: The Google Cloud region where the feature store is located
|
|
356
|
+
(e.g., 'us-central1', 'us-east1').
|
|
357
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
358
|
+
:param timeout: The timeout for this request.
|
|
359
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
360
|
+
"""
|
|
361
|
+
data_client = self.get_feature_online_store_service_client(location, endpoint_domain_name)
|
|
362
|
+
return data_client.fetch_feature_values(
|
|
363
|
+
request={
|
|
364
|
+
"id": entity_id,
|
|
365
|
+
"feature_view": f"projects/{project_id}/locations/{location}/featureOnlineStores/"
|
|
366
|
+
f"{feature_online_store_id}/featureViews/{feature_view_id}",
|
|
367
|
+
"data_key": data_key,
|
|
368
|
+
"data_format": data_format,
|
|
369
|
+
},
|
|
370
|
+
timeout=timeout,
|
|
371
|
+
retry=retry,
|
|
372
|
+
metadata=metadata,
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
376
|
+
def delete_feature_view(
|
|
377
|
+
self,
|
|
378
|
+
feature_view_id: str,
|
|
379
|
+
feature_online_store_id: str,
|
|
380
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
381
|
+
location: str | None = None,
|
|
382
|
+
timeout: float | _MethodDefault = DEFAULT,
|
|
383
|
+
retry: Retry | _MethodDefault | None = DEFAULT,
|
|
384
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
385
|
+
) -> Operation:
|
|
386
|
+
"""
|
|
387
|
+
Delete the Feature View.
|
|
388
|
+
|
|
389
|
+
This method deletes the Feature View from the Feature Online Store.
|
|
390
|
+
|
|
391
|
+
:param feature_view_id: The ID to use for the FeatureView, to be deleted.
|
|
392
|
+
:param feature_online_store_id: The ID of the online feature store.
|
|
393
|
+
:param project_id: The ID of the Google Cloud project that contains the
|
|
394
|
+
feature store. If not provided, will attempt to determine from the
|
|
395
|
+
environment.
|
|
396
|
+
:param location: The Google Cloud region where the feature store is located
|
|
397
|
+
(e.g., 'us-central1', 'us-east1').
|
|
398
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
399
|
+
:param timeout: The timeout for this request.
|
|
400
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
401
|
+
"""
|
|
402
|
+
client = self.get_feature_online_store_admin_service_client(location)
|
|
403
|
+
return client.delete_feature_view(
|
|
404
|
+
name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}"
|
|
405
|
+
f"/featureViews/{feature_view_id}",
|
|
406
|
+
timeout=timeout,
|
|
407
|
+
retry=retry,
|
|
408
|
+
metadata=metadata,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
412
|
+
def delete_feature_online_store(
|
|
413
|
+
self,
|
|
414
|
+
feature_online_store_id: str,
|
|
415
|
+
force: bool = False,
|
|
416
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
417
|
+
location: str | None = None,
|
|
418
|
+
timeout: float | _MethodDefault = DEFAULT,
|
|
419
|
+
retry: Retry | _MethodDefault | None = DEFAULT,
|
|
420
|
+
metadata: Sequence[tuple[str, str]] = (),
|
|
421
|
+
) -> Operation:
|
|
422
|
+
"""
|
|
423
|
+
Delete the FeatureOnlineStore.
|
|
424
|
+
|
|
425
|
+
This method deletes the Feature Online Store and all features data.
|
|
426
|
+
The FeatureOnlineStore must not contain any FeatureViews.
|
|
427
|
+
|
|
428
|
+
:param feature_online_store_id: The ID of the online feature store.
|
|
429
|
+
:param force: If set to true, any FeatureViews and Features for this FeatureOnlineStore
|
|
430
|
+
will also be deleted.
|
|
431
|
+
:param project_id: The ID of the Google Cloud project that contains the
|
|
432
|
+
feature store. If not provided, will attempt to determine from the
|
|
433
|
+
environment.
|
|
434
|
+
:param location: The Google Cloud region where the feature store is located
|
|
435
|
+
(e.g., 'us-central1', 'us-east1').
|
|
436
|
+
:param retry: Designation of what errors, if any, should be retried.
|
|
437
|
+
:param timeout: The timeout for this request.
|
|
438
|
+
:param metadata: Strings which should be sent along with the request as metadata.
|
|
439
|
+
"""
|
|
440
|
+
client = self.get_feature_online_store_admin_service_client(location)
|
|
441
|
+
return client.delete_feature_online_store(
|
|
442
|
+
name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
|
|
443
|
+
force=force,
|
|
444
|
+
timeout=timeout,
|
|
445
|
+
retry=retry,
|
|
446
|
+
metadata=metadata,
|
|
447
|
+
)
|
|
@@ -21,14 +21,15 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import time
|
|
23
23
|
from datetime import timedelta
|
|
24
|
-
from typing import TYPE_CHECKING
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
25
25
|
|
|
26
26
|
import vertexai
|
|
27
|
+
from google.cloud import aiplatform
|
|
27
28
|
from vertexai.generative_models import GenerativeModel
|
|
28
|
-
from vertexai.language_models import TextEmbeddingModel
|
|
29
|
+
from vertexai.language_models import TextEmbeddingModel
|
|
30
|
+
from vertexai.preview import generative_models as preview_generative_model
|
|
29
31
|
from vertexai.preview.caching import CachedContent
|
|
30
32
|
from vertexai.preview.evaluation import EvalResult, EvalTask
|
|
31
|
-
from vertexai.preview.generative_models import GenerativeModel as preview_generative_model
|
|
32
33
|
from vertexai.preview.tuning import sft
|
|
33
34
|
|
|
34
35
|
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
@@ -36,23 +37,12 @@ from airflow.providers.google.common.deprecated import deprecated
|
|
|
36
37
|
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
|
|
37
38
|
|
|
38
39
|
if TYPE_CHECKING:
|
|
39
|
-
from google.cloud.aiplatform_v1 import types as types_v1
|
|
40
40
|
from google.cloud.aiplatform_v1beta1 import types as types_v1beta1
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
class GenerativeModelHook(GoogleBaseHook):
|
|
44
44
|
"""Hook for Google Cloud Vertex AI Generative Model APIs."""
|
|
45
45
|
|
|
46
|
-
@deprecated(
|
|
47
|
-
planned_removal_date="April 09, 2025",
|
|
48
|
-
use_instead="GenerativeModelHook.get_generative_model",
|
|
49
|
-
category=AirflowProviderDeprecationWarning,
|
|
50
|
-
)
|
|
51
|
-
def get_text_generation_model(self, pretrained_model: str):
|
|
52
|
-
"""Return a Model Garden Model object based on Text Generation."""
|
|
53
|
-
model = TextGenerationModel.from_pretrained(pretrained_model)
|
|
54
|
-
return model
|
|
55
|
-
|
|
56
46
|
def get_text_embedding_model(self, pretrained_model: str):
|
|
57
47
|
"""Return a Model Garden Model object based on Text Embedding."""
|
|
58
48
|
model = TextEmbeddingModel.from_pretrained(pretrained_model)
|
|
@@ -61,7 +51,7 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
61
51
|
def get_generative_model(
|
|
62
52
|
self,
|
|
63
53
|
pretrained_model: str,
|
|
64
|
-
system_instruction:
|
|
54
|
+
system_instruction: Any | None = None,
|
|
65
55
|
generation_config: dict | None = None,
|
|
66
56
|
safety_settings: dict | None = None,
|
|
67
57
|
tools: list | None = None,
|
|
@@ -93,66 +83,18 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
93
83
|
def get_cached_context_model(
|
|
94
84
|
self,
|
|
95
85
|
cached_content_name: str,
|
|
96
|
-
) ->
|
|
86
|
+
) -> Any:
|
|
97
87
|
"""Return a Generative Model with Cached Context."""
|
|
98
88
|
cached_content = CachedContent(cached_content_name=cached_content_name)
|
|
99
89
|
|
|
100
|
-
cached_context_model = preview_generative_model.from_cached_content(cached_content)
|
|
90
|
+
cached_context_model = preview_generative_model.GenerativeModel.from_cached_content(cached_content)
|
|
101
91
|
return cached_context_model
|
|
102
92
|
|
|
103
93
|
@deprecated(
|
|
104
|
-
planned_removal_date="
|
|
105
|
-
use_instead="
|
|
94
|
+
planned_removal_date="January 3, 2026",
|
|
95
|
+
use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.embed_content",
|
|
106
96
|
category=AirflowProviderDeprecationWarning,
|
|
107
97
|
)
|
|
108
|
-
@GoogleBaseHook.fallback_to_default_project_id
|
|
109
|
-
def text_generation_model_predict(
|
|
110
|
-
self,
|
|
111
|
-
prompt: str,
|
|
112
|
-
pretrained_model: str,
|
|
113
|
-
temperature: float,
|
|
114
|
-
max_output_tokens: int,
|
|
115
|
-
top_p: float,
|
|
116
|
-
top_k: int,
|
|
117
|
-
location: str,
|
|
118
|
-
project_id: str = PROVIDE_PROJECT_ID,
|
|
119
|
-
) -> str:
|
|
120
|
-
"""
|
|
121
|
-
Use the Vertex AI PaLM API to generate natural language text.
|
|
122
|
-
|
|
123
|
-
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
124
|
-
:param location: Required. The ID of the Google Cloud location that the service belongs to.
|
|
125
|
-
:param prompt: Required. Inputs or queries that a user or a program gives
|
|
126
|
-
to the Vertex AI PaLM API, in order to elicit a specific response.
|
|
127
|
-
:param pretrained_model: A pre-trained model optimized for performing natural
|
|
128
|
-
language tasks such as classification, summarization, extraction, content
|
|
129
|
-
creation, and ideation.
|
|
130
|
-
:param temperature: Temperature controls the degree of randomness in token
|
|
131
|
-
selection.
|
|
132
|
-
:param max_output_tokens: Token limit determines the maximum amount of text
|
|
133
|
-
output.
|
|
134
|
-
:param top_p: Tokens are selected from most probable to least until the sum
|
|
135
|
-
of their probabilities equals the top_p value. Defaults to 0.8.
|
|
136
|
-
:param top_k: A top_k of 1 means the selected token is the most probable
|
|
137
|
-
among all tokens.
|
|
138
|
-
"""
|
|
139
|
-
vertexai.init(project=project_id, location=location, credentials=self.get_credentials())
|
|
140
|
-
|
|
141
|
-
parameters = {
|
|
142
|
-
"temperature": temperature,
|
|
143
|
-
"max_output_tokens": max_output_tokens,
|
|
144
|
-
"top_p": top_p,
|
|
145
|
-
"top_k": top_k,
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
model = self.get_text_generation_model(pretrained_model)
|
|
149
|
-
|
|
150
|
-
response = model.predict(
|
|
151
|
-
prompt=prompt,
|
|
152
|
-
**parameters,
|
|
153
|
-
)
|
|
154
|
-
return response.text
|
|
155
|
-
|
|
156
98
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
157
99
|
def text_embedding_model_get_embeddings(
|
|
158
100
|
self,
|
|
@@ -177,16 +119,21 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
177
119
|
|
|
178
120
|
return response.values
|
|
179
121
|
|
|
122
|
+
@deprecated(
|
|
123
|
+
planned_removal_date="January 3, 2026",
|
|
124
|
+
use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.generate_content",
|
|
125
|
+
category=AirflowProviderDeprecationWarning,
|
|
126
|
+
)
|
|
180
127
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
181
128
|
def generative_model_generate_content(
|
|
182
129
|
self,
|
|
183
130
|
contents: list,
|
|
184
131
|
location: str,
|
|
132
|
+
pretrained_model: str,
|
|
185
133
|
tools: list | None = None,
|
|
186
134
|
generation_config: dict | None = None,
|
|
187
135
|
safety_settings: dict | None = None,
|
|
188
136
|
system_instruction: str | None = None,
|
|
189
|
-
pretrained_model: str = "gemini-pro",
|
|
190
137
|
project_id: str = PROVIDE_PROJECT_ID,
|
|
191
138
|
) -> str:
|
|
192
139
|
"""
|
|
@@ -200,7 +147,7 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
200
147
|
:param safety_settings: Optional. Per request settings for blocking unsafe content.
|
|
201
148
|
:param tools: Optional. A list of tools available to the model during evaluation, such as a data store.
|
|
202
149
|
:param system_instruction: Optional. An instruction given to the model to guide its behavior.
|
|
203
|
-
:param pretrained_model:
|
|
150
|
+
:param pretrained_model: Required. Model,
|
|
204
151
|
supporting prompts with text-only input, including natural language
|
|
205
152
|
tasks, multi-turn text and code chat, and code generation. It can
|
|
206
153
|
output text and code.
|
|
@@ -219,6 +166,11 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
219
166
|
|
|
220
167
|
return response.text
|
|
221
168
|
|
|
169
|
+
@deprecated(
|
|
170
|
+
planned_removal_date="January 3, 2026",
|
|
171
|
+
use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.supervised_fine_tuning_train",
|
|
172
|
+
category=AirflowProviderDeprecationWarning,
|
|
173
|
+
)
|
|
222
174
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
223
175
|
def supervised_fine_tuning_train(
|
|
224
176
|
self,
|
|
@@ -228,10 +180,10 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
228
180
|
tuned_model_display_name: str | None = None,
|
|
229
181
|
validation_dataset: str | None = None,
|
|
230
182
|
epochs: int | None = None,
|
|
231
|
-
adapter_size:
|
|
183
|
+
adapter_size: Literal[1, 4, 8, 16] | None = None,
|
|
232
184
|
learning_rate_multiplier: float | None = None,
|
|
233
185
|
project_id: str = PROVIDE_PROJECT_ID,
|
|
234
|
-
) ->
|
|
186
|
+
) -> Any:
|
|
235
187
|
"""
|
|
236
188
|
Use the Supervised Fine Tuning API to create a tuning job.
|
|
237
189
|
|
|
@@ -272,12 +224,17 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
272
224
|
|
|
273
225
|
return sft_tuning_job
|
|
274
226
|
|
|
227
|
+
@deprecated(
|
|
228
|
+
planned_removal_date="January 3, 2026",
|
|
229
|
+
use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.count_tokens",
|
|
230
|
+
category=AirflowProviderDeprecationWarning,
|
|
231
|
+
)
|
|
275
232
|
@GoogleBaseHook.fallback_to_default_project_id
|
|
276
233
|
def count_tokens(
|
|
277
234
|
self,
|
|
278
235
|
contents: list,
|
|
279
236
|
location: str,
|
|
280
|
-
pretrained_model: str
|
|
237
|
+
pretrained_model: str,
|
|
281
238
|
project_id: str = PROVIDE_PROJECT_ID,
|
|
282
239
|
) -> types_v1beta1.CountTokensResponse:
|
|
283
240
|
"""
|
|
@@ -287,7 +244,7 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
287
244
|
:param location: Required. The ID of the Google Cloud location that the service belongs to.
|
|
288
245
|
:param contents: Required. The multi-part content of a message that a user or a program
|
|
289
246
|
gives to the generative model, in order to elicit a specific response.
|
|
290
|
-
:param pretrained_model:
|
|
247
|
+
:param pretrained_model: Required. Model,
|
|
291
248
|
supporting prompts with text-only input, including natural language
|
|
292
249
|
tasks, multi-turn text and code chat, and code generation. It can
|
|
293
250
|
output text and code.
|
|
@@ -359,13 +316,18 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
359
316
|
|
|
360
317
|
return eval_result
|
|
361
318
|
|
|
319
|
+
@deprecated(
|
|
320
|
+
planned_removal_date="January 3, 2026",
|
|
321
|
+
use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.create_cached_content",
|
|
322
|
+
category=AirflowProviderDeprecationWarning,
|
|
323
|
+
)
|
|
362
324
|
def create_cached_content(
|
|
363
325
|
self,
|
|
364
326
|
model_name: str,
|
|
365
327
|
location: str,
|
|
366
328
|
ttl_hours: float = 1,
|
|
367
|
-
system_instruction:
|
|
368
|
-
contents: list | None = None,
|
|
329
|
+
system_instruction: Any | None = None,
|
|
330
|
+
contents: list[Any] | None = None,
|
|
369
331
|
display_name: str | None = None,
|
|
370
332
|
project_id: str = PROVIDE_PROJECT_ID,
|
|
371
333
|
) -> str:
|
|
@@ -393,6 +355,11 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
393
355
|
|
|
394
356
|
return response.name
|
|
395
357
|
|
|
358
|
+
@deprecated(
|
|
359
|
+
planned_removal_date="January 3, 2026",
|
|
360
|
+
use_instead="airflow.providers.google.cloud.hooks.gen_ai.generative_model.GenAIGenerativeModelHook.generate_content",
|
|
361
|
+
category=AirflowProviderDeprecationWarning,
|
|
362
|
+
)
|
|
396
363
|
def generate_from_cached_content(
|
|
397
364
|
self,
|
|
398
365
|
location: str,
|
|
@@ -413,6 +380,9 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
413
380
|
:param generation_config: Optional. Generation configuration settings.
|
|
414
381
|
:param safety_settings: Optional. Per request settings for blocking unsafe content.
|
|
415
382
|
"""
|
|
383
|
+
# During run of the system test it was found out that names from xcom, e.g. 3402922389 can be
|
|
384
|
+
# treated as int and throw an error TypeError: expected string or bytes-like object, got 'int'
|
|
385
|
+
cached_content_name = str(cached_content_name)
|
|
416
386
|
vertexai.init(project=project_id, location=location, credentials=self.get_credentials())
|
|
417
387
|
|
|
418
388
|
cached_context_model = self.get_cached_context_model(cached_content_name=cached_content_name)
|
|
@@ -424,3 +394,37 @@ class GenerativeModelHook(GoogleBaseHook):
|
|
|
424
394
|
)
|
|
425
395
|
|
|
426
396
|
return response.text
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
@deprecated(
|
|
400
|
+
planned_removal_date="January 3, 2026",
|
|
401
|
+
use_instead="airflow.providers.google.cloud.hooks.vertex_ai.experiment_service.ExperimentRunHook",
|
|
402
|
+
category=AirflowProviderDeprecationWarning,
|
|
403
|
+
)
|
|
404
|
+
class ExperimentRunHook(GoogleBaseHook):
|
|
405
|
+
"""Use the Vertex AI SDK for Python to create and manage your experiment runs."""
|
|
406
|
+
|
|
407
|
+
@GoogleBaseHook.fallback_to_default_project_id
|
|
408
|
+
def delete_experiment_run(
|
|
409
|
+
self,
|
|
410
|
+
experiment_run_name: str,
|
|
411
|
+
experiment_name: str,
|
|
412
|
+
location: str,
|
|
413
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
|
414
|
+
delete_backing_tensorboard_run: bool = False,
|
|
415
|
+
) -> None:
|
|
416
|
+
"""
|
|
417
|
+
Delete experiment run from the experiment.
|
|
418
|
+
|
|
419
|
+
:param project_id: Required. The ID of the Google Cloud project that the service belongs to.
|
|
420
|
+
:param location: Required. The ID of the Google Cloud location that the service belongs to.
|
|
421
|
+
:param experiment_name: Required. The name of the evaluation experiment.
|
|
422
|
+
:param experiment_run_name: Required. The specific run name or ID for this experiment.
|
|
423
|
+
:param delete_backing_tensorboard_run: Whether to delete the backing Vertex AI TensorBoard run
|
|
424
|
+
that stores time series metrics for this run.
|
|
425
|
+
"""
|
|
426
|
+
self.log.info("Next experiment run will be deleted: %s", experiment_run_name)
|
|
427
|
+
experiment_run = aiplatform.ExperimentRun(
|
|
428
|
+
run_name=experiment_run_name, experiment=experiment_name, project=project_id, location=location
|
|
429
|
+
)
|
|
430
|
+
experiment_run.delete(delete_backing_tensorboard_run=delete_backing_tensorboard_run)
|