apache-airflow-providers-google 16.0.0rc1__py3-none-any.whl → 16.1.0__py3-none-any.whl

airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py

@@ -18,15 +18,32 @@
 
 from __future__ import annotations
 
+from collections.abc import Sequence
+from typing import (
+    TYPE_CHECKING,
+)
+
 from google.api_core.client_options import ClientOptions
+from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
 from google.cloud.aiplatform_v1beta1 import (
     FeatureOnlineStoreAdminServiceClient,
+    FeatureOnlineStoreServiceClient,
 )
 
 from airflow.exceptions import AirflowException
 from airflow.providers.google.common.consts import CLIENT_INFO
 from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
 
+if TYPE_CHECKING:
+    from google.api_core.operation import Operation
+    from google.api_core.retry import Retry
+    from google.cloud.aiplatform_v1beta1.types import (
+        FeatureOnlineStore,
+        FeatureView,
+        FeatureViewDataKey,
+        FetchFeatureValuesResponse,
+    )
+
 
 class FeatureStoreHook(GoogleBaseHook):
     """
@@ -48,6 +65,19 @@ class FeatureStoreHook(GoogleBaseHook):
         originating account.
     """
 
+    @staticmethod
+    def _get_client_options(
+        location: str | None = None,
+        custom_endpoint: str | None = None,
+    ) -> ClientOptions:
+        if custom_endpoint:
+            client_options = ClientOptions(api_endpoint=custom_endpoint)
+        elif location and location != "global":
+            client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
+        else:
+            client_options = ClientOptions()
+        return client_options
+
     def get_feature_online_store_admin_service_client(
         self,
         location: str | None = None,
@@ -62,12 +92,153 @@ class FeatureStoreHook(GoogleBaseHook):
             If provided and not 'global', the client will be configured to use the
             region-specific API endpoint.
         """
-        if location and location != "global":
-            client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
-        else:
-            client_options = ClientOptions()
         return FeatureOnlineStoreAdminServiceClient(
-            credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options
+            credentials=self.get_credentials(),
+            client_info=CLIENT_INFO,
+            client_options=self._get_client_options(location),
+        )
+
+    def get_feature_online_store_service_client(
+        self,
+        location: str | None = None,
+        custom_endpoint: str | None = None,
+    ) -> FeatureOnlineStoreServiceClient:
+        return FeatureOnlineStoreServiceClient(
+            credentials=self.get_credentials(),
+            client_info=CLIENT_INFO,
+            client_options=self._get_client_options(location, custom_endpoint),
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_feature_online_store(
+        self,
+        feature_online_store_id: str,
+        feature_online_store: FeatureOnlineStore,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Create and sends request for Feature Online store.
+
+        This method initiates VertexAI Feature Online Store creation request.
+        Feature Online Store aims to serve and manage features data as a part of VertexAI MLOps.
+
+        :param feature_online_store_id: The ID of the online feature store.
+        :param feature_online_store: The configuration of the online repository.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.create_feature_online_store(
+            request={
+                "parent": f"projects/{project_id}/locations/{location}",
+                "feature_online_store_id": feature_online_store_id,
+                "feature_online_store": feature_online_store,
+            },
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_feature_online_store(
+        self,
+        feature_online_store_id: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> FeatureOnlineStore:
+        """
+        Get Feature Online store data.
+
+        Get the FeatureOnlineStore details.
+        Vertex AI Feature Online Store provides a centralized repository for serving ML features
+        and embedding indexes at low latency.
+
+        :param feature_online_store_id: The ID of the online feature store.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.get_feature_online_store(
+            name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @staticmethod
+    def _get_featurestore_public_endpoint(feature_online_store: FeatureOnlineStore):
+        public_endpoint = None
+        featurestore_data = type(feature_online_store).to_dict(feature_online_store)
+        if "dedicated_serving_endpoint" in featurestore_data:
+            public_endpoint = featurestore_data["dedicated_serving_endpoint"].get(
+                "public_endpoint_domain_name"
+            )
+        return public_endpoint
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_feature_view(
+        self,
+        feature_view_id: str,
+        feature_view: FeatureView,
+        feature_online_store_id: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        run_sync_immediately: bool = False,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Create request for Feature View creation.
+
+        This method initiates VertexAI Feature View request for the existing Feature Online Store.
+        Feature View represents features and data according to the source provided.
+
+        :param feature_view_id: The ID to use for the FeatureView, which will
+            become the final component of the FeatureView's resource name.
+            This value may be up to 60 characters, and valid characters are ``[a-z0-9_]``.
+            The first character cannot be a number. The value must be unique within a FeatureOnlineStore.
+        :param feature_view: The configuration of the FeatureView to create.
+        :param feature_online_store_id: The ID of the online feature store.
+        :param run_sync_immediately: If set to true, one on demand sync will be run
+            immediately, regardless the FeatureView.sync_config.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.create_feature_view(
+            request={
+                "parent": f"projects/{project_id}/locations/"
+                f"{location}/featureOnlineStores/{feature_online_store_id}",
+                "feature_view_id": feature_view_id,
+                "feature_view": feature_view,
+                "run_sync_immediately": run_sync_immediately,
+            },
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
         )
 
     def get_feature_view_sync(
@@ -135,13 +306,142 @@ class FeatureStoreHook(GoogleBaseHook):
             environment.
         """
         client = self.get_feature_online_store_admin_service_client(location)
-        feature_view = f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}/featureViews/{feature_view_id}"
+        feature_view = (
+            f"projects/{project_id}/locations/{location}/featureOnlineStores/"
+            f"{feature_online_store_id}/featureViews/{feature_view_id}"
+        )
 
         try:
             response = client.sync_feature_view(feature_view=feature_view)
-
             return str(response.feature_view_sync)
 
         except Exception as e:
             self.log.error("Failed to sync feature view: %s", str(e))
             raise AirflowException(str(e))
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def fetch_feature_values(
+        self,
+        feature_view_id: str,
+        feature_online_store_id: str,
+        entity_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
+        endpoint_domain_name: str | None = None,
+        data_key: FeatureViewDataKey | None = None,
+        data_format: int | None = None,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> FetchFeatureValuesResponse:
+        """
+        Fetch data from the Feature View provided.
+
+        This method fetches data from existing Feature view, filtered by provided (or default) data_key.
+        Helps to retrieve actual features data hosted in the VertexAI Feature Store.
+
+        :param entity_id: Simple ID to identify Entity to fetch feature values for.
+        :param endpoint_domain_name: Optional. Public domain name, hosting the content of Optimized
+            Feature Online store. Should be omitted, if bigtable configuration provided for the FeatureStore,
+            and default feature store endpoint will be used, based on location provided.
+        :param feature_view_id: The FeatureView ID to fetch data from.
+        :param feature_online_store_id: The ID of the online feature store.
+        :param data_key: Optional. The request key to fetch feature values for.
+        :param data_format: Optional. Response data format. If not set, FeatureViewDataFormat.KEY_VALUE
+            will be used.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the
+            environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        data_client = self.get_feature_online_store_service_client(location, endpoint_domain_name)
+        return data_client.fetch_feature_values(
+            request={
+                "id": entity_id,
+                "feature_view": f"projects/{project_id}/locations/{location}/featureOnlineStores/"
+                f"{feature_online_store_id}/featureViews/{feature_view_id}",
+                "data_key": data_key,
+                "data_format": data_format,
+            },
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_feature_view(
+        self,
+        feature_view_id: str,
+        feature_online_store_id: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Delete the Feature View.
+
+        This method deletes the Feature View from the Feature Online Store.
+
+        :param feature_view_id: The ID to use for the FeatureView, to be deleted.
+        :param feature_online_store_id: The ID of the online feature store.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the
+            environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.delete_feature_view(
+            name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}"
+            f"/featureViews/{feature_view_id}",
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_feature_online_store(
+        self,
+        feature_online_store_id: str,
+        force: bool = False,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Delete the FeatureOnlineStore.
+
+        This method deletes the Feature Online Store and all features data.
+        The FeatureOnlineStore must not contain any FeatureViews.
+
+        :param feature_online_store_id: The ID of the online feature store.
+        :param force:  If set to true, any FeatureViews and Features for this FeatureOnlineStore
+            will also be deleted.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the
+            environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.delete_feature_online_store(
+            name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
+            force=force,
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )

airflow/providers/google/cloud/hooks/vertex_ai/generative_model.py

@@ -21,20 +21,20 @@ from __future__ import annotations
 
 import time
 from datetime import timedelta
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Literal
 
 import vertexai
+from google.cloud import aiplatform
 from vertexai.generative_models import GenerativeModel
 from vertexai.language_models import TextEmbeddingModel
+from vertexai.preview import generative_models as preview_generative_model
 from vertexai.preview.caching import CachedContent
 from vertexai.preview.evaluation import EvalResult, EvalTask
-from vertexai.preview.generative_models import GenerativeModel as preview_generative_model
 from vertexai.preview.tuning import sft
 
 from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
 
 if TYPE_CHECKING:
-    from google.cloud.aiplatform_v1 import types as types_v1
     from google.cloud.aiplatform_v1beta1 import types as types_v1beta1
 
 
@@ -49,7 +49,7 @@ class GenerativeModelHook(GoogleBaseHook):
     def get_generative_model(
         self,
         pretrained_model: str,
-        system_instruction: str | None = None,
+        system_instruction: Any | None = None,
         generation_config: dict | None = None,
         safety_settings: dict | None = None,
         tools: list | None = None,
@@ -81,11 +81,11 @@ class GenerativeModelHook(GoogleBaseHook):
     def get_cached_context_model(
         self,
         cached_content_name: str,
-    ) -> preview_generative_model:
+    ) -> Any:
         """Return a Generative Model with Cached Context."""
         cached_content = CachedContent(cached_content_name=cached_content_name)
 
-        cached_context_model = preview_generative_model.from_cached_content(cached_content)
+        cached_context_model = preview_generative_model.GenerativeModel.from_cached_content(cached_content)
         return cached_context_model
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -117,11 +117,11 @@ class GenerativeModelHook(GoogleBaseHook):
         self,
         contents: list,
         location: str,
+        pretrained_model: str,
         tools: list | None = None,
         generation_config: dict | None = None,
         safety_settings: dict | None = None,
         system_instruction: str | None = None,
-        pretrained_model: str = "gemini-pro",
         project_id: str = PROVIDE_PROJECT_ID,
     ) -> str:
         """
@@ -135,7 +135,7 @@ class GenerativeModelHook(GoogleBaseHook):
         :param safety_settings: Optional. Per request settings for blocking unsafe content.
         :param tools: Optional. A list of tools available to the model during evaluation, such as a data store.
         :param system_instruction: Optional. An instruction given to the model to guide its behavior.
-        :param pretrained_model: By default uses the pre-trained model `gemini-pro`,
+        :param pretrained_model: Required. Model,
             supporting prompts with text-only input, including natural language
             tasks, multi-turn text and code chat, and code generation. It can
             output text and code.
@@ -163,10 +163,10 @@ class GenerativeModelHook(GoogleBaseHook):
         tuned_model_display_name: str | None = None,
         validation_dataset: str | None = None,
         epochs: int | None = None,
-        adapter_size: int | None = None,
+        adapter_size: Literal[1, 4, 8, 16] | None = None,
         learning_rate_multiplier: float | None = None,
         project_id: str = PROVIDE_PROJECT_ID,
-    ) -> types_v1.TuningJob:
+    ) -> Any:
         """
         Use the Supervised Fine Tuning API to create a tuning job.
 
@@ -212,7 +212,7 @@ class GenerativeModelHook(GoogleBaseHook):
         self,
         contents: list,
         location: str,
-        pretrained_model: str = "gemini-pro",
+        pretrained_model: str,
         project_id: str = PROVIDE_PROJECT_ID,
     ) -> types_v1beta1.CountTokensResponse:
         """
@@ -222,7 +222,7 @@ class GenerativeModelHook(GoogleBaseHook):
         :param location: Required. The ID of the Google Cloud location that the service belongs to.
         :param contents: Required. The multi-part content of a message that a user or a program
             gives to the generative model, in order to elicit a specific response.
-        :param pretrained_model: By default uses the pre-trained model `gemini-pro`,
+        :param pretrained_model: Required. Model,
             supporting prompts with text-only input, including natural language
             tasks, multi-turn text and code chat, and code generation. It can
             output text and code.
@@ -299,8 +299,8 @@ class GenerativeModelHook(GoogleBaseHook):
         model_name: str,
         location: str,
         ttl_hours: float = 1,
-        system_instruction: str | None = None,
-        contents: list | None = None,
+        system_instruction: Any | None = None,
+        contents: list[Any] | None = None,
         display_name: str | None = None,
         project_id: str = PROVIDE_PROJECT_ID,
     ) -> str:
@@ -359,3 +359,32 @@ class GenerativeModelHook(GoogleBaseHook):
         )
 
         return response.text
+
+
+class ExperimentRunHook(GoogleBaseHook):
+    """Use the Vertex AI SDK for Python to create and manage your experiment runs."""
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_experiment_run(
+        self,
+        experiment_run_name: str,
+        experiment_name: str,
+        location: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        delete_backing_tensorboard_run: bool = False,
+    ) -> None:
+        """
+        Delete experiment run from the experiment.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        :param experiment_run_name: Required. The specific run name or ID for this experiment.
+        :param delete_backing_tensorboard_run: Whether to delete the backing Vertex AI TensorBoard run
+            that stores time series metrics for this run.
+        """
+        self.log.info("Next experiment run will be deleted: %s", experiment_run_name)
+        experiment_run = aiplatform.ExperimentRun(
+            run_name=experiment_run_name, experiment=experiment_name, project=project_id, location=location
+        )
+        experiment_run.delete(delete_backing_tensorboard_run=delete_backing_tensorboard_run)

airflow/providers/google/cloud/hooks/vertex_ai/ray.py

@@ -22,8 +22,17 @@ from __future__ import annotations
 import dataclasses
 from typing import Any
 
-import vertex_ray
-from google._upb._message import ScalarMapContainer  # type: ignore[attr-defined]
+from airflow.exceptions import AirflowOptionalProviderFeatureException
+
+try:
+    import vertex_ray
+    from google._upb._message import ScalarMapContainer  # type: ignore[attr-defined]
+except ImportError:
+    # Fallback for environments where the upb module is not available.
+    raise AirflowOptionalProviderFeatureException(
+        "google._upb._message.ScalarMapContainer is not available. "
+        "Please install the ray package to use this feature."
+    )
 from google.cloud import aiplatform
 from google.cloud.aiplatform.vertex_ray.util import resources
 from google.cloud.aiplatform_v1 import (

airflow/providers/google/cloud/hooks/vision.py

@@ -19,10 +19,10 @@
 
 from __future__ import annotations
 
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from copy import deepcopy
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Callable
+from typing import TYPE_CHECKING, Any
 
 from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
 from google.cloud.vision_v1 import (

airflow/providers/google/cloud/links/alloy_db.py

@@ -19,14 +19,8 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 from airflow.providers.google.cloud.links.base import BaseGoogleLink
 
-if TYPE_CHECKING:
-    from airflow.models import BaseOperator
-    from airflow.utils.context import Context
-
 ALLOY_DB_BASE_LINK = "/alloydb"
 ALLOY_DB_CLUSTER_LINK = (
     ALLOY_DB_BASE_LINK + "/locations/{location_id}/clusters/{cluster_id}?project={project_id}"
@@ -44,20 +38,6 @@ class AlloyDBClusterLink(BaseGoogleLink):
     key = "alloy_db_cluster"
     format_str = ALLOY_DB_CLUSTER_LINK
 
-    @staticmethod
-    def persist(
-        context: Context,
-        task_instance: BaseOperator,
-        location_id: str,
-        cluster_id: str,
-        project_id: str | None,
-    ):
-        task_instance.xcom_push(
-            context,
-            key=AlloyDBClusterLink.key,
-            value={"location_id": location_id, "cluster_id": cluster_id, "project_id": project_id},
-        )
-
 
 class AlloyDBUsersLink(BaseGoogleLink):
     """Helper class for constructing AlloyDB users Link."""
@@ -66,20 +46,6 @@ class AlloyDBUsersLink(BaseGoogleLink):
     key = "alloy_db_users"
     format_str = ALLOY_DB_USERS_LINK
 
-    @staticmethod
-    def persist(
-        context: Context,
-        task_instance: BaseOperator,
-        location_id: str,
-        cluster_id: str,
-        project_id: str | None,
-    ):
-        task_instance.xcom_push(
-            context,
-            key=AlloyDBUsersLink.key,
-            value={"location_id": location_id, "cluster_id": cluster_id, "project_id": project_id},
-        )
-
 
 class AlloyDBBackupsLink(BaseGoogleLink):
     """Helper class for constructing AlloyDB backups Link."""
@@ -87,15 +53,3 @@ class AlloyDBBackupsLink(BaseGoogleLink):
     name = "AlloyDB Backups"
     key = "alloy_db_backups"
     format_str = ALLOY_DB_BACKUPS_LINK
-
-    @staticmethod
-    def persist(
-        context: Context,
-        task_instance: BaseOperator,
-        project_id: str | None,
-    ):
-        task_instance.xcom_push(
-            context,
-            key=AlloyDBBackupsLink.key,
-            value={"project_id": project_id},
-        )