apache-airflow-providers-google 14.0.0__py3-none-any.whl → 19.1.0rc1__py3-none-any.whl

airflow/providers/google/cloud/hooks/vertex_ai/experiment_service.py

@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from google.cloud import aiplatform
+from google.cloud.aiplatform.compat.types import execution_v1 as gca_execution
+
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
+
+
+class ExperimentHook(GoogleBaseHook):
+    """Use the Vertex AI SDK for Python to manage your experiments."""
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_experiment(
+        self,
+        experiment_name: str,
+        location: str,
+        experiment_description: str = "",
+        project_id: str = PROVIDE_PROJECT_ID,
+        experiment_tensorboard: str | None = None,
+    ):
+        """
+        Create an experiment and, optionally, associate a Vertex AI TensorBoard instance using the Vertex AI SDK for Python.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        :param experiment_description: Optional. Description of the evaluation experiment.
+        :param experiment_tensorboard: Optional. The Vertex TensorBoard instance to use as a backing
+            TensorBoard for the provided experiment. If no TensorBoard is provided, a default Tensorboard
+            instance is created and used by this experiment.
+        """
+        aiplatform.init(
+            experiment=experiment_name,
+            experiment_description=experiment_description,
+            experiment_tensorboard=experiment_tensorboard if experiment_tensorboard else False,
+            project=project_id,
+            location=location,
+        )
+        self.log.info("Created experiment with name: %s", experiment_name)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_experiment(
+        self,
+        experiment_name: str,
+        location: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        delete_backing_tensorboard_runs: bool = False,
+    ) -> None:
+        """
+        Delete an experiment.
+
+        Deleting an experiment deletes that experiment and all experiment runs associated with the experiment.
+        The Vertex AI TensorBoard experiment associated with the experiment is not deleted.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        :param delete_backing_tensorboard_runs: Optional. If True will also delete the Vertex AI TensorBoard
+            runs associated with the experiment runs under this experiment that we used to store time series
+            metrics.
+        """
+        experiment = aiplatform.Experiment(
+            experiment_name=experiment_name, project=project_id, location=location
+        )
+
+        experiment.delete(delete_backing_tensorboard_runs=delete_backing_tensorboard_runs)
+
+
+class ExperimentRunHook(GoogleBaseHook):
+    """Use the Vertex AI SDK for Python to create and manage your experiment runs."""
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_experiment_run(
+        self,
+        experiment_run_name: str,
+        experiment_name: str,
+        location: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        experiment_run_tensorboard: str | None = None,
+        run_after_creation: bool = False,
+    ) -> None:
+        """
+        Create experiment run for the experiment.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        :param experiment_run_name: Required. The specific run name or ID for this experiment.
+        :param experiment_run_tensorboard: Optional. A backing TensorBoard resource to enable and store time
+            series metrics logged to this experiment run.
+        :param run_after_creation: Optional. Responsible for state after creation of experiment run.
+            If true experiment run will be created with state RUNNING.
+        """
+        experiment_run_state = (
+            gca_execution.Execution.State.NEW
+            if not run_after_creation
+            else gca_execution.Execution.State.RUNNING
+        )
+        experiment_run = aiplatform.ExperimentRun.create(
+            run_name=experiment_run_name,
+            experiment=experiment_name,
+            project=project_id,
+            location=location,
+            state=experiment_run_state,
+            tensorboard=experiment_run_tensorboard,
+        )
+        self.log.info(
+            "Created experiment run with name: %s and status: %s",
+            experiment_run.name,
+            experiment_run.state,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def list_experiment_runs(
+        self,
+        experiment_name: str,
+        location: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+    ) -> list[aiplatform.ExperimentRun]:
+        """
+        List experiment run for the experiment.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        """
+        experiment_runs = aiplatform.ExperimentRun.list(
+            experiment=experiment_name,
+            project=project_id,
+            location=location,
+        )
+        return experiment_runs
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def update_experiment_run_state(
+        self,
+        experiment_run_name: str,
+        experiment_name: str,
+        location: str,
+        new_state: gca_execution.Execution.State,
+        project_id: str = PROVIDE_PROJECT_ID,
+    ) -> None:
+        """
+        Update state of the experiment run.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        :param experiment_run_name: Required. The specific run name or ID for this experiment.
+        :param new_state: Required. New state of the experiment run.
+        """
+        experiment_run = aiplatform.ExperimentRun(
+            run_name=experiment_run_name,
+            experiment=experiment_name,
+            project=project_id,
+            location=location,
+        )
+        self.log.info("State of the %s before update is: %s", experiment_run.name, experiment_run.state)
+
+        experiment_run.update_state(new_state)
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_experiment_run(
+        self,
+        experiment_run_name: str,
+        experiment_name: str,
+        location: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        delete_backing_tensorboard_run: bool = False,
+    ) -> None:
+        """
+        Delete experiment run from the experiment.
+
+        :param project_id: Required. The ID of the Google Cloud project that the service belongs to.
+        :param location: Required. The ID of the Google Cloud location that the service belongs to.
+        :param experiment_name: Required. The name of the evaluation experiment.
+        :param experiment_run_name: Required. The specific run name or ID for this experiment.
+        :param delete_backing_tensorboard_run: Whether to delete the backing Vertex AI TensorBoard run
+            that stores time series metrics for this run.
+        """
+        self.log.info("Next experiment run will be deleted: %s", experiment_run_name)
+        experiment_run = aiplatform.ExperimentRun(
+            run_name=experiment_run_name, experiment=experiment_name, project=project_id, location=location
+        )
+        experiment_run.delete(delete_backing_tensorboard_run=delete_backing_tensorboard_run)

airflow/providers/google/cloud/hooks/vertex_ai/feature_store.py

@@ -18,14 +18,32 @@
 
 from __future__ import annotations
 
-from airflow.exceptions import AirflowException
-from airflow.providers.google.common.consts import CLIENT_INFO
-from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
+from collections.abc import Sequence
+from typing import (
+    TYPE_CHECKING,
+)
+
 from google.api_core.client_options import ClientOptions
+from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
 from google.cloud.aiplatform_v1beta1 import (
     FeatureOnlineStoreAdminServiceClient,
+    FeatureOnlineStoreServiceClient,
 )
 
+from airflow.exceptions import AirflowException
+from airflow.providers.google.common.consts import CLIENT_INFO
+from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID, GoogleBaseHook
+
+if TYPE_CHECKING:
+    from google.api_core.operation import Operation
+    from google.api_core.retry import Retry
+    from google.cloud.aiplatform_v1beta1.types import (
+        FeatureOnlineStore,
+        FeatureView,
+        FeatureViewDataKey,
+        FetchFeatureValuesResponse,
+    )
+
 
 class FeatureStoreHook(GoogleBaseHook):
     """
@@ -47,6 +65,19 @@ class FeatureStoreHook(GoogleBaseHook):
         originating account.
     """
 
+    @staticmethod
+    def _get_client_options(
+        location: str | None = None,
+        custom_endpoint: str | None = None,
+    ) -> ClientOptions:
+        if custom_endpoint:
+            client_options = ClientOptions(api_endpoint=custom_endpoint)
+        elif location and location != "global":
+            client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
+        else:
+            client_options = ClientOptions()
+        return client_options
+
     def get_feature_online_store_admin_service_client(
         self,
         location: str | None = None,
@@ -61,12 +92,153 @@ class FeatureStoreHook(GoogleBaseHook):
             If provided and not 'global', the client will be configured to use the
             region-specific API endpoint.
         """
-        if location and location != "global":
-            client_options = ClientOptions(api_endpoint=f"{location}-aiplatform.googleapis.com:443")
-        else:
-            client_options = ClientOptions()
         return FeatureOnlineStoreAdminServiceClient(
-            credentials=self.get_credentials(), client_info=CLIENT_INFO, client_options=client_options
+            credentials=self.get_credentials(),
+            client_info=CLIENT_INFO,
+            client_options=self._get_client_options(location),
+        )
+
+    def get_feature_online_store_service_client(
+        self,
+        location: str | None = None,
+        custom_endpoint: str | None = None,
+    ) -> FeatureOnlineStoreServiceClient:
+        return FeatureOnlineStoreServiceClient(
+            credentials=self.get_credentials(),
+            client_info=CLIENT_INFO,
+            client_options=self._get_client_options(location, custom_endpoint),
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_feature_online_store(
+        self,
+        feature_online_store_id: str,
+        feature_online_store: FeatureOnlineStore,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Create and sends request for Feature Online store.
+
+        This method initiates VertexAI Feature Online Store creation request.
+        Feature Online Store aims to serve and manage features data as a part of VertexAI MLOps.
+
+        :param feature_online_store_id: The ID of the online feature store.
+        :param feature_online_store: The configuration of the online repository.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.create_feature_online_store(
+            request={
+                "parent": f"projects/{project_id}/locations/{location}",
+                "feature_online_store_id": feature_online_store_id,
+                "feature_online_store": feature_online_store,
+            },
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def get_feature_online_store(
+        self,
+        feature_online_store_id: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> FeatureOnlineStore:
+        """
+        Get Feature Online store data.
+
+        Get the FeatureOnlineStore details.
+        Vertex AI Feature Online Store provides a centralized repository for serving ML features
+        and embedding indexes at low latency.
+
+        :param feature_online_store_id: The ID of the online feature store.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.get_feature_online_store(
+            name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @staticmethod
+    def _get_featurestore_public_endpoint(feature_online_store: FeatureOnlineStore):
+        public_endpoint = None
+        featurestore_data = type(feature_online_store).to_dict(feature_online_store)
+        if "dedicated_serving_endpoint" in featurestore_data:
+            public_endpoint = featurestore_data["dedicated_serving_endpoint"].get(
+                "public_endpoint_domain_name"
+            )
+        return public_endpoint
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def create_feature_view(
+        self,
+        feature_view_id: str,
+        feature_view: FeatureView,
+        feature_online_store_id: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        run_sync_immediately: bool = False,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Create request for Feature View creation.
+
+        This method initiates VertexAI Feature View request for the existing Feature Online Store.
+        Feature View represents features and data according to the source provided.
+
+        :param feature_view_id: The ID to use for the FeatureView, which will
+            become the final component of the FeatureView's resource name.
+            This value may be up to 60 characters, and valid characters are ``[a-z0-9_]``.
+            The first character cannot be a number. The value must be unique within a FeatureOnlineStore.
+        :param feature_view: The configuration of the FeatureView to create.
+        :param feature_online_store_id: The ID of the online feature store.
+        :param run_sync_immediately: If set to true, one on demand sync will be run
+            immediately, regardless the FeatureView.sync_config.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.create_feature_view(
+            request={
+                "parent": f"projects/{project_id}/locations/"
+                f"{location}/featureOnlineStores/{feature_online_store_id}",
+                "feature_view_id": feature_view_id,
+                "feature_view": feature_view,
+                "run_sync_immediately": run_sync_immediately,
+            },
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
         )
 
     def get_feature_view_sync(
@@ -134,13 +306,142 @@ class FeatureStoreHook(GoogleBaseHook):
             environment.
         """
         client = self.get_feature_online_store_admin_service_client(location)
-        feature_view = f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}/featureViews/{feature_view_id}"
+        feature_view = (
+            f"projects/{project_id}/locations/{location}/featureOnlineStores/"
+            f"{feature_online_store_id}/featureViews/{feature_view_id}"
+        )
 
         try:
             response = client.sync_feature_view(feature_view=feature_view)
-
             return str(response.feature_view_sync)
 
         except Exception as e:
             self.log.error("Failed to sync feature view: %s", str(e))
             raise AirflowException(str(e))
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def fetch_feature_values(
+        self,
+        feature_view_id: str,
+        feature_online_store_id: str,
+        entity_id: str | None = None,
+        project_id: str = PROVIDE_PROJECT_ID,
+        endpoint_domain_name: str | None = None,
+        data_key: FeatureViewDataKey | None = None,
+        data_format: int | None = None,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> FetchFeatureValuesResponse:
+        """
+        Fetch data from the Feature View provided.
+
+        This method fetches data from existing Feature view, filtered by provided (or default) data_key.
+        Helps to retrieve actual features data hosted in the VertexAI Feature Store.
+
+        :param entity_id: Simple ID to identify Entity to fetch feature values for.
+        :param endpoint_domain_name: Optional. Public domain name, hosting the content of Optimized
+            Feature Online store. Should be omitted, if bigtable configuration provided for the FeatureStore,
+            and default feature store endpoint will be used, based on location provided.
+        :param feature_view_id: The FeatureView ID to fetch data from.
+        :param feature_online_store_id: The ID of the online feature store.
+        :param data_key: Optional. The request key to fetch feature values for.
+        :param data_format: Optional. Response data format. If not set, FeatureViewDataFormat.KEY_VALUE
+            will be used.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the
+            environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        data_client = self.get_feature_online_store_service_client(location, endpoint_domain_name)
+        return data_client.fetch_feature_values(
+            request={
+                "id": entity_id,
+                "feature_view": f"projects/{project_id}/locations/{location}/featureOnlineStores/"
+                f"{feature_online_store_id}/featureViews/{feature_view_id}",
+                "data_key": data_key,
+                "data_format": data_format,
+            },
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_feature_view(
+        self,
+        feature_view_id: str,
+        feature_online_store_id: str,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Delete the Feature View.
+
+        This method deletes the Feature View from the Feature Online Store.
+
+        :param feature_view_id: The ID to use for the FeatureView, to be deleted.
+        :param feature_online_store_id: The ID of the online feature store.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the
+            environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.delete_feature_view(
+            name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}"
+            f"/featureViews/{feature_view_id}",
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def delete_feature_online_store(
+        self,
+        feature_online_store_id: str,
+        force: bool = False,
+        project_id: str = PROVIDE_PROJECT_ID,
+        location: str | None = None,
+        timeout: float | _MethodDefault = DEFAULT,
+        retry: Retry | _MethodDefault | None = DEFAULT,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> Operation:
+        """
+        Delete the FeatureOnlineStore.
+
+        This method deletes the Feature Online Store and all features data.
+        The FeatureOnlineStore must not contain any FeatureViews.
+
+        :param feature_online_store_id: The ID of the online feature store.
+        :param force:  If set to true, any FeatureViews and Features for this FeatureOnlineStore
+            will also be deleted.
+        :param project_id: The ID of the Google Cloud project that contains the
+            feature store. If not provided, will attempt to determine from the
+            environment.
+        :param location: The Google Cloud region where the feature store is located
+            (e.g., 'us-central1', 'us-east1').
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request as metadata.
+        """
+        client = self.get_feature_online_store_admin_service_client(location)
+        return client.delete_feature_online_store(
+            name=f"projects/{project_id}/locations/{location}/featureOnlineStores/{feature_online_store_id}",
+            force=force,
+            timeout=timeout,
+            retry=retry,
+            metadata=metadata,
+        )