azure-ai-evaluation 1.4.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

azure/ai/evaluation/_common/__init__.py

@@ -8,9 +8,14 @@
 from . import constants
 from .rai_service import evaluate_with_rai_service
 from .utils import get_harm_severity_level
+from .evaluation_onedp_client import EvaluationServiceOneDPClient
+from .onedp.models import EvaluationUpload, EvaluationResult
 
 __all__ = [
     "get_harm_severity_level",
     "evaluate_with_rai_service",
     "constants",
+    "EvaluationServiceOneDPClient",
+    "EvaluationResult",
+    "EvaluationUpload",
 ]

azure/ai/evaluation/_common/evaluation_onedp_client.py

@@ -0,0 +1,118 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import logging
+from typing import Union, Any, Dict
+from azure.core.credentials import AzureKeyCredential, TokenCredential
+from azure.ai.evaluation._common.onedp import AIProjectClient as RestEvaluationServiceClient
+from azure.ai.evaluation._common.onedp.models import (PendingUploadRequest, PendingUploadType, EvaluationResult,
+                                                      ResultType, AssetCredentialRequest, EvaluationUpload, InputDataset)
+from azure.storage.blob import ContainerClient
+from .utils import upload
+
+LOGGER = logging.getLogger(__name__)
+
+class EvaluationServiceOneDPClient:
+
+    def __init__(self, endpoint: str, credential: Union[AzureKeyCredential, "TokenCredential"], **kwargs: Any) -> None:
+        self.rest_client = RestEvaluationServiceClient(
+            endpoint=endpoint,
+            credential=credential,
+            **kwargs,
+        )
+
+    def create_evaluation_result(self, *, name: str, path: str, version=1, metrics: Dict[str, int]=None, **kwargs) -> EvaluationResult:
+        """Create and upload evaluation results to Azure evaluation service.
+
+        This method uploads evaluation results from a local path to Azure Blob Storage
+        and registers them with the evaluation service. The process involves:
+        1. Starting a pending upload with the evaluation service
+        2. Getting a SAS token for the blob container
+        3. Uploading the local evaluation results to the blob container
+        4. Creating a version record for the evaluation results
+
+        :param name: The name to identify the evaluation results
+        :type name: str
+        :param path: The local path to the evaluation results file or directory
+        :type path: str
+        :param version: The version number for the evaluation results, defaults to 1
+        :type version: int, optional
+        :param metrics: Metrics to be added to evaluation result
+        :type version: Dict[str, int], optional
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The response from creating the evaluation result version
+        :rtype: EvaluationResult
+        :raises: Various exceptions from the underlying API calls or upload process
+        """
+
+        LOGGER.debug(f"Creating evaluation result for {name} with version {version} from path {path}")
+        start_pending_upload_response = self.rest_client.evaluation_results.start_pending_upload(
+            name=name,
+            version=version,
+            body=PendingUploadRequest(pending_upload_type=PendingUploadType.TEMPORARY_BLOB_REFERENCE),
+            **kwargs
+        )
+
+        LOGGER.debug(f"Uploading {path} to {start_pending_upload_response.blob_reference_for_consumption.blob_uri}")
+        with ContainerClient.from_container_url(
+            start_pending_upload_response.blob_reference_for_consumption.credential.sas_uri) as container_client:
+            upload(path=path, container_client=container_client, logger=LOGGER)
+
+        LOGGER.debug(f"Creating evaluation result version for {name} with version {version}")
+        create_version_response = self.rest_client.evaluation_results.create_or_update_version(
+            body=EvaluationResult(
+                blob_uri=start_pending_upload_response.blob_reference_for_consumption.blob_uri,
+                result_type=ResultType.EVALUATION,
+                name=name,
+                version=version,
+                metrics=metrics,
+            ),
+            name=name,
+            version=version,
+            **kwargs
+        )
+
+        return create_version_response
+
+    def start_evaluation_run(self, *, evaluation: EvaluationUpload, **kwargs) -> EvaluationUpload:
+        """Start a new evaluation run in the Azure evaluation service.
+
+        This method creates a new evaluation run with the provided configuration details.
+
+        :param evaluation: The evaluation configuration to upload
+        :type evaluation: EvaluationUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The created evaluation run object
+        :rtype: EvaluationUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        upload_run_response = self.rest_client.evaluations.upload_run(
+            evaluation=evaluation,
+            **kwargs
+        )
+
+        return upload_run_response
+
+    def update_evaluation_run(self, *, name: str, evaluation: EvaluationUpload, **kwargs) -> EvaluationUpload:
+        """Update an existing evaluation run in the Azure evaluation service.
+
+        This method updates an evaluation run with new information such as status changes,
+        result references, or other metadata.
+
+        :param name: The identifier of the evaluation run to update
+        :type name: str
+        :param evaluation: The updated evaluation configuration
+        :type evaluation: EvaluationUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The updated evaluation run object
+        :rtype: EvaluationUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        update_run_response = self.rest_client.evaluations.upload_update_run(
+            name=name,
+            evaluation=evaluation,
+            **kwargs
+        )
+
+        return update_run_response

azure/ai/evaluation/_common/onedp/__init__.py

@@ -0,0 +1,32 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) Python Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+# pylint: disable=wrong-import-position
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ._patch import *  # pylint: disable=unused-wildcard-import
+
+from ._client import AIProjectClient  # type: ignore
+from ._version import VERSION
+
+__version__ = VERSION
+
+try:
+    from ._patch import __all__ as _patch_all
+    from ._patch import *
+except ImportError:
+    _patch_all = []
+from ._patch import patch_sdk as _patch_sdk
+
+__all__ = [
+    "AIProjectClient",
+]
+__all__.extend([p for p in _patch_all if p not in __all__])  # pyright: ignore
+
+_patch_sdk()

azure/ai/evaluation/_common/onedp/_client.py

@@ -0,0 +1,139 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) Python Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+
+from copy import deepcopy
+from typing import Any, TYPE_CHECKING, Union
+from typing_extensions import Self
+
+from azure.core import PipelineClient
+from azure.core.credentials import AzureKeyCredential
+from azure.core.pipeline import policies
+from azure.core.rest import HttpRequest, HttpResponse
+
+from ._configuration import AIProjectClientConfiguration
+from ._serialization import Deserializer, Serializer
+from .operations import (
+    ConnectionsOperations,
+    DatasetsOperations,
+    DeploymentsOperations,
+    EvaluationResultsOperations,
+    EvaluationsOperations,
+    IndexesOperations,
+    RedTeamsOperations,
+    ServicePatternsOperations,
+)
+
+if TYPE_CHECKING:
+    from azure.core.credentials import TokenCredential
+
+
+class AIProjectClient:  # pylint: disable=too-many-instance-attributes
+    """AIProjectClient.
+
+    :ivar service_patterns: ServicePatternsOperations operations
+    :vartype service_patterns: azure.ai.projects.onedp.operations.ServicePatternsOperations
+    :ivar connections: ConnectionsOperations operations
+    :vartype connections: azure.ai.projects.onedp.operations.ConnectionsOperations
+    :ivar evaluations: EvaluationsOperations operations
+    :vartype evaluations: azure.ai.projects.onedp.operations.EvaluationsOperations
+    :ivar datasets: DatasetsOperations operations
+    :vartype datasets: azure.ai.projects.onedp.operations.DatasetsOperations
+    :ivar indexes: IndexesOperations operations
+    :vartype indexes: azure.ai.projects.onedp.operations.IndexesOperations
+    :ivar deployments: DeploymentsOperations operations
+    :vartype deployments: azure.ai.projects.onedp.operations.DeploymentsOperations
+    :ivar red_teams: RedTeamsOperations operations
+    :vartype red_teams: azure.ai.projects.onedp.operations.RedTeamsOperations
+    :ivar evaluation_results: EvaluationResultsOperations operations
+    :vartype evaluation_results: azure.ai.projects.onedp.operations.EvaluationResultsOperations
+    :param endpoint: Project endpoint in the form of:
+     https://<aiservices-id>.services.ai.azure.com/api/projects/<project-name>. Required.
+    :type endpoint: str
+    :param credential: Credential used to authenticate requests to the service. Is either a key
+     credential type or a token credential type. Required.
+    :type credential: ~azure.core.credentials.AzureKeyCredential or
+     ~azure.core.credentials.TokenCredential
+    :keyword api_version: The API version to use for this operation. Default value is
+     "2025-05-15-preview". Note that overriding this default value may result in unsupported
+     behavior.
+    :paramtype api_version: str
+    """
+
+    def __init__(self, endpoint: str, credential: Union[AzureKeyCredential, "TokenCredential"], **kwargs: Any) -> None:
+        _endpoint = "{endpoint}"
+        self._config = AIProjectClientConfiguration(endpoint=endpoint, credential=credential, **kwargs)
+        _policies = kwargs.pop("policies", None)
+        if _policies is None:
+            _policies = [
+                policies.RequestIdPolicy(**kwargs),
+                self._config.headers_policy,
+                self._config.user_agent_policy,
+                self._config.proxy_policy,
+                policies.ContentDecodePolicy(**kwargs),
+                self._config.redirect_policy,
+                self._config.retry_policy,
+                self._config.authentication_policy,
+                self._config.custom_hook_policy,
+                self._config.logging_policy,
+                policies.DistributedTracingPolicy(**kwargs),
+                policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None,
+                self._config.http_logging_policy,
+            ]
+        self._client: PipelineClient = PipelineClient(base_url=_endpoint, policies=_policies, **kwargs)
+
+        self._serialize = Serializer()
+        self._deserialize = Deserializer()
+        self._serialize.client_side_validation = False
+        self.service_patterns = ServicePatternsOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
+        self.connections = ConnectionsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluations = EvaluationsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.datasets = DatasetsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.indexes = IndexesOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.deployments = DeploymentsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.red_teams = RedTeamsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluation_results = EvaluationResultsOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
+
+    def send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: Any) -> HttpResponse:
+        """Runs the network request through the client's chained policies.
+
+        >>> from azure.core.rest import HttpRequest
+        >>> request = HttpRequest("GET", "https://www.example.org/")
+        <HttpRequest [GET], url: 'https://www.example.org/'>
+        >>> response = client.send_request(request)
+        <HttpResponse: 200 OK>
+
+        For more information on this code flow, see https://aka.ms/azsdk/dpcodegen/python/send_request
+
+        :param request: The network request you want to make. Required.
+        :type request: ~azure.core.rest.HttpRequest
+        :keyword bool stream: Whether the response payload will be streamed. Defaults to False.
+        :return: The response of your network call. Does not do error handling on your response.
+        :rtype: ~azure.core.rest.HttpResponse
+        """
+
+        request_copy = deepcopy(request)
+        path_format_arguments = {
+            "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
+        }
+
+        request_copy.url = self._client.format_url(request_copy.url, **path_format_arguments)
+        return self._client.send_request(request_copy, stream=stream, **kwargs)  # type: ignore
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> Self:
+        self._client.__enter__()
+        return self
+
+    def __exit__(self, *exc_details: Any) -> None:
+        self._client.__exit__(*exc_details)

azure/ai/evaluation/_common/onedp/_configuration.py

@@ -0,0 +1,73 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) Python Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+
+from typing import Any, TYPE_CHECKING, Union
+
+from azure.core.credentials import AzureKeyCredential
+from azure.core.pipeline import policies
+
+from ._version import VERSION
+
+if TYPE_CHECKING:
+    from azure.core.credentials import TokenCredential
+
+
+class AIProjectClientConfiguration:  # pylint: disable=too-many-instance-attributes
+    """Configuration for AIProjectClient.
+
+    Note that all parameters used to create this instance are saved as instance
+    attributes.
+
+    :param endpoint: Project endpoint in the form of:
+     https://<aiservices-id>.services.ai.azure.com/api/projects/<project-name>. Required.
+    :type endpoint: str
+    :param credential: Credential used to authenticate requests to the service. Is either a key
+     credential type or a token credential type. Required.
+    :type credential: ~azure.core.credentials.AzureKeyCredential or
+     ~azure.core.credentials.TokenCredential
+    :keyword api_version: The API version to use for this operation. Default value is
+     "2025-05-15-preview". Note that overriding this default value may result in unsupported
+     behavior.
+    :paramtype api_version: str
+    """
+
+    def __init__(self, endpoint: str, credential: Union[AzureKeyCredential, "TokenCredential"], **kwargs: Any) -> None:
+        api_version: str = kwargs.pop("api_version", "2025-05-15-preview")
+
+        if endpoint is None:
+            raise ValueError("Parameter 'endpoint' must not be None.")
+        if credential is None:
+            raise ValueError("Parameter 'credential' must not be None.")
+
+        self.endpoint = endpoint
+        self.credential = credential
+        self.api_version = api_version
+        self.credential_scopes = kwargs.pop("credential_scopes", ["https://ai.azure.com/.default"])
+        kwargs.setdefault("sdk_moniker", "ai-projects-onedp/{}".format(VERSION))
+        self.polling_interval = kwargs.get("polling_interval", 30)
+        self._configure(**kwargs)
+
+    def _infer_policy(self, **kwargs):
+        if isinstance(self.credential, AzureKeyCredential):
+            return policies.AzureKeyCredentialPolicy(self.credential, "Authorization", prefix="Bearer", **kwargs)
+        if hasattr(self.credential, "get_token"):
+            return policies.BearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs)
+        raise TypeError(f"Unsupported credential: {self.credential}")
+
+    def _configure(self, **kwargs: Any) -> None:
+        self.user_agent_policy = kwargs.get("user_agent_policy") or policies.UserAgentPolicy(**kwargs)
+        self.headers_policy = kwargs.get("headers_policy") or policies.HeadersPolicy(**kwargs)
+        self.proxy_policy = kwargs.get("proxy_policy") or policies.ProxyPolicy(**kwargs)
+        self.logging_policy = kwargs.get("logging_policy") or policies.NetworkTraceLoggingPolicy(**kwargs)
+        self.http_logging_policy = kwargs.get("http_logging_policy") or policies.HttpLoggingPolicy(**kwargs)
+        self.custom_hook_policy = kwargs.get("custom_hook_policy") or policies.CustomHookPolicy(**kwargs)
+        self.redirect_policy = kwargs.get("redirect_policy") or policies.RedirectPolicy(**kwargs)
+        self.retry_policy = kwargs.get("retry_policy") or policies.RetryPolicy(**kwargs)
+        self.authentication_policy = kwargs.get("authentication_policy")
+        if self.credential and not self.authentication_policy:
+            self.authentication_policy = self._infer_policy(**kwargs)