azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_common/evaluation_onedp_client.py

@@ -0,0 +1,169 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import logging
+from typing import Union, Any, Dict
+from azure.core.credentials import AzureKeyCredential, TokenCredential
+from azure.ai.evaluation._common.onedp import ProjectsClient as RestEvaluationServiceClient
+from azure.ai.evaluation._common.onedp.models import (
+    PendingUploadRequest,
+    PendingUploadType,
+    EvaluationResult,
+    ResultType,
+    AssetCredentialRequest,
+    EvaluationUpload,
+    InputDataset,
+    RedTeamUpload,
+)
+from azure.storage.blob import ContainerClient
+from .utils import upload
+
+LOGGER = logging.getLogger(__name__)
+
+
+class EvaluationServiceOneDPClient:
+
+    def __init__(self, endpoint: str, credential: Union[AzureKeyCredential, "TokenCredential"], **kwargs: Any) -> None:
+        self.rest_client = RestEvaluationServiceClient(
+            endpoint=endpoint,
+            credential=credential,
+            **kwargs,
+        )
+
+    def create_evaluation_result(
+        self,
+        *,
+        name: str,
+        path: str,
+        version=1,
+        metrics: Dict[str, int] = None,
+        result_type: ResultType = ResultType.EVALUATION,
+        **kwargs,
+    ) -> EvaluationResult:
+        """Create and upload evaluation results to Azure evaluation service.
+
+        This method uploads evaluation results from a local path to Azure Blob Storage
+        and registers them with the evaluation service. The process involves:
+        1. Starting a pending upload with the evaluation service
+        2. Getting a SAS token for the blob container
+        3. Uploading the local evaluation results to the blob container
+        4. Creating a version record for the evaluation results
+
+        :param name: The name to identify the evaluation results
+        :type name: str
+        :param path: The local path to the evaluation results file or directory
+        :type path: str
+        :param version: The version number for the evaluation results, defaults to 1
+        :type version: int, optional
+        :param metrics: Metrics to be added to evaluation result
+        :type metrics: Dict[str, int], optional
+        :param result_type: Evaluation Result Type to create
+        :type result_type: ResultType, optional
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The response from creating the evaluation result version
+        :rtype: EvaluationResult
+        :raises: Various exceptions from the underlying API calls or upload process
+        """
+
+        LOGGER.debug(
+            f"Creating evaluation result for {name} with version {version} type {result_type} from path {path}"
+        )
+        start_pending_upload_response = self.rest_client.evaluation_results.start_pending_upload(
+            name=name,
+            version=str(version),
+            body=PendingUploadRequest(pending_upload_type=PendingUploadType.TEMPORARY_BLOB_REFERENCE),
+            **kwargs,
+        )
+
+        LOGGER.debug(f"Uploading {path} to {start_pending_upload_response.blob_reference_for_consumption.blob_uri}")
+        with ContainerClient.from_container_url(
+            start_pending_upload_response.blob_reference_for_consumption.credential.sas_uri
+        ) as container_client:
+            upload(path=path, container_client=container_client, logger=LOGGER)
+
+        LOGGER.debug(f"Creating evaluation result version for {name} with version {version}")
+        create_version_response = self.rest_client.evaluation_results.create_or_update_version(
+            evaluation_result=EvaluationResult(
+                blob_uri=start_pending_upload_response.blob_reference_for_consumption.blob_uri,
+                result_type=result_type,
+                name=name,
+                version=str(version),
+                metrics=metrics,
+            ),
+            name=name,
+            version=str(version),
+            **kwargs,
+        )
+
+        return create_version_response
+
+    def start_evaluation_run(self, *, evaluation: EvaluationUpload, **kwargs) -> EvaluationUpload:
+        """Start a new evaluation run in the Azure evaluation service.
+
+        This method creates a new evaluation run with the provided configuration details.
+
+        :param evaluation: The evaluation configuration to upload
+        :type evaluation: EvaluationUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The created evaluation run object
+        :rtype: EvaluationUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        upload_run_response = self.rest_client.evaluations.upload_run(evaluation=evaluation, **kwargs)
+
+        return upload_run_response
+
+    def update_evaluation_run(self, *, name: str, evaluation: EvaluationUpload, **kwargs) -> EvaluationUpload:
+        """Update an existing evaluation run in the Azure evaluation service.
+
+        This method updates an evaluation run with new information such as status changes,
+        result references, or other metadata.
+
+        :param name: The identifier of the evaluation run to update
+        :type name: str
+        :param evaluation: The updated evaluation configuration
+        :type evaluation: EvaluationUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The updated evaluation run object
+        :rtype: EvaluationUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        update_run_response = self.rest_client.evaluations.upload_update_run(name=name, evaluation=evaluation, **kwargs)
+
+        return update_run_response
+
+    def start_red_team_run(self, *, red_team: RedTeamUpload, **kwargs):
+        """Start a new red team run in the Azure evaluation service.
+
+        This method creates a new red team run with the provided configuration details.
+
+        :param red_team: The red team configuration to upload
+        :type red_team: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The created red team run object
+        :rtype: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        upload_run_response = self.rest_client.red_teams.upload_run(redteam=red_team, **kwargs)
+
+        return upload_run_response
+
+    def update_red_team_run(self, *, name: str, red_team: RedTeamUpload, **kwargs):
+        """Update an existing red team run in the Azure evaluation service.
+
+        This method updates a red team run with new information such as status changes,
+        result references, or other metadata.
+
+        :param name: The identifier of the red team run to update
+        :type name: str
+        :param red_team: The updated red team configuration
+        :type red_team: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :param kwargs: Additional keyword arguments to pass to the underlying API calls
+        :return: The updated red team run object
+        :rtype: ~azure.ai.evaluation._common.onedp.models.RedTeamUpload
+        :raises: Various exceptions from the underlying API calls
+        """
+        update_run_response = self.rest_client.red_teams.upload_update_run(name=name, redteam=red_team, **kwargs)
+
+        return update_run_response

azure/ai/evaluation/_common/math.py

@@ -0,0 +1,89 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import math
+from typing import List, Callable, Any
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+
+
+def list_sum(lst: List[float]) -> float:
+    """Given a list of floats, return the sum of the values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :return: The sum of the values in the list.
+    :rtype: float
+    """
+
+    return sum(lst)
+
+
+def list_mean(lst: List[float]) -> float:
+    """Given a list of floats, calculate the mean of the values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :return: The mean of the values in the list.
+    :rtype: float
+    """
+
+    return list_sum(lst) / len(lst)
+
+
+def list_mean_nan_safe(lst: List[float]) -> float:
+    """Given a list of floats, remove all nan or None values, then calculate the mean of the remaining values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :return: The mean of the values in the list.
+    :rtype: float
+    """
+
+    msg = "All score values are NaN. The mean cannot be calculated."
+    if all(math.isnan(l) for l in lst):
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.CONVERSATION,
+        )
+    return list_mean([l for l in lst if not is_none_or_nan(l)])
+
+
+def apply_transform_nan_safe(lst: List[float], transform_fn: Callable[[float], Any]) -> List[Any]:
+    """Given a list of floats, remove all nan values, then apply the inputted transform function
+    to the remaining values, and return the resulting list of outputted values.
+
+    :param lst: A list of floats.
+    :type lst: List[float]
+    :param transform_fn: A function that produces something when applied to a float.
+    :type transform_fn: Callable[[float], Any]
+    :return: A list of the transformed values.
+    :rtype: List[Any]
+    """
+
+    msg = "All score values are NaN. The mean cannot be calculated."
+    if all(math.isnan(l) for l in lst):
+        raise EvaluationException(
+            message=msg,
+            internal_message=msg,
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.CONVERSATION,
+        )
+    return [transform_fn(l) for l in lst if not is_none_or_nan(l)]
+
+
+def is_none_or_nan(val: float) -> bool:
+    """math.isnan raises an error if None is inputted. This is a more robust wrapper.
+
+    :param val: The value to check.
+    :type val: float
+    :return: Whether the value is None or NaN.
+    :rtype: bool
+    """
+
+    return val is None or math.isnan(val)

azure/ai/evaluation/_common/onedp/__init__.py

@@ -0,0 +1,32 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) Python Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+# pylint: disable=wrong-import-position
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ._patch import *  # pylint: disable=unused-wildcard-import
+
+from ._client import ProjectsClient  # type: ignore
+from ._version import VERSION
+
+__version__ = VERSION
+
+try:
+    from ._patch import __all__ as _patch_all
+    from ._patch import *
+except ImportError:
+    _patch_all = []
+from ._patch import patch_sdk as _patch_sdk
+
+__all__ = [
+    "ProjectsClient",
+]
+__all__.extend([p for p in _patch_all if p not in __all__])  # pyright: ignore
+
+_patch_sdk()

azure/ai/evaluation/_common/onedp/_client.py

@@ -0,0 +1,166 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) Python Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+
+from copy import deepcopy
+from typing import Any, TYPE_CHECKING
+from typing_extensions import Self
+
+from azure.core import PipelineClient
+from azure.core.pipeline import policies
+from azure.core.rest import HttpRequest, HttpResponse
+
+from ._configuration import ProjectsClientConfiguration
+from ._utils.serialization import Deserializer, Serializer
+from .operations import (
+    ConnectionsOperations,
+    DatasetsOperations,
+    DeploymentsOperations,
+    EvaluationResultsOperations,
+    EvaluationRulesOperations,
+    EvaluationTaxonomiesOperations,
+    EvaluationsOperations,
+    EvaluatorsOperations,
+    IndexesOperations,
+    InsightsOperations,
+    RedTeamsOperations,
+    SchedulesOperations,
+    SyncEvalsOperations,
+)
+
+if TYPE_CHECKING:
+    from azure.core.credentials import TokenCredential
+
+
+class ProjectsClient:  # pylint: disable=too-many-instance-attributes
+    """ProjectsClient.
+
+    :ivar connections: ConnectionsOperations operations
+    :vartype connections: azure.ai.projects.operations.ConnectionsOperations
+    :ivar sync_evals: SyncEvalsOperations operations
+    :vartype sync_evals: azure.ai.projects.operations.SyncEvalsOperations
+    :ivar evaluations: EvaluationsOperations operations
+    :vartype evaluations: azure.ai.projects.operations.EvaluationsOperations
+    :ivar evaluators: EvaluatorsOperations operations
+    :vartype evaluators: azure.ai.projects.operations.EvaluatorsOperations
+    :ivar datasets: DatasetsOperations operations
+    :vartype datasets: azure.ai.projects.operations.DatasetsOperations
+    :ivar indexes: IndexesOperations operations
+    :vartype indexes: azure.ai.projects.operations.IndexesOperations
+    :ivar insights: InsightsOperations operations
+    :vartype insights: azure.ai.projects.operations.InsightsOperations
+    :ivar deployments: DeploymentsOperations operations
+    :vartype deployments: azure.ai.projects.operations.DeploymentsOperations
+    :ivar red_teams: RedTeamsOperations operations
+    :vartype red_teams: azure.ai.projects.operations.RedTeamsOperations
+    :ivar evaluation_taxonomies: EvaluationTaxonomiesOperations operations
+    :vartype evaluation_taxonomies: azure.ai.projects.operations.EvaluationTaxonomiesOperations
+    :ivar schedules: SchedulesOperations operations
+    :vartype schedules: azure.ai.projects.operations.SchedulesOperations
+    :ivar evaluation_results: EvaluationResultsOperations operations
+    :vartype evaluation_results: azure.ai.projects.operations.EvaluationResultsOperations
+    :ivar evaluation_rules: EvaluationRulesOperations operations
+    :vartype evaluation_rules: azure.ai.projects.operations.EvaluationRulesOperations
+    :param endpoint: Project endpoint. In the form
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project>`_"
+     if your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the
+     form
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name>`_"
+     if you want to explicitly
+     specify the Foundry Project name. Required.
+    :type endpoint: str
+    :param credential: Credential used to authenticate requests to the service. Required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword api_version: The API version to use for this operation. Default value is
+     "2025-11-15-preview". Note that overriding this default value may result in unsupported
+     behavior.
+    :paramtype api_version: str
+    """
+
+    def __init__(self, endpoint: str, credential: "TokenCredential", **kwargs: Any) -> None:
+        _endpoint = "{endpoint}"
+        self._config = ProjectsClientConfiguration(endpoint=endpoint, credential=credential, **kwargs)
+
+        _policies = kwargs.pop("policies", None)
+        if _policies is None:
+            _policies = [
+                policies.RequestIdPolicy(**kwargs),
+                self._config.headers_policy,
+                self._config.user_agent_policy,
+                self._config.proxy_policy,
+                policies.ContentDecodePolicy(**kwargs),
+                self._config.redirect_policy,
+                self._config.retry_policy,
+                self._config.authentication_policy,
+                self._config.custom_hook_policy,
+                self._config.logging_policy,
+                policies.DistributedTracingPolicy(**kwargs),
+                policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None,
+                self._config.http_logging_policy,
+            ]
+        self._client: PipelineClient = PipelineClient(base_url=_endpoint, policies=_policies, **kwargs)
+
+        self._serialize = Serializer()
+        self._deserialize = Deserializer()
+        self._serialize.client_side_validation = False
+        self.connections = ConnectionsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.sync_evals = SyncEvalsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluations = EvaluationsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluators = EvaluatorsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.datasets = DatasetsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.indexes = IndexesOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.insights = InsightsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.deployments = DeploymentsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.red_teams = RedTeamsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluation_taxonomies = EvaluationTaxonomiesOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
+        self.schedules = SchedulesOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluation_results = EvaluationResultsOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
+        self.evaluation_rules = EvaluationRulesOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
+
+    def send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: Any) -> HttpResponse:
+        """Runs the network request through the client's chained policies.
+
+        >>> from azure.core.rest import HttpRequest
+        >>> request = HttpRequest("GET", "https://www.example.org/")
+        <HttpRequest [GET], url: 'https://www.example.org/'>
+        >>> response = client.send_request(request)
+        <HttpResponse: 200 OK>
+
+        For more information on this code flow, see https://aka.ms/azsdk/dpcodegen/python/send_request
+
+        :param request: The network request you want to make. Required.
+        :type request: ~azure.core.rest.HttpRequest
+        :keyword bool stream: Whether the response payload will be streamed. Defaults to False.
+        :return: The response of your network call. Does not do error handling on your response.
+        :rtype: ~azure.core.rest.HttpResponse
+        """
+
+        request_copy = deepcopy(request)
+        path_format_arguments = {
+            "endpoint": self._serialize.url("self._config.endpoint", self._config.endpoint, "str", skip_quote=True),
+        }
+
+        request_copy.url = self._client.format_url(request_copy.url, **path_format_arguments)
+        return self._client.send_request(request_copy, stream=stream, **kwargs)  # type: ignore
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> Self:
+        self._client.__enter__()
+        return self
+
+    def __exit__(self, *exc_details: Any) -> None:
+        self._client.__exit__(*exc_details)

azure/ai/evaluation/_common/onedp/_configuration.py

@@ -0,0 +1,72 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) Python Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+
+from typing import Any, TYPE_CHECKING
+
+from azure.core.pipeline import policies
+
+from ._version import VERSION
+
+if TYPE_CHECKING:
+    from azure.core.credentials import TokenCredential
+
+
+class ProjectsClientConfiguration:  # pylint: disable=too-many-instance-attributes
+    """Configuration for ProjectsClient.
+
+    Note that all parameters used to create this instance are saved as instance
+    attributes.
+
+    :param endpoint: Project endpoint. In the form
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project>`_"
+     if your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the
+     form
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name>`_"
+     if you want to explicitly
+     specify the Foundry Project name. Required.
+    :type endpoint: str
+    :param credential: Credential used to authenticate requests to the service. Required.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword api_version: The API version to use for this operation. Default value is
+     "2025-11-15-preview". Note that overriding this default value may result in unsupported
+     behavior.
+    :paramtype api_version: str
+    """
+
+    def __init__(self, endpoint: str, credential: "TokenCredential", **kwargs: Any) -> None:
+        api_version: str = kwargs.pop("api_version", "2025-11-15-preview")
+
+        if endpoint is None:
+            raise ValueError("Parameter 'endpoint' must not be None.")
+        if credential is None:
+            raise ValueError("Parameter 'credential' must not be None.")
+
+        self.endpoint = endpoint
+        self.credential = credential
+        self.api_version = api_version
+        self.credential_scopes = kwargs.pop("credential_scopes", ["https://ai.azure.com/.default"])
+        kwargs.setdefault("sdk_moniker", "ai-projects/{}".format(VERSION))
+        self.polling_interval = kwargs.get("polling_interval", 30)
+        self._configure(**kwargs)
+
+    def _configure(self, **kwargs: Any) -> None:
+        self.user_agent_policy = kwargs.get("user_agent_policy") or policies.UserAgentPolicy(**kwargs)
+        self.headers_policy = kwargs.get("headers_policy") or policies.HeadersPolicy(**kwargs)
+        self.proxy_policy = kwargs.get("proxy_policy") or policies.ProxyPolicy(**kwargs)
+        self.logging_policy = kwargs.get("logging_policy") or policies.NetworkTraceLoggingPolicy(**kwargs)
+        self.http_logging_policy = kwargs.get("http_logging_policy") or policies.HttpLoggingPolicy(**kwargs)
+        self.custom_hook_policy = kwargs.get("custom_hook_policy") or policies.CustomHookPolicy(**kwargs)
+        self.redirect_policy = kwargs.get("redirect_policy") or policies.RedirectPolicy(**kwargs)
+        self.retry_policy = kwargs.get("retry_policy") or policies.RetryPolicy(**kwargs)
+        self.authentication_policy = kwargs.get("authentication_policy")
+        if self.credential and not self.authentication_policy:
+            self.authentication_policy = policies.BearerTokenCredentialPolicy(
+                self.credential, *self.credential_scopes, **kwargs
+            )