azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_aoai/text_similarity_grader.py

@@ -0,0 +1,80 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Optional, Union
+
+from openai.types.graders import TextSimilarityGrader
+from typing_extensions import Literal
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
+
+from .aoai_grader import AzureOpenAIGrader
+
+
+@experimental
+class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
+    """Wrapper class for OpenAI's string check graders.
+
+    Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[
+            ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+            ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param evaluation_metric: The evaluation metric to use.
+    :type evaluation_metric: Literal["fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3",
+        "rouge_4", "rouge_5", "rouge_l", "cosine"]
+    :param input: The text being graded.
+    :type input: str
+    :param pass_threshold: A float score where a value greater than or equal indicates a passing grade.
+    :type pass_threshold: float
+    :param reference: The text being graded against.
+    :type reference: str
+    :param name: The name of the grader.
+    :type name: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/text_similarity_grader"
+    _type = "text_similarity"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        evaluation_metric: Literal[
+            "fuzzy_match",
+            "bleu",
+            "gleu",
+            "meteor",
+            "rouge_1",
+            "rouge_2",
+            "rouge_3",
+            "rouge_4",
+            "rouge_5",
+            "rouge_l",
+            "cosine",
+        ],
+        input: str,
+        pass_threshold: float,
+        reference: str,
+        name: str,
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any
+    ):
+        grader = TextSimilarityGrader(
+            evaluation_metric=evaluation_metric,
+            input=input,
+            pass_threshold=pass_threshold,
+            name=name,
+            reference=reference,
+            type=AzureOpenAITextSimilarityGrader._type,
+        )
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_azure/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

azure/ai/evaluation/_azure/_clients.py

@@ -0,0 +1,204 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from logging import Logger
+from typing import Any, Dict, Final, Optional, Set, Union, cast
+from threading import Lock
+from urllib.parse import quote
+from json.decoder import JSONDecodeError
+
+from azure.core.credentials import TokenCredential, AzureSasCredential, AccessToken
+from azure.core.rest import HttpResponse
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._http_utils import HttpPipeline, get_http_client
+from azure.ai.evaluation._azure._token_manager import AzureMLTokenManager
+from azure.ai.evaluation._constants import TokenScope
+from ._models import BlobStoreInfo, Workspace
+
+
+API_VERSION: Final[str] = "2024-07-01-preview"
+QUERY_KEY_API_VERSION: Final[str] = "api-version"
+PATH_ML_WORKSPACES = ("providers", "Microsoft.MachineLearningServices", "workspaces")
+
+
+class LiteMLClient:
+    """A lightweight Azure ML API client.
+
+    :param subscription_id: Azure subscription ID
+    :type subscription_id: str
+    :param resource_group: Azure resource group name
+    :type resource_group: str
+    :param logger: Logger object
+    :type logger: logging.Logger
+    :keyword credential: Azure credentials
+    :paramtype credential: TokenCredential
+    :keyword kwargs: Additional keyword arguments
+    :paramtype kwargs: Dict
+    :keyword str api_version: The API version. Default is 2024-10-01
+    """
+
+    def __init__(
+        self,
+        subscription_id: str,
+        resource_group: str,
+        logger: Logger,
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any,
+    ) -> None:
+        subscription_id = quote(subscription_id, safe="")
+        resource_group = quote(resource_group, safe="")
+
+        self._base_url: Final[str] = (
+            f"https://management.azure.com/subscriptions/{subscription_id}/resourceGroups/{resource_group}"
+        )
+        self._logger: Final[Logger] = logger
+        self._api_version: Final[str] = kwargs.get("api_version", API_VERSION)
+        self._http_client: Final[HttpPipeline] = get_http_client(**kwargs)
+        self._lock: Final[Lock] = Lock()
+
+        # things that can change under lock
+        self._token_manager: Optional[AzureMLTokenManager] = None
+        self._credential: Optional[TokenCredential] = credential
+
+    def get_token(self) -> AccessToken:
+        return self._get_token_manager().get_token()
+
+    def get_credential(self) -> TokenCredential:
+        # load the token manager to get the credential if needed
+        self._get_token_manager()
+        return cast(TokenCredential, self._credential)
+
+    def workspace_get_default_datastore(
+        self, workspace_name: str, *, include_credentials: bool = False, **kwargs: Any
+    ) -> BlobStoreInfo:
+        # 1. Get the default blob store
+        # REST API documentation:
+        # https://learn.microsoft.com/rest/api/azureml/datastores/list?view=rest-azureml-2024-10-01
+        url = self._generate_path(  # pylint: disable=specify-parameter-names-in-call
+            *PATH_ML_WORKSPACES, workspace_name, "datastores"
+        )
+        headers = self._get_headers()
+
+        stores_response = self._http_client.request(
+            method="GET",
+            url=url,
+            params={QUERY_KEY_API_VERSION: self._api_version, "isDefault": True, "count": 1, "orderByAsc": "false"},
+            headers=headers,
+        )
+        self._throw_on_http_error(stores_response, "list default workspace datastore")
+
+        json = stores_response.json()["value"][0]
+        props_json = json["properties"]
+        name = json["name"]
+        account_name = props_json["accountName"]
+        endpoint = props_json["endpoint"]
+        container_name = props_json["containerName"]
+        credential_type = props_json.get("credentials", {}).get("credentialsType")
+
+        # 2. Get the SAS token to use for accessing the blob store
+        # REST API documentation:
+        # https://learn.microsoft.com/rest/api/azureml/datastores/list-secrets?view=rest-azureml-2024-10-01
+        blob_store_credential: Optional[Union[AzureSasCredential, TokenCredential, str]]
+        if not include_credentials:
+            blob_store_credential = None
+        elif credential_type and credential_type.lower() == "none":
+            # If storage account key access is disabled, and only Microsoft Entra ID authentication is available,
+            # the credentialsType will be "None" and we should not attempt to get the secrets.
+            blob_store_credential = self.get_credential()
+        else:
+            url = self._generate_path(
+                *PATH_ML_WORKSPACES, workspace_name, "datastores", "workspaceblobstore", "listSecrets"
+            )
+            secrets_response = self._http_client.request(
+                method="POST",
+                url=url,
+                json={
+                    "expirableSecret": True,
+                    "expireAfterHours": int(kwargs.get("key_expiration_hours", 1)),
+                },
+                params={
+                    QUERY_KEY_API_VERSION: self._api_version,
+                },
+                headers=headers,
+            )
+            self._throw_on_http_error(secrets_response, "workspace datastore secrets")
+
+            secrets_json = secrets_response.json()
+            secrets_type = secrets_json["secretsType"].lower()
+
+            # As per this website, only SAS tokens, access tokens, or Entra IDs are valid for accessing blob data
+            # stores:
+            # https://learn.microsoft.com/rest/api/storageservices/authorize-requests-to-azure-storage.
+            if secrets_type == "sas":
+                blob_store_credential = AzureSasCredential(secrets_json["sasToken"])
+            elif secrets_type == "accountkey":
+                # To support older versions of azure-storage-blob better, we return a string here instead of
+                # an AzureNamedKeyCredential
+                blob_store_credential = secrets_json["key"]
+            else:
+                raise EvaluationException(
+                    message=f"The '{account_name}' blob store does not use a recognized credential type.",
+                    internal_message=f"The credential type is '{secrets_type}'",
+                    target=ErrorTarget.EVALUATE,
+                    category=ErrorCategory.INVALID_VALUE,
+                    blame=ErrorBlame.SYSTEM_ERROR,
+                )
+
+        return BlobStoreInfo(name, account_name, endpoint, container_name, blob_store_credential)
+
+    def workspace_get_info(self, workspace_name: str) -> Workspace:
+        # https://learn.microsoft.com/rest/api/azureml/workspaces/get?view=rest-azureml-2024-10-01
+        workspace_response = self._http_client.request(
+            "GET",
+            self._generate_path(*PATH_ML_WORKSPACES, workspace_name),
+            params={QUERY_KEY_API_VERSION: self._api_version},
+            headers=self._get_headers(),
+        )
+
+        self._throw_on_http_error(workspace_response, f"get '{workspace_name}' workspace")
+        workspace = Workspace.deserialize(workspace_response)
+        return workspace
+
+    def _get_token_manager(self) -> AzureMLTokenManager:
+        # Lazy init since getting credentials in the constructor can take a long time in some situations
+        if self._token_manager is None:
+            with self._lock:
+                if self._token_manager is None:
+                    self._token_manager = AzureMLTokenManager(
+                        TokenScope.DEFAULT_AZURE_MANAGEMENT.value, self._logger, credential=self._credential
+                    )
+                    self._credential = self._token_manager.credential
+
+        return self._token_manager
+
+    @staticmethod
+    def _throw_on_http_error(response: HttpResponse, description: str, valid_status: Optional[Set[int]] = None) -> None:
+        if valid_status and (response.status_code in valid_status):
+            return
+        if response.status_code >= 200 and response.status_code < 300:
+            # nothing to see here, move along
+            return
+
+        message = f"The {description} request failed with HTTP {response.status_code}"
+        try:
+            error_json = response.json()["error"]
+            additional_info = f"({error_json['code']}) {error_json['message']}"
+            message += f" - {additional_info}"
+        except (JSONDecodeError, ValueError, KeyError):
+            pass
+
+        raise EvaluationException(
+            message=message,
+            target=ErrorTarget.EVALUATE,
+            category=ErrorCategory.FAILED_EXECUTION,
+            blame=ErrorBlame.SYSTEM_ERROR,
+        )
+
+    def _generate_path(self, *paths: str) -> str:
+        sanitized_paths = [quote(path, safe="") for path in paths]
+        url = self._base_url + "/" + str.join("/", sanitized_paths)
+        return url
+
+    def _get_headers(self) -> Dict[str, str]:
+        return {"Authorization": f"Bearer {self.get_token().token}", "Content-Type": "application/json"}

azure/ai/evaluation/_azure/_envs.py

@@ -0,0 +1,207 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+# NOTE:
+# This is a simplified version of the original code from azure-ai-ml:
+# sdk\ml\azure-ai-ml\azure\ai\ml\_azure_environments.py
+
+import asyncio
+import os
+
+from typing import Any, Dict, Final, Mapping, Optional, Sequence, TypedDict
+
+from azure.core import AsyncPipelineClient
+from azure.core.configuration import Configuration
+from azure.core.rest import HttpRequest
+from azure.core.pipeline.policies import ProxyPolicy, AsyncRetryPolicy
+
+
+class AzureEnvironmentMetadata(TypedDict):
+    """Configuration for various Azure environments. All endpoints include a trailing slash."""
+
+    portal_endpoint: str
+    """The management portal for the Azure environment (e.g. https://portal.azure.com/)"""
+    resource_manager_endpoint: str
+    """The API endpoint for Azure control plan (e.g. https://management.azure.com/)"""
+    active_directory_endpoint: str
+    """The active directory endpoint used for authentication (e.g. https://login.microsoftonline.com/)"""
+    aml_resource_endpoint: str
+    """The endpoint for Azure Machine Learning resources (e.g. https://ml.azure.com/)"""
+    storage_suffix: str
+    """The suffix to use for storage endpoint URLs (e.g. core.windows.net)"""
+    registry_discovery_endpoint: str
+
+
+_ENV_ARM_CLOUD_METADATA_URL: Final[str] = "ARM_CLOUD_METADATA_URL"
+_ENV_DEFAULT_CLOUD_NAME: Final[str] = "AZUREML_CURRENT_CLOUD"
+_ENV_REGISTRY_DISCOVERY_URL: Final[str] = "REGISTRY_DISCOVERY_ENDPOINT_URL"
+_ENV_REGISTRY_DISCOVERY_REGION: Final[str] = "REGISTRY_DISCOVERY_ENDPOINT_REGION"
+_DEFAULT_REGISTRY_DISCOVERY_REGION: Final[str] = "west"
+_DEFAULT_AZURE_ENV_NAME: Final[str] = "AzureCloud"
+
+
+_ASYNC_LOCK = asyncio.Lock()
+_KNOWN_AZURE_ENVIRONMENTS: Dict[str, AzureEnvironmentMetadata] = {
+    _DEFAULT_AZURE_ENV_NAME: {
+        "portal_endpoint": "https://portal.azure.com/",
+        "resource_manager_endpoint": "https://management.azure.com/",
+        "active_directory_endpoint": "https://login.microsoftonline.com/",
+        "aml_resource_endpoint": "https://ml.azure.com/",
+        "storage_suffix": "core.windows.net",
+        "registry_discovery_endpoint": "https://eastus.api.azureml.ms/",
+    },
+    "AzureChinaCloud": {
+        "portal_endpoint": "https://portal.azure.cn/",
+        "resource_manager_endpoint": "https://management.chinacloudapi.cn/",
+        "active_directory_endpoint": "https://login.chinacloudapi.cn/",
+        "aml_resource_endpoint": "https://ml.azure.cn/",
+        "storage_suffix": "core.chinacloudapi.cn",
+        "registry_discovery_endpoint": "https://chinaeast2.api.ml.azure.cn/",
+    },
+    "AzureUSGovernment": {
+        "portal_endpoint": "https://portal.azure.us/",
+        "resource_manager_endpoint": "https://management.usgovcloudapi.net/",
+        "active_directory_endpoint": "https://login.microsoftonline.us/",
+        "aml_resource_endpoint": "https://ml.azure.us/",
+        "storage_suffix": "core.usgovcloudapi.net",
+        "registry_discovery_endpoint": "https://usgovarizona.api.ml.azure.us/",
+    },
+}
+
+
+class AzureEnvironmentClient:
+    DEFAULT_API_VERSION: Final[str] = "2019-05-01"
+    DEFAULT_AZURE_CLOUD_NAME: Final[str] = _DEFAULT_AZURE_ENV_NAME
+
+    def __init__(self, *, base_url: Optional[str] = None, **kwargs: Any) -> None:
+        base_url = base_url if base_url is not None else AzureEnvironmentClient.get_default_metadata_url()
+
+        config: Configuration = kwargs.pop("config", Configuration(**kwargs))
+        if config.retry_policy is None:
+            config.retry_policy = AsyncRetryPolicy(**kwargs)
+        if config.proxy_policy is None and "proxy" in kwargs:
+            config.proxy_policy = ProxyPolicy(proxies={"http": kwargs["proxy"], "https": kwargs["proxy"]})
+
+        self._async_client = AsyncPipelineClient(base_url, config=config, **kwargs)
+
+    async def get_default_cloud_name_async(self, *, update_cached: bool = True) -> str:
+        current_cloud_env = os.getenv(_ENV_DEFAULT_CLOUD_NAME)
+        if current_cloud_env is not None:
+            return current_cloud_env
+
+        arm_metadata_url = os.getenv(_ENV_ARM_CLOUD_METADATA_URL)
+        if arm_metadata_url is None:
+            return _DEFAULT_AZURE_ENV_NAME
+
+        # load clouds from metadata url
+        clouds = await self.get_clouds_async(metadata_url=arm_metadata_url, update_cached=update_cached)
+        matched = next(filter(lambda t: t[1]["resource_manager_endpoint"] in arm_metadata_url, clouds.items()), None)
+        if matched is None:
+            return _DEFAULT_AZURE_ENV_NAME
+
+        os.environ[_ENV_DEFAULT_CLOUD_NAME] = matched[0]
+        return matched[0]
+
+    async def get_cloud_async(self, name: str, *, update_cached: bool = True) -> Optional[AzureEnvironmentMetadata]:
+        default_endpoint: Optional[str]
+
+        def case_insensitive_match(d: Mapping[str, Any], key: str) -> Optional[Any]:
+            key = key.strip().lower()
+            return next((v for k, v in d.items() if k.strip().lower() == key), None)
+
+        async with _ASYNC_LOCK:
+            cloud = _KNOWN_AZURE_ENVIRONMENTS.get(name) or case_insensitive_match(_KNOWN_AZURE_ENVIRONMENTS, name)
+            if cloud:
+                return cloud
+            default_endpoint = _KNOWN_AZURE_ENVIRONMENTS.get(_DEFAULT_AZURE_ENV_NAME, {}).get(
+                "resource_manager_endpoint"
+            )
+
+        metadata_url = self.get_default_metadata_url(default_endpoint)
+        clouds = await self.get_clouds_async(metadata_url=metadata_url, update_cached=update_cached)
+        cloud_metadata = clouds.get(name) or case_insensitive_match(clouds, name)
+
+        return cloud_metadata
+
+    async def get_clouds_async(
+        self, *, metadata_url: Optional[str] = None, update_cached: bool = True
+    ) -> Mapping[str, AzureEnvironmentMetadata]:
+        metadata_url = metadata_url or self.get_default_metadata_url()
+
+        clouds: Mapping[str, AzureEnvironmentMetadata]
+        async with self._async_client.send_request(HttpRequest("GET", metadata_url)) as response:  # type: ignore
+            response.raise_for_status()
+            clouds = await self._parse_cloud_endpoints_async(response.json())
+
+        if update_cached:
+            async with _ASYNC_LOCK:
+                recursive_update(_KNOWN_AZURE_ENVIRONMENTS, clouds)
+        return clouds
+
+    async def close(self) -> None:
+        await self._async_client.close()
+
+    @staticmethod
+    def get_default_metadata_url(default_endpoint: Optional[str] = None) -> str:
+        default_endpoint = default_endpoint or "https://management.azure.com/"
+        metadata_url = os.getenv(
+            _ENV_ARM_CLOUD_METADATA_URL,
+            f"{default_endpoint}metadata/endpoints?api-version={AzureEnvironmentClient.DEFAULT_API_VERSION}",
+        )
+        return metadata_url
+
+    @staticmethod
+    async def _get_registry_discovery_url_async(cloud_name: str, cloud_suffix: str) -> str:
+        async with _ASYNC_LOCK:
+            discovery_url = _KNOWN_AZURE_ENVIRONMENTS.get(cloud_name, {}).get("registry_discovery_endpoint")
+            if discovery_url:
+                return discovery_url
+
+        discovery_url = os.getenv(_ENV_REGISTRY_DISCOVERY_URL)
+        if discovery_url is not None:
+            return discovery_url
+
+        region = os.getenv(_ENV_REGISTRY_DISCOVERY_REGION, _DEFAULT_REGISTRY_DISCOVERY_REGION)
+        return f"https://{cloud_name.lower()}{region}.api.ml.azure.{cloud_suffix}/"
+
+    @staticmethod
+    async def _parse_cloud_endpoints_async(data: Any) -> Mapping[str, AzureEnvironmentMetadata]:
+        # If there is only one cloud, you will get a dict, otherwise a list of dicts
+        cloud_data: Sequence[Mapping[str, Any]] = data if not isinstance(data, dict) else [data]
+        clouds: Dict[str, AzureEnvironmentMetadata] = {}
+
+        def append_trailing_slash(url: str) -> str:
+            return url if url.endswith("/") else f"{url}/"
+
+        for cloud in cloud_data:
+            try:
+                name: str = cloud["name"]
+                portal_endpoint: str = cloud["portal"]
+                cloud_suffix = ".".join(portal_endpoint.split(".")[2:]).replace("/", "")
+                discovery_url = await AzureEnvironmentClient._get_registry_discovery_url_async(name, cloud_suffix)
+                clouds[name] = {
+                    "portal_endpoint": append_trailing_slash(portal_endpoint),
+                    "resource_manager_endpoint": append_trailing_slash(cloud["resourceManager"]),
+                    "active_directory_endpoint": append_trailing_slash(cloud["authentication"]["loginEndpoint"]),
+                    "aml_resource_endpoint": append_trailing_slash(f"https://ml.azure.{cloud_suffix}/"),
+                    "storage_suffix": cloud["suffixes"]["storage"],
+                    "registry_discovery_endpoint": append_trailing_slash(discovery_url),
+                }
+            except KeyError:
+                continue
+
+        return clouds
+
+
+def recursive_update(d: Dict, u: Mapping) -> None:
+    """Recursively update a dictionary.
+
+    :param Dict d: The dictionary to update.
+    :param Mapping u: The mapping to update from.
+    """
+    for k, v in u.items():
+        if isinstance(v, Dict) and k in d:
+            recursive_update(d[k], v)
+        else:
+            d[k] = v