azure-ai-evaluation 1.5.0__tar.gz → 1.7.0__tar.gz

{azure_ai_evaluation-1.5.0 → azure_ai_evaluation-1.7.0}/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Release History
 
+## 1.7.0 (2025-05-12)
+
+### Bugs Fixed
+- azure-ai-evaluation failed with module not found [#40992](https://github.com/Azure/azure-sdk-for-python/issues/40992)
+
+## 1.6.0 (2025-05-07)
+
+### Features Added
+- New `<evaluator>.binary_aggregate` field added to evaluation result metrics. This field contains the aggregated binary evaluation results for each evaluator, providing a summary of the evaluation outcomes.
+- Added support for Azure Open AI evaluation via 4 new 'grader' classes, which serve as wrappers around Azure Open AI grader configurations. These new grader objects can be supplied to the main `evaluate` method as if they were normal callable evaluators. The new classes are:
+    - AzureOpenAIGrader (general class for experienced users)
+    - AzureOpenAILabelGrader
+    - AzureOpenAIStringCheckGrader
+    - AzureOpenAITextSimilarityGrader
+
+### Breaking Changes
+- In the experimental RedTeam's scan method, the `data_only` param has been replaced with `skip_evals` and if you do not want data to be uploaded, use the `skip_upload` flag.
+
+### Bugs Fixed
+- Fixed error in `evaluate` where data fields could not contain numeric characters. Previously, a data file with schema:
+    ```
+    "query1": "some query", "response": "some response"
+    ```
+    throws error when passed into `evaluator_config` as `{"evaluator_name": {"column_mapping": {"query": "${data.query1}", "response": "${data.response}"}},}`.
+    Now, users may import data containing fields with numeric characters. 
+
+
 ## 1.5.0 (2025-04-04)
 
 ### Features Added

{azure_ai_evaluation-1.5.0/azure_ai_evaluation.egg-info → azure_ai_evaluation-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: azure-ai-evaluation
-Version: 1.5.0
+Version: 1.7.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -30,9 +30,11 @@ Requires-Dist: nltk>=3.9.1
 Requires-Dist: azure-storage-blob>=12.10.0
 Requires-Dist: httpx>=0.25.1
 Requires-Dist: pandas<3.0.0,>=2.1.2
-Requires-Dist: openai>=1.40.0
+Requires-Dist: openai>=1.78.0
 Requires-Dist: ruamel.yaml<1.0.0,>=0.17.10
 Requires-Dist: msrest>=0.6.21
+Requires-Dist: Jinja2>=3.1.6
+Requires-Dist: aiohttp>=3.0
 Provides-Extra: redteam
 Requires-Dist: pyrit==0.8.1; extra == "redteam"
 
@@ -114,13 +116,23 @@ result = relevance_evaluator(
     response="The capital of Japan is Tokyo."
 )
 
-# AI assisted safety evaluator
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project Details 
 azure_ai_project = {
     "subscription_id": "<subscription_id>",
     "resource_group_name": "<resource_group_name>",
     "project_name": "<project_name>",
 }
 
+violence_evaluator = ViolenceEvaluator(azure_ai_project)
+result = violence_evaluator(
+    query="What is the capital of France?",
+    response="Paris."
+)
+
+# Option # 2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 violence_evaluator = ViolenceEvaluator(azure_ai_project)
 result = violence_evaluator(
     query="What is the capital of France?",
@@ -271,11 +283,18 @@ with open("simulator_output.jsonl", "w") as f:
 ```python
 from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario
 from azure.identity import DefaultAzureCredential
+
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project 
 azure_ai_project = {
     "subscription_id": <subscription_id>,
     "resource_group_name": <resource_group_name>,
     "project_name": <project_name>
 }
+
+# Option #2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 scenario = AdversarialScenario.ADVERSARIAL_QA
 simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
 
@@ -381,6 +400,33 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.7.0 (2025-05-12)
+
+### Bugs Fixed
+- azure-ai-evaluation failed with module not found [#40992](https://github.com/Azure/azure-sdk-for-python/issues/40992)
+
+## 1.6.0 (2025-05-07)
+
+### Features Added
+- New `<evaluator>.binary_aggregate` field added to evaluation result metrics. This field contains the aggregated binary evaluation results for each evaluator, providing a summary of the evaluation outcomes.
+- Added support for Azure Open AI evaluation via 4 new 'grader' classes, which serve as wrappers around Azure Open AI grader configurations. These new grader objects can be supplied to the main `evaluate` method as if they were normal callable evaluators. The new classes are:
+    - AzureOpenAIGrader (general class for experienced users)
+    - AzureOpenAILabelGrader
+    - AzureOpenAIStringCheckGrader
+    - AzureOpenAITextSimilarityGrader
+
+### Breaking Changes
+- In the experimental RedTeam's scan method, the `data_only` param has been replaced with `skip_evals` and if you do not want data to be uploaded, use the `skip_upload` flag.
+
+### Bugs Fixed
+- Fixed error in `evaluate` where data fields could not contain numeric characters. Previously, a data file with schema:
+    ```
+    "query1": "some query", "response": "some response"
+    ```
+    throws error when passed into `evaluator_config` as `{"evaluator_name": {"column_mapping": {"query": "${data.query1}", "response": "${data.response}"}},}`.
+    Now, users may import data containing fields with numeric characters. 
+
+
 ## 1.5.0 (2025-04-04)
 
 ### Features Added

{azure_ai_evaluation-1.5.0 → azure_ai_evaluation-1.7.0}/README.md

@@ -76,13 +76,23 @@ result = relevance_evaluator(
     response="The capital of Japan is Tokyo."
 )
 
-# AI assisted safety evaluator
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project Details 
 azure_ai_project = {
     "subscription_id": "<subscription_id>",
     "resource_group_name": "<resource_group_name>",
     "project_name": "<project_name>",
 }
 
+violence_evaluator = ViolenceEvaluator(azure_ai_project)
+result = violence_evaluator(
+    query="What is the capital of France?",
+    response="Paris."
+)
+
+# Option # 2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 violence_evaluator = ViolenceEvaluator(azure_ai_project)
 result = violence_evaluator(
     query="What is the capital of France?",
@@ -233,11 +243,18 @@ with open("simulator_output.jsonl", "w") as f:
 ```python
 from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario
 from azure.identity import DefaultAzureCredential
+
+# There are two ways to provide Azure AI Project.
+# Option #1 : Using Azure AI Project 
 azure_ai_project = {
     "subscription_id": <subscription_id>,
     "resource_group_name": <resource_group_name>,
     "project_name": <project_name>
 }
+
+# Option #2 : Using Azure AI Project Url 
+azure_ai_project = "https://{resource_name}.services.ai.azure.com/api/projects/{project_name}"
+
 scenario = AdversarialScenario.ADVERSARIAL_QA
 simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
 

{azure_ai_evaluation-1.5.0 → azure_ai_evaluation-1.7.0}/TROUBLESHOOTING.md

@@ -6,11 +6,18 @@ This guide walks you through how to investigate failures, common errors in the `
 
 - [Handle Evaluate API Errors](#handle-evaluate-api-errors)
   - [Troubleshoot Remote Tracking Issues](#troubleshoot-remote-tracking-issues)
+  - [Troubleshoot Column Mapping Issues](#troubleshoot-column-mapping-issues)
   - [Troubleshoot Safety Evaluator Issues](#troubleshoot-safety-evaluator-issues)
+  - [Troubleshoot Quality Evaluator Issues](#troubleshoot-quality-evaluator-issues)
 - [Handle Simulation Errors](#handle-simulation-errors)
   - [Adversarial Simulation Supported Regions](#adversarial-simulation-supported-regions)
+  - [Need to generate simulations for specific harm type](#need-to-generate-simulations-for-specific-harm-type)
+  - [Simulator is slow](#simulator-is-slow)
+- [Handle RedTeam Errors](#handle-redteam-errors)
+  - [Target resource not found](#target-resource-not-found)
+  - [Insufficient Storage Permissions](#insufficient-storage-permissions)
 - [Logging](#logging)
-- [Get additional help](#get-additional-help)
+- [Get Additional Help](#get-additional-help)
 
 ## Handle Evaluate API Errors
 
@@ -30,11 +37,18 @@ This guide walks you through how to investigate failures, common errors in the `
 
 - Additionally, if you're using a virtual network or private link, and your evaluation run upload fails because of that, check out this [guide](https://docs.microsoft.com/azure/machine-learning/how-to-enable-studio-virtual-network#access-data-using-the-studio).
 
+### Troubleshoot Column Mapping Issues
+
+- When using `column_mapping` parameter in evaluators, ensure all keys and values are non-empty strings and contain only alphanumeric characters. Empty strings, non-string values, or non-alphanumeric characters can cause serialization errors and issues in downstream applications. Example of valid mapping: `{"query": "${data.query}", "response": "${data.response}"}`.
+
 ### Troubleshoot Safety Evaluator Issues
 
 - Risk and safety evaluators depend on the Azure AI Studio safety evaluation backend service. For a list of supported regions, please refer to the documentation [here](https://aka.ms/azureaisafetyeval-regionsupport).
 - If you encounter a 403 Unauthorized error when using safety evaluators, verify that you have the `Contributor` role assigned to your Azure AI project. `Contributor` role is currently required to run safety evaluations.
 
+### Troubleshoot Quality Evaluator Issues
+- For `ToolCallAccuracyEvaluator`, if your input did not have a tool to evaluate, the current behavior is to output `null`.
+
 ## Handle Simulation Errors
 
 ### Adversarial Simulation Supported Regions
@@ -51,6 +65,30 @@ The Adversarial simulator does not support selecting individual harms, instead w
 Identify the type of simulations being run (adversarial or non-adversarial).
 Adjust parameters such as `api_call_retry_sleep_sec`, `api_call_delay_sec`, and `concurrent_async_task`. Please note that rate limits to llm calls can be both tokens per minute and requests per minute.
 
+## Handle RedTeam errors
+
+### Target resource not found
+When initializing an Azure OpenAI model directly as `target` for a `RedTeam` scan, ensure `azure_endpoint` is specified in the format `https://<hub>.openai.azure.com/openai/deployments/<deployment_name>/chat/completions?api-version=2025-01-01-preview`. If using `AzureOpenAI`, `endpoint` should be specified in the format `https://<hub>.openai.azure.com/`. 
+
+### Insufficient Storage Permissions
+If you see an error like `WARNING: Failed to log artifacts to MLFlow: (UserError) Failed to upload evaluation run to the cloud due to insufficient permission to access the storage`, you need to ensure that proper permissions are assigned to the storage account linked to your Azure AI Project.
+
+To fix this issue:
+1. Open the associated resource group being used in your Azure AI Project in the Azure Portal
+2. Look up the storage accounts associated with that resource group
+3. Open each storage account and click on "Access control (IAM)" on the left side navigation
+4. Add permissions for the desired users with the "Storage Blob Data Contributor" role
+
+If you have Azure CLI, you can use the following command:
+
+```Shell
+# <mySubscriptionID>: Subscription ID of the Azure AI Studio hub's linked storage account (available in Azure AI hub resource view in Azure Portal).
+# <myResourceGroupName>: Resource group of the Azure AI Studio hub's linked storage account.
+# <user-id>: User object ID for role assignment (retrieve with "az ad user show" command).
+
+az role assignment create --role "Storage Blob Data Contributor" --scope /subscriptions/<mySubscriptionID>/resourceGroups/<myResourceGroupName> --assignee-principal-type User --assignee-object-id "<user-id>"
+```
+
 ## Logging
 
 You can set logging level via environment variable `PF_LOGGING_LEVEL`, valid values includes `CRITICAL`, `ERROR`, `WARNING`, `INFO`, `DEBUG`, default to `INFO`.

{azure_ai_evaluation-1.5.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/__init__.py

@@ -31,6 +31,7 @@ from ._evaluators._xpia import IndirectAttackEvaluator
 from ._evaluators._code_vulnerability import CodeVulnerabilityEvaluator
 from ._evaluators._ungrounded_attributes import UngroundedAttributesEvaluator
 from ._evaluators._tool_call_accuracy import ToolCallAccuracyEvaluator
+from ._evaluators._document_retrieval import DocumentRetrievalEvaluator
 from ._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
@@ -40,6 +41,11 @@ from ._model_configurations import (
     Message,
     OpenAIModelConfiguration,
 )
+from ._aoai.aoai_grader import AzureOpenAIGrader
+from ._aoai.label_grader import AzureOpenAILabelGrader
+from ._aoai.string_check_grader import AzureOpenAIStringCheckGrader
+from ._aoai.text_similarity_grader import AzureOpenAITextSimilarityGrader
+
 
 _patch_all = []
 
@@ -89,6 +95,10 @@ __all__ = [
     "CodeVulnerabilityEvaluator",
     "UngroundedAttributesEvaluator",
     "ToolCallAccuracyEvaluator",
+    "AzureOpenAIGrader",
+    "AzureOpenAILabelGrader",
+    "AzureOpenAIStringCheckGrader",
+    "AzureOpenAITextSimilarityGrader",
 ]
 
 __all__.extend([p for p in _patch_all if p not in __all__])

azure_ai_evaluation-1.7.0/azure/ai/evaluation/_aoai/__init__.py

@@ -0,0 +1,10 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+
+from .aoai_grader import AzureOpenAIGrader
+
+__all__ = [
+    "AzureOpenAIGrader",
+]

azure_ai_evaluation-1.7.0/azure/ai/evaluation/_aoai/aoai_grader.py

@@ -0,0 +1,89 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+
+from azure.ai.evaluation._constants import DEFAULT_AOAI_API_VERSION
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from typing import Any, Dict, Union
+from azure.ai.evaluation._common._experimental import experimental
+
+
+@experimental
+class AzureOpenAIGrader():
+    """
+    Base class for Azure OpenAI grader wrappers, recommended only for use by experienced OpenAI API users.
+    Combines a model configuration and any grader configuration
+    into a singular object that can be used in evaluations.
+
+    Supplying an AzureOpenAIGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[
+        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration
+    ]
+    :param grader_config: The grader configuration to use for the grader. This is expected
+        to be formatted as a dictionary that matches the specifications of the sub-types of
+        the TestingCriterion alias specified in (OpenAI's SDK)[https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L151].
+    :type grader_config: Dict[str, Any]
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+
+
+    """
+
+    id = "aoai://general"
+
+    def __init__(self, *, model_config : Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration], grader_config: Dict[str, Any], **kwargs: Any):
+        self._model_config = model_config
+        self._grader_config = grader_config
+
+        if kwargs.get("validate", True):
+            self._validate_model_config()
+            self._validate_grader_config()
+
+
+
+    def _validate_model_config(self) -> None:
+        """Validate the model configuration that this grader wrapper is using."""
+        if "api_key" not in self._model_config or not self._model_config.get("api_key"):
+            msg = f"{type(self).__name__}: Requires an api_key in the supplied model_config."
+            raise EvaluationException(
+                message=msg,
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.INVALID_VALUE,
+                target=ErrorTarget.AOAI_GRADER,
+            )
+    
+    def _validate_grader_config(self) -> None:
+        """Validate the grader configuration that this grader wrapper is using."""
+
+        return
+
+    def get_client(self) -> Any:
+        """Construct an appropriate OpenAI client using this grader's model configuration.
+        Returns a slightly different client depending on whether or not this grader's model
+        configuration is for Azure OpenAI or OpenAI.
+
+        :return: The OpenAI client.
+        :rtype: [~openai.OpenAI, ~openai.AzureOpenAI]
+        """
+        if "azure_endpoint" in self._model_config:
+           from openai import AzureOpenAI
+           # TODO set default values?
+           return AzureOpenAI(
+                azure_endpoint=self._model_config["azure_endpoint"],
+                api_key=self._model_config.get("api_key", None), # Default-style access to appease linters.
+                api_version=DEFAULT_AOAI_API_VERSION, # Force a known working version
+                azure_deployment=self._model_config.get("azure_deployment", ""),
+            )
+        from openai import OpenAI
+        # TODO add default values for base_url and organization?
+        return OpenAI(
+            api_key=self._model_config["api_key"],
+            base_url=self._model_config.get("base_url", ""),
+            organization=self._model_config.get("organization", ""),
+        )

azure_ai_evaluation-1.7.0/azure/ai/evaluation/_aoai/label_grader.py

@@ -0,0 +1,66 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Union, List
+
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from openai.types.graders import LabelModelGrader
+from azure.ai.evaluation._common._experimental import experimental
+
+from .aoai_grader import AzureOpenAIGrader
+
+@experimental
+class AzureOpenAILabelGrader(AzureOpenAIGrader):
+    """
+    Wrapper class for OpenAI's label model graders.
+
+    Supplying a LabelGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[
+        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration
+    ]
+    :param input: The list of label-based testing criterion for this grader. Individual
+        values of this list are expected to be dictionaries that match the format of any of the valid
+        (TestingCriterionLabelModelInput)[https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L125C1-L125C32]
+        subtypes.
+    :type input: List[Dict[str, str]]
+    :param labels: A list of strings representing the classification labels of this grader.
+    :type labels: List[str]
+    :param model: The model to use for the evaluation. Must support structured outputs.
+    :type model: str
+    :param name: The name of the grader.
+    :type name: str
+    :param passing_labels: The labels that indicate a passing result. Must be a subset of labels.
+    :type passing_labels: List[str]
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+
+
+    """
+
+    id = "aoai://label_model"
+
+    def __init__(
+        self,
+        *,
+        model_config : Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        input: List[Dict[str, str]],
+        labels: List[str],
+        model: str,
+        name: str,
+        passing_labels: List[str],
+        **kwargs: Any
+    ):
+        grader = LabelModelGrader(
+            input=input,
+            labels=labels,
+            model=model,
+            name=name,
+            passing_labels=passing_labels,
+            type="label_model",
+        )
+        super().__init__(model_config=model_config, grader_config=grader, **kwargs)

azure_ai_evaluation-1.7.0/azure/ai/evaluation/_aoai/string_check_grader.py

@@ -0,0 +1,65 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Union
+from typing_extensions import Literal
+
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from openai.types.graders import StringCheckGrader
+from azure.ai.evaluation._common._experimental import experimental
+
+from .aoai_grader import AzureOpenAIGrader
+
+@experimental
+class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
+    """
+    Wrapper class for OpenAI's string check graders.
+
+    Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[
+        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration
+    ]
+    :param input: The input text. This may include template strings.
+    :type input: str
+    :param name: The name of the grader.
+    :type name: str
+    :param operation: The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`.
+    :type operation: Literal["eq", "ne", "like", "ilike"]
+    :param reference: The reference text. This may include template strings.
+    :type reference: str
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+
+
+    """
+
+    id = "aoai://string_check"
+
+    def __init__(
+        self,
+        *,
+        model_config : Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        input: str,
+        name: str,
+        operation: Literal[
+            "eq",
+            "ne",
+            "like",
+            "ilike",
+        ],
+        reference: str,
+        **kwargs: Any
+    ):
+        grader = StringCheckGrader(
+            input=input,
+            name=name,
+            operation=operation,
+            reference=reference,
+            type="string_check",
+        )
+        super().__init__(model_config=model_config, grader_config=grader, **kwargs)

azure_ai_evaluation-1.7.0/azure/ai/evaluation/_aoai/text_similarity_grader.py

@@ -0,0 +1,88 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Union
+from typing_extensions import Literal
+
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from openai.types.graders import TextSimilarityGrader
+from azure.ai.evaluation._common._experimental import experimental
+
+from .aoai_grader import AzureOpenAIGrader
+
+@experimental
+class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
+    """
+    Wrapper class for OpenAI's string check graders.
+
+    Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[
+        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration
+    ]
+    :param evaluation_metric: The evaluation metric to use.
+    :type evaluation_metric: Literal[
+            "fuzzy_match",
+            "bleu",
+            "gleu",
+            "meteor",
+            "rouge_1",
+            "rouge_2",
+            "rouge_3",
+            "rouge_4",
+            "rouge_5",
+            "rouge_l",
+            "cosine",
+        ]
+    :param input: The text being graded.
+    :type input: str
+    :param pass_threshold: A float score where a value greater than or equal indicates a passing grade.
+    :type pass_threshold: float
+    :param reference: The text being graded against.
+    :type reference: str
+    :param name: The name of the grader.
+    :type name: str
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+
+
+    """
+
+    id = "aoai://text_similarity"
+
+    def __init__(
+        self,
+        *,
+        model_config : Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        evaluation_metric: Literal[
+            "fuzzy_match",
+            "bleu",
+            "gleu",
+            "meteor",
+            "rouge_1",
+            "rouge_2",
+            "rouge_3",
+            "rouge_4",
+            "rouge_5",
+            "rouge_l",
+            "cosine",
+        ],
+        input: str,
+        pass_threshold: float,
+        reference: str,
+        name: str,
+        **kwargs: Any
+    ):
+        grader = TextSimilarityGrader(
+            evaluation_metric=evaluation_metric,
+            input=input,
+            pass_threshold=pass_threshold,
+            name=name,
+            reference=reference,
+            type="text_similarity",
+        )
+        super().__init__(model_config=model_config, grader_config=grader, **kwargs)

{azure_ai_evaluation-1.5.0 → azure_ai_evaluation-1.7.0}/azure/ai/evaluation/_azure/_clients.py

@@ -8,12 +8,12 @@ from threading import Lock
 from urllib.parse import quote
 from json.decoder import JSONDecodeError
 
-from azure.core.credentials import TokenCredential, AzureSasCredential
+from azure.core.credentials import TokenCredential, AzureSasCredential, AccessToken
 from azure.core.rest import HttpResponse
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import HttpPipeline, get_http_client
 from azure.ai.evaluation._azure._token_manager import AzureMLTokenManager
-from azure.ai.evaluation.simulator._model_tools._identity_manager import TokenScope
+from azure.ai.evaluation._constants import TokenScope
 from ._models import BlobStoreInfo, Workspace
 
 
@@ -61,7 +61,7 @@ class LiteMLClient:
         self._token_manager: Optional[AzureMLTokenManager] = None
         self._credential: Optional[TokenCredential] = credential
 
-    def get_token(self) -> str:
+    def get_token(self) -> AccessToken:
         return self._get_token_manager().get_token()
 
     def get_credential(self) -> TokenCredential:
@@ -201,4 +201,4 @@ class LiteMLClient:
         return url
 
     def _get_headers(self) -> Dict[str, str]:
-        return {"Authorization": f"Bearer {self.get_token()}", "Content-Type": "application/json"}
+        return {"Authorization": f"Bearer {self.get_token().token}", "Content-Type": "application/json"}