azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/__init__.py

@@ -12,27 +12,27 @@ from ._evaluators._content_safety import (
     SexualEvaluator,
     ViolenceEvaluator,
 )
-from ._evaluators._multimodal._content_safety_multimodal import (
-    ContentSafetyMultimodalEvaluator,
-    HateUnfairnessMultimodalEvaluator,
-    SelfHarmMultimodalEvaluator,
-    SexualMultimodalEvaluator,
-    ViolenceMultimodalEvaluator,
-)
-from ._evaluators._multimodal._protected_material import ProtectedMaterialMultimodalEvaluator
 from ._evaluators._f1_score import F1ScoreEvaluator
 from ._evaluators._fluency import FluencyEvaluator
 from ._evaluators._gleu import GleuScoreEvaluator
 from ._evaluators._groundedness import GroundednessEvaluator
 from ._evaluators._service_groundedness import GroundednessProEvaluator
+from ._evaluators._intent_resolution import IntentResolutionEvaluator
 from ._evaluators._meteor import MeteorScoreEvaluator
 from ._evaluators._protected_material import ProtectedMaterialEvaluator
 from ._evaluators._qa import QAEvaluator
+from ._evaluators._response_completeness import ResponseCompletenessEvaluator
+from ._evaluators._task_adherence import TaskAdherenceEvaluator
 from ._evaluators._relevance import RelevanceEvaluator
 from ._evaluators._retrieval import RetrievalEvaluator
 from ._evaluators._rouge import RougeScoreEvaluator, RougeType
 from ._evaluators._similarity import SimilarityEvaluator
 from ._evaluators._xpia import IndirectAttackEvaluator
+from ._evaluators._code_vulnerability import CodeVulnerabilityEvaluator
+from ._evaluators._ungrounded_attributes import UngroundedAttributesEvaluator
+from ._evaluators._tool_call_accuracy import ToolCallAccuracyEvaluator
+from ._evaluators._document_retrieval import DocumentRetrievalEvaluator
+from ._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator
 from ._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
@@ -42,6 +42,59 @@ from ._model_configurations import (
     Message,
     OpenAIModelConfiguration,
 )
+from ._aoai.aoai_grader import AzureOpenAIGrader
+from ._aoai.label_grader import AzureOpenAILabelGrader
+from ._aoai.string_check_grader import AzureOpenAIStringCheckGrader
+from ._aoai.text_similarity_grader import AzureOpenAITextSimilarityGrader
+from ._aoai.score_model_grader import AzureOpenAIScoreModelGrader
+from ._aoai.python_grader import AzureOpenAIPythonGrader
+
+
+_patch_all = []
+
+# The converter from the AI service to the evaluator schema requires a dependency on
+# ai.projects, but we also don't want to force users installing ai.evaluations to pull
+# in ai.projects. So we only import it if it's available and the user has ai.projects.
+# We use lazy loading to avoid printing messages during import unless the classes are actually used.
+_lazy_imports = {}
+
+
+def _create_lazy_import(class_name, module_path, dependency_name):
+    """Create a lazy import function for optional dependencies.
+
+    Args:
+        class_name: Name of the class to import
+        module_path: Module path to import from
+        dependency_name: Name of the dependency package for error message
+
+    Returns:
+        A function that performs the lazy import when called
+    """
+
+    def lazy_import():
+        try:
+            module = __import__(module_path, fromlist=[class_name])
+            cls = getattr(module, class_name)
+            _patch_all.append(class_name)
+            return cls
+        except ImportError:
+            raise ImportError(
+                f"Could not import {class_name}. Please install the dependency with `pip install {dependency_name}`."
+            )
+
+    return lazy_import
+
+
+_lazy_imports["AIAgentConverter"] = _create_lazy_import(
+    "AIAgentConverter",
+    "azure.ai.evaluation._converters._ai_services",
+    "azure-ai-projects",
+)
+_lazy_imports["SKAgentConverter"] = _create_lazy_import(
+    "SKAgentConverter",
+    "azure.ai.evaluation._converters._sk_services",
+    "semantic-kernel",
+)
 
 __all__ = [
     "evaluate",
@@ -50,6 +103,9 @@ __all__ = [
     "FluencyEvaluator",
     "GroundednessEvaluator",
     "GroundednessProEvaluator",
+    "ResponseCompletenessEvaluator",
+    "TaskAdherenceEvaluator",
+    "IntentResolutionEvaluator",
     "RelevanceEvaluator",
     "SimilarityEvaluator",
     "QAEvaluator",
@@ -73,10 +129,23 @@ __all__ = [
     "Conversation",
     "Message",
     "EvaluationResult",
-    "ContentSafetyMultimodalEvaluator",
-    "HateUnfairnessMultimodalEvaluator",
-    "SelfHarmMultimodalEvaluator",
-    "SexualMultimodalEvaluator",
-    "ViolenceMultimodalEvaluator",
-    "ProtectedMaterialMultimodalEvaluator",
+    "CodeVulnerabilityEvaluator",
+    "UngroundedAttributesEvaluator",
+    "ToolCallAccuracyEvaluator",
+    "_ToolOutputUtilizationEvaluator",
+    "AzureOpenAIGrader",
+    "AzureOpenAILabelGrader",
+    "AzureOpenAIStringCheckGrader",
+    "AzureOpenAITextSimilarityGrader",
+    "AzureOpenAIScoreModelGrader",
+    "AzureOpenAIPythonGrader",
 ]
+
+__all__.extend([p for p in _patch_all if p not in __all__])
+
+
+def __getattr__(name):
+    """Handle lazy imports for optional dependencies."""
+    if name in _lazy_imports:
+        return _lazy_imports[name]()
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

azure/ai/evaluation/_aoai/__init__.py

@@ -0,0 +1,10 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+
+from .aoai_grader import AzureOpenAIGrader
+
+__all__ = [
+    "AzureOpenAIGrader",
+]

azure/ai/evaluation/_aoai/aoai_grader.py

@@ -0,0 +1,140 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+
+from typing_extensions import TypeIs
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._constants import DEFAULT_AOAI_API_VERSION, TokenScope
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.ai.evaluation._user_agent import UserAgentSingleton
+from azure.core.credentials import TokenCredential
+
+if TYPE_CHECKING:
+    from openai.lib.azure import AzureADTokenProvider
+
+
+@experimental
+class AzureOpenAIGrader:
+    """Base class for Azure OpenAI grader wrappers.
+
+    Recommended only for use by experienced OpenAI API users.
+    Combines a model configuration and any grader configuration
+    into a singular object that can be used in evaluations.
+
+    Supplying an AzureOpenAIGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param grader_config: The grader configuration to use for the grader. This is expected
+        to be formatted as a dictionary that matches the specifications of the sub-types of
+        the TestingCriterion alias specified in `OpenAI's SDK <https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L151>`_.
+    :type grader_config: Dict[str, Any]
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/custom_grader"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        grader_config: Dict[str, Any],
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any,
+    ):
+        self._model_config = model_config
+        self._grader_config = grader_config
+        self._credential = credential
+
+        if kwargs.get("validate", True):
+            self._validate_model_config()
+            self._validate_grader_config()
+
+    def _validate_model_config(self) -> None:
+        """Validate the model configuration that this grader wrapper is using."""
+        msg = None
+        if self._is_azure_model_config(self._model_config):
+            if not any(auth for auth in (self._model_config.get("api_key"), self._credential)):
+                msg = (
+                    f"{type(self).__name__}: Requires an api_key in the supplied model_config, "
+                    + "or providing a credential to the grader's __init__ method. "
+                )
+
+        else:
+            if "api_key" not in self._model_config or not self._model_config.get("api_key"):
+                msg = f"{type(self).__name__}: Requires an api_key in the supplied model_config."
+
+        if msg is None:
+            return
+
+        raise EvaluationException(
+            message=msg,
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.AOAI_GRADER,
+        )
+
+    def _validate_grader_config(self) -> None:
+        """Validate the grader configuration that this grader wrapper is using."""
+
+        return
+
+    @staticmethod
+    def _is_azure_model_config(
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    ) -> TypeIs[AzureOpenAIModelConfiguration]:
+        return "azure_endpoint" in model_config
+
+    def get_client(self) -> Any:
+        """Construct an appropriate OpenAI client using this grader's model configuration.
+        Returns a slightly different client depending on whether or not this grader's model
+        configuration is for Azure OpenAI or OpenAI.
+
+        :return: The OpenAI client.
+        :rtype: [~openai.OpenAI, ~openai.AzureOpenAI]
+        """
+        default_headers = {"User-Agent": UserAgentSingleton().value}
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration] = self._model_config
+        api_key: Optional[str] = model_config.get("api_key")
+
+        if self._is_azure_model_config(model_config):
+            from openai import AzureOpenAI
+
+            # TODO set default values?
+            return AzureOpenAI(
+                azure_endpoint=model_config["azure_endpoint"],
+                api_key=api_key,  # Default-style access to appease linters.
+                api_version=DEFAULT_AOAI_API_VERSION,  # Force a known working version
+                azure_deployment=model_config.get("azure_deployment", ""),
+                azure_ad_token_provider=self._get_token_provider(self._credential) if not api_key else None,
+                default_headers=default_headers,
+            )
+        from openai import OpenAI
+
+        # TODO add default values for base_url and organization?
+        return OpenAI(
+            api_key=api_key,
+            base_url=model_config.get("base_url", ""),
+            organization=model_config.get("organization", ""),
+            default_headers=default_headers,
+        )
+
+    @staticmethod
+    def _get_token_provider(cred: TokenCredential) -> "AzureADTokenProvider":
+        """Get the token provider the AzureOpenAI client.
+
+        :param TokenCredential cred: The Azure authentication credential.
+        :return: The token provider if a credential is provided, otherwise None.
+        :rtype: openai.lib.azure.AzureADTokenProvider
+        """
+
+        return lambda: cred.get_token(TokenScope.COGNITIVE_SERVICES_MANAGEMENT).token

azure/ai/evaluation/_aoai/label_grader.py

@@ -0,0 +1,68 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, List, Optional, Union
+
+from openai.types.graders import LabelModelGrader
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
+
+from .aoai_grader import AzureOpenAIGrader
+
+
+@experimental
+class AzureOpenAILabelGrader(AzureOpenAIGrader):
+    """Wrapper class for OpenAI's label model graders.
+
+    Supplying a LabelGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param input: The list of label-based testing criterion for this grader. Individual
+        values of this list are expected to be dictionaries that match the format of any of the valid
+        `TestingCriterionLabelModelInput <https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L125C1-L125C32>`_
+        subtypes.
+    :type input: List[Dict[str, str]]
+    :param labels: A list of strings representing the classification labels of this grader.
+    :type labels: List[str]
+    :param model: The model to use for the evaluation. Must support structured outputs.
+    :type model: str
+    :param name: The name of the grader.
+    :type name: str
+    :param passing_labels: The labels that indicate a passing result. Must be a subset of labels.
+    :type passing_labels: List[str]
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/label_grader"
+    _type = "label_model"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        input: List[Dict[str, str]],
+        labels: List[str],
+        model: str,
+        name: str,
+        passing_labels: List[str],
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any
+    ):
+        grader = LabelModelGrader(
+            input=input,
+            labels=labels,
+            model=model,
+            name=name,
+            passing_labels=passing_labels,
+            type=AzureOpenAILabelGrader._type,
+        )
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/python_grader.py

@@ -0,0 +1,86 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Optional, Union
+
+from openai.types.graders import PythonGrader
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
+
+from .aoai_grader import AzureOpenAIGrader
+
+
+@experimental
+class AzureOpenAIPythonGrader(AzureOpenAIGrader):
+    """Wrapper class for OpenAI's Python code graders.
+
+    Enables custom Python-based evaluation logic with flexible scoring and
+    pass/fail thresholds. The grader executes user-provided Python code
+    to evaluate outputs against custom criteria.
+
+    Supplying a PythonGrader to the `evaluate` method will cause an
+    asynchronous request to evaluate the grader via the OpenAI API. The
+    results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param name: The name of the grader.
+    :type name: str
+    :param image_tag: The image tag for the Python execution environment.
+    :type image_tag: str
+    :param pass_threshold: Score threshold for pass/fail classification. Scores >= threshold are considered passing.
+    :type pass_threshold: float
+    :param source: Python source code containing the grade function.
+        Must define: def grade(sample: dict, item: dict) -> float
+    :type source: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_common.py
+            :start-after: [START python_grader_example]
+            :end-before: [END python_grader_example]
+            :language: python
+            :dedent: 8
+            :caption: Using AzureOpenAIPythonGrader for custom evaluation logic.
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/python_grader"
+    _type = "python"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        name: str,
+        pass_threshold: float,
+        source: str,
+        image_tag: Optional[str] = None,
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any,
+    ):
+        # Validate pass_threshold
+        if not 0.0 <= pass_threshold <= 1.0:
+            raise ValueError("pass_threshold must be between 0.0 and 1.0")
+
+        # Store pass_threshold as instance attribute for potential future use
+        self.pass_threshold = pass_threshold
+
+        # Create OpenAI PythonGrader instance
+        grader = PythonGrader(
+            name=name,
+            image_tag=image_tag,
+            pass_threshold=pass_threshold,
+            source=source,
+            type=AzureOpenAIPythonGrader._type,
+        )
+
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/score_model_grader.py

@@ -0,0 +1,94 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, List, Optional, Union
+
+from openai.types.graders import ScoreModelGrader
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
+
+from .aoai_grader import AzureOpenAIGrader
+
+
+@experimental
+class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
+    """Wrapper class for OpenAI's score model graders.
+
+    Enables continuous scoring evaluation with custom prompts and flexible
+    conversation-style inputs. Supports configurable score ranges and
+    pass thresholds for binary classification.
+
+    Supplying a ScoreModelGrader to the `evaluate` method will cause an
+    asynchronous request to evaluate the grader via the OpenAI API. The
+    results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param input: The input messages for the grader. List of conversation
+        messages with role and content.
+    :type input: List[Dict[str, str]]
+    :param model: The model to use for the evaluation.
+    :type model: str
+    :param name: The name of the grader.
+    :type name: str
+    :param range: The range of the score. Defaults to [0, 1].
+    :type range: Optional[List[float]]
+    :param pass_threshold: Score threshold for pass/fail classification.
+        Defaults to midpoint of range.
+    :type pass_threshold: Optional[float]
+    :param sampling_params: The sampling parameters for the model.
+    :type sampling_params: Optional[Dict[str, Any]]
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/score_model_grader"
+    _type = "score_model"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        input: List[Dict[str, str]],
+        model: str,
+        name: str,
+        range: Optional[List[float]] = None,
+        pass_threshold: Optional[float] = None,
+        sampling_params: Optional[Dict[str, Any]] = None,
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any,
+    ):
+        # Validate range and pass_threshold
+        if range is not None:
+            if len(range) != 2 or range[0] >= range[1]:
+                raise ValueError("range must be a list of two numbers [min, max] where min < max")
+        else:
+            range = [0.0, 1.0]  # Default range
+
+        if pass_threshold is not None:
+            if range and (pass_threshold < range[0] or pass_threshold > range[1]):
+                raise ValueError(f"pass_threshold {pass_threshold} must be within range {range}")
+        else:
+            pass_threshold = (range[0] + range[1]) / 2  # Default to midpoint
+
+        # Store pass_threshold as instance attribute
+        self.pass_threshold = pass_threshold
+
+        # Create OpenAI ScoreModelGrader instance
+        grader_kwargs = {"input": input, "model": model, "name": name, "type": AzureOpenAIScoreModelGrader._type}
+
+        if range is not None:
+            grader_kwargs["range"] = range
+        if sampling_params is not None:
+            grader_kwargs["sampling_params"] = sampling_params
+        grader_kwargs["pass_threshold"] = self.pass_threshold
+
+        grader = ScoreModelGrader(**grader_kwargs)
+
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/string_check_grader.py

@@ -0,0 +1,66 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Optional, Union
+
+from openai.types.graders import StringCheckGrader
+from typing_extensions import Literal
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
+
+from .aoai_grader import AzureOpenAIGrader
+
+
+@experimental
+class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
+    """Wrapper class for OpenAI's string check graders.
+
+    Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param input: The input text. This may include template strings.
+    :type input: str
+    :param name: The name of the grader.
+    :type name: str
+    :param operation: The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`.
+    :type operation: Literal["eq", "ne", "like", "ilike"]
+    :param reference: The reference text. This may include template strings.
+    :type reference: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/string_check_grader"
+    _type = "string_check"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        input: str,
+        name: str,
+        operation: Literal[
+            "eq",
+            "ne",
+            "like",
+            "ilike",
+        ],
+        reference: str,
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any
+    ):
+        grader = StringCheckGrader(
+            input=input,
+            name=name,
+            operation=operation,
+            reference=reference,
+            type=AzureOpenAIStringCheckGrader._type,
+        )
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/text_similarity_grader.py

@@ -0,0 +1,80 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from typing import Any, Dict, Optional, Union
+
+from openai.types.graders import TextSimilarityGrader
+from typing_extensions import Literal
+
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
+
+from .aoai_grader import AzureOpenAIGrader
+
+
+@experimental
+class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
+    """Wrapper class for OpenAI's string check graders.
+
+    Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
+    the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
+    evaluation results.
+
+    :param model_config: The model configuration to use for the grader.
+    :type model_config: Union[
+            ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+            ~azure.ai.evaluation.OpenAIModelConfiguration]
+    :param evaluation_metric: The evaluation metric to use.
+    :type evaluation_metric: Literal["fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3",
+        "rouge_4", "rouge_5", "rouge_l", "cosine"]
+    :param input: The text being graded.
+    :type input: str
+    :param pass_threshold: A float score where a value greater than or equal indicates a passing grade.
+    :type pass_threshold: float
+    :param reference: The text being graded against.
+    :type reference: str
+    :param name: The name of the grader.
+    :type name: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :param kwargs: Additional keyword arguments to pass to the grader.
+    :type kwargs: Any
+    """
+
+    id = "azureai://built-in/evaluators/azure-openai/text_similarity_grader"
+    _type = "text_similarity"
+
+    def __init__(
+        self,
+        *,
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        evaluation_metric: Literal[
+            "fuzzy_match",
+            "bleu",
+            "gleu",
+            "meteor",
+            "rouge_1",
+            "rouge_2",
+            "rouge_3",
+            "rouge_4",
+            "rouge_5",
+            "rouge_l",
+            "cosine",
+        ],
+        input: str,
+        pass_threshold: float,
+        reference: str,
+        name: str,
+        credential: Optional[TokenCredential] = None,
+        **kwargs: Any
+    ):
+        grader = TextSimilarityGrader(
+            evaluation_metric=evaluation_metric,
+            input=input,
+            pass_threshold=pass_threshold,
+            name=name,
+            reference=reference,
+            type=AzureOpenAITextSimilarityGrader._type,
+        )
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)