azure-ai-evaluation 1.11.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/__init__.py

@@ -32,6 +32,7 @@ from ._evaluators._code_vulnerability import CodeVulnerabilityEvaluator
 from ._evaluators._ungrounded_attributes import UngroundedAttributesEvaluator
 from ._evaluators._tool_call_accuracy import ToolCallAccuracyEvaluator
 from ._evaluators._document_retrieval import DocumentRetrievalEvaluator
+from ._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator
 from ._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
@@ -131,6 +132,7 @@ __all__ = [
     "CodeVulnerabilityEvaluator",
     "UngroundedAttributesEvaluator",
     "ToolCallAccuracyEvaluator",
+    "_ToolOutputUtilizationEvaluator",
     "AzureOpenAIGrader",
     "AzureOpenAILabelGrader",
     "AzureOpenAIStringCheckGrader",

azure/ai/evaluation/_aoai/aoai_grader.py

@@ -1,19 +1,26 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+
+from typing_extensions import TypeIs
 
-from azure.ai.evaluation._constants import DEFAULT_AOAI_API_VERSION
+from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._constants import DEFAULT_AOAI_API_VERSION, TokenScope
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from azure.ai.evaluation._user_agent import UserAgentSingleton
-from typing import Any, Dict, Union
-from azure.ai.evaluation._common._experimental import experimental
+from azure.core.credentials import TokenCredential
+
+if TYPE_CHECKING:
+    from openai.lib.azure import AzureADTokenProvider
 
 
 @experimental
 class AzureOpenAIGrader:
-    """
-    Base class for Azure OpenAI grader wrappers, recommended only for use by experienced OpenAI API users.
+    """Base class for Azure OpenAI grader wrappers.
+
+    Recommended only for use by experienced OpenAI API users.
     Combines a model configuration and any grader configuration
     into a singular object that can be used in evaluations.
 
@@ -22,18 +29,16 @@ class AzureOpenAIGrader:
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param grader_config: The grader configuration to use for the grader. This is expected
         to be formatted as a dictionary that matches the specifications of the sub-types of
-        the TestingCriterion alias specified in (OpenAI's SDK)[https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L151].
+        the TestingCriterion alias specified in `OpenAI's SDK <https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L151>`_.
     :type grader_config: Dict[str, Any]
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/custom_grader"
@@ -43,10 +48,12 @@ class AzureOpenAIGrader:
         *,
         model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
         grader_config: Dict[str, Any],
+        credential: Optional[TokenCredential] = None,
         **kwargs: Any,
     ):
         self._model_config = model_config
         self._grader_config = grader_config
+        self._credential = credential
 
         if kwargs.get("validate", True):
             self._validate_model_config()
@@ -54,20 +61,39 @@ class AzureOpenAIGrader:
 
     def _validate_model_config(self) -> None:
         """Validate the model configuration that this grader wrapper is using."""
-        if "api_key" not in self._model_config or not self._model_config.get("api_key"):
-            msg = f"{type(self).__name__}: Requires an api_key in the supplied model_config."
-            raise EvaluationException(
-                message=msg,
-                blame=ErrorBlame.USER_ERROR,
-                category=ErrorCategory.INVALID_VALUE,
-                target=ErrorTarget.AOAI_GRADER,
-            )
+        msg = None
+        if self._is_azure_model_config(self._model_config):
+            if not any(auth for auth in (self._model_config.get("api_key"), self._credential)):
+                msg = (
+                    f"{type(self).__name__}: Requires an api_key in the supplied model_config, "
+                    + "or providing a credential to the grader's __init__ method. "
+                )
+
+        else:
+            if "api_key" not in self._model_config or not self._model_config.get("api_key"):
+                msg = f"{type(self).__name__}: Requires an api_key in the supplied model_config."
+
+        if msg is None:
+            return
+
+        raise EvaluationException(
+            message=msg,
+            blame=ErrorBlame.USER_ERROR,
+            category=ErrorCategory.INVALID_VALUE,
+            target=ErrorTarget.AOAI_GRADER,
+        )
 
     def _validate_grader_config(self) -> None:
         """Validate the grader configuration that this grader wrapper is using."""
 
         return
 
+    @staticmethod
+    def _is_azure_model_config(
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    ) -> TypeIs[AzureOpenAIModelConfiguration]:
+        return "azure_endpoint" in model_config
+
     def get_client(self) -> Any:
         """Construct an appropriate OpenAI client using this grader's model configuration.
         Returns a slightly different client depending on whether or not this grader's model
@@ -77,23 +103,38 @@ class AzureOpenAIGrader:
         :rtype: [~openai.OpenAI, ~openai.AzureOpenAI]
         """
         default_headers = {"User-Agent": UserAgentSingleton().value}
-        if "azure_endpoint" in self._model_config:
+        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration] = self._model_config
+        api_key: Optional[str] = model_config.get("api_key")
+
+        if self._is_azure_model_config(model_config):
             from openai import AzureOpenAI
 
             # TODO set default values?
             return AzureOpenAI(
-                azure_endpoint=self._model_config["azure_endpoint"],
-                api_key=self._model_config.get("api_key", None),  # Default-style access to appease linters.
+                azure_endpoint=model_config["azure_endpoint"],
+                api_key=api_key,  # Default-style access to appease linters.
                 api_version=DEFAULT_AOAI_API_VERSION,  # Force a known working version
-                azure_deployment=self._model_config.get("azure_deployment", ""),
+                azure_deployment=model_config.get("azure_deployment", ""),
+                azure_ad_token_provider=self._get_token_provider(self._credential) if not api_key else None,
                 default_headers=default_headers,
             )
         from openai import OpenAI
 
         # TODO add default values for base_url and organization?
         return OpenAI(
-            api_key=self._model_config["api_key"],
-            base_url=self._model_config.get("base_url", ""),
-            organization=self._model_config.get("organization", ""),
+            api_key=api_key,
+            base_url=model_config.get("base_url", ""),
+            organization=model_config.get("organization", ""),
             default_headers=default_headers,
         )
+
+    @staticmethod
+    def _get_token_provider(cred: TokenCredential) -> "AzureADTokenProvider":
+        """Get the token provider the AzureOpenAI client.
+
+        :param TokenCredential cred: The Azure authentication credential.
+        :return: The token provider if a credential is provided, otherwise None.
+        :rtype: openai.lib.azure.AzureADTokenProvider
+        """
+
+        return lambda: cred.get_token(TokenScope.COGNITIVE_SERVICES_MANAGEMENT).token

azure/ai/evaluation/_aoai/label_grader.py

@@ -1,32 +1,31 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Any, Dict, Union, List
+from typing import Any, Dict, List, Optional, Union
 
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from openai.types.graders import LabelModelGrader
+
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
 
 from .aoai_grader import AzureOpenAIGrader
 
 
 @experimental
 class AzureOpenAILabelGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's label model graders.
+    """Wrapper class for OpenAI's label model graders.
 
     Supplying a LabelGrader to the `evaluate` method will cause an asynchronous request to evaluate
     the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param input: The list of label-based testing criterion for this grader. Individual
         values of this list are expected to be dictionaries that match the format of any of the valid
-        (TestingCriterionLabelModelInput)[https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L125C1-L125C32]
+        `TestingCriterionLabelModelInput <https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L125C1-L125C32>`_
         subtypes.
     :type input: List[Dict[str, str]]
     :param labels: A list of strings representing the classification labels of this grader.
@@ -37,13 +36,14 @@ class AzureOpenAILabelGrader(AzureOpenAIGrader):
     :type name: str
     :param passing_labels: The labels that indicate a passing result. Must be a subset of labels.
     :type passing_labels: List[str]
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/label_grader"
+    _type = "label_model"
 
     def __init__(
         self,
@@ -54,6 +54,7 @@ class AzureOpenAILabelGrader(AzureOpenAIGrader):
         model: str,
         name: str,
         passing_labels: List[str],
+        credential: Optional[TokenCredential] = None,
         **kwargs: Any
     ):
         grader = LabelModelGrader(
@@ -62,6 +63,6 @@ class AzureOpenAILabelGrader(AzureOpenAIGrader):
             model=model,
             name=name,
             passing_labels=passing_labels,
-            type="label_model",
+            type=AzureOpenAILabelGrader._type,
         )
-        super().__init__(model_config=model_config, grader_config=grader, **kwargs)
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/python_grader.py

@@ -1,19 +1,20 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Any, Dict, Union, Optional
+from typing import Any, Dict, Optional, Union
 
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from openai.types.graders import PythonGrader
+
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
 
 from .aoai_grader import AzureOpenAIGrader
 
 
 @experimental
 class AzureOpenAIPythonGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's Python code graders.
+    """Wrapper class for OpenAI's Python code graders.
 
     Enables custom Python-based evaluation logic with flexible scoring and
     pass/fail thresholds. The grader executes user-provided Python code
@@ -25,20 +26,19 @@ class AzureOpenAIPythonGrader(AzureOpenAIGrader):
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param name: The name of the grader.
     :type name: str
     :param image_tag: The image tag for the Python execution environment.
     :type image_tag: str
-    :param pass_threshold: Score threshold for pass/fail classification.
-        Scores >= threshold are considered passing.
+    :param pass_threshold: Score threshold for pass/fail classification. Scores >= threshold are considered passing.
     :type pass_threshold: float
     :param source: Python source code containing the grade function.
         Must define: def grade(sample: dict, item: dict) -> float
     :type source: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
 
@@ -54,15 +54,17 @@ class AzureOpenAIPythonGrader(AzureOpenAIGrader):
     """
 
     id = "azureai://built-in/evaluators/azure-openai/python_grader"
+    _type = "python"
 
     def __init__(
         self,
         *,
         model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
         name: str,
-        image_tag: str,
         pass_threshold: float,
         source: str,
+        image_tag: Optional[str] = None,
+        credential: Optional[TokenCredential] = None,
         **kwargs: Any,
     ):
         # Validate pass_threshold
@@ -78,7 +80,7 @@ class AzureOpenAIPythonGrader(AzureOpenAIGrader):
             image_tag=image_tag,
             pass_threshold=pass_threshold,
             source=source,
-            type="python",
+            type=AzureOpenAIPythonGrader._type,
         )
 
-        super().__init__(model_config=model_config, grader_config=grader, **kwargs)
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/score_model_grader.py

@@ -1,19 +1,20 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Any, Dict, Union, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from openai.types.graders import ScoreModelGrader
+
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
 
 from .aoai_grader import AzureOpenAIGrader
 
 
 @experimental
 class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's score model graders.
+    """Wrapper class for OpenAI's score model graders.
 
     Enables continuous scoring evaluation with custom prompts and flexible
     conversation-style inputs. Supports configurable score ranges and
@@ -25,10 +26,8 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param input: The input messages for the grader. List of conversation
         messages with role and content.
     :type input: List[Dict[str, str]]
@@ -43,11 +42,14 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
     :type pass_threshold: Optional[float]
     :param sampling_params: The sampling parameters for the model.
     :type sampling_params: Optional[Dict[str, Any]]
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
     """
 
     id = "azureai://built-in/evaluators/azure-openai/score_model_grader"
+    _type = "score_model"
 
     def __init__(
         self,
@@ -59,6 +61,7 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
         range: Optional[List[float]] = None,
         pass_threshold: Optional[float] = None,
         sampling_params: Optional[Dict[str, Any]] = None,
+        credential: Optional[TokenCredential] = None,
         **kwargs: Any,
     ):
         # Validate range and pass_threshold
@@ -78,7 +81,7 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
         self.pass_threshold = pass_threshold
 
         # Create OpenAI ScoreModelGrader instance
-        grader_kwargs = {"input": input, "model": model, "name": name, "type": "score_model"}
+        grader_kwargs = {"input": input, "model": model, "name": name, "type": AzureOpenAIScoreModelGrader._type}
 
         if range is not None:
             grader_kwargs["range"] = range
@@ -88,4 +91,4 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
 
         grader = ScoreModelGrader(**grader_kwargs)
 
-        super().__init__(model_config=model_config, grader_config=grader, **kwargs)
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/string_check_grader.py

@@ -1,30 +1,28 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Any, Dict, Union
-from typing_extensions import Literal
+from typing import Any, Dict, Optional, Union
 
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from openai.types.graders import StringCheckGrader
+from typing_extensions import Literal
+
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
 
 from .aoai_grader import AzureOpenAIGrader
 
 
 @experimental
 class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's string check graders.
+    """Wrapper class for OpenAI's string check graders.
 
     Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
     the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,~azure.ai.evaluation.OpenAIModelConfiguration]
     :param input: The input text. This may include template strings.
     :type input: str
     :param name: The name of the grader.
@@ -33,13 +31,14 @@ class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
     :type operation: Literal["eq", "ne", "like", "ilike"]
     :param reference: The reference text. This may include template strings.
     :type reference: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/string_check_grader"
+    _type = "string_check"
 
     def __init__(
         self,
@@ -54,6 +53,7 @@ class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
             "ilike",
         ],
         reference: str,
+        credential: Optional[TokenCredential] = None,
         **kwargs: Any
     ):
         grader = StringCheckGrader(
@@ -61,6 +61,6 @@ class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
             name=name,
             operation=operation,
             reference=reference,
-            type="string_check",
+            type=AzureOpenAIStringCheckGrader._type,
         )
-        super().__init__(model_config=model_config, grader_config=grader, **kwargs)
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_aoai/text_similarity_grader.py

@@ -1,20 +1,21 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Any, Dict, Union
-from typing_extensions import Literal
+from typing import Any, Dict, Optional, Union
 
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from openai.types.graders import TextSimilarityGrader
+from typing_extensions import Literal
+
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.core.credentials import TokenCredential
 
 from .aoai_grader import AzureOpenAIGrader
 
 
 @experimental
 class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's string check graders.
+    """Wrapper class for OpenAI's string check graders.
 
     Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
     the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
@@ -22,23 +23,11 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
 
     :param model_config: The model configuration to use for the grader.
     :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+            ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+            ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param evaluation_metric: The evaluation metric to use.
-    :type evaluation_metric: Literal[
-            "fuzzy_match",
-            "bleu",
-            "gleu",
-            "meteor",
-            "rouge_1",
-            "rouge_2",
-            "rouge_3",
-            "rouge_4",
-            "rouge_5",
-            "rouge_l",
-            "cosine",
-        ]
+    :type evaluation_metric: Literal["fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3",
+        "rouge_4", "rouge_5", "rouge_l", "cosine"]
     :param input: The text being graded.
     :type input: str
     :param pass_threshold: A float score where a value greater than or equal indicates a passing grade.
@@ -47,13 +36,14 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
     :type reference: str
     :param name: The name of the grader.
     :type name: str
+    :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
+    :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/text_similarity_grader"
+    _type = "text_similarity"
 
     def __init__(
         self,
@@ -76,6 +66,7 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
         pass_threshold: float,
         reference: str,
         name: str,
+        credential: Optional[TokenCredential] = None,
         **kwargs: Any
     ):
         grader = TextSimilarityGrader(
@@ -84,6 +75,6 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
             pass_threshold=pass_threshold,
             name=name,
             reference=reference,
-            type="text_similarity",
+            type=AzureOpenAITextSimilarityGrader._type,
         )
-        super().__init__(model_config=model_config, grader_config=grader, **kwargs)
+        super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

azure/ai/evaluation/_common/__init__.py

@@ -6,7 +6,7 @@
 # that would have otherwise been a relative import scoped to single evaluator directories.
 
 from . import constants
-from .rai_service import evaluate_with_rai_service
+from .rai_service import evaluate_with_rai_service, evaluate_with_rai_service_sync
 from .utils import get_harm_severity_level
 from .evaluation_onedp_client import EvaluationServiceOneDPClient
 from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, ResultType
@@ -14,6 +14,7 @@ from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, Res
 __all__ = [
     "get_harm_severity_level",
     "evaluate_with_rai_service",
+    "evaluate_with_rai_service_sync",
     "constants",
     "EvaluationServiceOneDPClient",
     "EvaluationResult",