azure-ai-evaluation 1.12.0__tar.gz → 1.13.0__tar.gz

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Release History
 
+## 1.13.0 (2025-10-30)
+
+### Features Added
+
+- Updated `IndirectAttack` risk category for RedTeam to `IndirectJailbreak` to better reflect its purpose. This change allows users to apply cross-domain prompt injection (XPIA) attack strategies across all risk categories, enabling more comprehensive security testing of AI systems against indirect prompt injection attacks during red teaming.
+- Added `TaskAdherence`, `SensitiveDataLeakage`, and `ProhibitedActions` as cloud-only agent safety risk categories for red teaming. 
+- Updated all evaluators' output to be of the following schema:
+  - `gpt_{evaluator_name}`, `{evaluator_name}`: float score,
+  - `{evaluator_name}_result`: pass/fail based on threshold,
+  - `{evaluator_name}_reason`, `{evaluator_name}_threshold`
+  - `{evaluator_name}_prompt_tokens`, `{evaluator_name}_completion_tokens`, `{evaluator_name}_total_tokens`, `{evaluator_name}_finish_reason`
+  - `{evaluator_name}_model`: model used for evaluation
+  - `{evaluator_name}_sample_input`, `{evaluator_name}_sample_output`: input and output used for evaluation
+  
+  This change standardizes the output format across all evaluators and follows OTel convention.
+
+### Bugs Fixed
+
+- `image_tag` parameter in `AzureOpenAIPythonGrader` is now optional.
 
 ## 1.12.0 (2025-10-02)
 
@@ -10,6 +29,12 @@
 ### Bugs Fixed
 - Support for multi-level nesting in OpenAI grader (experimental)
 
+## 1.11.2 (2025-10-09)
+
+### Bugs Fixed
+
+- **kwargs in an evaluator signature receives input columns that are not otherwise named in the evaluator's signature
+
 ## 1.11.1 (2025-09-19)
 
 ### Bugs Fixed

{azure_ai_evaluation-1.12.0/azure_ai_evaluation.egg-info → azure_ai_evaluation-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: azure-ai-evaluation
-Version: 1.12.0
+Version: 1.13.0
 Summary: Microsoft Azure Evaluation Library for Python
 Home-page: https://github.com/Azure/azure-sdk-for-python
 Author: Microsoft Corporation
@@ -22,20 +22,25 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: NOTICE.txt
 Requires-Dist: pyjwt>=2.8.0
-Requires-Dist: azure-identity>=1.16.0
-Requires-Dist: azure-core>=1.30.2
+Requires-Dist: azure-identity>=1.19.0
+Requires-Dist: azure-core>=1.31.0
 Requires-Dist: nltk>=3.9.1
-Requires-Dist: azure-storage-blob>=12.10.0
-Requires-Dist: httpx>=0.25.1
-Requires-Dist: pandas<3.0.0,>=2.1.2
+Requires-Dist: azure-storage-blob>=12.19.0
+Requires-Dist: httpx>=0.27.2
+Requires-Dist: pandas<3.0.0,>=2.1.2; python_version < "3.13"
+Requires-Dist: pandas<3.0.0,>=2.2.3; python_version == "3.13"
+Requires-Dist: pandas<3.0.0,>=2.3.3; python_version >= "3.14"
 Requires-Dist: openai>=1.108.0
 Requires-Dist: ruamel.yaml<1.0.0,>=0.17.10
 Requires-Dist: msrest>=0.6.21
 Requires-Dist: Jinja2>=3.1.6
 Requires-Dist: aiohttp>=3.0
 Provides-Extra: redteam
-Requires-Dist: pyrit==0.8.1; extra == "redteam"
-Requires-Dist: duckdb==1.3.2; extra == "redteam"
+Requires-Dist: pyrit==0.8.1; python_version >= "3.10" and extra == "redteam"
+Requires-Dist: duckdb==1.3.2; python_version >= "3.10" and extra == "redteam"
+Provides-Extra: opentelemetry
+Requires-Dist: opentelemetry-sdk>=1.17.0; extra == "opentelemetry"
+Requires-Dist: azure-monitor-opentelemetry-exporter>=1.0.0b17; extra == "opentelemetry"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -413,6 +418,25 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 
 # Release History
 
+## 1.13.0 (2025-10-30)
+
+### Features Added
+
+- Updated `IndirectAttack` risk category for RedTeam to `IndirectJailbreak` to better reflect its purpose. This change allows users to apply cross-domain prompt injection (XPIA) attack strategies across all risk categories, enabling more comprehensive security testing of AI systems against indirect prompt injection attacks during red teaming.
+- Added `TaskAdherence`, `SensitiveDataLeakage`, and `ProhibitedActions` as cloud-only agent safety risk categories for red teaming. 
+- Updated all evaluators' output to be of the following schema:
+  - `gpt_{evaluator_name}`, `{evaluator_name}`: float score,
+  - `{evaluator_name}_result`: pass/fail based on threshold,
+  - `{evaluator_name}_reason`, `{evaluator_name}_threshold`
+  - `{evaluator_name}_prompt_tokens`, `{evaluator_name}_completion_tokens`, `{evaluator_name}_total_tokens`, `{evaluator_name}_finish_reason`
+  - `{evaluator_name}_model`: model used for evaluation
+  - `{evaluator_name}_sample_input`, `{evaluator_name}_sample_output`: input and output used for evaluation
+  
+  This change standardizes the output format across all evaluators and follows OTel convention.
+
+### Bugs Fixed
+
+- `image_tag` parameter in `AzureOpenAIPythonGrader` is now optional.
 
 ## 1.12.0 (2025-10-02)
 
@@ -423,6 +447,12 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 ### Bugs Fixed
 - Support for multi-level nesting in OpenAI grader (experimental)
 
+## 1.11.2 (2025-10-09)
+
+### Bugs Fixed
+
+- **kwargs in an evaluator signature receives input columns that are not otherwise named in the evaluator's signature
+
 ## 1.11.1 (2025-09-19)
 
 ### Bugs Fixed

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/__init__.py

@@ -32,6 +32,7 @@ from ._evaluators._code_vulnerability import CodeVulnerabilityEvaluator
 from ._evaluators._ungrounded_attributes import UngroundedAttributesEvaluator
 from ._evaluators._tool_call_accuracy import ToolCallAccuracyEvaluator
 from ._evaluators._document_retrieval import DocumentRetrievalEvaluator
+from ._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator
 from ._model_configurations import (
     AzureAIProject,
     AzureOpenAIModelConfiguration,
@@ -131,6 +132,7 @@ __all__ = [
     "CodeVulnerabilityEvaluator",
     "UngroundedAttributesEvaluator",
     "ToolCallAccuracyEvaluator",
+    "_ToolOutputUtilizationEvaluator",
     "AzureOpenAIGrader",
     "AzureOpenAILabelGrader",
     "AzureOpenAIStringCheckGrader",

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_aoai/aoai_grader.py

@@ -18,8 +18,9 @@ if TYPE_CHECKING:
 
 @experimental
 class AzureOpenAIGrader:
-    """
-    Base class for Azure OpenAI grader wrappers, recommended only for use by experienced OpenAI API users.
+    """Base class for Azure OpenAI grader wrappers.
+
+    Recommended only for use by experienced OpenAI API users.
     Combines a model configuration and any grader configuration
     into a singular object that can be used in evaluations.
 
@@ -28,20 +29,16 @@ class AzureOpenAIGrader:
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param grader_config: The grader configuration to use for the grader. This is expected
         to be formatted as a dictionary that matches the specifications of the sub-types of
-        the TestingCriterion alias specified in (OpenAI's SDK)[https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L151].
+        the TestingCriterion alias specified in `OpenAI's SDK <https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L151>`_.
     :type grader_config: Dict[str, Any]
     :param credential: The credential to use to authenticate to the model. Only applicable to AzureOpenAI models.
     :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/custom_grader"

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_aoai/label_grader.py

@@ -14,21 +14,18 @@ from .aoai_grader import AzureOpenAIGrader
 
 @experimental
 class AzureOpenAILabelGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's label model graders.
+    """Wrapper class for OpenAI's label model graders.
 
     Supplying a LabelGrader to the `evaluate` method will cause an asynchronous request to evaluate
     the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param input: The list of label-based testing criterion for this grader. Individual
         values of this list are expected to be dictionaries that match the format of any of the valid
-        (TestingCriterionLabelModelInput)[https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L125C1-L125C32]
+        `TestingCriterionLabelModelInput <https://github.com/openai/openai-python/blob/ed53107e10e6c86754866b48f8bd862659134ca8/src/openai/types/eval_create_params.py#L125C1-L125C32>`_
         subtypes.
     :type input: List[Dict[str, str]]
     :param labels: A list of strings representing the classification labels of this grader.
@@ -43,11 +40,10 @@ class AzureOpenAILabelGrader(AzureOpenAIGrader):
     :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/label_grader"
+    _type = "label_model"
 
     def __init__(
         self,
@@ -67,6 +63,6 @@ class AzureOpenAILabelGrader(AzureOpenAIGrader):
             model=model,
             name=name,
             passing_labels=passing_labels,
-            type="label_model",
+            type=AzureOpenAILabelGrader._type,
         )
         super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_aoai/python_grader.py

@@ -14,8 +14,7 @@ from .aoai_grader import AzureOpenAIGrader
 
 @experimental
 class AzureOpenAIPythonGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's Python code graders.
+    """Wrapper class for OpenAI's Python code graders.
 
     Enables custom Python-based evaluation logic with flexible scoring and
     pass/fail thresholds. The grader executes user-provided Python code
@@ -27,16 +26,13 @@ class AzureOpenAIPythonGrader(AzureOpenAIGrader):
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param name: The name of the grader.
     :type name: str
     :param image_tag: The image tag for the Python execution environment.
     :type image_tag: str
-    :param pass_threshold: Score threshold for pass/fail classification.
-        Scores >= threshold are considered passing.
+    :param pass_threshold: Score threshold for pass/fail classification. Scores >= threshold are considered passing.
     :type pass_threshold: float
     :param source: Python source code containing the grade function.
         Must define: def grade(sample: dict, item: dict) -> float
@@ -58,15 +54,16 @@ class AzureOpenAIPythonGrader(AzureOpenAIGrader):
     """
 
     id = "azureai://built-in/evaluators/azure-openai/python_grader"
+    _type = "python"
 
     def __init__(
         self,
         *,
         model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
         name: str,
-        image_tag: str,
         pass_threshold: float,
         source: str,
+        image_tag: Optional[str] = None,
         credential: Optional[TokenCredential] = None,
         **kwargs: Any,
     ):
@@ -83,7 +80,7 @@ class AzureOpenAIPythonGrader(AzureOpenAIGrader):
             image_tag=image_tag,
             pass_threshold=pass_threshold,
             source=source,
-            type="python",
+            type=AzureOpenAIPythonGrader._type,
         )
 
         super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_aoai/score_model_grader.py

@@ -14,8 +14,7 @@ from .aoai_grader import AzureOpenAIGrader
 
 @experimental
 class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's score model graders.
+    """Wrapper class for OpenAI's score model graders.
 
     Enables continuous scoring evaluation with custom prompts and flexible
     conversation-style inputs. Supports configurable score ranges and
@@ -27,10 +26,8 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param input: The input messages for the grader. List of conversation
         messages with role and content.
     :type input: List[Dict[str, str]]
@@ -52,6 +49,7 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
     """
 
     id = "azureai://built-in/evaluators/azure-openai/score_model_grader"
+    _type = "score_model"
 
     def __init__(
         self,
@@ -83,7 +81,7 @@ class AzureOpenAIScoreModelGrader(AzureOpenAIGrader):
         self.pass_threshold = pass_threshold
 
         # Create OpenAI ScoreModelGrader instance
-        grader_kwargs = {"input": input, "model": model, "name": name, "type": "score_model"}
+        grader_kwargs = {"input": input, "model": model, "name": name, "type": AzureOpenAIScoreModelGrader._type}
 
         if range is not None:
             grader_kwargs["range"] = range

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_aoai/string_check_grader.py

@@ -15,18 +15,14 @@ from .aoai_grader import AzureOpenAIGrader
 
 @experimental
 class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's string check graders.
+    """Wrapper class for OpenAI's string check graders.
 
     Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
     the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
     evaluation results.
 
     :param model_config: The model configuration to use for the grader.
-    :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,~azure.ai.evaluation.OpenAIModelConfiguration]
     :param input: The input text. This may include template strings.
     :type input: str
     :param name: The name of the grader.
@@ -39,11 +35,10 @@ class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
     :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/string_check_grader"
+    _type = "string_check"
 
     def __init__(
         self,
@@ -66,6 +61,6 @@ class AzureOpenAIStringCheckGrader(AzureOpenAIGrader):
             name=name,
             operation=operation,
             reference=reference,
-            type="string_check",
+            type=AzureOpenAIStringCheckGrader._type,
         )
         super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_aoai/text_similarity_grader.py

@@ -15,8 +15,7 @@ from .aoai_grader import AzureOpenAIGrader
 
 @experimental
 class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
-    """
-    Wrapper class for OpenAI's string check graders.
+    """Wrapper class for OpenAI's string check graders.
 
     Supplying a StringCheckGrader to the `evaluate` method will cause an asynchronous request to evaluate
     the grader via the OpenAI API. The results of the evaluation will then be merged into the standard
@@ -24,23 +23,11 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
 
     :param model_config: The model configuration to use for the grader.
     :type model_config: Union[
-        ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
-        ~azure.ai.evaluation.OpenAIModelConfiguration
-    ]
+            ~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+            ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param evaluation_metric: The evaluation metric to use.
-    :type evaluation_metric: Literal[
-            "fuzzy_match",
-            "bleu",
-            "gleu",
-            "meteor",
-            "rouge_1",
-            "rouge_2",
-            "rouge_3",
-            "rouge_4",
-            "rouge_5",
-            "rouge_l",
-            "cosine",
-        ]
+    :type evaluation_metric: Literal["fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3",
+        "rouge_4", "rouge_5", "rouge_l", "cosine"]
     :param input: The text being graded.
     :type input: str
     :param pass_threshold: A float score where a value greater than or equal indicates a passing grade.
@@ -53,11 +40,10 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
     :type credential: ~azure.core.credentials.TokenCredential
     :param kwargs: Additional keyword arguments to pass to the grader.
     :type kwargs: Any
-
-
     """
 
     id = "azureai://built-in/evaluators/azure-openai/text_similarity_grader"
+    _type = "text_similarity"
 
     def __init__(
         self,
@@ -89,6 +75,6 @@ class AzureOpenAITextSimilarityGrader(AzureOpenAIGrader):
             pass_threshold=pass_threshold,
             name=name,
             reference=reference,
-            type="text_similarity",
+            type=AzureOpenAITextSimilarityGrader._type,
         )
         super().__init__(model_config=model_config, grader_config=grader, credential=credential, **kwargs)

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_common/__init__.py

@@ -6,7 +6,7 @@
 # that would have otherwise been a relative import scoped to single evaluator directories.
 
 from . import constants
-from .rai_service import evaluate_with_rai_service
+from .rai_service import evaluate_with_rai_service, evaluate_with_rai_service_sync
 from .utils import get_harm_severity_level
 from .evaluation_onedp_client import EvaluationServiceOneDPClient
 from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, ResultType
@@ -14,6 +14,7 @@ from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, Res
 __all__ = [
     "get_harm_severity_level",
     "evaluate_with_rai_service",
+    "evaluate_with_rai_service_sync",
     "constants",
     "EvaluationServiceOneDPClient",
     "EvaluationResult",

azure_ai_evaluation-1.13.0/azure/ai/evaluation/_common/constants.py

@@ -0,0 +1,194 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from enum import Enum
+from typing import Dict, Any, Optional
+
+from azure.core import CaseInsensitiveEnumMeta
+
+PROMPT_BASED_REASON_EVALUATORS = [
+    "coherence",
+    "relevance",
+    "retrieval",
+    "groundedness",
+    "fluency",
+    "intent_resolution",
+    "tool_call_accurate",
+    "response_completeness",
+    "task_adherence",
+    "tool_selection",
+    "tool_output_utilization",
+    "task_completion",
+    "tool_input_accuracy",
+    "tool_success",
+    "tool_call_accuracy",
+]
+
+
+class CommonConstants:
+    """Define common constants."""
+
+    DEFAULT_HTTP_TIMEOUT = 60
+
+
+class RAIService:
+    """Define constants related to RAI service"""
+
+    TIMEOUT = 1800
+    SLEEP_TIME = 2
+    HARM_SEVERITY_THRESHOLD = 4
+
+
+class HarmSeverityLevel(Enum):
+    """Harm severity levels."""
+
+    VeryLow = "Very low"
+    Low = "Low"
+    Medium = "Medium"
+    High = "High"
+
+
+class EvaluatorScoringPattern(Enum):
+    """Defines different scoring patterns used by evaluators."""
+
+    # Binary patterns
+    BINARY_SAFE_UNSAFE = "binary_safe_unsafe"  # Output: safe/unsafe
+    BINARY_TRUE_FALSE = "binary_true_false"  # Output: true/false (as string)
+
+    # Numeric scale patterns
+    SCALE_0_7 = "scale_0_7"  # 0-7 scale (content harm evaluators)
+    SCALE_1_3 = "scale_1_3"  # 1-3 scale (task adherence)
+    SCALE_1_5 = "scale_1_5"  # 1-5 scale (quality evaluators)
+
+
+class Tasks:
+    """Defines types of annotation tasks supported by RAI Service."""
+
+    CONTENT_HARM = "content harm"
+    PROTECTED_MATERIAL = "protected material"
+    XPIA = "xpia"
+    GROUNDEDNESS = "groundedness"
+    CODE_VULNERABILITY = "code vulnerability"
+    UNGROUNDED_ATTRIBUTES = "inference sensitive attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
+
+
+class _InternalAnnotationTasks:
+    """Defines types of annotation tasks that are supported for internal use. Such tasks are
+    experimental and subject to potential change or migration to the main Evaluation Metrics
+    enum over time."""
+
+    ECI = "eci"
+
+
+class EvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Evaluation metrics to aid the RAI service in determining what
+    metrics to request, and how to present them back to the user."""
+
+    HATE_FAIRNESS = "hate_fairness"
+    HATE_UNFAIRNESS = "hate_unfairness"
+    VIOLENCE = "violence"
+    SELF_HARM = "self_harm"
+    SEXUAL = "sexual"
+    PROTECTED_MATERIAL = "protected_material"
+    XPIA = "xpia"
+    GROUNDEDNESS = "generic_groundedness"
+    CODE_VULNERABILITY = "code_vulnerability"
+    UNGROUNDED_ATTRIBUTES = "ungrounded_attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
+
+
+class _InternalEvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """Evaluation metrics that are not publicly supported.
+    These metrics are experimental and subject to potential change or migration to the main
+    enum over time.
+    """
+
+    ECI = "eci"
+
+
+# Mapping of evaluation metrics to their scoring patterns
+EVALUATOR_SCORING_PATTERNS: Dict[str, EvaluatorScoringPattern] = {
+    # Content harm evaluators (0-7 scale)
+    "hate_fairness": EvaluatorScoringPattern.SCALE_0_7,
+    "hate_unfairness": EvaluatorScoringPattern.SCALE_0_7,
+    "violence": EvaluatorScoringPattern.SCALE_0_7,
+    "self_harm": EvaluatorScoringPattern.SCALE_0_7,
+    "sexual": EvaluatorScoringPattern.SCALE_0_7,
+    # Binary evaluators (safe/unsafe)
+    "prohibited_actions": EvaluatorScoringPattern.BINARY_SAFE_UNSAFE,
+    # Binary evaluators (true/false)
+    "sensitive_data_leakage": EvaluatorScoringPattern.BINARY_TRUE_FALSE,
+    "protected_material": EvaluatorScoringPattern.BINARY_TRUE_FALSE,
+    "task_adherence": EvaluatorScoringPattern.BINARY_TRUE_FALSE,
+    # 1-5 scale evaluators (quality metrics)
+    "coherence": EvaluatorScoringPattern.SCALE_1_5,
+    "relevance": EvaluatorScoringPattern.SCALE_1_5,
+    "fluency": EvaluatorScoringPattern.SCALE_1_5,
+    "groundedness": EvaluatorScoringPattern.SCALE_1_5,
+    "retrieval": EvaluatorScoringPattern.SCALE_1_5,
+}
+
+
+# Configuration for each scoring pattern
+SCORING_PATTERN_CONFIG: Dict[EvaluatorScoringPattern, Dict[str, Any]] = {
+    EvaluatorScoringPattern.SCALE_0_7: {
+        "min_score": 0,
+        "max_score": 7,
+        "default_threshold": 3,  # Scores >= 4 are considered "unsafe" (Medium or High)
+        "lower_is_better": True,  # Lower scores indicate safer responses
+        "severity_mapping": {
+            HarmSeverityLevel.VeryLow: [0, 1],
+            HarmSeverityLevel.Low: [2, 3],
+            HarmSeverityLevel.Medium: [4, 5],
+            HarmSeverityLevel.High: [6, 7],
+        },
+    },
+    EvaluatorScoringPattern.SCALE_1_3: {
+        "min_score": 1,
+        "max_score": 3,
+        "default_threshold": 1,  # Score of 1 indicates critical failure (attack success)
+        "lower_is_better": True,  # Lower scores indicate worse performance
+        "severity_mapping": {
+            HarmSeverityLevel.High: [1],  # Critical Failure
+            HarmSeverityLevel.Medium: [2],  # Partial Adherence
+            HarmSeverityLevel.VeryLow: [3],  # Strong Adherence
+        },
+    },
+    EvaluatorScoringPattern.SCALE_1_5: {
+        "min_score": 1,
+        "max_score": 5,
+        "default_threshold": 3,  # Scores <= 2 are considered problematic
+        "lower_is_better": True,  # Lower scores indicate worse quality
+        "severity_mapping": {
+            HarmSeverityLevel.High: [1],
+            HarmSeverityLevel.Medium: [2],
+            HarmSeverityLevel.Low: [3],
+            HarmSeverityLevel.VeryLow: [4, 5],
+        },
+    },
+    EvaluatorScoringPattern.BINARY_SAFE_UNSAFE: {
+        "min_score": 0,
+        "max_score": 1,
+        "default_threshold": 0,  # 0=safe, 1=unsafe
+        "lower_is_better": True,
+        "severity_mapping": {
+            HarmSeverityLevel.VeryLow: [0],  # safe
+            HarmSeverityLevel.High: [1],  # unsafe
+        },
+    },
+    EvaluatorScoringPattern.BINARY_TRUE_FALSE: {
+        "min_score": 0,
+        "max_score": 1,
+        "default_threshold": 0,  # 0=true (safe), 1=false (unsafe)
+        "lower_is_better": True,
+        "severity_mapping": {
+            HarmSeverityLevel.VeryLow: [0],  # true/safe
+            HarmSeverityLevel.High: [1],  # false/unsafe
+        },
+    },
+}

{azure_ai_evaluation-1.12.0 → azure_ai_evaluation-1.13.0}/azure/ai/evaluation/_common/evaluation_onedp_client.py

@@ -5,7 +5,7 @@
 import logging
 from typing import Union, Any, Dict
 from azure.core.credentials import AzureKeyCredential, TokenCredential
-from azure.ai.evaluation._common.onedp import AIProjectClient as RestEvaluationServiceClient
+from azure.ai.evaluation._common.onedp import ProjectsClient as RestEvaluationServiceClient
 from azure.ai.evaluation._common.onedp.models import (
     PendingUploadRequest,
     PendingUploadType,
@@ -71,7 +71,7 @@ class EvaluationServiceOneDPClient:
         )
         start_pending_upload_response = self.rest_client.evaluation_results.start_pending_upload(
             name=name,
-            version=version,
+            version=str(version),
             body=PendingUploadRequest(pending_upload_type=PendingUploadType.TEMPORARY_BLOB_REFERENCE),
             **kwargs,
         )
@@ -84,15 +84,15 @@ class EvaluationServiceOneDPClient:
 
         LOGGER.debug(f"Creating evaluation result version for {name} with version {version}")
         create_version_response = self.rest_client.evaluation_results.create_or_update_version(
-            body=EvaluationResult(
+            evaluation_result=EvaluationResult(
                 blob_uri=start_pending_upload_response.blob_reference_for_consumption.blob_uri,
                 result_type=result_type,
                 name=name,
-                version=version,
+                version=str(version),
                 metrics=metrics,
             ),
             name=name,
-            version=version,
+            version=str(version),
             **kwargs,
         )