azure-ai-evaluation 1.12.0__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_common/utils.py

@@ -21,34 +21,122 @@ from azure.ai.evaluation._model_configurations import (
 )
 
 from . import constants
+from .constants import EvaluatorScoringPattern, EVALUATOR_SCORING_PATTERNS, SCORING_PATTERN_CONFIG
 
 _nltk_data_download_lock = threading.Lock()
 
 T_TypedDict = TypeVar("T_TypedDict", bound=Mapping[Any, Any])
 
 
-def get_harm_severity_level(harm_score: Union[float, int]) -> Union[str, float]:
+def get_harm_severity_level(harm_score: Union[float, int], evaluator: Optional[str] = None) -> Union[str, float]:
     """Generate harm severity level based on harm score.
 
     :param harm_score: The harm score to be evaluated.
-    :type harm_score: int
+    :type harm_score: Union[float, int]
+    :param evaluator: Optional evaluator name to determine scoring pattern. If not provided, uses default 0-7 scale.
+    :type evaluator: Optional[str]
     :return: The harm severity level. If harm score is None or numpy.nan, returns numpy.nan.
-    :rtype: str
+    :rtype: Union[str, float]
     """
-    HARM_SEVERITY_LEVEL_MAPPING = {
-        constants.HarmSeverityLevel.VeryLow: [0, 1],
-        constants.HarmSeverityLevel.Low: [2, 3],
-        constants.HarmSeverityLevel.Medium: [4, 5],
-        constants.HarmSeverityLevel.High: [6, 7],
-    }
-    if math.isnan(harm_score) or harm_score is None:
+    if harm_score is None or (isinstance(harm_score, float) and math.isnan(harm_score)):
+        return math.nan
+
+    # Get the scoring pattern for this evaluator
+    pattern = get_evaluator_scoring_pattern(evaluator)
+    config = SCORING_PATTERN_CONFIG.get(pattern)
+
+    if not config:
+        # Fallback to default 0-7 mapping
+        HARM_SEVERITY_LEVEL_MAPPING = {
+            constants.HarmSeverityLevel.VeryLow: [0, 1],
+            constants.HarmSeverityLevel.Low: [2, 3],
+            constants.HarmSeverityLevel.Medium: [4, 5],
+            constants.HarmSeverityLevel.High: [6, 7],
+        }
+        for harm_level, harm_score_range in HARM_SEVERITY_LEVEL_MAPPING.items():
+            if harm_score_range[0] <= harm_score <= harm_score_range[1]:
+                return harm_level.value
         return math.nan
-    for harm_level, harm_score_range in HARM_SEVERITY_LEVEL_MAPPING.items():
-        if harm_score_range[0] <= harm_score <= harm_score_range[1]:
+
+    # Use pattern-specific severity mapping
+    severity_mapping = config.get("severity_mapping", {})
+    for harm_level, score_range in severity_mapping.items():
+        if score_range[0] <= harm_score <= score_range[-1]:
             return harm_level.value
+
     return math.nan
 
 
+def get_evaluator_scoring_pattern(evaluator: Optional[str]) -> EvaluatorScoringPattern:
+    """Get the scoring pattern for a given evaluator.
+
+    :param evaluator: The name of the evaluator.
+    :type evaluator: Optional[str]
+    :return: The scoring pattern for the evaluator, defaults to SCALE_0_7 if not found.
+    :rtype: EvaluatorScoringPattern
+    """
+    if evaluator is None:
+        return EvaluatorScoringPattern.SCALE_0_7
+
+    # Normalize evaluator name (remove underscores, lowercase)
+    normalized_name = evaluator.lower().replace("_", "")
+
+    # Try exact match first
+    if evaluator in EVALUATOR_SCORING_PATTERNS:
+        return EVALUATOR_SCORING_PATTERNS[evaluator]
+
+    # Try normalized match
+    for key, pattern in EVALUATOR_SCORING_PATTERNS.items():
+        if key.replace("_", "").lower() == normalized_name:
+            return pattern
+
+    # Default to 0-7 scale for unknown evaluators
+    return EvaluatorScoringPattern.SCALE_0_7
+
+
+def get_default_threshold_for_evaluator(evaluator: str) -> int:
+    """Get the default attack success threshold for a given evaluator.
+
+    :param evaluator: The name of the evaluator.
+    :type evaluator: str
+    :return: The default threshold value.
+    :rtype: int
+    """
+    pattern = get_evaluator_scoring_pattern(evaluator)
+    config = SCORING_PATTERN_CONFIG.get(pattern, {})
+    return config.get("default_threshold", 3)
+
+
+def convert_binary_to_numeric(value: Union[str, bool, int]) -> int:
+    """Convert binary evaluator outputs to numeric scores.
+
+    :param value: The binary value ("safe"/"unsafe", "true"/"false", True/False, 0/1).
+    :type value: Union[str, bool, int]
+    :return: 0 for safe/true, 1 for unsafe/false.
+    :rtype: int
+    """
+    if isinstance(value, bool):
+        return 0 if value else 1
+
+    if isinstance(value, int):
+        return value
+
+    if isinstance(value, str):
+        value_lower = value.lower().strip()
+        # For "safe"/"unsafe" pattern
+        if value_lower == "safe":
+            return 0
+        if value_lower == "unsafe":
+            return 1
+        # For "true"/"false" pattern
+        if value_lower == "true":
+            return 0
+        if value_lower == "false":
+            return 1
+
+    raise ValueError(f"Unable to convert value '{value}' to numeric score")
+
+
 def ensure_nltk_data_downloaded():
     """Download NLTK data packages if not already downloaded."""
     nltk_data = [
@@ -492,36 +580,69 @@ def _extract_text_from_content(content):
     return text
 
 
-def _get_conversation_history(query, include_system_messages=False):
-    all_user_queries = []
-    cur_user_query = []
-    all_agent_responses = []
-    cur_agent_response = []
+def filter_to_used_tools(tool_definitions, msgs_lists, logger=None):
+    """Filters the tool definitions to only include those that were actually used in the messages lists."""
+    try:
+        used_tool_names = set()
+        any_tools_used = False
+        for msgs in msgs_lists:
+            for msg in msgs:
+                if msg.get("role") == "assistant" and "content" in msg:
+                    for content in msg.get("content", []):
+                        if content.get("type") == "tool_call":
+                            any_tools_used = True
+                            if "tool_call" in content and "function" in content["tool_call"]:
+                                used_tool_names.add(content["tool_call"]["function"])
+                            elif "name" in content:
+                                used_tool_names.add(content["name"])
+
+        filtered_tools = [tool for tool in tool_definitions if tool.get("name") in used_tool_names]
+        if any_tools_used and not filtered_tools:
+            if logger:
+                logger.warning("No tool definitions matched the tools used in the messages. Returning original list.")
+            filtered_tools = tool_definitions
+
+        return filtered_tools
+    except Exception as e:
+        if logger:
+            logger.warning(f"Failed to filter tool definitions, returning original list. Error: {e}")
+        return tool_definitions
+
+
+def _get_conversation_history(query, include_system_messages=False, include_tool_messages=False):
+    all_user_queries, all_agent_responses = [], []
+    cur_user_query, cur_agent_response = [], []
     system_message = None
+
     for msg in query:
-        if not "role" in msg:
+        role = msg.get("role")
+        if not role:
             continue
-        if include_system_messages and msg["role"] == "system" and "content" in msg:
+        if include_system_messages and role == "system":
             system_message = msg.get("content", "")
-        if msg["role"] == "user" and "content" in msg:
-            if cur_agent_response != []:
-                all_agent_responses.append(cur_agent_response)
+
+        elif role == "user" and "content" in msg:
+            if cur_agent_response:
+                formatted_agent_response = _get_agent_response(
+                    cur_agent_response, include_tool_messages=include_tool_messages
+                )
+                all_agent_responses.append([formatted_agent_response])
                 cur_agent_response = []
             text_in_msg = _extract_text_from_content(msg["content"])
             if text_in_msg:
                 cur_user_query.append(text_in_msg)
 
-        if msg["role"] == "assistant" and "content" in msg:
-            if cur_user_query != []:
+        elif role in ("assistant", "tool"):
+            if cur_user_query:
                 all_user_queries.append(cur_user_query)
                 cur_user_query = []
-            text_in_msg = _extract_text_from_content(msg["content"])
-            if text_in_msg:
-                cur_agent_response.append(text_in_msg)
-    if cur_user_query != []:
+            cur_agent_response.append(msg)
+
+    if cur_user_query:
         all_user_queries.append(cur_user_query)
-    if cur_agent_response != []:
-        all_agent_responses.append(cur_agent_response)
+    if cur_agent_response:
+        formatted_agent_response = _get_agent_response(cur_agent_response, include_tool_messages=include_tool_messages)
+        all_agent_responses.append([formatted_agent_response])
 
     if len(all_user_queries) != len(all_agent_responses) + 1:
         raise EvaluationException(
@@ -531,8 +652,9 @@ def _get_conversation_history(query, include_system_messages=False):
             category=ErrorCategory.INVALID_VALUE,
             blame=ErrorBlame.USER_ERROR,
         )
+
     result = {"user_queries": all_user_queries, "agent_responses": all_agent_responses}
-    if include_system_messages:
+    if include_system_messages and system_message:
         result["system_message"] = system_message
     return result
 
@@ -540,7 +662,7 @@ def _get_conversation_history(query, include_system_messages=False):
 def _pretty_format_conversation_history(conversation_history):
     """Formats the conversation history for better readability."""
     formatted_history = ""
-    if "system_message" in conversation_history and conversation_history["system_message"] is not None:
+    if conversation_history.get("system_message"):
         formatted_history += "SYSTEM_PROMPT:\n"
         formatted_history += "  " + conversation_history["system_message"] + "\n\n"
     for i, (user_query, agent_response) in enumerate(
@@ -548,22 +670,34 @@ def _pretty_format_conversation_history(conversation_history):
     ):
         formatted_history += f"User turn {i+1}:\n"
         for msg in user_query:
-            formatted_history += "  " + "\n  ".join(msg)
-        formatted_history += "\n\n"
+            if isinstance(msg, list):
+                for submsg in msg:
+                    formatted_history += "  " + "\n  ".join(submsg.split("\n")) + "\n"
+            else:
+                formatted_history += "  " + "\n  ".join(msg.split("\n")) + "\n"
+        formatted_history += "\n"
         if agent_response:
             formatted_history += f"Agent turn {i+1}:\n"
             for msg in agent_response:
-                formatted_history += "  " + "\n  ".join(msg)
-            formatted_history += "\n\n"
+                if isinstance(msg, list):
+                    for submsg in msg:
+                        formatted_history += "  " + "\n  ".join(submsg.split("\n")) + "\n"
+                else:
+                    formatted_history += "  " + "\n  ".join(msg.split("\n")) + "\n"
+            formatted_history += "\n"
     return formatted_history
 
 
-def reformat_conversation_history(query, logger=None, include_system_messages=False):
+def reformat_conversation_history(query, logger=None, include_system_messages=False, include_tool_messages=False):
     """Reformats the conversation history to a more compact representation."""
     try:
-        conversation_history = _get_conversation_history(query, include_system_messages=include_system_messages)
+        conversation_history = _get_conversation_history(
+            query,
+            include_system_messages=include_system_messages,
+            include_tool_messages=include_tool_messages,
+        )
         return _pretty_format_conversation_history(conversation_history)
-    except:
+    except Exception as e:
         # If the conversation history cannot be parsed for whatever reason (e.g. the converter format changed), the original query is returned
         # This is a fallback to ensure that the evaluation can still proceed. However the accuracy of the evaluation will be affected.
         # From our tests the negative impact on IntentResolution is:
@@ -659,6 +793,74 @@ def reformat_tool_definitions(tool_definitions, logger=None):
         return tool_definitions
 
 
+def simplify_messages(messages, drop_system=True, drop_tool_calls=False, logger=None):
+    """
+    Simplify a list of conversation messages by keeping only role and content.
+    Optionally filter out system messages and/or tool calls.
+
+    :param messages: List of message dicts (e.g., from query or response)
+    :param drop_system: If True, remove system role messages
+    :param drop_tool_calls: If True, remove tool_call items from assistant content
+    :return: New simplified list of messages
+    """
+    if isinstance(messages, str):
+        return messages
+    try:
+        # Validate input is a list
+        if not isinstance(messages, list):
+            return messages
+
+        simplified_msgs = []
+        for msg in messages:
+            # Ensure msg is a dict
+            if not isinstance(msg, dict):
+                simplified_msgs.append(msg)
+                continue
+
+            role = msg.get("role")
+            content = msg.get("content", [])
+
+            # Drop system message (if should)
+            if drop_system and role == "system":
+                continue
+
+            # Simplify user messages
+            if role == "user":
+                simplified_msg = {
+                    "role": role,
+                    "content": _extract_text_from_content(content),
+                }
+                simplified_msgs.append(simplified_msg)
+                continue
+
+            # Drop tool results (if should)
+            if drop_tool_calls and role == "tool":
+                continue
+
+            # Simplify assistant messages
+            if role == "assistant":
+                simplified_content = _extract_text_from_content(content)
+                # Check if message has content
+                if simplified_content:
+                    simplified_msg = {"role": role, "content": simplified_content}
+                    simplified_msgs.append(simplified_msg)
+                    continue
+
+                # Drop tool calls (if should)
+                if drop_tool_calls and any(c.get("type") == "tool_call" for c in content if isinstance(c, dict)):
+                    continue
+
+            # If we reach here, it means we want to keep the message
+            simplified_msgs.append(msg)
+
+        return simplified_msgs
+
+    except Exception as ex:
+        if logger:
+            logger.debug(f"Error simplifying messages: {str(ex)}. Returning original messages.")
+        return messages
+
+
 def upload(path: str, container_client: ContainerClient, logger=None):
     """Upload files or directories to Azure Blob Storage using a container client.
 

azure/ai/evaluation/_constants.py

@@ -28,6 +28,9 @@ class EvaluationMetrics:
     XPIA = "xpia"
     CODE_VULNERABILITY = "code_vulnerability"
     UNGROUNDED_ATTRIBUTES = "ungrounded_attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
 
 
 class _InternalEvaluationMetrics:
@@ -90,6 +93,100 @@ class TokenScope(str, enum.Enum):
     AZURE_ML = "https://ml.azure.com/.default"
 
 
+class _EvaluatorMetricMapping:
+    """
+    Static mapping of evaluator names to their metric names, based on assets.json.
+    The 'builtin.' prefix is removed from the evaluator name keys.
+    """
+
+    EVALUATOR_NAME_METRICS_MAPPINGS = {
+        "bleu_score": ["bleu"],
+        "coherence": ["coherence"],
+        "document_retrieval": [
+            "ndcg@3",
+            "xdcg@3",
+            "fidelity",
+            "top1_relevance",
+            "top3_max_relevance",
+            "holes",
+            "holes_ratio",
+            "total_retrieved_documents",
+            "total_ground_truth_documents",
+        ],
+        "f1_score": ["f1_score"],
+        "fluency": ["fluency"],
+        "gleu_score": ["gleu"],
+        "meteor_score": ["meteor"],
+        "relevance": ["relevance"],
+        "response_completeness": ["response_completeness"],
+        "rouge_score": ["rouge_precision", "rouge_recall", "rouge_f1_score"],
+        "groundedness_pro": ["groundedness_pro"],
+        "similarity": ["similarity"],
+        "intent_resolution": ["intent_resolution"],
+        "retrieval": ["retrieval"],
+        "task_adherence": ["task_adherence"],
+        "tool_call_accuracy": ["tool_call_accuracy"],
+        "groundedness": ["groundedness"],
+        "code_vulnerability": ["code_vulnerability"],
+        "eci": ["eci"],
+        "protected_material": ["protected_material"],
+        "ungrounded_attributes": ["ungrounded_attributes"],
+        "indirect_attack": ["xpia", "xpia_manipulated_content", "xpia_intrusion", "xpia_information_gathering"],
+        "label_grader": ["label_model"],
+        "string_check_grader": ["string_check"],
+        "text_similarity_grader": ["similarity"],
+        "score_model_grader": ["score_model"],
+        "sexual": ["sexual"],
+        "self_harm": ["self_harm"],
+        "violence": ["violence"],
+        "hate_unfairness": ["hate_unfairness"],
+        "tool_input_accuracy": ["tool_input_accuracy"],
+        "task_completion": ["task_completion"],
+        "tool_success": ["tool_success"],
+        "tool_selection": ["tool_selection"],
+        "tool_output_utilization": ["tool_output_utilization"],
+        "task_navigation_efficiency": ["task_navigation_efficiency"],
+        "text_similarity": ["similarity"],
+        "string_check": ["string_check"],
+        "sensitive_data_leakage": ["prohibited_actions"],
+        "score_model": ["score_model"],
+        "label_model": ["label_model"],
+        "prohibited_actions": ["prohibited_actions"],
+    }
+
+    EVAL_CLASS_NAME_MAP = {
+        "BleuScoreEvaluator": "bleu_score",
+        "CodeVulnerabilityEvaluator": "code_vulnerability",
+        "CoherenceEvaluator": "coherence",
+        "ContentSafetyEvaluator": "content_safety",
+        "DocumentRetrievalEvaluator": "document_retrieval",
+        "ECIEvaluator": "eci",
+        "F1ScoreEvaluator": "f1_score",
+        "FluencyEvaluator": "fluency",
+        "GleuScoreEvaluator": "gleu_score",
+        "GroundednessEvaluator": "groundedness",
+        "GroundednessProEvaluator": "groundedness_pro",
+        "HateUnfairnessEvaluator": "hate_unfairness",
+        "IndirectAttackEvaluator": "indirect_attack",
+        "IntentResolutionEvaluator": "intent_resolution",
+        "MeteorScoreEvaluator": "meteor_score",
+        "ProtectedMaterialEvaluator": "protected_material",
+        "QAEvaluator": "qa",
+        "RelevanceEvaluator": "relevance",
+        "ResponseCompletenessEvaluator": "response_completeness",
+        "RetrievalEvaluator": "retrieval",
+        "RougeScoreEvaluator": "rouge_score",
+        "SelfHarmEvaluator": "self_harm",
+        "SexualEvaluator": "sexual",
+        "SimilarityEvaluator": "similarity",
+        "TaskAdherenceEvaluator": "task_adherence",
+        "TaskCompletionEvaluator": "task_completion",
+        "ToolCallAccuracyEvaluator": "tool_call_accuracy",
+        "UngroundedAttributesEvaluator": "ungrounded_attributes",
+        "ViolenceEvaluator": "violence",
+    }
+
+
 DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json"
 
 CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT = 4
@@ -116,3 +213,6 @@ BINARY_AGGREGATE_SUFFIX = "binary_aggregate"
 AOAI_COLUMN_NAME = "aoai"
 DEFAULT_OAI_EVAL_RUN_NAME = "AI_SDK_EVAL_RUN"
 DEFAULT_AOAI_API_VERSION = "2025-04-01-preview"  # Unfortunately relying on preview version for now.
+
+# OpenTelemetry event names
+EVALUATION_EVENT_NAME = "gen_ai.evaluation.result"

azure/ai/evaluation/_eval_mapping.py

@@ -11,7 +11,11 @@
 
 # Import all evals
 from azure.ai.evaluation._evaluators._eci._eci import ECIEvaluator
-from azure.ai.evaluation._evaluators._task_success import TaskSuccessEvaluator
+from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator
+from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator
+from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator
+from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator
+from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator
 from azure.ai.evaluation import (
     BleuScoreEvaluator,
     CodeVulnerabilityEvaluator,
@@ -68,8 +72,12 @@ EVAL_CLASS_MAP = {
     SexualEvaluator: "sexual",
     SimilarityEvaluator: "similarity",
     TaskAdherenceEvaluator: "task_adherence",
-    TaskSuccessEvaluator: "task_success",
+    _TaskCompletionEvaluator: "task_completion",
+    _TaskNavigationEfficiencyEvaluator: "task_navigation_efficiency",
     ToolCallAccuracyEvaluator: "tool_call_accuracy",
+    _ToolInputAccuracyEvaluator: "tool_input_accuracy",
+    _ToolSelectionEvaluator: "tool_selection",
+    _ToolSuccessEvaluator: "tool_success",
     UngroundedAttributesEvaluator: "ungrounded_attributes",
     ViolenceEvaluator: "violence",
 }

azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -159,6 +159,16 @@ class RunSubmitterClient:
             "completed_lines": total_lines - failed_lines,
             "failed_lines": failed_lines,
             "log_path": None,
+            "error_message": (
+                f"({run.result.error.blame.value}) {run.result.error.message}"
+                if run.result and run.result.error and run.result.error.blame
+                else None
+            ),
+            "error_code": (
+                f"{run.result.error.category.value}"
+                if run.result and run.result.error and run.result.error.category
+                else None
+            ),
         }
 
     @staticmethod