azure-ai-evaluation 1.11.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_constants.py

@@ -28,6 +28,9 @@ class EvaluationMetrics:
     XPIA = "xpia"
     CODE_VULNERABILITY = "code_vulnerability"
     UNGROUNDED_ATTRIBUTES = "ungrounded_attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
 
 
 class _InternalEvaluationMetrics:
@@ -90,6 +93,100 @@ class TokenScope(str, enum.Enum):
     AZURE_ML = "https://ml.azure.com/.default"
 
 
+class _EvaluatorMetricMapping:
+    """
+    Static mapping of evaluator names to their metric names, based on assets.json.
+    The 'builtin.' prefix is removed from the evaluator name keys.
+    """
+
+    EVALUATOR_NAME_METRICS_MAPPINGS = {
+        "bleu_score": ["bleu"],
+        "coherence": ["coherence"],
+        "document_retrieval": [
+            "ndcg@3",
+            "xdcg@3",
+            "fidelity",
+            "top1_relevance",
+            "top3_max_relevance",
+            "holes",
+            "holes_ratio",
+            "total_retrieved_documents",
+            "total_ground_truth_documents",
+        ],
+        "f1_score": ["f1_score"],
+        "fluency": ["fluency"],
+        "gleu_score": ["gleu"],
+        "meteor_score": ["meteor"],
+        "relevance": ["relevance"],
+        "response_completeness": ["response_completeness"],
+        "rouge_score": ["rouge_precision", "rouge_recall", "rouge_f1_score"],
+        "groundedness_pro": ["groundedness_pro"],
+        "similarity": ["similarity"],
+        "intent_resolution": ["intent_resolution"],
+        "retrieval": ["retrieval"],
+        "task_adherence": ["task_adherence"],
+        "tool_call_accuracy": ["tool_call_accuracy"],
+        "groundedness": ["groundedness"],
+        "code_vulnerability": ["code_vulnerability"],
+        "eci": ["eci"],
+        "protected_material": ["protected_material"],
+        "ungrounded_attributes": ["ungrounded_attributes"],
+        "indirect_attack": ["xpia", "xpia_manipulated_content", "xpia_intrusion", "xpia_information_gathering"],
+        "label_grader": ["label_model"],
+        "string_check_grader": ["string_check"],
+        "text_similarity_grader": ["similarity"],
+        "score_model_grader": ["score_model"],
+        "sexual": ["sexual"],
+        "self_harm": ["self_harm"],
+        "violence": ["violence"],
+        "hate_unfairness": ["hate_unfairness"],
+        "tool_input_accuracy": ["tool_input_accuracy"],
+        "task_completion": ["task_completion"],
+        "tool_success": ["tool_success"],
+        "tool_selection": ["tool_selection"],
+        "tool_output_utilization": ["tool_output_utilization"],
+        "task_navigation_efficiency": ["task_navigation_efficiency"],
+        "text_similarity": ["similarity"],
+        "string_check": ["string_check"],
+        "sensitive_data_leakage": ["prohibited_actions"],
+        "score_model": ["score_model"],
+        "label_model": ["label_model"],
+        "prohibited_actions": ["prohibited_actions"],
+    }
+
+    EVAL_CLASS_NAME_MAP = {
+        "BleuScoreEvaluator": "bleu_score",
+        "CodeVulnerabilityEvaluator": "code_vulnerability",
+        "CoherenceEvaluator": "coherence",
+        "ContentSafetyEvaluator": "content_safety",
+        "DocumentRetrievalEvaluator": "document_retrieval",
+        "ECIEvaluator": "eci",
+        "F1ScoreEvaluator": "f1_score",
+        "FluencyEvaluator": "fluency",
+        "GleuScoreEvaluator": "gleu_score",
+        "GroundednessEvaluator": "groundedness",
+        "GroundednessProEvaluator": "groundedness_pro",
+        "HateUnfairnessEvaluator": "hate_unfairness",
+        "IndirectAttackEvaluator": "indirect_attack",
+        "IntentResolutionEvaluator": "intent_resolution",
+        "MeteorScoreEvaluator": "meteor_score",
+        "ProtectedMaterialEvaluator": "protected_material",
+        "QAEvaluator": "qa",
+        "RelevanceEvaluator": "relevance",
+        "ResponseCompletenessEvaluator": "response_completeness",
+        "RetrievalEvaluator": "retrieval",
+        "RougeScoreEvaluator": "rouge_score",
+        "SelfHarmEvaluator": "self_harm",
+        "SexualEvaluator": "sexual",
+        "SimilarityEvaluator": "similarity",
+        "TaskAdherenceEvaluator": "task_adherence",
+        "TaskCompletionEvaluator": "task_completion",
+        "ToolCallAccuracyEvaluator": "tool_call_accuracy",
+        "UngroundedAttributesEvaluator": "ungrounded_attributes",
+        "ViolenceEvaluator": "violence",
+    }
+
+
 DEFAULT_EVALUATION_RESULTS_FILE_NAME = "evaluation_results.json"
 
 CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT = 4
@@ -116,3 +213,6 @@ BINARY_AGGREGATE_SUFFIX = "binary_aggregate"
 AOAI_COLUMN_NAME = "aoai"
 DEFAULT_OAI_EVAL_RUN_NAME = "AI_SDK_EVAL_RUN"
 DEFAULT_AOAI_API_VERSION = "2025-04-01-preview"  # Unfortunately relying on preview version for now.
+
+# OpenTelemetry event names
+EVALUATION_EVENT_NAME = "gen_ai.evaluation.result"

azure/ai/evaluation/_eval_mapping.py

@@ -11,6 +11,11 @@
 
 # Import all evals
 from azure.ai.evaluation._evaluators._eci._eci import ECIEvaluator
+from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator
+from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator
+from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator
+from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator
+from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator
 from azure.ai.evaluation import (
     BleuScoreEvaluator,
     CodeVulnerabilityEvaluator,
@@ -67,7 +72,12 @@ EVAL_CLASS_MAP = {
     SexualEvaluator: "sexual",
     SimilarityEvaluator: "similarity",
     TaskAdherenceEvaluator: "task_adherence",
+    _TaskCompletionEvaluator: "task_completion",
+    _TaskNavigationEfficiencyEvaluator: "task_navigation_efficiency",
     ToolCallAccuracyEvaluator: "tool_call_accuracy",
+    _ToolInputAccuracyEvaluator: "tool_input_accuracy",
+    _ToolSelectionEvaluator: "tool_selection",
+    _ToolSuccessEvaluator: "tool_success",
     UngroundedAttributesEvaluator: "ungrounded_attributes",
     ViolenceEvaluator: "violence",
 }

azure/ai/evaluation/_evaluate/_batch_run/_run_submitter_client.py

@@ -159,6 +159,16 @@ class RunSubmitterClient:
             "completed_lines": total_lines - failed_lines,
             "failed_lines": failed_lines,
             "log_path": None,
+            "error_message": (
+                f"({run.result.error.blame.value}) {run.result.error.message}"
+                if run.result and run.result.error and run.result.error.blame
+                else None
+            ),
+            "error_code": (
+                f"{run.result.error.category.value}"
+                if run.result and run.result.error and run.result.error.category
+                else None
+            ),
         }
 
     @staticmethod