azure-ai-evaluation 1.11.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_task_navigation_efficiency/_task_navigation_efficiency.py

@@ -0,0 +1,384 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+from enum import Enum
+from collections import Counter
+import json
+from typing import Dict, List, Union, Any, Tuple
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._exceptions import (
+    ErrorCategory,
+    ErrorTarget,
+    EvaluationException,
+)
+
+
+class _TaskNavigationEfficiencyMatchingMode(str, Enum):
+    """
+    Enumeration of task navigation efficiency matching mode.
+
+    This enum allows you to specify which single matching technique should be used when evaluating
+    the efficiency of an agent's tool calls sequence against a ground truth path.
+    """
+
+    EXACT_MATCH = "exact_match"
+    """
+    Binary metric indicating whether the agent's tool calls exactly match the ground truth.
+
+    Returns True only if the agent's tool calls sequence is identical to the expected sequence
+    in both order and content (no extra steps, no missing steps, correct order).
+    """
+
+    IN_ORDER_MATCH = "in_order_match"
+    """
+    Binary metric allowing extra steps but requiring correct order of required tool calls.
+    
+    Returns True if all ground truth steps appear in the agent's sequence in the correct
+    order, even if there are additional steps interspersed.
+    """
+
+    ANY_ORDER_MATCH = "any_order_match"
+    """
+    Binary metric allowing both extra steps and different ordering.
+    
+    Returns True if all ground truth steps appear in the agent's sequence with sufficient
+    frequency, regardless of order. Most lenient matching criterion.
+    """
+
+
+class _TaskNavigationEfficiencyEvaluator(EvaluatorBase):
+    """
+    Evaluates whether an agent's sequence of actions is efficient and follows optimal decision-making patterns.
+
+    The Task Navigation Efficiency Evaluator returns binary matching results between the agent's tool usage trajectory and the ground truth expected steps.
+    It has three matching techniques: exact match, in-order match (allows extra steps), and any-order match (allows extra steps and ignores order).
+    It also returns precision, recall, and F1 scores in properties bag.
+
+    :param matching_mode: The matching mode to use. Default is "exact_match".
+    :type matching_mode: enum[str, _TaskNavigationEfficiencyMatchingMode]
+
+    .. admonition:: Example:
+
+        .. code-block:: python
+
+            from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator
+
+            task_navigation_efficiency_eval = _TaskNavigationEfficiencyEvaluator(
+                matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH
+            )
+
+            # Example 1: Using simple tool names list
+            result = path_efficiency_eval(
+                response=[
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_1", "name": "identify_tools_to_call", "arguments": {}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_2", "name": "call_tool_A", "arguments": {}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_3", "name": "call_tool_B", "arguments": {}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_4", "name": "response_synthesis", "arguments": {}}]}
+                ],
+                ground_truth=["identify_tools_to_call", ""call_tool_A", "call_tool_B", "response_synthesis"]
+            )
+
+            # Example 2: Using tool names with parameters (exact parameter matching required)
+            result = path_efficiency_eval(
+                response=[
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_1", "name": "search", "arguments": {"query": "weather", "location": "NYC"}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_2", "name": "format_result", "arguments": {"format": "json"}}]}
+                ],
+                ground_truth=(
+                    ["search", "format_result"],
+                    {
+                        "search": {"query": "weather", "location": "NYC"},
+                        "format_result": {"format": "json"}
+                    }
+                )
+            )
+    """
+
+    id = "azureai://built-in/evaluators/task_navigation_efficiency"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    matching_mode: _TaskNavigationEfficiencyMatchingMode
+    """The matching mode to use."""
+
+    @override
+    def __init__(
+        self,
+        *,
+        matching_mode: Union[
+            str, _TaskNavigationEfficiencyMatchingMode
+        ] = _TaskNavigationEfficiencyMatchingMode.EXACT_MATCH,
+    ):
+        # Type checking for metric parameter
+        if isinstance(matching_mode, str):
+            try:
+                self.matching_mode = _TaskNavigationEfficiencyMatchingMode(matching_mode)
+            except ValueError:
+                raise ValueError(
+                    f"matching_mode must be one of {[m.value for m in _TaskNavigationEfficiencyMatchingMode]}, got '{matching_mode}'"
+                )
+        elif isinstance(matching_mode, _TaskNavigationEfficiencyMatchingMode):
+            self.matching_mode = matching_mode
+        else:
+            raise EvaluationException(
+                f"matching_mode must be a string with one of {[m.value for m in _TaskNavigationEfficiencyMatchingMode]} or _TaskNavigationEfficiencyMatchingMode enum, got {type(matching_mode)}",
+                internal_message=str(matching_mode),
+                target=ErrorTarget.TASK_NAVIGATION_EFFICIENCY_EVALUATOR,
+                category=ErrorCategory.INVALID_VALUE,
+            )
+
+        super().__init__()
+
+    def _prepare_steps_for_comparison(
+        self,
+        agent_tool_pairs: List[Tuple[str, Dict[str, Any]]],
+        ground_truth: List[str],
+        ground_truth_params: Dict[str, Dict[str, Any]],
+        use_parameter_matching: bool,
+    ) -> Tuple[
+        List[Union[str, Tuple[str, Tuple]]],
+        List[Union[str, Tuple[str, Tuple]]],
+    ]:
+        """Prepare agent and ground truth steps for comparison based on parameter matching mode."""
+        agent_steps: List[Union[str, Tuple[str, Tuple]]] = []
+        ground_truth_steps: List[Union[str, Tuple[str, Tuple]]] = []
+        if use_parameter_matching:
+            # When parameter matching is enabled, we need to match both tool name and parameters
+            agent_steps = [(pair[0], tuple(sorted(pair[1].items()))) for pair in agent_tool_pairs]
+            ground_truth_steps = [
+                (name, tuple(sorted(ground_truth_params.get(name, {}).items()))) for name in ground_truth
+            ]
+        else:
+            # When parameter matching is disabled, only compare tool names
+            agent_steps = [name for name, _ in agent_tool_pairs]
+            ground_truth_steps = [step for step in ground_truth]
+
+        return agent_steps, ground_truth_steps
+
+    def _calculate_precision_recall_f1_scores(self, agent_steps: List, ground_truth_steps: List) -> Dict[str, float]:
+        """Calculate precision, recall, and F1 scores."""
+        if not agent_steps:
+            return {"precision_score": 0.0, "recall_score": 0.0, "f1_score": 0.0}
+
+        # Count occurrences of each step in both lists to handle duplicates
+        agent_steps_counts = Counter(agent_steps)
+        ground_truth_counts = Counter(ground_truth_steps)
+
+        # Calculate true positives by taking the minimum count for each common element
+        # For each step, count the intersection (min count) of agent and ground truth steps
+        true_positives = sum(
+            min(agent_steps_counts[step], ground_truth_counts[step])
+            for step in agent_steps_counts
+            if step in ground_truth_counts
+        )
+
+        # Calculate false positives (agent steps not in ground truth or excess occurrences)
+        # For each step, count the excess occurrences of agent steps not in (minus) ground truth
+        # or zero (agent steps minus agent steps) if agent steps is less than ground truth
+        false_positives = sum(
+            agent_steps_counts[step] - min(agent_steps_counts[step], ground_truth_counts.get(step, 0))
+            for step in agent_steps_counts
+        )
+
+        # Calculate false negatives (ground truth steps not in agent or missing occurrences)
+        # For each step, count the excess occurrences of ground truth steps not in (minus) agent steps
+        # or zero (ground truth steps minus ground truth steps) if ground truth steps is less than agent steps
+        false_negatives = sum(
+            ground_truth_counts[step] - min(ground_truth_counts[step], agent_steps_counts.get(step, 0))
+            for step in ground_truth_counts
+        )
+
+        # Calculate precision, recall, F1
+        precision = (
+            true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0.0
+        )
+        recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0.0
+        f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+
+        return {
+            "precision_score": precision,
+            "recall_score": recall,
+            "f1_score": f1_score,
+        }
+
+    def _calculate_exact_match(self, agent_steps: List, ground_truth_steps: List) -> bool:
+        """Check if agent steps exactly match ground truth (order and content)."""
+        return agent_steps == ground_truth_steps
+
+    def _calculate_in_order_match(self, agent_steps: List, ground_truth_steps: List) -> bool:
+        """Check if all ground truth steps appear in agent steps in correct order (extra steps allowed)."""
+        if not ground_truth_steps:
+            return True
+
+        gt_index = 0
+        for step in agent_steps:
+            if gt_index < len(ground_truth_steps) and step == ground_truth_steps[gt_index]:
+                gt_index += 1
+
+        return gt_index == len(ground_truth_steps)
+
+    def _calculate_any_order_match(self, agent_steps: List, ground_truth_steps: List) -> bool:
+        """Check if all ground truth steps appear in agent steps with sufficient frequency (any order, extra steps allowed)."""
+        # Count occurrences of each step in both lists to handle duplicates
+        agent_counts = Counter(agent_steps)
+        ground_truth_counts = Counter(ground_truth_steps)
+
+        # Check if agent has at least as many occurrences of each ground truth step
+        return all(agent_counts[step] >= ground_truth_counts[step] for step in ground_truth_counts)
+
+    _TASK_NAVIGATION_EFFICIENCY_MATCHING_MODE_TO_FUNCTIONS = {
+        _TaskNavigationEfficiencyMatchingMode.EXACT_MATCH: _calculate_exact_match,
+        _TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH: _calculate_in_order_match,
+        _TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH: _calculate_any_order_match,
+    }
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str, Dict[str, float]]]:
+        """Produce a path efficiency evaluation result.
+
+        :param eval_input: The input to the evaluation function. Must contain "response" and "ground_truth".
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
+        """
+        response = eval_input["response"]
+        ground_truth = eval_input["ground_truth"]
+
+        # Value and type checking for ground truth steps
+        if not ground_truth:
+            raise ValueError("ground_truth cannot be empty")
+
+        # Check if ground_truth is a tuple (tool names + parameters) or list (tool names only)
+        use_parameter_matching = False
+        ground_truth_names = []
+        ground_truth_params_dict: Dict[str, Dict[str, Any]] = {}
+
+        if isinstance(ground_truth, tuple) and len(ground_truth) == 2:
+            # Tuple format: (tool_names, parameters_dict)
+            tool_names_list, params_dict = ground_truth
+
+            if not isinstance(tool_names_list, list) or not all(isinstance(name, str) for name in tool_names_list):
+                raise TypeError("ground_truth tuple first element must be a list of strings (tool names)")
+
+            if not isinstance(params_dict, dict):
+                raise TypeError(
+                    "ground_truth tuple second element must be a dictionary mapping tool names to parameters"
+                )
+
+            # Validate that all values in params_dict are dictionaries with string keys and values
+            for tool_name, params in params_dict.items():
+                if not isinstance(tool_name, str):
+                    raise TypeError("ground_truth parameters dictionary keys must be strings (tool names)")
+                if not isinstance(params, dict):
+                    raise TypeError(f"ground_truth parameters for tool '{tool_name}' must be a dictionary")
+                for k, v in params.items():
+                    if not isinstance(k, str):
+                        raise TypeError(f"ground_truth parameters for tool '{tool_name}' must have string keys")
+                    try:
+                        json.dumps(v)
+                    except (TypeError, ValueError):
+                        raise TypeError(
+                            f"ground_truth parameters for tool '{tool_name}' must have JSON-serializable values (got type {type(v)} for key '{k}')"
+                        )
+
+            ground_truth_names = [name.strip() for name in tool_names_list]
+            ground_truth_params_dict = params_dict
+            use_parameter_matching = True
+        elif isinstance(ground_truth, list) and all(isinstance(step, str) for step in ground_truth):
+            # List format: just tool names
+            ground_truth_names = [step.strip() for step in ground_truth]
+            use_parameter_matching = False
+        else:
+            raise TypeError(
+                "ground_truth must be a list of strings or a tuple of (list[str], dict[str, dict[str, str]])"
+            )
+
+        # Extract tool information from the response
+        agent_tool_pairs = self._extract_tool_names_and_params_from_response(response)
+
+        # Prepare steps for comparison
+        agent_steps, ground_truth_steps = self._prepare_steps_for_comparison(
+            agent_tool_pairs,
+            ground_truth_names,
+            ground_truth_params_dict,
+            use_parameter_matching,
+        )
+
+        # Calculate precision, recall, and F1 scores
+        additional_properties_metrics = self._calculate_precision_recall_f1_scores(agent_steps, ground_truth_steps)
+
+        # Convert metrics to floats, using nan for None or non-convertible values
+        for metric, score in additional_properties_metrics.items():
+            additional_properties_metrics[metric] = float(score) if score is not None else float("nan")
+
+        if self.matching_mode in self._TASK_NAVIGATION_EFFICIENCY_MATCHING_MODE_TO_FUNCTIONS:
+            # Calculate binary match metrics
+            match_result = self._TASK_NAVIGATION_EFFICIENCY_MATCHING_MODE_TO_FUNCTIONS[self.matching_mode](
+                self, agent_steps, ground_truth_steps
+            )
+
+            return {
+                "task_navigation_efficiency_label": match_result,
+                "task_navigation_efficiency_result": EVALUATION_PASS_FAIL_MAPPING[match_result],
+                "task_navigation_efficiency_details": additional_properties_metrics,
+            }
+        else:
+            raise EvaluationException(
+                f"Unsupported matching_mode '{self.matching_mode}'",
+                internal_message=str(self.matching_mode),
+                target=ErrorTarget.TASK_NAVIGATION_EFFICIENCY_EVALUATOR,
+                category=ErrorCategory.INVALID_VALUE,
+            )
+
+    @overload
+    def __call__(  # type: ignore
+        self, *, response: Union[str, List[Dict[str, Any]]], ground_truth: List[str]
+    ) -> Dict[str, Union[float, str, Dict[str, float]]]:
+        """
+        Evaluate the task navigation efficiency of an agent's action sequence.
+
+        :keyword response: The agent's response containing tool calls.
+        :paramtype response: Union[str, List[Dict[str, Any]]]
+        :keyword ground_truth: List of expected tool/action steps.
+        :paramtype ground_truth: List[str]
+        :return: The task navigation efficiency scores and results.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
+        """
+
+    @overload
+    def __call__(  # type: ignore
+        self,
+        *,
+        response: Union[str, List[Dict[str, Any]]],
+        ground_truth: Tuple[List[str], Dict[str, Dict[str, str]]],
+    ) -> Dict[str, Union[float, str, Dict[str, float]]]:
+        """
+        Evaluate the task navigation efficiency of an agent's action sequence with tool parameters.
+
+        :keyword response: The agent's response containing tool calls.
+        :paramtype response: Union[str, List[Dict[str, Any]]]
+        :keyword ground_truth: Tuple of (tool names list, parameters dict) where parameters must match exactly.
+        :paramtype ground_truth: Tuple[List[str], Dict[str, Dict[str, str]]]
+        :return: The task navigation efficiency scores and results.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
+        """
+
+    @override
+    def __call__(
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate task navigation efficiency.
+
+        :keyword response: The agent's response containing tool calls.
+        :paramtype response: Union[str, List[Dict[str, Any]]]
+        :keyword ground_truth: List of expected tool/action steps or tuple of (tool names, parameters dict).
+        :paramtype ground_truth: Union[List[str], Tuple[List[str], Dict[str, Dict[str, str]]]]
+        :return: The task navigation efficiency scores and results.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py

@@ -1,7 +1,6 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from itertools import chain
 import math
 import os
 import logging
@@ -17,46 +16,12 @@ from azure.ai.evaluation._exceptions import (
 )
 from ..._common.utils import check_score_is_valid
 from azure.ai.evaluation._common._experimental import experimental
-from ..._converters._models import (
-    _BUILT_IN_DESCRIPTIONS,
-    _BUILT_IN_PARAMS,
-)
 
 logger = logging.getLogger(__name__)
 
 T_EvalValue = TypeVar("T_EvalValue")
 
 
-def _get_built_in_definition(tool_name: str):
-    """Get the definition for the built-in tool."""
-    if tool_name in _BUILT_IN_DESCRIPTIONS:
-        return {
-            "type": tool_name,
-            "description": _BUILT_IN_DESCRIPTIONS[tool_name],
-            "name": tool_name,
-            "parameters": _BUILT_IN_PARAMS.get(tool_name, {}),
-        }
-    return None
-
-
-def _get_needed_built_in_definitions(tool_calls: List[Dict]) -> List[Dict]:
-    """Extract tool definitions needed for the given built-in tool calls."""
-    needed_definitions = []
-    for tool_call in tool_calls:
-        if isinstance(tool_call, dict):
-            tool_type = tool_call.get("type")
-
-            # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}}
-            if tool_type == "tool_call":
-                tool_name = tool_call.get("name")
-                if tool_name in _BUILT_IN_DESCRIPTIONS:
-                    built_in_def = _get_built_in_definition(tool_name)
-                    if built_in_def and built_in_def not in needed_definitions:
-                        needed_definitions.append(built_in_def)
-
-    return needed_definitions
-
-
 @experimental
 class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:
@@ -100,9 +65,12 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
 
     .. note::
 
+        The output field "details" has been renamed to "tool_call_accuracy_details" for clarity.
+
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
         To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+
     """
 
     _PROMPTY_FILE = "tool_call_accuracy.prompty"
@@ -132,6 +100,7 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
             credential=credential,
+            threshold=threshold,
             **kwargs,
         )
 
@@ -207,7 +176,9 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             tool_definitions = [tool_definitions] if tool_definitions else []
 
         try:
-            needed_tool_definitions = self._extract_needed_tool_definitions(tool_calls, tool_definitions)
+            needed_tool_definitions = self._extract_needed_tool_definitions(
+                tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+            )
         except EvaluationException as e:
             # Check if this is because no tool definitions were provided at all
             if len(tool_definitions) == 0:
@@ -235,8 +206,8 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         :rtype: Dict
         """
         # Single LLM call for all tool calls
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
-
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", {})
         if isinstance(llm_output, dict):
             score = llm_output.get(self._LLM_SCORE_KEY, None)
             if not score or not check_score_is_valid(
@@ -248,6 +219,7 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                     message=f"Invalid score value: {score}. Expected a number in range [{ToolCallAccuracyEvaluator._MIN_TOOL_CALL_ACCURACY_SCORE}, {ToolCallAccuracyEvaluator._MAX_TOOL_CALL_ACCURACY_SCORE}].",
                     internal_message="Invalid score value.",
                     category=ErrorCategory.FAILED_EXECUTION,
+                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
                     blame=ErrorBlame.SYSTEM_ERROR,
                 )
 
@@ -257,10 +229,18 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             score_result = "pass" if score >= self.threshold else "fail"
             response_dict = {
                 self._result_key: score,
+                f"gpt_{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_threshold": self._threshold,
                 f"{self._result_key}_reason": reason,
-                "details": llm_output.get("details", {}),
+                f"{self._result_key}_details": llm_output.get("details", {}),
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
             return response_dict
 
@@ -275,105 +255,21 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     async def _real_call(self, **kwargs):
         """The asynchronous call where real end-to-end evaluation logic is performed.
 
-        :keyword kwargs: The inputs to evaluate.
+        :keyword kwargs: The inputs to evaluate
         :type kwargs: Dict
-        :return: The evaluation result.
+        :return: The evaluation result
         :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
         """
         # Convert inputs into list of evaluable inputs.
         eval_input = self._convert_kwargs_to_eval_input(**kwargs)
         if isinstance(eval_input, dict) and eval_input.get("error_message"):
             # If there is an error message, return not applicable result
-            return self._not_applicable_result(eval_input.get("error_message"))
+            return self._not_applicable_result(eval_input.get("error_message"), self.threshold)
         # Do the evaluation
         result = await self._do_eval(eval_input)
         # Return the result
         return result
 
-    def _not_applicable_result(self, error_message):
-        """Return a result indicating that the tool call is not applicable for evaluation.
-        :param eval_input: The input to the evaluator.
-        :type eval_input: Dict
-        :return: A dictionary containing the result of the evaluation.
-        :rtype: Dict[str, Union[str, float]]
-        """
-        # If no tool calls were made or tool call type is not supported, return not applicable result
-        return {
-            self._result_key: self._NOT_APPLICABLE_RESULT,
-            f"{self._result_key}_result": "pass",
-            f"{self._result_key}_threshold": self.threshold,
-            f"{self._result_key}_reason": error_message,
-            "details": {},
-        }
-
-    def _extract_needed_tool_definitions(self, tool_calls, tool_definitions):
-        """Extract the tool definitions that are needed for the provided tool calls."""
-        needed_tool_definitions = []
-
-        # Add all user-provided tool definitions
-        needed_tool_definitions.extend(tool_definitions)
-
-        # Add the needed built-in tool definitions (if they are called)
-        built_in_definitions = _get_needed_built_in_definitions(tool_calls)
-        needed_tool_definitions.extend(built_in_definitions)
-
-        # OpenAPI tool is a collection of functions, so we need to expand it
-        tool_definitions_expanded = list(
-            chain.from_iterable(
-                tool.get("functions", []) if tool.get("type") == "openapi" else [tool]
-                for tool in needed_tool_definitions
-            )
-        )
-
-        # Validate that all tool calls have corresponding definitions
-        for tool_call in tool_calls:
-            if isinstance(tool_call, dict):
-                tool_type = tool_call.get("type")
-
-                if tool_type == "tool_call":
-                    tool_name = tool_call.get("name")
-                    if tool_name and tool_name in _BUILT_IN_DESCRIPTIONS:
-                        # This is a built-in tool from converter, already handled above
-                        continue
-                    elif tool_name:
-                        # This is a regular function tool from converter
-                        tool_definition_exists = any(
-                            tool.get("name") == tool_name and tool.get("type", "function") == "function"
-                            for tool in tool_definitions_expanded
-                        )
-                        if not tool_definition_exists:
-                            raise EvaluationException(
-                                message=f"Tool definition for {tool_name} not found",
-                                blame=ErrorBlame.USER_ERROR,
-                                category=ErrorCategory.INVALID_VALUE,
-                                target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                            )
-                    else:
-                        raise EvaluationException(
-                            message=f"Tool call missing name: {tool_call}",
-                            blame=ErrorBlame.USER_ERROR,
-                            category=ErrorCategory.INVALID_VALUE,
-                            target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                        )
-                else:
-                    # Unsupported tool format - only converter format is supported
-                    raise EvaluationException(
-                        message=f"Unsupported tool call format. Only converter format is supported: {tool_call}",
-                        blame=ErrorBlame.USER_ERROR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                    )
-            else:
-                # Tool call is not a dictionary
-                raise EvaluationException(
-                    message=f"Tool call is not a dictionary: {tool_call}",
-                    blame=ErrorBlame.USER_ERROR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                )
-
-        return needed_tool_definitions
-
     @override
     def __call__(  # pylint: disable=docstring-missing-param
         self,

azure/ai/evaluation/_evaluators/_tool_input_accuracy/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._tool_input_accuracy import _ToolInputAccuracyEvaluator
+
+__all__ = [
+    "_ToolInputAccuracyEvaluator",
+]