azure-ai-evaluation 1.12.0__py3-none-any.whl → 1.13.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_task_navigation_efficiency/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode
+
+__all__ = ["_TaskNavigationEfficiencyEvaluator", "_TaskNavigationEfficiencyMatchingMode"]

azure/ai/evaluation/_evaluators/{_path_efficiency/_path_efficiency.py → _task_navigation_efficiency/_task_navigation_efficiency.py}

@@ -1,40 +1,73 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-import json
+from enum import Enum
 from collections import Counter
+import json
 from typing import Dict, List, Union, Any, Tuple
 from typing_extensions import overload, override
 
-from azure.ai.evaluation._evaluators._common import EvaluatorBase
 from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._exceptions import (
+    ErrorCategory,
+    ErrorTarget,
+    EvaluationException,
+)
+
+
+class _TaskNavigationEfficiencyMatchingMode(str, Enum):
+    """
+    Enumeration of task navigation efficiency matching mode.
+
+    This enum allows you to specify which single matching technique should be used when evaluating
+    the efficiency of an agent's tool calls sequence against a ground truth path.
+    """
+
+    EXACT_MATCH = "exact_match"
+    """
+    Binary metric indicating whether the agent's tool calls exactly match the ground truth.
+
+    Returns True only if the agent's tool calls sequence is identical to the expected sequence
+    in both order and content (no extra steps, no missing steps, correct order).
+    """
+
+    IN_ORDER_MATCH = "in_order_match"
+    """
+    Binary metric allowing extra steps but requiring correct order of required tool calls.
+    
+    Returns True if all ground truth steps appear in the agent's sequence in the correct
+    order, even if there are additional steps interspersed.
+    """
+
+    ANY_ORDER_MATCH = "any_order_match"
+    """
+    Binary metric allowing both extra steps and different ordering.
+    
+    Returns True if all ground truth steps appear in the agent's sequence with sufficient
+    frequency, regardless of order. Most lenient matching criterion.
+    """
 
 
-class PathEfficiencyEvaluator(EvaluatorBase):
+class _TaskNavigationEfficiencyEvaluator(EvaluatorBase):
     """
     Evaluates whether an agent's sequence of actions is efficient and follows optimal decision-making patterns.
 
-    The Path Efficiency Evaluator calculates precision, recall, and F1 scores based on the comparison
-    between the agent's tool usage trajectory and the ground truth expected steps. It also provides
-    three binary match metrics: exact match, in-order match (allows extra steps), and any-order match (allows extra steps and ignores order).
+    The Task Navigation Efficiency Evaluator returns binary matching results between the agent's tool usage trajectory and the ground truth expected steps.
+    It has three matching techniques: exact match, in-order match (allows extra steps), and any-order match (allows extra steps and ignores order).
+    It also returns precision, recall, and F1 scores in properties bag.
 
-    :param precision_threshold: The threshold value to determine if the precision evaluation passes or fails. Default is 0.5.
-    :type precision_threshold: float
-    :param recall_threshold: The threshold value to determine if the recall evaluation passes or fails. Default is 0.5.
-    :type recall_threshold: float
-    :param f1_score_threshold: The threshold value to determine if the F1 score evaluation passes or fails. Default is 0.5.
-    :type f1_score_threshold: float
+    :param matching_mode: The matching mode to use. Default is "exact_match".
+    :type matching_mode: enum[str, _TaskNavigationEfficiencyMatchingMode]
 
     .. admonition:: Example:
 
         .. code-block:: python
 
-            from azure.ai.evaluation import PathEfficiencyEvaluator
+            from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator
 
-            path_efficiency_eval = PathEfficiencyEvaluator(
-                precision_threshold=0.7,
-                recall_threshold=0.8,
-                f1_score_threshold=0.75
+            task_navigation_efficiency_eval = _TaskNavigationEfficiencyEvaluator(
+                matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH
             )
 
             # Example 1: Using simple tool names list
@@ -64,36 +97,39 @@ class PathEfficiencyEvaluator(EvaluatorBase):
             )
     """
 
-    _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD = 0.5
-
-    id = "azureai://built-in/evaluators/path_efficiency"
+    id = "azureai://built-in/evaluators/task_navigation_efficiency"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
+    matching_mode: _TaskNavigationEfficiencyMatchingMode
+    """The matching mode to use."""
+
     @override
     def __init__(
         self,
         *,
-        precision_threshold: float = _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD,
-        recall_threshold: float = _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD,
-        f1_score_threshold: float = _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD,
+        matching_mode: Union[
+            str, _TaskNavigationEfficiencyMatchingMode
+        ] = _TaskNavigationEfficiencyMatchingMode.EXACT_MATCH,
     ):
-        self._higher_is_better = True
-        super().__init__()
+        # Type checking for metric parameter
+        if isinstance(matching_mode, str):
+            try:
+                self.matching_mode = _TaskNavigationEfficiencyMatchingMode(matching_mode)
+            except ValueError:
+                raise ValueError(
+                    f"matching_mode must be one of {[m.value for m in _TaskNavigationEfficiencyMatchingMode]}, got '{matching_mode}'"
+                )
+        elif isinstance(matching_mode, _TaskNavigationEfficiencyMatchingMode):
+            self.matching_mode = matching_mode
+        else:
+            raise EvaluationException(
+                f"matching_mode must be a string with one of {[m.value for m in _TaskNavigationEfficiencyMatchingMode]} or _TaskNavigationEfficiencyMatchingMode enum, got {type(matching_mode)}",
+                internal_message=str(matching_mode),
+                target=ErrorTarget.TASK_NAVIGATION_EFFICIENCY_EVALUATOR,
+                category=ErrorCategory.INVALID_VALUE,
+            )
 
-        # Type checking for threshold parameters
-        for name, value in [
-            ("precision_threshold", precision_threshold),
-            ("recall_threshold", recall_threshold),
-            ("f1_score_threshold", f1_score_threshold),
-        ]:
-            if not isinstance(value, float):
-                raise TypeError(f"{name} must be a float, got {type(value)}")
-
-        self._threshold = {
-            "path_efficiency_precision": precision_threshold,
-            "path_efficiency_recall": recall_threshold,
-            "path_efficiency_f1": f1_score_threshold,
-        }
+        super().__init__()
 
     def _prepare_steps_for_comparison(
         self,
@@ -192,14 +228,20 @@ class PathEfficiencyEvaluator(EvaluatorBase):
         # Check if agent has at least as many occurrences of each ground truth step
         return all(agent_counts[step] >= ground_truth_counts[step] for step in ground_truth_counts)
 
+    _TASK_NAVIGATION_EFFICIENCY_MATCHING_MODE_TO_FUNCTIONS = {
+        _TaskNavigationEfficiencyMatchingMode.EXACT_MATCH: _calculate_exact_match,
+        _TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH: _calculate_in_order_match,
+        _TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH: _calculate_any_order_match,
+    }
+
     @override
-    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str, Dict[str, float]]]:
         """Produce a path efficiency evaluation result.
 
         :param eval_input: The input to the evaluation function. Must contain "response" and "ground_truth".
         :type eval_input: Dict
         :return: The evaluation result.
-        :rtype: Dict[str, Union[float, str]]
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
         """
         response = eval_input["response"]
         ground_truth = eval_input["ground_truth"]
@@ -244,12 +286,10 @@ class PathEfficiencyEvaluator(EvaluatorBase):
             ground_truth_names = [name.strip() for name in tool_names_list]
             ground_truth_params_dict = params_dict
             use_parameter_matching = True
-
         elif isinstance(ground_truth, list) and all(isinstance(step, str) for step in ground_truth):
             # List format: just tool names
             ground_truth_names = [step.strip() for step in ground_truth]
             use_parameter_matching = False
-
         else:
             raise TypeError(
                 "ground_truth must be a list of strings or a tuple of (list[str], dict[str, dict[str, str]])"
@@ -267,42 +307,44 @@ class PathEfficiencyEvaluator(EvaluatorBase):
         )
 
         # Calculate precision, recall, and F1 scores
-        metrics = self._calculate_precision_recall_f1_scores(agent_steps, ground_truth_steps)
-
-        # Calculate binary match metrics
-        exact_match = self._calculate_exact_match(agent_steps, ground_truth_steps)
-        in_order_match = self._calculate_in_order_match(agent_steps, ground_truth_steps)
-        any_order_match = self._calculate_any_order_match(agent_steps, ground_truth_steps)
+        additional_properties_metrics = self._calculate_precision_recall_f1_scores(agent_steps, ground_truth_steps)
 
         # Convert metrics to floats, using nan for None or non-convertible values
-        path_efficiency_precision = (
-            float(metrics["precision_score"]) if metrics["precision_score"] is not None else float("nan")
-        )
-        path_efficiency_recall = float(metrics["recall_score"]) if metrics["recall_score"] is not None else float("nan")
-        path_efficiency_f1_score = float(metrics["f1_score"]) if metrics["f1_score"] is not None else float("nan")
+        for metric, score in additional_properties_metrics.items():
+            additional_properties_metrics[metric] = float(score) if score is not None else float("nan")
 
-        return {
-            "path_efficiency_precision_score": path_efficiency_precision,
-            "path_efficiency_recall_score": path_efficiency_recall,
-            "path_efficiency_f1_score": path_efficiency_f1_score,
-            "path_efficiency_exact_match_result": EVALUATION_PASS_FAIL_MAPPING[exact_match],
-            "path_efficiency_in_order_match_result": EVALUATION_PASS_FAIL_MAPPING[in_order_match],
-            "path_efficiency_any_order_match_result": EVALUATION_PASS_FAIL_MAPPING[any_order_match],
-        }
+        if self.matching_mode in self._TASK_NAVIGATION_EFFICIENCY_MATCHING_MODE_TO_FUNCTIONS:
+            # Calculate binary match metrics
+            match_result = self._TASK_NAVIGATION_EFFICIENCY_MATCHING_MODE_TO_FUNCTIONS[self.matching_mode](
+                self, agent_steps, ground_truth_steps
+            )
+
+            return {
+                "task_navigation_efficiency_label": match_result,
+                "task_navigation_efficiency_result": EVALUATION_PASS_FAIL_MAPPING[match_result],
+                "task_navigation_efficiency_details": additional_properties_metrics,
+            }
+        else:
+            raise EvaluationException(
+                f"Unsupported matching_mode '{self.matching_mode}'",
+                internal_message=str(self.matching_mode),
+                target=ErrorTarget.TASK_NAVIGATION_EFFICIENCY_EVALUATOR,
+                category=ErrorCategory.INVALID_VALUE,
+            )
 
     @overload
     def __call__(  # type: ignore
         self, *, response: Union[str, List[Dict[str, Any]]], ground_truth: List[str]
-    ) -> Dict[str, Union[float, str]]:
+    ) -> Dict[str, Union[float, str, Dict[str, float]]]:
         """
-        Evaluate the path efficiency of an agent's action sequence.
+        Evaluate the task navigation efficiency of an agent's action sequence.
 
         :keyword response: The agent's response containing tool calls.
         :paramtype response: Union[str, List[Dict[str, Any]]]
         :keyword ground_truth: List of expected tool/action steps.
         :paramtype ground_truth: List[str]
-        :return: The path efficiency scores and results.
-        :rtype: Dict[str, Union[float, str]]
+        :return: The task navigation efficiency scores and results.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
         """
 
     @overload
@@ -311,16 +353,16 @@ class PathEfficiencyEvaluator(EvaluatorBase):
         *,
         response: Union[str, List[Dict[str, Any]]],
         ground_truth: Tuple[List[str], Dict[str, Dict[str, str]]],
-    ) -> Dict[str, Union[float, str]]:
+    ) -> Dict[str, Union[float, str, Dict[str, float]]]:
         """
-        Evaluate the path efficiency of an agent's action sequence with tool parameters.
+        Evaluate the task navigation efficiency of an agent's action sequence with tool parameters.
 
         :keyword response: The agent's response containing tool calls.
         :paramtype response: Union[str, List[Dict[str, Any]]]
         :keyword ground_truth: Tuple of (tool names list, parameters dict) where parameters must match exactly.
         :paramtype ground_truth: Tuple[List[str], Dict[str, Dict[str, str]]]
-        :return: The path efficiency scores and results.
-        :rtype: Dict[str, Union[float, str]]
+        :return: The task navigation efficiency scores and results.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
         """
 
     @override
@@ -330,13 +372,13 @@ class PathEfficiencyEvaluator(EvaluatorBase):
         **kwargs,
     ):
         """
-        Evaluate path efficiency.
+        Evaluate task navigation efficiency.
 
         :keyword response: The agent's response containing tool calls.
         :paramtype response: Union[str, List[Dict[str, Any]]]
         :keyword ground_truth: List of expected tool/action steps or tuple of (tool names, parameters dict).
         :paramtype ground_truth: Union[List[str], Tuple[List[str], Dict[str, Dict[str, str]]]]
-        :return: The path efficiency scores and results.
-        :rtype: Dict[str, Union[float, str]]
+        :return: The task navigation efficiency scores and results.
+        :rtype: Dict[str, Union[float, str, Dict[str, float]]]
         """
         return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py

@@ -1,7 +1,6 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from itertools import chain
 import math
 import os
 import logging
@@ -17,46 +16,12 @@ from azure.ai.evaluation._exceptions import (
 )
 from ..._common.utils import check_score_is_valid
 from azure.ai.evaluation._common._experimental import experimental
-from ..._converters._models import (
-    _BUILT_IN_DESCRIPTIONS,
-    _BUILT_IN_PARAMS,
-)
 
 logger = logging.getLogger(__name__)
 
 T_EvalValue = TypeVar("T_EvalValue")
 
 
-def _get_built_in_definition(tool_name: str):
-    """Get the definition for the built-in tool."""
-    if tool_name in _BUILT_IN_DESCRIPTIONS:
-        return {
-            "type": tool_name,
-            "description": _BUILT_IN_DESCRIPTIONS[tool_name],
-            "name": tool_name,
-            "parameters": _BUILT_IN_PARAMS.get(tool_name, {}),
-        }
-    return None
-
-
-def _get_needed_built_in_definitions(tool_calls: List[Dict]) -> List[Dict]:
-    """Extract tool definitions needed for the given built-in tool calls."""
-    needed_definitions = []
-    for tool_call in tool_calls:
-        if isinstance(tool_call, dict):
-            tool_type = tool_call.get("type")
-
-            # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}}
-            if tool_type == "tool_call":
-                tool_name = tool_call.get("name")
-                if tool_name in _BUILT_IN_DESCRIPTIONS:
-                    built_in_def = _get_built_in_definition(tool_name)
-                    if built_in_def and built_in_def not in needed_definitions:
-                        needed_definitions.append(built_in_def)
-
-    return needed_definitions
-
-
 @experimental
 class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:
@@ -100,9 +65,12 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
 
     .. note::
 
+        The output field "details" has been renamed to "tool_call_accuracy_details" for clarity.
+
         To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
         To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+
     """
 
     _PROMPTY_FILE = "tool_call_accuracy.prompty"
@@ -132,6 +100,7 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             prompty_file=prompty_path,
             result_key=self._RESULT_KEY,
             credential=credential,
+            threshold=threshold,
             **kwargs,
         )
 
@@ -207,7 +176,9 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             tool_definitions = [tool_definitions] if tool_definitions else []
 
         try:
-            needed_tool_definitions = self._extract_needed_tool_definitions(tool_calls, tool_definitions)
+            needed_tool_definitions = self._extract_needed_tool_definitions(
+                tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+            )
         except EvaluationException as e:
             # Check if this is because no tool definitions were provided at all
             if len(tool_definitions) == 0:
@@ -235,8 +206,8 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         :rtype: Dict
         """
         # Single LLM call for all tool calls
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
-
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", {})
         if isinstance(llm_output, dict):
             score = llm_output.get(self._LLM_SCORE_KEY, None)
             if not score or not check_score_is_valid(
@@ -248,6 +219,7 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
                     message=f"Invalid score value: {score}. Expected a number in range [{ToolCallAccuracyEvaluator._MIN_TOOL_CALL_ACCURACY_SCORE}, {ToolCallAccuracyEvaluator._MAX_TOOL_CALL_ACCURACY_SCORE}].",
                     internal_message="Invalid score value.",
                     category=ErrorCategory.FAILED_EXECUTION,
+                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
                     blame=ErrorBlame.SYSTEM_ERROR,
                 )
 
@@ -257,10 +229,18 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             score_result = "pass" if score >= self.threshold else "fail"
             response_dict = {
                 self._result_key: score,
+                f"gpt_{self._result_key}": score,
                 f"{self._result_key}_result": score_result,
-                f"{self._result_key}_threshold": self.threshold,
+                f"{self._result_key}_threshold": self._threshold,
                 f"{self._result_key}_reason": reason,
-                "details": llm_output.get("details", {}),
+                f"{self._result_key}_details": llm_output.get("details", {}),
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
             }
             return response_dict
 
@@ -275,105 +255,21 @@ class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     async def _real_call(self, **kwargs):
         """The asynchronous call where real end-to-end evaluation logic is performed.
 
-        :keyword kwargs: The inputs to evaluate.
+        :keyword kwargs: The inputs to evaluate
         :type kwargs: Dict
-        :return: The evaluation result.
+        :return: The evaluation result
         :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
         """
         # Convert inputs into list of evaluable inputs.
         eval_input = self._convert_kwargs_to_eval_input(**kwargs)
         if isinstance(eval_input, dict) and eval_input.get("error_message"):
             # If there is an error message, return not applicable result
-            return self._not_applicable_result(eval_input.get("error_message"))
+            return self._not_applicable_result(eval_input.get("error_message"), self.threshold)
         # Do the evaluation
         result = await self._do_eval(eval_input)
         # Return the result
         return result
 
-    def _not_applicable_result(self, error_message):
-        """Return a result indicating that the tool call is not applicable for evaluation.
-        :param eval_input: The input to the evaluator.
-        :type eval_input: Dict
-        :return: A dictionary containing the result of the evaluation.
-        :rtype: Dict[str, Union[str, float]]
-        """
-        # If no tool calls were made or tool call type is not supported, return not applicable result
-        return {
-            self._result_key: self._NOT_APPLICABLE_RESULT,
-            f"{self._result_key}_result": "pass",
-            f"{self._result_key}_threshold": self.threshold,
-            f"{self._result_key}_reason": error_message,
-            "details": {},
-        }
-
-    def _extract_needed_tool_definitions(self, tool_calls, tool_definitions):
-        """Extract the tool definitions that are needed for the provided tool calls."""
-        needed_tool_definitions = []
-
-        # Add all user-provided tool definitions
-        needed_tool_definitions.extend(tool_definitions)
-
-        # Add the needed built-in tool definitions (if they are called)
-        built_in_definitions = _get_needed_built_in_definitions(tool_calls)
-        needed_tool_definitions.extend(built_in_definitions)
-
-        # OpenAPI tool is a collection of functions, so we need to expand it
-        tool_definitions_expanded = list(
-            chain.from_iterable(
-                tool.get("functions", []) if tool.get("type") == "openapi" else [tool]
-                for tool in needed_tool_definitions
-            )
-        )
-
-        # Validate that all tool calls have corresponding definitions
-        for tool_call in tool_calls:
-            if isinstance(tool_call, dict):
-                tool_type = tool_call.get("type")
-
-                if tool_type == "tool_call":
-                    tool_name = tool_call.get("name")
-                    if tool_name and tool_name in _BUILT_IN_DESCRIPTIONS:
-                        # This is a built-in tool from converter, already handled above
-                        continue
-                    elif tool_name:
-                        # This is a regular function tool from converter
-                        tool_definition_exists = any(
-                            tool.get("name") == tool_name and tool.get("type", "function") == "function"
-                            for tool in tool_definitions_expanded
-                        )
-                        if not tool_definition_exists:
-                            raise EvaluationException(
-                                message=f"Tool definition for {tool_name} not found",
-                                blame=ErrorBlame.USER_ERROR,
-                                category=ErrorCategory.INVALID_VALUE,
-                                target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                            )
-                    else:
-                        raise EvaluationException(
-                            message=f"Tool call missing name: {tool_call}",
-                            blame=ErrorBlame.USER_ERROR,
-                            category=ErrorCategory.INVALID_VALUE,
-                            target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                        )
-                else:
-                    # Unsupported tool format - only converter format is supported
-                    raise EvaluationException(
-                        message=f"Unsupported tool call format. Only converter format is supported: {tool_call}",
-                        blame=ErrorBlame.USER_ERROR,
-                        category=ErrorCategory.INVALID_VALUE,
-                        target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                    )
-            else:
-                # Tool call is not a dictionary
-                raise EvaluationException(
-                    message=f"Tool call is not a dictionary: {tool_call}",
-                    blame=ErrorBlame.USER_ERROR,
-                    category=ErrorCategory.INVALID_VALUE,
-                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
-                )
-
-        return needed_tool_definitions
-
     @override
     def __call__(  # pylint: disable=docstring-missing-param
         self,

azure/ai/evaluation/_evaluators/_tool_input_accuracy/__init__.py

@@ -0,0 +1,9 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._tool_input_accuracy import _ToolInputAccuracyEvaluator
+
+__all__ = [
+    "_ToolInputAccuracyEvaluator",
+]