azure-ai-evaluation 1.0.1__py3-none-any.whl → 1.13.5__py3-none-any.whl

azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py

@@ -0,0 +1,298 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import math
+import os
+import logging
+import re
+from typing import Dict, List, Union, TypeVar, Optional
+from typing_extensions import overload, override
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from azure.ai.evaluation._exceptions import (
+    ErrorBlame,
+    ErrorCategory,
+    ErrorTarget,
+    EvaluationException,
+)
+from ..._common.utils import check_score_is_valid
+from azure.ai.evaluation._common._experimental import experimental
+
+logger = logging.getLogger(__name__)
+
+T_EvalValue = TypeVar("T_EvalValue")
+
+
+@experimental
+class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:
+        - Relevance to the conversation.
+        - Parameter correctness according to tool definitions.
+        - Parameter value extraction from the conversation.
+
+    The evaluator uses a scoring rubric of 1 to 5:
+        - Score 1: The tool calls are irrelevant
+        - Score 2: The tool calls are partially relevant, but not enough tools were called or the parameters were not correctly passed.
+        - Score 3: The tool calls are relevant, but there were unnecessary, excessive tool calls made.
+        - Score 4: The tool calls are relevant, but some tools returned errors and agent retried calling them again and succeeded.
+        - Score 5: The tool calls are relevant, and all parameters were correctly passed.
+
+    This evaluation focuses on measuring whether tool calls meaningfully contribute to addressing
+    user needs while properly following tool definitions and using information present in the
+    conversation history.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START tool_call_accuracy_evaluator]
+            :end-before: [END tool_call_accuracy_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a ToolCallAccuracyEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START tool_call_accuracy_evaluator]
+            :end-before: [END tool_call_accuracy_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call ToolCallAccuracyEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    .. note::
+
+        The output field "details" has been renamed to "tool_call_accuracy_details" for clarity.
+
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+
+    """
+
+    _PROMPTY_FILE = "tool_call_accuracy.prompty"
+    _RESULT_KEY = "tool_call_accuracy"
+
+    _MAX_TOOL_CALL_ACCURACY_SCORE = 5
+    _MIN_TOOL_CALL_ACCURACY_SCORE = 1
+    _DEFAULT_TOOL_CALL_ACCURACY_SCORE = 3
+
+    _NO_TOOL_CALLS_MESSAGE = "No tool calls found in response or provided tool_calls."
+    _NO_TOOL_DEFINITIONS_MESSAGE = "Tool definitions must be provided."
+    _TOOL_DEFINITIONS_MISSING_MESSAGE = "Tool definitions for all tool calls must be provided."
+    _INVALID_SCORE_MESSAGE = "Tool call accuracy score must be between 1 and 5."
+
+    _LLM_SCORE_KEY = "tool_calls_success_level"
+
+    id = "azureai://built-in/evaluators/tool_call_accuracy"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config, *, threshold=_DEFAULT_TOOL_CALL_ACCURACY_SCORE, credential=None, **kwargs):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        self.threshold = threshold
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            credential=credential,
+            threshold=threshold,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: Union[str, List[dict]],
+        tool_definitions: Union[dict, List[dict]],
+        tool_calls: Union[dict, List[dict]] = None,
+        response: Union[str, List[dict]] = None,
+    ) -> Dict[str, Union[str, float]]:
+        """
+        Evaluate tool call accuracy. Accepts a query, tool definitions, and tool calls for evaluation.
+
+        :keyword query: Query or Chat history up to the message that has the tool call being evaluated.
+        :paramtype query: Union[str, List[dict]]
+        :keyword tool_definitions: List of tool definitions whose calls are being evaluated.
+        :paramtype tool_definitions: Union[dict, List[dict]]
+        :keyword tool_calls: Optional List of tool calls to evaluate. If not provided response should be provided and should have
+            tool call(s) in it.
+        :paramtype tool_calls: Union[dict, List[dict]]
+        :keyword response: Optional response to be evaluated alongside the tool calls.
+            If provided all tool calls in response will be evaluated when tool_calls parameter is not provided.
+            If provided and tool_calls parameter is provided, only the tool calls in tool_calls parameter will be evaluated.
+                If response has extra tool calls they will not be evaluated, response will be used to extract any tool calls that are needed for evaluating a certain tool call.
+            Recommended to provide it when there are tool calls that depend on output of a previous tool call.
+        :paramtype response: Union[str, List[dict]]
+        :return: The tool selection evaluation results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
+    def _convert_kwargs_to_eval_input(self, **kwargs):
+        """Convert an arbitrary input into a list of inputs for evaluators.
+        It is assumed that evaluators generally make use of their inputs in one of two ways.
+        Either they receive a collection of keyname inputs that are all single values
+        (like a query and response), or they receive conversation that iss a list of dictionary
+        values.
+
+        The self._singleton_inputs list assigned during initialization is used to find and extract
+        singleton keywords, and self._allow_conversation_input is used to determine if a conversation
+        is a valid input.
+
+        If both conversations and singletons are allowed, the function will raise an exception if both
+        are inputted.
+
+        This function must be overridden by child classes IF they need to both a conversation and
+        other inputs to be passed in.
+
+        :keyword kwargs: The inputs to convert.
+        :type kwargs: Dict
+        :return: A list of arbitrary values that are valid inputs for this evaluator's do_eval function.
+        :rtype: List
+        """
+        # TODO add warning that only tool calls of type function are supported
+        # Collect inputs
+        tool_calls = kwargs.get("tool_calls")
+        tool_definitions = kwargs.get("tool_definitions", [])  # Default to empty list
+        query = kwargs.get("query")
+        response = kwargs.get("response")
+        # TODO : Support classes that represents tool calls, messages etc once client side definitions are available
+        if response:
+            parsed_tool_calls = self._parse_tools_from_response(response)
+            if parsed_tool_calls:
+                tool_calls = parsed_tool_calls
+
+        if not tool_calls:
+            return {"error_message": self._NO_TOOL_CALLS_MESSAGE}
+
+        if not isinstance(tool_calls, list):
+            tool_calls = [tool_calls]
+        if not isinstance(tool_definitions, list):
+            tool_definitions = [tool_definitions] if tool_definitions else []
+
+        try:
+            needed_tool_definitions = self._extract_needed_tool_definitions(
+                tool_calls, tool_definitions, ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR
+            )
+        except EvaluationException as e:
+            # Check if this is because no tool definitions were provided at all
+            if len(tool_definitions) == 0:
+                return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
+            else:
+                return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+
+        if len(needed_tool_definitions) == 0:
+            return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
+
+        return {
+            "query": query,
+            "tool_calls": tool_calls,
+            "tool_definitions": needed_tool_definitions,
+        }
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
+        """Do a tool call accuracy evaluation.
+        :param eval_input: The input to the evaluator. Expected to contain
+        whatever inputs are needed for the _flow method, including context
+        and other fields depending on the child class.
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        # Single LLM call for all tool calls
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", {})
+        if isinstance(llm_output, dict):
+            score = llm_output.get(self._LLM_SCORE_KEY, None)
+            if not score or not check_score_is_valid(
+                score,
+                ToolCallAccuracyEvaluator._MIN_TOOL_CALL_ACCURACY_SCORE,
+                ToolCallAccuracyEvaluator._MAX_TOOL_CALL_ACCURACY_SCORE,
+            ):
+                raise EvaluationException(
+                    message=f"Invalid score value: {score}. Expected a number in range [{ToolCallAccuracyEvaluator._MIN_TOOL_CALL_ACCURACY_SCORE}, {ToolCallAccuracyEvaluator._MAX_TOOL_CALL_ACCURACY_SCORE}].",
+                    internal_message="Invalid score value.",
+                    category=ErrorCategory.FAILED_EXECUTION,
+                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                    blame=ErrorBlame.SYSTEM_ERROR,
+                )
+
+            # Format the output
+            reason = llm_output.get("chain_of_thought", "")
+            score = float(score)
+            score_result = "pass" if score >= self.threshold else "fail"
+            response_dict = {
+                self._result_key: score,
+                f"gpt_{self._result_key}": score,
+                f"{self._result_key}_result": score_result,
+                f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_details": llm_output.get("details", {}),
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
+            }
+            return response_dict
+
+        else:
+            raise EvaluationException(
+                message="Tool call accuracy evaluator returned invalid output.",
+                blame=ErrorBlame.SYSTEM_ERROR,
+                category=ErrorCategory.FAILED_EXECUTION,
+                target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+            )
+
+    async def _real_call(self, **kwargs):
+        """The asynchronous call where real end-to-end evaluation logic is performed.
+
+        :keyword kwargs: The inputs to evaluate
+        :type kwargs: Dict
+        :return: The evaluation result
+        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
+        """
+        # Convert inputs into list of evaluable inputs.
+        eval_input = self._convert_kwargs_to_eval_input(**kwargs)
+        if isinstance(eval_input, dict) and eval_input.get("error_message"):
+            # If there is an error message, return not applicable result
+            return self._not_applicable_result(eval_input.get("error_message"), self.threshold)
+        # Do the evaluation
+        result = await self._do_eval(eval_input)
+        # Return the result
+        return result
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate tool call accuracy. Accepts a query, tool definitions, and tool calls for evaluation.
+
+        :keyword query: Query or Chat history up to the message that has the tool call being evaluated.
+        :paramtype query: Union[str, List[dict]]
+        :keyword tool_definitions: List of tool definitions whose calls are being evaluated.
+        :paramtype tool_definitions: Union[dict, List[dict]]
+        :keyword tool_calls: Optional List of tool calls to evaluate. If not provided response should be provided and should have
+            tool call(s) in it.
+        :paramtype tool_calls: Union[dict, List[dict]]
+        :keyword response: Optional response to be evaluated alongside the tool calls.
+            If provided all tool calls in response will be evaluated when tool_calls parameter is not provided.
+            If provided and tool_calls parameter is provided, only the tool calls in tool_calls parameter will be evaluated.
+                If response has extra tool calls they will not be evaluated, response will be used to extract any tool calls that are needed for evaluating a certain tool call.
+            Recommended to provide it when there are tool calls that depend on output of a previous tool call.
+        :paramtype response: Union[str, List[dict]]
+        :return: The tool selection evaluation results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_tool_call_accuracy/tool_call_accuracy.prompty

@@ -0,0 +1,166 @@
+---
+name: Tool Call Accuracy
+description: Evaluates Tool Call Accuracy for tool used by agent
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 3000
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+
+inputs:
+  query:
+    type: List
+  tool_calls:
+    type: List
+  tool_definitions:
+    type: Dict
+
+---
+system:
+# Instruction
+## Goal
+### Your are an expert in evaluating the accuracy of a tool call considering relevance and potential usefulness including syntactic and semantic correctness of a proposed tool call from an intelligent system based on provided definition and data. Your goal will involve answering the questions below using the information provided.
+- **Definition**: You are given a definition of the communication trait that is being evaluated to help guide your Score.
+- **Data**: Your input data include CONVERSATION , TOOL CALL and TOOL DEFINITION.
+- **Tasks**: To complete your evaluation you will be asked to evaluate the Data in different ways, and you need to be very precise in your evaluation.
+
+user:
+# Definition
+# Definition
+**Tool Call Accuracy** refers to the overall effectiveness of ALL TOOL CALLS made by an agent in response to a user's query within an ongoing CONVERSATION.
+
+# EVALUATION CRITERIA
+Evaluate based on these factors:
+
+1. **Collective Relevance**: Do the tool calls, taken together, appropriately address the user's query?
+2. **Parameter Correctness**: Are all parameter values extracted from or reasonably inferred from the CONVERSATION? 
+   - *Fabricated parameters automatically result in Level 2*
+3. **Completeness**: Did the agent make all necessary tool calls available in the tool definitions?
+   - *Failed calls don't count as missing*
+4. **Efficiency**: Did the agent avoid unnecessary duplicate tool calls with identical parameters?
+   - *Don't penalize single tools returning multiple results (like file_search)*
+5. **Execution Success**: Were tool calls executed successfully or recovered from errors appropriately?
+6. **Scope Limitation**: ONLY evaluate tool calls in the "TOOL CALLS TO BE EVALUATED" section.
+   - Tool calls in the CONVERSATION section are for context only
+   - Focus exclusively on the agent's response to the user's LAST query
+   - Use conversation history only to verify parameter correctness and context
+
+**Success Criteria**: Tools should retrieve relevant data to help answer the query. Complete final answers are not required from individual tools.
+
+**Tool Assessment**: Focus solely on appropriate use of available tools, not on capabilities beyond what tools can provide.
+
+
+# Ratings
+## [Tool Call Accuracy: 1] (Irrelevant)
+**Definition:**
+Tool calls were not relevant to the user's query, resulting in an irrelevant or unhelpful final output.
+
+**Example:**
+ User asks for distance between two cities -> Agent calls a weather function to get the weather in the two cities.
+
+
+## [Tool Call Accuracy: 2] (Partially Relevant - Wrong Execution)
+**Definition:**
+Tool calls were somewhat related to the user's query, but the agent was not able to reach information that helps address the user query due to one or more of the following:
+  •	Parameters passed to the tool were incorrect.
+  •	Not enough tools (available in the tool definitions) were called to fully help address the query (missing tool calls).
+  •	Tools returned errors, and no retrials for the tool call were successful.
+
+
+**Example:**
+  The user asks for the coordinates of Chicago. The agent calls the tool that gets the coordinates but passes 'New York' instead of Chicago as parameter.
+
+**Example:**
+  The user asks for the coordinates of Chicago. The agent calls the tool that gets the coordinates and passes 'Chicago' as the tool parameter, but the tool returns an error.
+
+**Example:**
+  The user asks a question that needs 3 tool calls for it to be answered. The agent calls only one of the three required tool calls. So this case is a Level 2.
+
+
+## [Tool Call Accuracy: 3] (Relevant but Inefficient)
+**Definition:**
+Tool calls were relevant, correct and grounded parameters were passed so that led to a correct output. However, multiple excessive, unnecessary tool calls were made.
+
+**Important**: Do NOT penalize built-in tools like file_search that naturally return multiple results in a single call. Only penalize when there are actually multiple separate tool call objects.
+
+**Example:**
+  The user asked to do a modification in the database. The agent called the tool multiple times, resulting in multiple modifications in the database instead of one.
+
+**Example:**
+  The user asked for popular hotels in a certain place. The agent calls the same tool with the same parameters multiple times, even though a single tool call that returns an output is sufficient. So there were unnecessary tool calls.
+
+
+## [Tool Call Accuracy: 4] (Correct with Retrials)
+**Definition:**
+Tool calls were fully relevant and efficient:
+•	Correct tools were called with the correct and grounded parameters, whether they are extracted from the conversation history or the current user query.
+•	A tool returned an error, but the agent retried calling the tool and successfully got an output.
+
+**Example:**
+  The user asks for the weather forecast in a certain place. The agent calls the correct tool that retrieves the weather forecast, but the tool returns an error. The agent re-calls the tool once again and it returns the correct output. This is a Level 4.
+
+
+## [Tool Call Accuracy: 5] (Optimal Solution)
+**Definition:**
+Tool calls were fully relevant and efficient:
+  •	Correct tools were called with the correct and grounded parameters, whether they are extracted from the conversation history or the current user query.
+  •	No unnecessary or excessive tool calls were made.
+  •	No errors occurred in any of the tools.
+  •	The tool calls made helped the agent address the user's query without facing any issues.
+
+**Example:**
+  The user asks for the distance between two places. The agent correctly calls the tools that retrieve the coordinates for the two places respectively, then calls the tool that calculates the distance between the two sets of coordinates, passing the correct arguments to all the tools, without calling other tools excessively or unnecessarily. This is the optimal solution for the user's query.
+
+**Example:**
+  The user asks for the distance between two places. The agent retrieves the needed coordinates from the outputs of the tool calls in the conversation history, and then correctly passes these coordinates to the tool that calculates the distance to output it to the user. This is also an optimal solution for the user's query.
+
+**Example:**
+  The user asked to summarize a file on their SharePoint. The agent calls the sharepoint_grounding tool to retrieve the file. This retrieved file will help the agent fulfill the task of summarization. This is a Level 5.
+
+
+## Chain of Thought Structure
+Structure your reasoning as follows:
+1. **Start with the user's last query**: Understand well what the last message that is sent by the user is.
+2. **Identify relevant available tools**: Look into the TOOL DEFINITIONS and analyze which tools could help answer the user's last query in the conversation.
+3. **Analyze the actual tool calls made**: Compare what was done in the TOOL CALLS TO BE EVALUATED section vs. What should've been done by the agent.
+4. **Check parameter grounding** - Ensure all parameters are grounded from the CONVERSATION section and are not hallucinated.
+5. **Determine the appropriate level** - Be VERY precise and follow the level definitions exactly.
+
+# Data
+CONVERSATION : {{query}}
+TOOL CALLS TO BE EVALUATED: {{tool_calls}}
+TOOL DEFINITIONS: {{tool_definitions}}
+
+
+# Tasks
+## Please provide your evaluation for the assistant RESPONSE in relation to the user QUERY and tool definitions based on the Definitions and examples above.
+Your output should consist only of a JSON object, as provided in the examples, that has the following keys:
+  - chain_of_thought: a string that explains your thought process to decide on the tool call accuracy level, based on the Chain of Thought structure. Start this string with 'Let's think step by step:'.
+  - tool_calls_success_level: a integer value between 1 and 5 that represents the level of tool call success, based on the level definitions mentioned before. You need to be very precise when deciding on this level. Ensure you are correctly following the rating system based on the description of each level.
+  - details: a dictionary that contains the following keys:
+        - tool_calls_made_by_agent: total number of tool calls made by the agent
+        - correct_tool_calls_made_by_agent: total number of correct tool calls made by the agent
+        - per_tool_call_details: a list of dictionaries, each containing:
+          - tool_name: name of the tool
+          - total_calls_required: total number of calls required for the tool
+          - correct_calls_made_by_agent: number of correct calls made by the agent
+          - correct_tool_percentage: percentage of correct calls made by the agent for this tool. It is a value between 0.0 and 1.0
+          - tool_call_errors: number of errors encountered during the tool call
+          - tool_success_result: 'pass' or 'fail' based on the evaluation of the tool call accuracy for this tool
+        - excess_tool_calls: a dictionary with the following keys:
+            - total: total number of excess, unnecessary tool calls made by the agent
+            - details: a list of dictionaries, each containing:
+              - tool_name: name of the tool
+              - excess_count: number of excess calls made for this query
+        - missing_tool_calls: a dictionary with the following keys:
+            - total: total number of missing tool calls that should have been made by the agent to be able to answer the query, but were not made by the agent at all.
+            - details: a list of dictionaries, each containing:
+              - tool_name: name of the tool
+              - missing_count: number of missing calls for this query
+
+# Output

azure/ai/evaluation/_evaluators/_tool_call_success/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._tool_call_success import _ToolCallSuccessEvaluator
+
+__all__ = ["_ToolCallSuccessEvaluator"]