azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_evaluators/_tool_success/_tool_success.py

@@ -0,0 +1,301 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import math
+import os
+import logging
+from typing import Dict, Union, List, Optional
+from typing_extensions import overload, override
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from azure.ai.evaluation._common._experimental import experimental
+
+
+logger = logging.getLogger(__name__)
+
+
+@experimental
+class _ToolSuccessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """The Tool Success evaluator determines whether tool calls done by an AI agent includes failures or not.
+
+    This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to
+    the agent, conversation history and agent's final response. Although tool definitions is optional,
+    providing them can help the evaluator better understand the context of the tool calls made by the
+    agent. Please note that this evaluator validates tool calls for potential technical failures like
+    errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a
+    failure). It does not assess the correctness or the tool result itself, like mathematical errors and
+    unrealistic field values like name="668656".
+
+    Scoring is binary:
+    - TRUE: All tool calls were successful
+    - FALSE: At least one tool call failed
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START tool_success_evaluator]
+            :end-before: [END tool_success_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a _ToolSuccessEvaluator with a tool definitions and response.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+    .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+        :start-after: [START tool_success_evaluator]
+        :end-before: [END tool_success_evaluator]
+        :language: python
+        :dedent: 8
+        :caption: Initialize and call a _ToolSuccessEvaluator using Azure AI Project URL in the following
+            format https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    """
+
+    _PROMPTY_FILE = "tool_success.prompty"
+    _RESULT_KEY = "tool_success"
+    _OPTIONAL_PARAMS = ["tool_definitions"]
+
+    id = "azureai://built-in/evaluators/tool_success"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config, *, credential=None, **kwargs):
+        """Initialize the Tool Success evaluator."""
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            threshold=1,
+            credential=credential,
+            _higher_is_better=True,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        response: Union[str, List[dict]],
+        tool_definitions: Optional[Union[dict, List[dict]]] = None,
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate tool call success for a given response, and optionally tool definitions.
+
+        Example with list of messages:
+            evaluator = _ToolSuccessEvaluator(model_config)
+            response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant',
+            'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}]
+
+            result = evaluator(response=response, )
+
+        :keyword response: The response being evaluated, either a string or a list of messages (full agent
+            response potentially including tool calls)
+        :paramtype response: Union[str, List[dict]]
+        :keyword tool_definitions: Optional tool definitions to use for evaluation.
+        :paramtype tool_definitions: Union[dict, List[dict]]
+        :return: A dictionary with the tool success evaluation results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Invoke the instance using the overloaded __call__ signature.
+
+        For detailed parameter types and return value documentation, see the overloaded __call__ definition.
+        """
+        return super().__call__(*args, **kwargs)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]:  # type: ignore[override]
+        """Do Tool Success evaluation.
+
+        :param eval_input: The input to the evaluator. Expected to contain whatever inputs are
+        needed for the _flow method
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        if "response" not in eval_input:
+            raise EvaluationException(
+                message="response is a required input to the Tool Success evaluator.",
+                internal_message="response is a required input to the Tool Success evaluator.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.MISSING_FIELD,
+                target=ErrorTarget.TOOL_SUCCESS_EVALUATOR,
+            )
+        if eval_input["response"] is None or eval_input["response"] == []:
+            raise EvaluationException(
+                message="response cannot be None or empty for the Tool Success evaluator.",
+                internal_message="response cannot be None or empty for the Tool Success evaluator.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.INVALID_VALUE,
+                target=ErrorTarget.TOOL_SUCCESS_EVALUATOR,
+            )
+
+        eval_input["tool_calls"] = _reformat_tool_calls_results(eval_input["response"], logger)
+
+        if "tool_definitions" in eval_input:
+            tool_definitions = eval_input["tool_definitions"]
+            filtered_tool_definitions = _filter_to_used_tools(
+                tool_definitions=tool_definitions,
+                msgs_list=eval_input["response"],
+                logger=logger,
+            )
+            eval_input["tool_definitions"] = _reformat_tool_definitions(filtered_tool_definitions, logger)
+
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", "")
+
+        if isinstance(llm_output, dict):
+            success = llm_output.get("success", False)
+            if isinstance(success, str):
+                success = success.upper() == "TRUE"
+
+            success_result = "pass" if success else "fail"
+            reason = llm_output.get("explanation", "")
+            return {
+                f"{self._result_key}": success * 1.0,
+                f"{self._result_key}_result": success_result,
+                f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_reason": f"{reason} {llm_output.get('details', '')}",
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
+            }
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
+
+        score = math.nan
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }
+
+
+def _filter_to_used_tools(tool_definitions, msgs_list, logger=None):
+    """Filter the tool definitions to only include those that were actually used in the messages lists."""
+    try:
+        used_tool_names = set()
+        any_tools_used = False
+
+        for msg in msgs_list:
+            if msg.get("role") == "assistant" and "content" in msg:
+                for content in msg.get("content", []):
+                    if content.get("type") == "tool_call":
+                        any_tools_used = True
+                        if "tool_call" in content and "function" in content["tool_call"]:
+                            used_tool_names.add(content["tool_call"]["function"])
+                        elif "name" in content:
+                            used_tool_names.add(content["name"])
+
+        filtered_tools = [tool for tool in tool_definitions if tool.get("name") in used_tool_names]
+        if any_tools_used and not filtered_tools:
+            if logger:
+                logger.warning("No tool definitions matched the tools used in the messages. Returning original list.")
+            filtered_tools = tool_definitions
+
+        return filtered_tools
+    except Exception as e:
+        if logger:
+            logger.warning(f"Failed to filter tool definitions, returning original list. Error: {e}")
+        return tool_definitions
+
+
+def _get_tool_calls_results(agent_response_msgs):
+    """Extract formatted agent tool calls and results from response."""
+    agent_response_text = []
+    tool_results = {}
+
+    # First pass: collect tool results
+
+    for msg in agent_response_msgs:
+        if msg.get("role") == "tool" and "tool_call_id" in msg:
+            for content in msg.get("content", []):
+                if content.get("type") == "tool_result":
+                    result = content.get("tool_result")
+                    tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {result}"
+
+    # Second pass: parse assistant messages and tool calls
+    for msg in agent_response_msgs:
+        if "role" in msg and msg.get("role") == "assistant" and "content" in msg:
+
+            for content in msg.get("content", []):
+
+                if content.get("type") == "tool_call":
+                    if "tool_call" in content and "function" in content.get("tool_call", {}):
+                        tc = content.get("tool_call", {})
+                        func_name = tc.get("function", {}).get("name", "")
+                        args = tc.get("function", {}).get("arguments", {})
+                        tool_call_id = tc.get("id")
+                    else:
+                        tool_call_id = content.get("tool_call_id")
+                        func_name = content.get("name", "")
+                        args = content.get("arguments", {})
+                    args_str = ", ".join(f'{k}="{v}"' for k, v in args.items())
+                    call_line = f"[TOOL_CALL] {func_name}({args_str})"
+                    agent_response_text.append(call_line)
+                    if tool_call_id in tool_results:
+                        agent_response_text.append(tool_results[tool_call_id])
+
+    return agent_response_text
+
+
+def _reformat_tool_calls_results(response, logger=None):
+    try:
+        if response is None or response == []:
+            return ""
+        agent_response = _get_tool_calls_results(response)
+        if agent_response == []:
+            # If no message could be extracted, likely the format changed,
+            # fallback to the original response in that case
+            if logger:
+                logger.warning(
+                    f"Empty agent response extracted, likely due to input schema change. "
+                    f"Falling back to using the original response: {response}"
+                )
+            return response
+        return "\n".join(agent_response)
+    except Exception:
+        # If the agent response cannot be parsed for whatever
+        # reason (e.g. the converter format changed), the original response is returned
+        # This is a fallback to ensure that the evaluation can still proceed.
+        # See comments on reformat_conversation_history for more details.
+        if logger:
+            logger.warning(f"Agent response could not be parsed, falling back to original response: {response}")
+        return response
+
+
+def _reformat_tool_definitions(tool_definitions, logger=None):
+    try:
+        output_lines = ["TOOL_DEFINITIONS:"]
+        for tool in tool_definitions:
+            name = tool.get("name", "unnamed_tool")
+            desc = tool.get("description", "").strip()
+            params = tool.get("parameters", {}).get("properties", {})
+            param_names = ", ".join(params.keys()) if params else "no parameters"
+            output_lines.append(f"- {name}: {desc} (inputs: {param_names})")
+        return "\n".join(output_lines)
+    except Exception:
+        # If the tool definitions cannot be parsed for whatever reason, the original tool definitions are returned
+        # This is a fallback to ensure that the evaluation can still proceed.
+        # See comments on reformat_conversation_history for more details.
+        if logger:
+            logger.warning(
+                f"Tool definitions could not be parsed, falling back to original definitions: {tool_definitions}"
+            )
+        return tool_definitions

azure/ai/evaluation/_evaluators/_tool_success/tool_success.prompty

@@ -0,0 +1,321 @@
+---
+name: Tool Success
+description: Evaluates whether a Tool call was successful or resulted in a technical error
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 1500
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+inputs:
+  tool_calls:
+    type: List
+  tool_definitions:
+    type: Dict
+---
+system:
+
+You are an expert evaluator with strong software development background. You are required to extract the tool result for every tool call then decide for each tool result whether it indicates that the tool succeeded or failed
+
+user:
+ROLE
+====
+You are a judge on tool call success who assesses **each tool call made by an AI agent and decide if the result of the tool call indicates a success or failure**. You only care about technical errors , failures and exceptions , not the business correctness of the tool implementation.
+
+You are NOT evaluating:
+- The parameters passed to the tool
+- The rationale behind choosing this tool
+- Whether the successfully returned result from the tool is correct or not business-wise given the tool name and definition
+
+You **ARE ONLY** evaluating:
+-Whether tool results indicate the presence of a technical error
+
+**INPUT**
+=====
+TOOL_DEFINITIONS: {{tool_definitions}}
+TOOL_CALLS: {{tool_calls}}
+
+
+
+TOOL_CALLS is a list of tool calls that were produced by the AI agent. It includes calls together with the result of every tool call.
+TOOL_DEFINITIONS is a list of definitions for the tools that were called. This definition can contain a description of functionality provided by the tool, the parameters that the tool accepts and the expected return of the tool. This definition can contribute to the assessment of whether a tool call succeeded or failed. 
+
+
+EVALUATION FRAMEWORK
+====================
+
+A. Iterate on the list of tool calls
+B. Examine tool result and definition for the tool being called to check whether the call **succeeded** or **failed** as in the following steps explain:
+  1. A tool result is **failed** if **any** of the following ERROR-CASES applies to it:
+    ERROR-CASES:
+    ===========
+    - The tool call resulted in an error or exception
+    - The tool call failed to run or failed to return
+    - The tool call returned a result that indicates an error or failure
+    - The tool call returned an object or JSON string that has one or more of its fields indicating an error
+    - The tool timed-out or returned a result that indicate a time-out
+    - The tool result does not make sense, from technical perspective, not business perspective, given the definition of that tool, if the definition is present
+  2. If none of the error cases apply to the tool result , it is considered **succeeded** even if the tool result itself indicates a business mistake
+C. If one or more tool result are **failed** , then you the **evaluation process** has **failed**, otherwise , the **evaluation process** has **succeeded**
+D. You are required to return your **output** in the following format:
+  {
+      "explanation": "<15-60 words explaining the logic flow of your decision>",
+        "details": {
+          "failed_tools": "<comma-separated list for the tools that has failed results if any>",
+        },
+        "success": <True or False based on whether the **evaluation process** has **succeeded** or **failed**>
+  }
+E. If no tool calls found at all , the TOOL_CALLS input is empty or the TOOL_CALLS input is not passed , the **evaluation process** has **succeeded**
+
+
+## Successful Evaluation Process Examples
+========================================
+
+### Example - Succeeded
+
+[TOOL_CALLS]
+[TOOL_CALL] get_account_balances(user_id="USER456")
+[TOOL_RESULT] {'accounts': [{'account_id': 'CHK001', 'type': 'checking', 'balance': 1250.75}, {'account_id': 'SAV001', 'type': 'savings', 'balance': 3400.20}]}
+[TOOL_CALL] get_weather_info()
+[TOOL_RESULT] "the temperature is 23 C and it is cloudy"
+
+EXPECTED OUTPUT
+{
+     "explanation": "None of the results indicate an error",
+     "details": {
+       "failed_tools": "",
+     },
+     "success": True
+} 
+
+### Example - Succeeded
+
+[TOOL_CALLS]
+[TOOL_CALL] get_employee_info(employee_id="EMP2568")
+[TOOL_RESULT] {"name":"David", "Age":32}
+
+EXPECTED OUTPUT
+{
+     "explanation": "None of the results indicate an error",
+     "details": {
+       "failed_tools": "",
+     },
+     "success": True
+} 
+
+### Example - Succeeded
+
+[TOOL_DEFINITIONS] [get_sqrt] gets the square root of the input parameter
+[TOOL_CALLS]
+[TOOL_CALL] get_sqrt(4)
+[TOOL_RESULT] {"value":7}
+
+EXPECTED OUTPUT
+{
+     "explanation": "Although the returned value 7 is not the square root of 4, this is a business mistake in the tool. The tool did not return a result indicating a technical error",
+     "details": {
+       "failed_tools": "",
+     },
+     "success": True
+} 
+
+
+### Example - Succeeded
+
+[TOOL_DEFINITIONS] [get_blocked_usernames] gets comma-separated list of usernames for blocked users
+[TOOL_CALLS]
+[TOOL_CALL] get_blocked_usernames()
+[TOOL_RESULT] "david33;amr_master;phantom5"
+
+EXPECTED OUTPUT
+{
+     "explanation": "The tool returned a semicolon separated list of names. Although the description in the definition says it should return comma-separated list , this formatting mistake is a business mistake of the tool, not a technical failure. The tool did not return an error",
+     "details": {
+       "failed_tools": "",
+     },
+     "success": True
+} 
+
+
+
+### Example - Succeeded
+
+[TOOL_DEFINITIONS] [update_user_email] Updates the email of the given user id to the new email specified in the parameters
+[TOOL_CALLS]
+[TOOL_CALL] update_user_email(userId:2251 , newEmail:"david235@mydomain.com")
+[TOOL_RESULT] {}
+
+EXPECTED OUTPUT
+{
+     "explanation": "The tool returned empty response which is accepted given that this tool functionality does not include returning data to the caller",
+     "details": {
+       "failed_tools": "",
+     },
+     "success": True
+} 
+
+
+## Failed Evaluation Process Examples
+========================================
+
+### Example - Failed
+
+[TOOL_DEFINITIONS] [get_weather_info] return today's the weather information of the specified city 
+[TOOL_CALLS]
+[TOOL_CALL] get_weather_info(city:"London")
+[TOOL_RESULT] ""
+
+EXPECTED OUTPUT
+{
+     "explanation": "The tool returned empty response , however , given the tool definition , it should never return empty response because there should be weather info at any given point in time. An empty response here is considered a technical failure. The conclusion is the get_weather_info failed",
+     "details": {
+       "failed_tools": "get_weather_info",
+     },
+     "success": False
+} 
+
+
+### Example - Failed
+
+[TOOL_CALLS]
+[TOOL_CALL] get_current_user_Info()
+[TOOL_RESULT] "failed to get current user information"
+
+EXPECTED OUTPUT
+{
+     "explanation": "The tool returned a string indicating that it failed",
+     "details": {
+       "failed_tools": "get_current_user_Info",
+     },
+     "success": False
+} 
+
+
+### Example - Failed
+
+[TOOL_CALLS]
+[TOOL_CALL] get_current_user_Info()
+[TOOL_RESULT] {"UserName":"", "UserEmail":"", "Message":"failed to get current user information"}
+
+EXPECTED OUTPUT
+{
+     "explanation": "The tool returned an object with empty fields and a string indicating that it failed",
+     "details": {
+       "failed_tools": "get_current_user_Info",
+     },
+     "success": False
+} 
+
+
+### Example - Failed
+
+[TOOL_CALLS]
+[TOOL_CALL] GetWeatherInfo()
+[TOOL_RESULT] {temp:""}
+
+EXPECTED OUTPUT
+ {
+      "explanation": "The call for GetWeatherInfo returned an object containing single property 'temp' that is an empty string. This means the call to GetWeatherInfo returned empty result while weather info should be available at any time",
+      "details": {
+        "failed_tools": "GetWeatherInfo",
+      },
+      "success": False
+    } 
+
+
+### Example - Failed
+
+[TOOL_CALLS]
+[TOOL_CALL] get_day_of_week(date:"1/1/2023")
+[TOOL_RESULT] time out
+
+EXPECTED OUTPUT
+{
+     "explanation": "the returned result indicates that the call to get_day_of_week timed out",
+     "details": {
+       "failed_tools": "get_day_of_week",
+     },
+     "success": False
+} 
+
+
+
+### Example - Failed
+
+[TOOL_DEFINITIONS] [get_day_of_week] Takes date as an input and returns the day of week that this day represents 
+[TOOL_CALLS]
+[TOOL_CALL] get_day_of_week(date:"1/1/2023")
+[TOOL_RESULT] null
+
+EXPECTED OUTPUT
+{
+     "explanation": "null indicates an empty result which cannot be an accepted output of the tool given the tool definition since any given date represents a day of week",
+     "details": {
+       "failed_tools": "get_day_of_week",
+     },
+     "success": False
+} 
+
+
+
+### Example - Failed
+
+[TOOL_DEFINITIONS] [get_day_of_week] Takes date as an input and returns the day of week that this day represents 
+[TOOL_CALLS]
+[TOOL_CALL] get_day_of_week(date:"1/1/2023")
+[TOOL_RESULT] {}
+
+EXPECTED OUTPUT
+{
+     "explanation": "Empty object cannot be an accepted output of the tool given the tool definition since any given date should represent a day of week",
+     "details": {
+       "failed_tools": "get_day_of_week",
+     },
+     "success": False
+} 
+
+
+
+### Example - Failed
+
+[TOOL_CALLS]
+[TOOL_CALL] GetWeatherInfo()
+[TOOL_RESULT] {temp:"" }
+[TOOL_CALL] BookTicket(flightId:"FL23" , Seat:"A17")
+[TOOL_RESULT] "Failed to book the ticket"
+
+EXPECTED OUTPUT
+ {
+      "explanation": "GetWeatherInfo returned an empty response while it should return the weather info and BookTicket returned an error.Both tools failed.",
+      "details": {
+        "failed_tools": "GetWeatherInfo,BookTicket",
+      },
+      "success": False
+    } 
+
+
+### Example - Failed
+
+[TOOL_CALLS]
+[TOOL_CALL] GetWeatherInfo()
+[TOOL_RESULT] {temp:"23 C" }
+[TOOL_CALL] BookTicket(flightId:"FL23" , Seat:"A17")
+[TOOL_RESULT] "Failed to book the ticket"
+
+EXPECTED OUTPUT
+ {
+      "explanation": "Although GetWeatherInfo succeeded, BookTicket returned an error. The final result is failure because one of the tool calls has failed",
+      "details": {
+        "failed_tools": "BookTicket",
+      },
+      "success": False
+    } 
+
+
+
+Now given the **INPUT** you received generate the output 
+# Output

azure/ai/evaluation/_evaluators/_ungrounded_attributes/__init__.py

@@ -0,0 +1,5 @@
+from ._ungrounded_attributes import UngroundedAttributesEvaluator
+
+__all__ = [
+    "UngroundedAttributesEvaluator",
+]