azure-ai-evaluation 1.0.0b2__py3-none-any.whl → 1.13.3__py3-none-any.whl

azure/ai/evaluation/_evaluators/_tool_input_accuracy/_tool_input_accuracy.py

@@ -0,0 +1,263 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import logging
+import math
+import json
+from typing import Dict, List, Union, TypeVar, Optional, cast
+from typing_extensions import override
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from azure.ai.evaluation._exceptions import (
+    ErrorBlame,
+    ErrorCategory,
+    ErrorTarget,
+    EvaluationException,
+)
+from ..._common.utils import reformat_conversation_history, _get_agent_response
+from azure.ai.evaluation._common._experimental import experimental
+
+logger = logging.getLogger(__name__)
+
+
+@experimental
+class _ToolInputAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """The Tool Input Accuracy evaluator performs a strict binary evaluation (PASS/FAIL) of parameters
+    passed to tool calls. It ensures that ALL parameters meet ALL criteria:
+
+        - Parameter grounding: All parameters must be derived from conversation history/query
+        - Type compliance: All parameters must match exact types specified in tool definitions
+        - Format compliance: All parameters must follow exact format and structure requirements
+        - Completeness: All required parameters must be provided
+        - No unexpected parameters: Only defined parameters are allowed
+
+    The evaluator uses strict binary evaluation:
+        - 1: Only when ALL criteria are satisfied perfectly for ALL parameters
+        - 0: When ANY criterion fails for ANY parameter
+
+    This evaluation focuses on ensuring tool call parameters are completely correct without any tolerance
+    for partial correctness.
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START tool_input_accuracy_evaluator]
+            :end-before: [END tool_input_accuracy_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a _ToolInputAccuracyEvaluator.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START tool_input_accuracy_evaluator]
+            :end-before: [END tool_input_accuracy_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call _ToolInputAccuracyEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    .. note::
+
+        To align with our support of a diverse set of models, an output key without the `gpt_` prefix has been added.
+        To maintain backwards compatibility, the old key with the `gpt_` prefix is still be present in the output;
+        however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
+    """
+
+    _PROMPTY_FILE = "tool_input_accuracy.prompty"
+    _RESULT_KEY = "tool_input_accuracy"
+
+    _NO_TOOL_CALLS_MESSAGE = "No tool calls found in response or provided tool_calls."
+    _NO_TOOL_DEFINITIONS_MESSAGE = "Tool definitions must be provided."
+    _TOOL_DEFINITIONS_MISSING_MESSAGE = "Tool definitions for all tool calls must be provided."
+
+    def __init__(
+        self,
+        model_config,
+        *,
+        credential=None,
+        **kwargs,
+    ):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            threshold=1,
+            credential=credential,
+            **kwargs,
+        )
+
+    def _convert_kwargs_to_eval_input(self, **kwargs):
+        """Convert kwargs to evaluation input format.
+
+        :keyword kwargs: The inputs to convert.
+        :type kwargs: Dict
+        :return: The formatted evaluation input.
+        :rtype: Dict
+        """
+        # Collect inputs
+        tool_definitions = kwargs.get("tool_definitions", [])  # Default to empty list
+        query = kwargs.get("query")
+        response = kwargs.get("response")
+
+        # Extract tool calls from response
+        if not response:
+            return {"error_message": "Response parameter is required to extract tool calls."}
+
+        tool_calls = self._parse_tools_from_response(response)
+        if not tool_calls:
+            return {"error_message": self._NO_TOOL_CALLS_MESSAGE}
+
+        if not isinstance(tool_calls, list):
+            tool_calls = [tool_calls]
+        if not isinstance(tool_definitions, list):
+            tool_definitions = [tool_definitions] if tool_definitions else []
+
+        try:
+            # Type cast to satisfy static type checker
+            tool_calls_typed = cast(List[Dict], tool_calls)
+            needed_tool_definitions = self._extract_needed_tool_definitions(
+                tool_calls_typed, tool_definitions, ErrorTarget.TOOL_INPUT_ACCURACY_EVALUATOR
+            )
+        except EvaluationException as e:
+            # Check if this is because no tool definitions were provided at all
+            if len(tool_definitions) == 0:
+                return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
+            else:
+                return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+
+        if len(needed_tool_definitions) == 0:
+            return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
+
+        # Get agent response with tool calls and results using _get_agent_response
+        agent_response_with_tools = _get_agent_response(response, include_tool_messages=True)
+
+        return {
+            "query": query,
+            "tool_calls": agent_response_with_tools,
+            "tool_definitions": needed_tool_definitions,
+        }
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
+        """Do Tool Input Accuracy evaluation.
+
+        :param eval_input: The input to the evaluator.
+        :type eval_input: Dict
+        :return: A dictionary containing the result of the evaluation.
+        :rtype: Dict[str, Union[str, float]]
+        """
+        # Format conversation history for cleaner evaluation
+        if "query" in eval_input:
+            eval_input["query"] = reformat_conversation_history(
+                eval_input["query"], logger, include_system_messages=True, include_tool_messages=True
+            )
+
+        # Call the LLM to evaluate
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", {})
+
+        if isinstance(llm_output, dict):
+            result = llm_output.get("result", None)
+            if result not in [0, 1]:
+                raise EvaluationException(
+                    message=f"Invalid result value: {result}. Expected 0 or 1.",
+                    internal_message="Invalid result value.",
+                    category=ErrorCategory.FAILED_EXECUTION,
+                    blame=ErrorBlame.SYSTEM_ERROR,
+                )
+
+            # Add parameter extraction accuracy post-processing
+            details = llm_output.get("details", {})
+            if details:
+                parameter_extraction_accuracy = self._calculate_parameter_extraction_accuracy(details)
+                details["parameter_extraction_accuracy"] = parameter_extraction_accuracy
+
+            # Format the output
+            explanation = llm_output.get("chain_of_thought", "")
+            score_result = "pass" if result == 1 else "fail"
+            response_dict = {
+                self._result_key: result,
+                f"{self._result_key}_result": score_result,
+                f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_reason": explanation,
+                f"{self._result_key}_details": details,
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
+            }
+            return response_dict
+
+        else:
+            raise EvaluationException(
+                message="Tool input accuracy evaluator returned invalid output.",
+                blame=ErrorBlame.SYSTEM_ERROR,
+                category=ErrorCategory.FAILED_EXECUTION,
+                target=ErrorTarget.TOOL_INPUT_ACCURACY_EVALUATOR,
+            )
+
+    async def _real_call(self, **kwargs):
+        """The asynchronous call where real end-to-end evaluation logic is performed.
+
+        :keyword kwargs: The inputs to evaluate.
+        :type kwargs: Dict
+        :return: The evaluation result.
+        :rtype: Union[DoEvalResult[T_EvalValue], AggregateResult[T_EvalValue]]
+        """
+        # Convert inputs into list of evaluable inputs.
+        eval_input = self._convert_kwargs_to_eval_input(**kwargs)
+        if isinstance(eval_input, dict) and eval_input.get("error_message"):
+            # If there is an error message, return not applicable result
+            error_message = eval_input.get("error_message", "Unknown error")
+            return self._not_applicable_result(error_message, 1)
+        # Do the evaluation
+        result = await self._do_eval(eval_input)
+        # Return the result
+        return result
+
+    def _calculate_parameter_extraction_accuracy(self, details):
+        """Calculate parameter extraction accuracy from the evaluation details.
+
+        :param details: The details dictionary from the LLM evaluation output
+        :type details: Dict
+        :return: Parameter extraction accuracy as a percentage
+        :rtype: float
+        """
+        total_parameters = details.get("total_parameters_passed", 0)
+        correct_parameters = details.get("correct_parameters_passed", 0)
+
+        if total_parameters == 0:
+            return 100.0  # If no parameters were passed, accuracy is 100%
+
+        accuracy = (correct_parameters / total_parameters) * 100
+        return round(accuracy, 2)
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate parameter correctness of tool calls.
+
+        :keyword query: Query or Chat history up to the message that has the tool call being evaluated.
+        :paramtype query: Union[str, List[dict]]
+        :keyword tool_definitions: List of tool definitions whose calls are being evaluated.
+        :paramtype tool_definitions: Union[dict, List[dict]]
+        :keyword response: Response containing tool calls to be evaluated.
+        :paramtype response: Union[str, List[dict]]
+        :return: The tool input accuracy evaluation results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_tool_input_accuracy/tool_input_accuracy.prompty

@@ -0,0 +1,76 @@
+---
+name: Tool Input Accuracy
+description: Evaluates the accuracy of all inputs/parameters passed to the tools by the agent
+model:
+  api: chat
+  parameters:
+    temperature: 0.0
+    max_tokens: 1000
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+
+inputs:
+  query:
+    type: List
+  tool_calls:
+    type: List
+  tool_definitions:
+    type: Dict
+---
+
+# system:
+You are an AI system designed to evaluate the correctness of parameters passed to tool calls. Your task is to perform a strict binary evaluation (PASS/FAIL) based on whether ALL parameters are correct.
+
+The evaluation must check ALL of the following criteria. If ANY criterion fails, the overall result is FAIL:
+1. **Parameter Groundedness**: ALL parameters must be derived from or supported by information in the conversation history/query. NO fabricated or unsupported values.
+2. **Type Compliance**: ALL parameters must match the exact type specified in the tool definitions (string, number, boolean, array, object, etc.).
+3. **Format Compliance**: ALL parameters must follow the exact format, structure, and constraints specified in the tool definitions.
+4. **Required Parameters**: ALL required parameters must be provided. Missing any required parameter results in FAIL.
+5. **Unexpected Parameters**: NO parameters should be provided that are not defined in the tool definition. Any extra/unexpected parameters result in FAIL.
+6. **Value Appropriateness**: ALL parameter values must be contextually appropriate and meaningful for the tool's purpose.
+
+## Evaluation Rules
+
+**PASS**: Only when ALL criteria above are satisfied perfectly. Every single parameter must be:
+- Properly grounded in conversation history/query
+- Correct type according to tool definition
+- Proper format and structure
+- Required parameters all present
+- No unexpected/undefined parameters
+- Contextually appropriate values
+
+**FAIL**: When ANY of the above criteria fails, including:
+- Any parameter lacks grounding in conversation history
+- Any parameter has wrong type
+- Any parameter has wrong format/structure
+- Any required parameter is missing
+- Any unexpected parameter is present
+- Any parameter value is inappropriate for the context
+
+## Task
+Analyze each tool call and its parameters against the provided tool definitions and conversation context. Provide your evaluation in the following JSON format:
+
+{
+  "chain_of_thought": "Step-by-step analysis for all parameters passed to all the tools to check for the criteria mentioned above",
+  "details": {
+    "total_parameters_passed": <number of total parameters that were passed to all tools>,
+    "correct_parameters_passed": <number of correct parameters that were passed to all tools in the agent's response>,
+    "incorrect_parameters": ["list of incorrect parameters passed with reasons"]
+  },
+  "result": <0 for FAIL, 1 for PASS>
+}
+
+
+## Conversation History/Query:
+{{query}}
+
+## Tool Calls Made:
+{{tool_calls}}
+
+## Tool Definitions:
+{{tool_definitions}}
+
+# Output

azure/ai/evaluation/_evaluators/_tool_output_utilization/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._tool_output_utilization import _ToolOutputUtilizationEvaluator
+
+__all__ = ["_ToolOutputUtilizationEvaluator"]

azure/ai/evaluation/_evaluators/_tool_output_utilization/_tool_output_utilization.py

@@ -0,0 +1,225 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import math
+import logging
+from typing import Dict, Union, List, Optional
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._exceptions import (
+    EvaluationException,
+    ErrorBlame,
+    ErrorCategory,
+    ErrorTarget,
+)
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from ..._common.utils import (
+    reformat_conversation_history,
+    reformat_agent_response,
+    reformat_tool_definitions,
+    filter_to_used_tools,
+)
+from azure.ai.evaluation._model_configurations import Message
+from azure.ai.evaluation._common._experimental import experimental
+
+logger = logging.getLogger(__name__)
+
+
+@experimental
+class _ToolOutputUtilizationEvaluator(PromptyEvaluatorBase[Union[str, float]]):
+    """The Tool Output Utilization Evaluator assesses how effectively an AI agent utilizes the outputs from tools and whether it accurately incorporates this information into its responses.
+
+    Scoring is based on two levels:
+    1. Pass - The agent effectively utilizes tool outputs and accurately incorporates the information into its response.
+    2. Fail - The agent fails to properly utilize tool outputs or incorrectly incorporates the information into its response.
+
+    The evaluation includes the score, a brief explanation, and a final pass/fail result.
+
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START tool_output_utilization_evaluator]
+            :end-before: [END tool_output_utilization_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a _ToolOutputUtilizationEvaluator with a query and response.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START tool_output_utilization_evaluator]
+            :end-before: [END tool_output_utilization_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call _ToolOutputUtilizationEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    """
+
+    _PROMPTY_FILE = "tool_output_utilization.prompty"
+    _RESULT_KEY = "tool_output_utilization"
+    _OPTIONAL_PARAMS = ["tool_definitions"]
+
+    _DEFAULT_TOOL_OUTPUT_UTILIZATION_SCORE = 1
+
+    id = "azureai://built-in/evaluators/tool_output_utilization"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(
+        self,
+        model_config,
+        *,
+        credential=None,
+        **kwargs,
+    ):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            threshold=self._DEFAULT_TOOL_OUTPUT_UTILIZATION_SCORE,
+            credential=credential,
+            _higher_is_better=True,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: Union[str, List[dict]],
+        response: Union[str, List[dict]],
+        tool_definitions: Union[dict, List[dict]],
+    ) -> Dict[str, Union[str, float]]:
+        """Evaluate tool output utilization for a given query, response, and optional tool defintions.
+        The query and response can be either a string or a list of messages.
+
+
+        Example with string inputs and no tools:
+            evaluator = _ToolOutputUtilizationEvaluator(model_config)
+            query = "What is the weather today?"
+            response = "The weather is sunny."
+
+            result = evaluator(query=query, response=response)
+
+        Example with list of messages:
+            evaluator = _ToolOutputUtilizationEvaluator(model_config)
+            query = [{'role': 'system', 'content': 'You are a friendly and helpful customer service agent.'}, {'createdAt': 1700000060, 'role': 'user', 'content': [{'type': 'text', 'text': 'Hi, I need help with the last 2 orders on my account #888. Could you please update me on their status?'}]}]
+            response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': 'Hello! Let me quickly look up your account details.'}]}, {'createdAt': 1700000075, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'tool_call', 'tool_call': {'id': 'tool_call_20250310_001', 'type': 'function', 'function': {'name': 'get_orders', 'arguments': {'account_number': '888'}}}}]}, {'createdAt': 1700000080, 'run_id': '0', 'tool_call_id': 'tool_call_20250310_001', 'role': 'tool', 'content': [{'type': 'tool_result', 'tool_result': '[{ "order_id": "123" }, { "order_id": "124" }]'}]}, {'createdAt': 1700000085, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': 'Thanks for your patience. I see two orders on your account. Let me fetch the details for both.'}]}, {'createdAt': 1700000090, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'tool_call', 'tool_call': {'id': 'tool_call_20250310_002', 'type': 'function', 'function': {'name': 'get_order', 'arguments': {'order_id': '123'}}}}, {'type': 'tool_call', 'tool_call': {'id': 'tool_call_20250310_003', 'type': 'function', 'function': {'name': 'get_order', 'arguments': {'order_id': '124'}}}}]}, {'createdAt': 1700000095, 'run_id': '0', 'tool_call_id': 'tool_call_20250310_002', 'role': 'tool', 'content': [{'type': 'tool_result', 'tool_result': '{ "order": { "id": "123", "status": "shipped", "delivery_date": "2025-03-15" } }'}]}, {'createdAt': 1700000100, 'run_id': '0', 'tool_call_id': 'tool_call_20250310_003', 'role': 'tool', 'content': [{'type': 'tool_result', 'tool_result': '{ "order": { "id": "124", "status": "delayed", "expected_delivery": "2025-03-20" } }'}]}, {'createdAt': 1700000105, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': 'The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025. Is there anything else I can help you with?'}]}]
+            tool_definitions = [{'name': 'get_orders', 'description': 'Get the list of orders for a given account number.', 'parameters': {'type': 'object', 'properties': {'account_number': {'type': 'string', 'description': 'The account number to get the orders for.'}}}}, {'name': 'get_order', 'description': 'Get the details of a specific order.', 'parameters': {'type': 'object', 'properties': {'order_id': {'type': 'string', 'description': 'The order ID to get the details for.'}}}}, {'name': 'initiate_return', 'description': 'Initiate the return process for an order.', 'parameters': {'type': 'object', 'properties': {'order_id': {'type': 'string', 'description': 'The order ID for the return process.'}}}}, {'name': 'update_shipping_address', 'description': 'Update the shipping address for a given account.', 'parameters': {'type': 'object', 'properties': {'account_number': {'type': 'string', 'description': 'The account number to update.'}, 'new_address': {'type': 'string', 'description': 'The new shipping address.'}}}}]
+
+            result = evaluator(query=query, response=response, tool_definitions=tool_definitions)
+
+        :keyword query: The query being evaluated, either a string or a list of messages.
+        :paramtype query: Union[str, List[dict]]
+        :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls)
+        :paramtype response: Union[str, List[dict]]
+        :keyword tool_definitions: An optional list of messages containing the tool definitions the agent is aware of.
+        :paramtype tool_definitions: Union[dict, List[dict]]
+        :return: A dictionary with the tool output utilization evaluation results.
+        :rtype: Dict[str, Union[str, float]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Invokes the instance using the overloaded __call__ signature.
+
+        For detailed parameter types and return value documentation, see the overloaded __call__ definition.
+        """
+        return super().__call__(*args, **kwargs)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # type: ignore[override]
+        """Do Tool Output Utilization evaluation.
+        :param eval_input: The input to the evaluator. Expected to contain whatever inputs are needed for the _flow method
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        # we override the _do_eval method as we want the output to be a dictionary,
+        # which is a different schema than _base_prompty_eval.py
+        if ("query" not in eval_input) and ("response" not in eval_input) and ("tool_definitions" not in eval_input):
+            raise EvaluationException(
+                message="Query, response, and tool_definitions are required inputs to the Tool Output Utilization evaluator.",
+                internal_message="Query, response, and tool_definitions are required inputs to the Tool Output Utilization evaluator.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.MISSING_FIELD,
+                target=ErrorTarget.TOOL_OUTPUT_UTILIZATION_EVALUATOR,
+            )
+
+        tool_definitions = eval_input["tool_definitions"]
+        filtered_tool_definitions = filter_to_used_tools(
+            tool_definitions=tool_definitions,
+            msgs_lists=[eval_input["query"], eval_input["response"]],
+            logger=logger,
+        )
+        eval_input["tool_definitions"] = reformat_tool_definitions(filtered_tool_definitions, logger)
+
+        eval_input["query"] = reformat_conversation_history(
+            eval_input["query"],
+            logger,
+            include_system_messages=True,
+            include_tool_messages=True,
+        )
+        eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
+
+        prompty_output_dict = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = prompty_output_dict.get("llm_output", "")
+        if isinstance(llm_output, dict):
+            output_label = llm_output.get("label", None)
+            if output_label is None:
+                if logger:
+                    logger.warning("LLM output does not contain 'label' key, returning NaN for the score.")
+                output_label = "fail"
+
+            output_label = output_label.lower()
+            if output_label not in ["pass", "fail"]:
+                if logger:
+                    logger.warning(
+                        f"LLM output label is not 'pass' or 'fail' (got '{output_label}'), returning NaN for the score."
+                    )
+
+            score = 1.0 if output_label == "pass" else 0.0
+            score_result = output_label
+            reason = llm_output.get("reason", "")
+
+            faulty_details = llm_output.get("faulty_details", [])
+            if faulty_details:
+                reason += " Issues found: " + "; ".join(faulty_details)
+
+            return {
+                f"{self._result_key}": score,
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_result": score_result,
+                f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_prompt_tokens": prompty_output_dict.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": prompty_output_dict.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": prompty_output_dict.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": prompty_output_dict.get("finish_reason", ""),
+                f"{self._result_key}_model": prompty_output_dict.get("model_id", ""),
+                f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
+                f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
+            }
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning NaN for the score.")
+
+        score = math.nan
+        binary_result = self._get_binary_result(score)
+        return {
+            self._result_key: float(score),
+            f"{self._result_key}_result": binary_result,
+            f"{self._result_key}_threshold": self._threshold,
+        }