azure-ai-evaluation 1.11.1__py3-none-any.whl → 1.12.0__py3-none-any.whl

azure/ai/evaluation/_evaluators/_path_efficiency/_path_efficiency.py

@@ -0,0 +1,342 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import json
+from collections import Counter
+from typing import Dict, List, Union, Any, Tuple
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
+from azure.ai.evaluation._constants import EVALUATION_PASS_FAIL_MAPPING
+
+
+class PathEfficiencyEvaluator(EvaluatorBase):
+    """
+    Evaluates whether an agent's sequence of actions is efficient and follows optimal decision-making patterns.
+
+    The Path Efficiency Evaluator calculates precision, recall, and F1 scores based on the comparison
+    between the agent's tool usage trajectory and the ground truth expected steps. It also provides
+    three binary match metrics: exact match, in-order match (allows extra steps), and any-order match (allows extra steps and ignores order).
+
+    :param precision_threshold: The threshold value to determine if the precision evaluation passes or fails. Default is 0.5.
+    :type precision_threshold: float
+    :param recall_threshold: The threshold value to determine if the recall evaluation passes or fails. Default is 0.5.
+    :type recall_threshold: float
+    :param f1_score_threshold: The threshold value to determine if the F1 score evaluation passes or fails. Default is 0.5.
+    :type f1_score_threshold: float
+
+    .. admonition:: Example:
+
+        .. code-block:: python
+
+            from azure.ai.evaluation import PathEfficiencyEvaluator
+
+            path_efficiency_eval = PathEfficiencyEvaluator(
+                precision_threshold=0.7,
+                recall_threshold=0.8,
+                f1_score_threshold=0.75
+            )
+
+            # Example 1: Using simple tool names list
+            result = path_efficiency_eval(
+                response=[
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_1", "name": "identify_tools_to_call", "arguments": {}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_2", "name": "call_tool_A", "arguments": {}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_3", "name": "call_tool_B", "arguments": {}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_4", "name": "response_synthesis", "arguments": {}}]}
+                ],
+                ground_truth=["identify_tools_to_call", ""call_tool_A", "call_tool_B", "response_synthesis"]
+            )
+
+            # Example 2: Using tool names with parameters (exact parameter matching required)
+            result = path_efficiency_eval(
+                response=[
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_1", "name": "search", "arguments": {"query": "weather", "location": "NYC"}}]},
+                    {"role": "assistant", "content": [{"type": "tool_call", "tool_call_id": "call_2", "name": "format_result", "arguments": {"format": "json"}}]}
+                ],
+                ground_truth=(
+                    ["search", "format_result"],
+                    {
+                        "search": {"query": "weather", "location": "NYC"},
+                        "format_result": {"format": "json"}
+                    }
+                )
+            )
+    """
+
+    _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD = 0.5
+
+    id = "azureai://built-in/evaluators/path_efficiency"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(
+        self,
+        *,
+        precision_threshold: float = _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD,
+        recall_threshold: float = _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD,
+        f1_score_threshold: float = _DEFAULT_PATH_EFFICIENCY_SCORE_THRESHOLD,
+    ):
+        self._higher_is_better = True
+        super().__init__()
+
+        # Type checking for threshold parameters
+        for name, value in [
+            ("precision_threshold", precision_threshold),
+            ("recall_threshold", recall_threshold),
+            ("f1_score_threshold", f1_score_threshold),
+        ]:
+            if not isinstance(value, float):
+                raise TypeError(f"{name} must be a float, got {type(value)}")
+
+        self._threshold = {
+            "path_efficiency_precision": precision_threshold,
+            "path_efficiency_recall": recall_threshold,
+            "path_efficiency_f1": f1_score_threshold,
+        }
+
+    def _prepare_steps_for_comparison(
+        self,
+        agent_tool_pairs: List[Tuple[str, Dict[str, Any]]],
+        ground_truth: List[str],
+        ground_truth_params: Dict[str, Dict[str, Any]],
+        use_parameter_matching: bool,
+    ) -> Tuple[
+        List[Union[str, Tuple[str, Tuple]]],
+        List[Union[str, Tuple[str, Tuple]]],
+    ]:
+        """Prepare agent and ground truth steps for comparison based on parameter matching mode."""
+        agent_steps: List[Union[str, Tuple[str, Tuple]]] = []
+        ground_truth_steps: List[Union[str, Tuple[str, Tuple]]] = []
+        if use_parameter_matching:
+            # When parameter matching is enabled, we need to match both tool name and parameters
+            agent_steps = [(pair[0], tuple(sorted(pair[1].items()))) for pair in agent_tool_pairs]
+            ground_truth_steps = [
+                (name, tuple(sorted(ground_truth_params.get(name, {}).items()))) for name in ground_truth
+            ]
+        else:
+            # When parameter matching is disabled, only compare tool names
+            agent_steps = [name for name, _ in agent_tool_pairs]
+            ground_truth_steps = [step for step in ground_truth]
+
+        return agent_steps, ground_truth_steps
+
+    def _calculate_precision_recall_f1_scores(self, agent_steps: List, ground_truth_steps: List) -> Dict[str, float]:
+        """Calculate precision, recall, and F1 scores."""
+        if not agent_steps:
+            return {"precision_score": 0.0, "recall_score": 0.0, "f1_score": 0.0}
+
+        # Count occurrences of each step in both lists to handle duplicates
+        agent_steps_counts = Counter(agent_steps)
+        ground_truth_counts = Counter(ground_truth_steps)
+
+        # Calculate true positives by taking the minimum count for each common element
+        # For each step, count the intersection (min count) of agent and ground truth steps
+        true_positives = sum(
+            min(agent_steps_counts[step], ground_truth_counts[step])
+            for step in agent_steps_counts
+            if step in ground_truth_counts
+        )
+
+        # Calculate false positives (agent steps not in ground truth or excess occurrences)
+        # For each step, count the excess occurrences of agent steps not in (minus) ground truth
+        # or zero (agent steps minus agent steps) if agent steps is less than ground truth
+        false_positives = sum(
+            agent_steps_counts[step] - min(agent_steps_counts[step], ground_truth_counts.get(step, 0))
+            for step in agent_steps_counts
+        )
+
+        # Calculate false negatives (ground truth steps not in agent or missing occurrences)
+        # For each step, count the excess occurrences of ground truth steps not in (minus) agent steps
+        # or zero (ground truth steps minus ground truth steps) if ground truth steps is less than agent steps
+        false_negatives = sum(
+            ground_truth_counts[step] - min(ground_truth_counts[step], agent_steps_counts.get(step, 0))
+            for step in ground_truth_counts
+        )
+
+        # Calculate precision, recall, F1
+        precision = (
+            true_positives / (true_positives + false_positives) if (true_positives + false_positives) > 0 else 0.0
+        )
+        recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0.0
+        f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+
+        return {
+            "precision_score": precision,
+            "recall_score": recall,
+            "f1_score": f1_score,
+        }
+
+    def _calculate_exact_match(self, agent_steps: List, ground_truth_steps: List) -> bool:
+        """Check if agent steps exactly match ground truth (order and content)."""
+        return agent_steps == ground_truth_steps
+
+    def _calculate_in_order_match(self, agent_steps: List, ground_truth_steps: List) -> bool:
+        """Check if all ground truth steps appear in agent steps in correct order (extra steps allowed)."""
+        if not ground_truth_steps:
+            return True
+
+        gt_index = 0
+        for step in agent_steps:
+            if gt_index < len(ground_truth_steps) and step == ground_truth_steps[gt_index]:
+                gt_index += 1
+
+        return gt_index == len(ground_truth_steps)
+
+    def _calculate_any_order_match(self, agent_steps: List, ground_truth_steps: List) -> bool:
+        """Check if all ground truth steps appear in agent steps with sufficient frequency (any order, extra steps allowed)."""
+        # Count occurrences of each step in both lists to handle duplicates
+        agent_counts = Counter(agent_steps)
+        ground_truth_counts = Counter(ground_truth_steps)
+
+        # Check if agent has at least as many occurrences of each ground truth step
+        return all(agent_counts[step] >= ground_truth_counts[step] for step in ground_truth_counts)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
+        """Produce a path efficiency evaluation result.
+
+        :param eval_input: The input to the evaluation function. Must contain "response" and "ground_truth".
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        response = eval_input["response"]
+        ground_truth = eval_input["ground_truth"]
+
+        # Value and type checking for ground truth steps
+        if not ground_truth:
+            raise ValueError("ground_truth cannot be empty")
+
+        # Check if ground_truth is a tuple (tool names + parameters) or list (tool names only)
+        use_parameter_matching = False
+        ground_truth_names = []
+        ground_truth_params_dict: Dict[str, Dict[str, Any]] = {}
+
+        if isinstance(ground_truth, tuple) and len(ground_truth) == 2:
+            # Tuple format: (tool_names, parameters_dict)
+            tool_names_list, params_dict = ground_truth
+
+            if not isinstance(tool_names_list, list) or not all(isinstance(name, str) for name in tool_names_list):
+                raise TypeError("ground_truth tuple first element must be a list of strings (tool names)")
+
+            if not isinstance(params_dict, dict):
+                raise TypeError(
+                    "ground_truth tuple second element must be a dictionary mapping tool names to parameters"
+                )
+
+            # Validate that all values in params_dict are dictionaries with string keys and values
+            for tool_name, params in params_dict.items():
+                if not isinstance(tool_name, str):
+                    raise TypeError("ground_truth parameters dictionary keys must be strings (tool names)")
+                if not isinstance(params, dict):
+                    raise TypeError(f"ground_truth parameters for tool '{tool_name}' must be a dictionary")
+                for k, v in params.items():
+                    if not isinstance(k, str):
+                        raise TypeError(f"ground_truth parameters for tool '{tool_name}' must have string keys")
+                    try:
+                        json.dumps(v)
+                    except (TypeError, ValueError):
+                        raise TypeError(
+                            f"ground_truth parameters for tool '{tool_name}' must have JSON-serializable values (got type {type(v)} for key '{k}')"
+                        )
+
+            ground_truth_names = [name.strip() for name in tool_names_list]
+            ground_truth_params_dict = params_dict
+            use_parameter_matching = True
+
+        elif isinstance(ground_truth, list) and all(isinstance(step, str) for step in ground_truth):
+            # List format: just tool names
+            ground_truth_names = [step.strip() for step in ground_truth]
+            use_parameter_matching = False
+
+        else:
+            raise TypeError(
+                "ground_truth must be a list of strings or a tuple of (list[str], dict[str, dict[str, str]])"
+            )
+
+        # Extract tool information from the response
+        agent_tool_pairs = self._extract_tool_names_and_params_from_response(response)
+
+        # Prepare steps for comparison
+        agent_steps, ground_truth_steps = self._prepare_steps_for_comparison(
+            agent_tool_pairs,
+            ground_truth_names,
+            ground_truth_params_dict,
+            use_parameter_matching,
+        )
+
+        # Calculate precision, recall, and F1 scores
+        metrics = self._calculate_precision_recall_f1_scores(agent_steps, ground_truth_steps)
+
+        # Calculate binary match metrics
+        exact_match = self._calculate_exact_match(agent_steps, ground_truth_steps)
+        in_order_match = self._calculate_in_order_match(agent_steps, ground_truth_steps)
+        any_order_match = self._calculate_any_order_match(agent_steps, ground_truth_steps)
+
+        # Convert metrics to floats, using nan for None or non-convertible values
+        path_efficiency_precision = (
+            float(metrics["precision_score"]) if metrics["precision_score"] is not None else float("nan")
+        )
+        path_efficiency_recall = float(metrics["recall_score"]) if metrics["recall_score"] is not None else float("nan")
+        path_efficiency_f1_score = float(metrics["f1_score"]) if metrics["f1_score"] is not None else float("nan")
+
+        return {
+            "path_efficiency_precision_score": path_efficiency_precision,
+            "path_efficiency_recall_score": path_efficiency_recall,
+            "path_efficiency_f1_score": path_efficiency_f1_score,
+            "path_efficiency_exact_match_result": EVALUATION_PASS_FAIL_MAPPING[exact_match],
+            "path_efficiency_in_order_match_result": EVALUATION_PASS_FAIL_MAPPING[in_order_match],
+            "path_efficiency_any_order_match_result": EVALUATION_PASS_FAIL_MAPPING[any_order_match],
+        }
+
+    @overload
+    def __call__(  # type: ignore
+        self, *, response: Union[str, List[Dict[str, Any]]], ground_truth: List[str]
+    ) -> Dict[str, Union[float, str]]:
+        """
+        Evaluate the path efficiency of an agent's action sequence.
+
+        :keyword response: The agent's response containing tool calls.
+        :paramtype response: Union[str, List[Dict[str, Any]]]
+        :keyword ground_truth: List of expected tool/action steps.
+        :paramtype ground_truth: List[str]
+        :return: The path efficiency scores and results.
+        :rtype: Dict[str, Union[float, str]]
+        """
+
+    @overload
+    def __call__(  # type: ignore
+        self,
+        *,
+        response: Union[str, List[Dict[str, Any]]],
+        ground_truth: Tuple[List[str], Dict[str, Dict[str, str]]],
+    ) -> Dict[str, Union[float, str]]:
+        """
+        Evaluate the path efficiency of an agent's action sequence with tool parameters.
+
+        :keyword response: The agent's response containing tool calls.
+        :paramtype response: Union[str, List[Dict[str, Any]]]
+        :keyword ground_truth: Tuple of (tool names list, parameters dict) where parameters must match exactly.
+        :paramtype ground_truth: Tuple[List[str], Dict[str, Dict[str, str]]]
+        :return: The path efficiency scores and results.
+        :rtype: Dict[str, Union[float, str]]
+        """
+
+    @override
+    def __call__(
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate path efficiency.
+
+        :keyword response: The agent's response containing tool calls.
+        :paramtype response: Union[str, List[Dict[str, Any]]]
+        :keyword ground_truth: List of expected tool/action steps or tuple of (tool names, parameters dict).
+        :paramtype ground_truth: Union[List[str], Tuple[List[str], Dict[str, Dict[str, str]]]]
+        :return: The path efficiency scores and results.
+        :rtype: Dict[str, Union[float, str]]
+        """
+        return super().__call__(*args, **kwargs)

azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -35,6 +35,11 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the relevance evaluator. Default is 3.
     :type threshold: int
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -79,7 +84,7 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, credential=None, threshold=3):
+    def __init__(self, model_config, *, credential=None, threshold=3, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._threshold = threshold
@@ -91,6 +96,7 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
             threshold=threshold,
             credential=credential,
             _higher_is_better=self._higher_is_better,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py

@@ -33,6 +33,11 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the evaluation. Default is 3.
     :type threshold: float
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
     :return: A function that evaluates and generates metrics for "chat" scenario.
     :rtype: Callable
 
@@ -78,7 +83,7 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold: float = 3, credential=None):
+    def __init__(self, model_config, *, threshold: float = 3, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._threshold = threshold
@@ -90,6 +95,7 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
             threshold=threshold,
             credential=credential,
             _higher_is_better=self._higher_is_better,
+            **kwargs,
         )
 
     @overload

azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -30,6 +30,11 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
         ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param threshold: The threshold for the similarity evaluator. Default is 3.
     :type threshold: int
+    :param credential: The credential for authenticating to Azure AI service.
+    :type credential: ~azure.core.credentials.TokenCredential
+    :keyword is_reasoning_model: If True, the evaluator will use reasoning model configuration (o1/o3 models).
+        This will adjust parameters like max_completion_tokens and remove unsupported parameters. Default is False.
+    :paramtype is_reasoning_model: bool
 
     .. admonition:: Example:
 
@@ -75,7 +80,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
     @override
-    def __init__(self, model_config, *, threshold=3, credential=None):
+    def __init__(self, model_config, *, threshold=3, credential=None, **kwargs):
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
         self._threshold = threshold
@@ -87,6 +92,7 @@ class SimilarityEvaluator(PromptyEvaluatorBase):
             threshold=threshold,
             credential=credential,
             _higher_is_better=self._higher_is_better,
+            **kwargs,
         )
 
     # Ignoring a mypy error about having only 1 overload function.

azure/ai/evaluation/_evaluators/_task_success/__init__.py

@@ -0,0 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._task_success import TaskSuccessEvaluator
+
+__all__ = ["TaskSuccessEvaluator"]

azure/ai/evaluation/_evaluators/_task_success/_task_success.py

@@ -0,0 +1,168 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import os
+import math
+import logging
+from typing import Dict, Union, List, Optional
+
+from typing_extensions import overload, override
+
+from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
+from ..._common.utils import reformat_conversation_history, reformat_agent_response, reformat_tool_definitions
+from azure.ai.evaluation._model_configurations import Message
+from azure.ai.evaluation._common._experimental import experimental
+
+logger = logging.getLogger(__name__)
+
+
+@experimental
+class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]):
+    """The Task Success evaluator determines whether an AI agent successfully completed the requested task based on:
+
+        - Final outcome and deliverable of the task
+        - Completeness of task requirements
+
+    This evaluator focuses solely on task completion and success, not on task adherence or intent understanding.
+
+    Scoring is binary:
+    - TRUE: Task fully completed with usable deliverable that meets all user requirements
+    - FALSE: Task incomplete, partially completed, or deliverable does not meet requirements
+
+    The evaluation includes task requirement analysis, outcome assessment, and completion gap identification.
+
+
+    :param model_config: Configuration for the Azure OpenAI model.
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
+
+    .. admonition:: Example:
+        .. literalinclude:: ../samples/evaluation_samples_evaluate.py
+            :start-after: [START task_success_evaluator]
+            :end-before: [END task_success_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call a TaskSuccessEvaluator with a query and response.
+
+    .. admonition:: Example using Azure AI Project URL:
+
+        .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py
+            :start-after: [START task_success_evaluator]
+            :end-before: [END task_success_evaluator]
+            :language: python
+            :dedent: 8
+            :caption: Initialize and call TaskSuccessEvaluator using Azure AI Project URL in the following format
+                https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    """
+
+    _PROMPTY_FILE = "task_success.prompty"
+    _RESULT_KEY = "task_success"
+    _OPTIONAL_PARAMS = ["tool_definitions"]
+
+    id = "azureai://built-in/evaluators/task_success"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
+    @override
+    def __init__(self, model_config, *, credential=None, **kwargs):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(
+            model_config=model_config,
+            prompty_file=prompty_path,
+            result_key=self._RESULT_KEY,
+            credential=credential,
+            **kwargs,
+        )
+
+    @overload
+    def __call__(
+        self,
+        *,
+        query: Union[str, List[dict]],
+        response: Union[str, List[dict]],
+        tool_definitions: Optional[Union[dict, List[dict]]] = None,
+    ) -> Dict[str, Union[str, bool]]:
+        """Evaluate task success for a given query, response, and optionally tool definitions.
+        The query and response can be either a string or a list of messages.
+
+
+        Example with string inputs and no tools:
+            evaluator = TaskSuccessEvaluator(model_config)
+            query = "Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine."
+            response = "**Day 1:** Morning: Louvre Museum, Lunch: Le Comptoir du Relais..."
+
+            result = evaluator(query=query, response=response)
+
+        Example with list of messages:
+            evaluator = TaskSuccessEvaluator(model_config)
+            query = [{'role': 'system', 'content': 'You are a helpful travel planning assistant.'}, {'createdAt': 1700000060, 'role': 'user', 'content': [{'type': 'text', 'text': 'Plan a 3-day Paris itinerary with cultural landmarks and cuisine'}]}]
+            response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}]
+            tool_definitions = [{'name': 'get_attractions', 'description': 'Get tourist attractions for a city.', 'parameters': {'type': 'object', 'properties': {'city': {'type': 'string', 'description': 'The city name.'}}}}]
+
+            result = evaluator(query=query, response=response, tool_definitions=tool_definitions)
+
+        :keyword query: The query being evaluated, either a string or a list of messages.
+        :paramtype query: Union[str, List[dict]]
+        :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls)
+        :paramtype response: Union[str, List[dict]]
+        :keyword tool_definitions: An optional list of messages containing the tool definitions the agent is aware of.
+        :paramtype tool_definitions: Optional[Union[dict, List[dict]]]
+        :return: A dictionary with the task success evaluation results.
+        :rtype: Dict[str, Union[str, bool]]
+        """
+
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Invokes the instance using the overloaded __call__ signature.
+
+        For detailed parameter types and return value documentation, see the overloaded __call__ definition.
+        """
+        return super().__call__(*args, **kwargs)
+
+    @override
+    async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[bool, str]]:  # type: ignore[override]
+        """Do Task Success evaluation.
+        :param eval_input: The input to the evaluator. Expected to contain whatever inputs are needed for the _flow method
+        :type eval_input: Dict
+        :return: The evaluation result.
+        :rtype: Dict
+        """
+        # we override the _do_eval method as we want the output to be a dictionary,
+        # which is a different schema than _base_prompty_eval.py
+        if "query" not in eval_input and "response" not in eval_input:
+            raise EvaluationException(
+                message=f"Both query and response must be provided as input to the Task Success evaluator.",
+                internal_message=f"Both query and response must be provided as input to the Task Success evaluator.",
+                blame=ErrorBlame.USER_ERROR,
+                category=ErrorCategory.MISSING_FIELD,
+                target=ErrorTarget.TASK_SUCCESS_EVALUATOR,
+            )
+        eval_input["query"] = reformat_conversation_history(eval_input["query"], logger, include_system_messages=True)
+        eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True)
+        if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None:
+            eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger)
+
+        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        if isinstance(llm_output, dict):
+            success = llm_output.get("success", False)
+            if isinstance(success, str):
+                success = success.upper() == "TRUE"
+
+            success_result = "pass" if success == True else "fail"
+            reason = llm_output.get("explanation", "")
+            return {
+                f"{self._result_key}": success,
+                f"{self._result_key}_result": success_result,
+                f"{self._result_key}_reason": reason,
+                f"{self._result_key}_details": llm_output.get("details", ""),
+            }
+        if logger:
+            logger.warning("LLM output is not a dictionary, returning False for the success.")
+        return {self._result_key: False}