deepeval 3.6.7__py3-none-any.whl → 3.6.9__py3-none-any.whl

deepeval/metrics/step_efficiency/template.py

@@ -0,0 +1,256 @@
+import textwrap
+import json
+from deepeval.tracing.utils import make_json_serializable
+
+
+class StepEfficiencyTemplate:
+
+    @staticmethod
+    def extract_task_from_trace(trace: dict) -> str:
+        return textwrap.dedent(
+            f"""You are a **trace analyst** tasked with extracting the **user's original goal or task** from a complete nested execution trace of an AI agent.
+
+                YOUR OBJECTIVE:
+
+                Identify and describe **exactly what the user asked the agent to do**, based only on the user's explicit input and any unambiguous contextual details present in the trace.
+
+                Your goal is to produce a **concise, fact-based statement** that captures the *intended user task* — not the agent's plan, actions, reasoning, or assumptions.
+
+                STRICT EXTRACTION RULES:
+
+                1. Primary Source: Root-Level User Input
+                - The user's task must be derived **directly and primarily** from the root agent's `"input"` field.  
+                - If that field contains nested `"input"` or `"messages"`, extract the true user instruction or request text from within it.
+
+                2. Secondary Context: Subtasks as Clarifiers (Optional)
+                - You may use child spans (tools, retrievers, LLMs) **only** to clarify or disambiguate what the user explicitly asked for — 
+                    e.g., to confirm that the task involves multiple subtasks the user clearly implied (like booking and planning steps for a trip).
+                - You may **NOT** infer new goals that the user did not state or imply.
+
+                3. No Hallucination
+                - Do **NOT** invent goals, assumptions, or implied needs beyond what is explicitly or clearly inferable from the input.
+                - If the user's request is vague, preserve that vagueness — do not expand it.
+
+                4. Agent-Agnostic Rule
+                - Ignore the agent's tools, methods, reasoning, or internal operations.
+                - The task reflects **what the user wanted**, not how the agent chose to approach it.
+
+                5. Perspective
+                - Express the extracted task **from the user's perspective**, as if restating what they asked the system to do.
+                - Avoid any meta or evaluative phrasing (“The user wanted the agent to…”).
+
+                6. Fallback Condition
+                - If the only available information about the task is the raw user input text, return that input verbatim without modification.
+
+                OUTPUT FORMAT:
+
+                Return **only** a JSON object of this form:
+
+                {{
+                    "task": "<a single clear sentence summarizing the user's explicit goal>"
+                }}
+
+                - The `"task"` value should be a single, coherent natural language sentence or two at most.
+                - Do not include commentary, metadata, or any additional fields.
+
+                EXAMPLES:
+
+                Example Trace: {{ 
+                    "name": "trip_planner", 
+                    "type": "agent", 
+                    "input": {{ 
+                        "input": "Help me plan a business trip to Chicago next week." 
+                    }}, 
+                    "children": [ 
+                        {{ 
+                            "name": "flight_tool", 
+                            "type": "tool", 
+                            "input": {{ 
+                                "inputParameters": {{ 
+                                    "destination": "Chicago", 
+                                    "date": "2024-07-10" 
+                                }} }}, 
+                                "output": {{ 
+                                    "flights": ["Flight 101", "Flight 202"] 
+                                }}, 
+                                "children": [] 
+                        }}, 
+                        {{ 
+                            "name": "hotel_tool", 
+                            "type": "tool", 
+                            "input": {{ 
+                                "inputParameters": {{ 
+                                    "location": "Chicago", 
+                                    "check_in": "2024-07-10", 
+                                    "check_out": "2024-07-12" 
+                                }} }}, 
+                                "output": {{ 
+                                    "hotels": ["The Grand Chicago", "Lakeview Inn"] 
+                                }}, 
+                                "children": [] 
+                        }}, 
+                        {{ 
+                            "name": "agenda_llm", 
+                            "type": "llm", 
+                            "input": {{ 
+                                "prompt": "Draft a meeting agenda", 
+                                "input": [ 
+                                    {{ 
+                                        "role": "system", 
+                                        "content": "You are an executive assistant." 
+                                    }}, 
+                                    {{ 
+                                        "role": "user", 
+                                        "content": "Create an agenda for a client strategy meeting." 
+                                    }} 
+                                ] 
+                            }}, 
+                        "output": "1. Q2 review\\n2. Client feedback\\n3. Strategy planning", 
+                        "children": [] 
+                        }} 
+                    ] 
+                }} 
+                
+                Expected JSON: 
+                {{ 
+                    "task": "Plan a business trip to Chicago next week, including booking a flight, reserving a hotel, and drafting a client meeting agenda." 
+                }}
+
+                IMPORTANT ENFORCEMENT RULES:
+
+                - If multiple user inputs exist, identify the overall task that user has in mind.
+                - Do not include execution details, tools, function names, or reasoning text.
+                - Avoid restating or paraphrasing beyond clarity; preserve the user's intent exactly.
+                - When uncertain, extract **less rather than more** — prefer minimal, factual phrasing over speculative completion.
+
+                TRACE DATA:
+
+                {json.dumps(trace, default=make_json_serializable, indent=2)}
+
+                ---
+
+                ### JSON:
+            """
+        )
+
+    @staticmethod
+    def get_execution_efficiency(task: str, trace: dict) -> str:
+        return textwrap.dedent(
+            f"""You are an **efficiency auditor** evaluating how economically an AI agent executed a task.
+
+                OBJECTIVE:
+
+                Determine how **efficiently** the agent executed the given task based on its full execution trace.
+                Efficiency means achieving the user's goal using the **fewest, simplest, and most direct** actions possible.
+
+                You must assign a score from **0.0 to 1.0** that reflects how close the execution came to the *minimal necessary sequence of actions*.
+
+                **Important:** You are not evaluating correctness, completeness, creativity, or helpfulness — only the *efficiency* of the execution.
+
+                STRICT EVALUATION RULES:
+
+                1. Zero-Tolerance for Unnecessary Actions
+                - Every step, tool call, LLM query, or retrieval must be **strictly required** to fulfill the task.  
+                - If a single tool, retrieval, or reasoning step is superfluous, speculative, repetitive, or stylistic, 
+                    the score must be as low as possible, regardless of outcome quality.
+                - Adding “helpful” or “contextual” actions that were not explicitly necessary is an inefficiency.
+
+                2. Minimal Action Principle
+                - The ideal execution performs the **exact minimum number of steps** needed to complete the task.  
+                - Each step must directly contribute to completing the task, not to exploration, confirmation, or elaboration.
+
+                3. No Speculation or Enrichment
+                - Any activity aimed at *enhancing*, *expanding*, or *beautifying* the answer (e.g., extra retrievals, style edits, rephrasings) 
+                    reduces the score sharply (≤ 0.25).  
+                - Efficiency is about restraint — **doing exactly what's required, nothing more**.
+
+                4. Directness and Focus
+                - Steps must appear in a logically minimal sequence from input to goal.  
+                - Repetition, re-querying, nested reasoning loops, or tool reuse when not needed 
+                    indicate inefficiency.
+
+                5. Resource Economy
+                - Use of multiple LLM calls, retrievers, or tools when one would suffice must be penalized.
+                - Avoided resources (if the agent achieved the task through simpler means) improve efficiency.
+
+                6. When in Doubt
+                - If it is unclear whether an action was required or not, **assume it was unnecessary** and lower the score.
+                - Err on the side of penalizing over generosity.
+
+                SCORING SCALE (STRICT)
+
+                - **1.0 — Perfectly efficient**
+                - Only essential steps taken.  
+                - Each action was directly necessary for task completion.  
+                - No speculative, redundant, or decorative work.
+
+                - **0.75 — Strong efficiency**
+                - Mostly minimal execution with one small redundant or stylistic step.  
+                - Slight overuse of a tool or repeated call, but otherwise tight.
+
+                - **0.5 — Moderate efficiency**
+                - Noticeable inefficiency: extra steps, unnecessary tool calls, or indirect methods.  
+                - The same task could clearly have been completed faster or with fewer actions.
+
+                - **0.25 — Low efficiency**
+                - Multiple irrelevant or unjustified actions taken.  
+                - Execution path significantly longer or more complex than needed.
+
+                - **0.0 — Highly inefficient**
+                - Execution was verbose, exploratory, speculative, or wasteful.  
+                - Most actions were unnecessary or unrelated to achieving the core task.
+
+                *When uncertain, always assign the lower score.*
+
+                OUTPUT FORMAT:
+
+                Return a single JSON object in this exact format:
+
+                {{
+                    "score": 0.0,
+                    "reason": "1-3 concise factual sentences describing where inefficiencies occurred."
+                }}
+
+                The `reason` must:
+                - Identify specific inefficient actions (e.g., redundant LLM call, unnecessary retrieval, speculative tool use).
+                - Avoid subjective phrasing (“reasonable”, “seems okay”, “somewhat efficient”).
+                - Be direct and concrete: “Extra retrieval used for enrichment”, “Multiple summarizations of same data”, etc.
+
+                EXAMPLES
+
+                **Example 1:**
+                Task: "Summarize the given text."
+                Trace: Agent calls an LLM twice, then performs an extra web search.
+
+                → Output:
+                {{
+                    "score": 0.25,
+                    "reason": "The agent used redundant LLM calls and performed an unnecessary web search. Only one LLM call was required for the summary."
+                }}
+
+                **Example 2:**
+                Task: "Convert a date to ISO format."
+                Trace: Agent performs one computation directly.
+
+                → Output:
+                {{
+                    "score": 1.0,
+                    "reason": "The agent completed the task with one minimal action and no unnecessary steps."
+                }}
+
+                FINAL REMINDERS
+
+                - Efficiency = minimality. Any extra work, enrichment, or indirect approach must lower the score.
+                - Do not consider correctness, helpfulness, or reasoning quality.
+                - A “good answer” can still score **0.0** if it was achieved inefficiently.
+                - This metric is adversarial: assign the lowest score possible unless execution was provably minimal.
+
+                TASK:
+                {task}
+
+                TRACE:
+                {json.dumps(trace, indent=2, default=str)}
+
+                JSON:
+            """
+        )

deepeval/metrics/task_completion/task_completion.py

@@ -44,6 +44,7 @@ class TaskCompletionMetric(BaseMetric):
         self.async_mode = async_mode
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
+        self.requires_trace = True
 
     def measure(
         self,

deepeval/metrics/tool_correctness/schema.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class ToolSelectionScore(BaseModel):
+    score: float
+    reason: str

deepeval/metrics/tool_correctness/template.py

@@ -0,0 +1,88 @@
+import textwrap
+import json
+
+
+class ToolCorrectnessTemplate:
+
+    @staticmethod
+    def get_tool_selection_score(
+        user_input: str, tools_called: list, available_tools: list
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are an expert evaluator assessing the **Tool Selection** quality of an AI agent.
+
+            You are given:
+            - The **user input** that defines the user's goal / task.
+            - A list of **available tools**, each with a name and description.
+            - A list of **tool calls made** by the agent during execution, including tool name and parameters.
+
+            Your job is to assign a **Tool Selection score** from 0.0 to 1.0 based on how appropriate and well-matched the agent's chosen tools were to the task's requirements.
+
+            ---
+
+            DEFINITION:
+
+            Tool Selection evaluates how suitable the agent's tool choices were in addressing the task and sub-tasks.
+
+            This metric does **not** consider:
+            - How well the tools were used (execution quality)
+            - Whether the agent adhered to a plan
+            - Whether the output was correct or efficient
+
+            It only assesses whether the **right tools** were selected, based on their stated descriptions and the demands of the task.
+
+            ---
+
+            INSTRUCTIONS:
+
+            Step 1: Read the **user task** to understand what needed to be accomplished.
+
+            Step 2: Examine the **available tools** and their descriptions to understand the intended purpose of each.
+
+            Step 3: Review the **tool calls made by the agent**:
+            - Were the selected tools well-aligned with the task?
+            - Were any obviously better-suited tools ignored?
+            - Were any tools misapplied or used unnecessarily?
+
+            Step 4: Identify selection issues:
+            - **Correct Selection**: Tool(s) chosen directly and appropriately matched the subtask.
+            - **Over-selection**: More tools were selected than necessary, despite availability of a simpler or more direct option.
+            - **Under-selection**: Key tools that were well-suited were omitted.
+            - **Mis-selection**: Tools were chosen that were poorly matched to their purpose or the subtask.
+
+            ---
+
+            SCORING GUIDE:
+
+            - **1.0** → All selected tools were appropriate and necessary. No better-suited tools were omitted.
+            - **0.75** → Tool choices were mostly appropriate, with minor omissions or unnecessary use.
+            - **0.5** → Mixed tool selection. Some useful tools ignored or some inappropriate ones used.
+            - **0.25** → Poor tool selection. Better alternatives were available and ignored.
+            - **0.0** → Tool selection was clearly misaligned with task requirements.
+
+            ---
+
+            OUTPUT FORMAT:
+
+            Return a valid JSON object with this exact structure:
+            {{
+                "score": float between 0.0 and 1.0,
+                "reason": "1-3 concise, factual sentences explaining the score. Reference specific tool names and descriptions when relevant."
+            }}
+
+            Do not include any additional commentary or output outside the JSON object.
+
+            ---
+
+            USER INPUT:
+            {user_input}
+
+            ALL AVAILABLE TOOLS:
+            {available_tools}
+
+            TOOL CALLS MADE BY AGENT:
+            {tools_called}
+
+            JSON:
+            """
+        )

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -1,10 +1,15 @@
-from typing import List, Dict
+from typing import List, Dict, Optional, Union
 
 from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
+    trimAndLoadJson,
+    initialize_model,
+    print_tools_called,
 )
+from deepeval.models import DeepEvalBaseLLM
 from deepeval.test_case import (
     LLMTestCase,
     LLMTestCaseParams,
@@ -13,6 +18,8 @@ from deepeval.test_case import (
 )
 from deepeval.metrics import BaseMetric
 from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics.tool_correctness.template import ToolCorrectnessTemplate
+from deepeval.metrics.tool_correctness.schema import ToolSelectionScore
 
 
 class ToolCorrectnessMetric(BaseMetric):
@@ -25,15 +32,21 @@ class ToolCorrectnessMetric(BaseMetric):
 
     def __init__(
         self,
+        available_tools: List[ToolCall] = None,
         threshold: float = 0.5,
         evaluation_params: List[ToolCallParams] = [],
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         include_reason: bool = True,
+        async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
         should_exact_match: bool = False,
         should_consider_ordering: bool = False,
     ):
+        self.available_tools = available_tools
         self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.async_mode = async_mode
         self.include_reason = include_reason
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
@@ -51,14 +64,140 @@ class ToolCorrectnessMetric(BaseMetric):
 
         check_llm_test_case_params(test_case, self._required_params, self)
         self.test_case = test_case
+        self.evaluation_cost = 0 if self.using_native_model else None
+
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                self.tools_called: List[ToolCall] = test_case.tools_called
+                self.expected_tools: List[ToolCall] = test_case.expected_tools
+                tool_calling_score = self._calculate_score()
+                if self.available_tools:
+                    tool_selection_score = self._get_tool_selection_score(
+                        test_case.input,
+                        test_case.tools_called,
+                        self.available_tools,
+                    )
+                else:
+                    tool_selection_score = tool_selection_score = (
+                        ToolSelectionScore(
+                            score=1,
+                            reason="No available tools were provided to assess tool selection criteria",
+                        )
+                    )
+                score = min(tool_calling_score, tool_selection_score.score)
+                self.score = (
+                    0 if self.strict_mode and score < self.threshold else score
+                )
+                tool_calling_reason = self._generate_reason()
+                self.reason = self._construct_final_reason(
+                    tool_calling_reason, tool_selection_score.reason
+                )
+                self.success = self.score >= self.threshold
+
+                expected_tools_formatted = (
+                    "Expected Tools:\n[\n"
+                    + ",\n".join(
+                        self.indent_multiline_string(
+                            repr(tool_call), indent_level=4
+                        )
+                        for tool_call in self.expected_tools
+                    )
+                    + "\n]"
+                )
+                tools_called_formatted = (
+                    "Tools Called:\n[\n"
+                    + ",\n".join(
+                        self.indent_multiline_string(
+                            repr(tool_call), indent_level=4
+                        )
+                        for tool_call in self.tools_called
+                    )
+                    + "\n]"
+                )
+                available_tools_formatted = (
+                    (
+                        "Available Tools:\n[\n"
+                        + ",\n".join(
+                            self.indent_multiline_string(
+                                repr(tool_call), indent_level=4
+                            )
+                            for tool_call in self.available_tools
+                        )
+                        + "\n]"
+                    )
+                    if self.available_tools
+                    else "Available Tools: []"
+                )
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"{expected_tools_formatted}",
+                        f"{tools_called_formatted}",
+                        f"{available_tools_formatted}",
+                        f"Tool Selection Score: {tool_selection_score.score}",
+                        f"Tool Selection Reason: {tool_selection_score.reason}",
+                        f"Final Score: {self.score}\nFinal Reason: {self.reason}",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             self.tools_called: List[ToolCall] = test_case.tools_called
             self.expected_tools: List[ToolCall] = test_case.expected_tools
-            self.score = self._calculate_score()
-            self.reason = self._generate_reason()
+            tool_calling_score = self._calculate_score()
+            if self.available_tools:
+                tool_selection_score = await self._a_get_tool_selection_score(
+                    test_case.input,
+                    test_case.tools_called,
+                    self.available_tools,
+                )
+            else:
+                tool_selection_score = ToolSelectionScore(
+                    score=1,
+                    reason="No available tools were provided to assess tool selection criteria",
+                )
+            score = min(tool_calling_score, tool_selection_score.score)
+            self.score = (
+                0 if self.strict_mode and score < self.threshold else score
+            )
+            tool_calling_reason = self._generate_reason()
+            self.reason = self._construct_final_reason(
+                tool_calling_reason, tool_selection_score.reason
+            )
             self.success = self.score >= self.threshold
+
             expected_tools_formatted = (
                 "Expected Tools:\n[\n"
                 + ",\n".join(
@@ -79,12 +218,31 @@ class ToolCorrectnessMetric(BaseMetric):
                 )
                 + "\n]"
             )
-            steps = [
-                f"{expected_tools_formatted}",
-                f"{tools_called_formatted}",
-            ]
-            steps.append(f"Score: {self.score}\nReason: {self.reason}")
-            self.verbose_logs = construct_verbose_logs(self, steps=steps)
+            available_tools_formatted = (
+                (
+                    "Available Tools:\n[\n"
+                    + ",\n".join(
+                        self.indent_multiline_string(
+                            repr(tool_call), indent_level=4
+                        )
+                        for tool_call in self.available_tools
+                    )
+                    + "\n]"
+                )
+                if self.available_tools
+                else "Available Tools: []"
+            )
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"{expected_tools_formatted}",
+                    f"{tools_called_formatted}",
+                    f"{available_tools_formatted}",
+                    f"Tool Selection Score: {tool_selection_score.score}",
+                    f"Tool Selection Reason: {tool_selection_score.reason}",
+                    f"Final Score: {self.score}\nFinal Reason: {self.reason}",
+                ],
+            )
 
             if _log_metric_to_confident:
                 metric_data_manager.post_metric_if_enabled(
@@ -92,19 +250,6 @@ class ToolCorrectnessMetric(BaseMetric):
                 )
             return self.score
 
-    async def a_measure(
-        self,
-        test_case: LLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        return self.measure(
-            test_case,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        )
-
     ##################################################
     ### Tool Correctness (Tool) ######################
     ##################################################
@@ -154,10 +299,69 @@ class ToolCorrectnessMetric(BaseMetric):
             else:
                 return f"Incomplete tool usage: missing tools {list(missing)}; expected {expected_tools_names}, called {tools_called_names}. See more details above."
 
+    def _construct_final_reason(
+        self,
+        tool_calling_reason,
+        tool_selection_reason,
+    ):
+        final_reason = "[\n"
+        final_reason += "\t Tool Calling Reason: " + tool_calling_reason + "\n"
+        final_reason += (
+            "\t Tool Selection Reason: " + tool_selection_reason + "\n"
+        )
+        final_reason += "]\n"
+        return final_reason
+
     ##################################################
     ### Score Helper Functions #######################
     ##################################################
 
+    def _get_tool_selection_score(
+        self, user_input, tools_called, available_tools
+    ):
+        tools_called_formatted = print_tools_called(tools_called)
+        available_tools_formatted = print_tools_called(available_tools)
+        prompt = ToolCorrectnessTemplate.get_tool_selection_score(
+            user_input, tools_called_formatted, available_tools_formatted
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=ToolSelectionScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = self.model.generate(prompt, schema=ToolSelectionScore)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return ToolSelectionScore(**data)
+
+    async def _a_get_tool_selection_score(
+        self, user_input, tools_called, available_tools
+    ):
+        tools_called_formatted = print_tools_called(tools_called)
+        available_tools_formatted = print_tools_called(available_tools)
+        prompt = ToolCorrectnessTemplate.get_tool_selection_score(
+            user_input, tools_called_formatted, available_tools_formatted
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=ToolSelectionScore
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = await self.model.a_generate(
+                    prompt, schema=ToolSelectionScore
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return ToolSelectionScore(**data)
+
     # Calculate score
     def _calculate_score(self):
         if self.should_exact_match:

deepeval/metrics/tool_use/__init__.py

@@ -0,0 +1 @@
+from .tool_use import ToolUseMetric