deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -1,10 +1,15 @@
-from typing import List, Dict
+from typing import List, Dict, Optional, Union
 
 from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
+    trimAndLoadJson,
+    initialize_model,
+    print_tools_called,
 )
+from deepeval.models import DeepEvalBaseLLM
 from deepeval.test_case import (
     LLMTestCase,
     LLMTestCaseParams,
@@ -12,6 +17,9 @@ from deepeval.test_case import (
     ToolCall,
 )
 from deepeval.metrics import BaseMetric
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics.tool_correctness.template import ToolCorrectnessTemplate
+from deepeval.metrics.tool_correctness.schema import ToolSelectionScore
 
 
 class ToolCorrectnessMetric(BaseMetric):
@@ -24,15 +32,21 @@ class ToolCorrectnessMetric(BaseMetric):
 
     def __init__(
         self,
+        available_tools: List[ToolCall] = None,
         threshold: float = 0.5,
         evaluation_params: List[ToolCallParams] = [],
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         include_reason: bool = True,
+        async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
         should_exact_match: bool = False,
         should_consider_ordering: bool = False,
     ):
+        self.available_tools = available_tools
         self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.async_mode = async_mode
         self.include_reason = include_reason
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
@@ -45,18 +59,145 @@ class ToolCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
         self.test_case = test_case
+        self.evaluation_cost = 0 if self.using_native_model else None
+
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                self.tools_called: List[ToolCall] = test_case.tools_called
+                self.expected_tools: List[ToolCall] = test_case.expected_tools
+                tool_calling_score = self._calculate_score()
+                if self.available_tools:
+                    tool_selection_score = self._get_tool_selection_score(
+                        test_case.input,
+                        test_case.tools_called,
+                        self.available_tools,
+                    )
+                else:
+                    tool_selection_score = tool_selection_score = (
+                        ToolSelectionScore(
+                            score=1,
+                            reason="No available tools were provided to assess tool selection criteria",
+                        )
+                    )
+                score = min(tool_calling_score, tool_selection_score.score)
+                self.score = (
+                    0 if self.strict_mode and score < self.threshold else score
+                )
+                tool_calling_reason = self._generate_reason()
+                self.reason = self._construct_final_reason(
+                    tool_calling_reason, tool_selection_score.reason
+                )
+                self.success = self.score >= self.threshold
+
+                expected_tools_formatted = (
+                    "Expected Tools:\n[\n"
+                    + ",\n".join(
+                        self.indent_multiline_string(
+                            repr(tool_call), indent_level=4
+                        )
+                        for tool_call in self.expected_tools
+                    )
+                    + "\n]"
+                )
+                tools_called_formatted = (
+                    "Tools Called:\n[\n"
+                    + ",\n".join(
+                        self.indent_multiline_string(
+                            repr(tool_call), indent_level=4
+                        )
+                        for tool_call in self.tools_called
+                    )
+                    + "\n]"
+                )
+                available_tools_formatted = (
+                    (
+                        "Available Tools:\n[\n"
+                        + ",\n".join(
+                            self.indent_multiline_string(
+                                repr(tool_call), indent_level=4
+                            )
+                            for tool_call in self.available_tools
+                        )
+                        + "\n]"
+                    )
+                    if self.available_tools
+                    else "Available Tools: []"
+                )
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"{expected_tools_formatted}",
+                        f"{tools_called_formatted}",
+                        f"{available_tools_formatted}",
+                        f"Tool Selection Score: {tool_selection_score.score}",
+                        f"Tool Selection Reason: {tool_selection_score.reason}",
+                        f"Final Score: {self.score}\nFinal Reason: {self.reason}",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             self.tools_called: List[ToolCall] = test_case.tools_called
             self.expected_tools: List[ToolCall] = test_case.expected_tools
-            self.score = self._calculate_score()
-            self.reason = self._generate_reason()
+            tool_calling_score = self._calculate_score()
+            if self.available_tools:
+                tool_selection_score = await self._a_get_tool_selection_score(
+                    test_case.input,
+                    test_case.tools_called,
+                    self.available_tools,
+                )
+            else:
+                tool_selection_score = ToolSelectionScore(
+                    score=1,
+                    reason="No available tools were provided to assess tool selection criteria",
+                )
+            score = min(tool_calling_score, tool_selection_score.score)
+            self.score = (
+                0 if self.strict_mode and score < self.threshold else score
+            )
+            tool_calling_reason = self._generate_reason()
+            self.reason = self._construct_final_reason(
+                tool_calling_reason, tool_selection_score.reason
+            )
             self.success = self.score >= self.threshold
+
             expected_tools_formatted = (
                 "Expected Tools:\n[\n"
                 + ",\n".join(
@@ -77,25 +218,37 @@ class ToolCorrectnessMetric(BaseMetric):
                 )
                 + "\n]"
             )
-            steps = [
-                f"{expected_tools_formatted}",
-                f"{tools_called_formatted}",
-            ]
-            steps.append(f"Score: {self.score}\nReason: {self.reason}")
-            self.verbose_logs = construct_verbose_logs(self, steps=steps)
-            return self.score
+            available_tools_formatted = (
+                (
+                    "Available Tools:\n[\n"
+                    + ",\n".join(
+                        self.indent_multiline_string(
+                            repr(tool_call), indent_level=4
+                        )
+                        for tool_call in self.available_tools
+                    )
+                    + "\n]"
+                )
+                if self.available_tools
+                else "Available Tools: []"
+            )
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"{expected_tools_formatted}",
+                    f"{tools_called_formatted}",
+                    f"{available_tools_formatted}",
+                    f"Tool Selection Score: {tool_selection_score.score}",
+                    f"Tool Selection Reason: {tool_selection_score.reason}",
+                    f"Final Score: {self.score}\nFinal Reason: {self.reason}",
+                ],
+            )
 
-    async def a_measure(
-        self,
-        test_case: LLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-    ) -> float:
-        return self.measure(
-            test_case,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+            return self.score
 
     ##################################################
     ### Tool Correctness (Tool) ######################
@@ -146,10 +299,69 @@ class ToolCorrectnessMetric(BaseMetric):
             else:
                 return f"Incomplete tool usage: missing tools {list(missing)}; expected {expected_tools_names}, called {tools_called_names}. See more details above."
 
+    def _construct_final_reason(
+        self,
+        tool_calling_reason,
+        tool_selection_reason,
+    ):
+        final_reason = "[\n"
+        final_reason += "\t Tool Calling Reason: " + tool_calling_reason + "\n"
+        final_reason += (
+            "\t Tool Selection Reason: " + tool_selection_reason + "\n"
+        )
+        final_reason += "]\n"
+        return final_reason
+
     ##################################################
     ### Score Helper Functions #######################
     ##################################################
 
+    def _get_tool_selection_score(
+        self, user_input, tools_called, available_tools
+    ):
+        tools_called_formatted = print_tools_called(tools_called)
+        available_tools_formatted = print_tools_called(available_tools)
+        prompt = ToolCorrectnessTemplate.get_tool_selection_score(
+            user_input, tools_called_formatted, available_tools_formatted
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=ToolSelectionScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = self.model.generate(prompt, schema=ToolSelectionScore)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return ToolSelectionScore(**data)
+
+    async def _a_get_tool_selection_score(
+        self, user_input, tools_called, available_tools
+    ):
+        tools_called_formatted = print_tools_called(tools_called)
+        available_tools_formatted = print_tools_called(available_tools)
+        prompt = ToolCorrectnessTemplate.get_tool_selection_score(
+            user_input, tools_called_formatted, available_tools_formatted
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=ToolSelectionScore
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res = await self.model.a_generate(
+                    prompt, schema=ToolSelectionScore
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return ToolSelectionScore(**data)
+
     # Calculate score
     def _calculate_score(self):
         if self.should_exact_match:

deepeval/metrics/tool_use/__init__.py

@@ -0,0 +1 @@
+from .tool_use import ToolUseMetric

deepeval/metrics/tool_use/schema.py

@@ -0,0 +1,19 @@
+from pydantic import BaseModel
+
+
+class UserInputAndTools(BaseModel):
+    user_messages: str
+    assistant_messages: str
+    tools_called: str
+    available_tools: str
+    tools_used: bool
+
+
+class ToolSelectionScore(BaseModel):
+    score: float
+    reason: str
+
+
+class ArgumentCorrectnessScore(BaseModel):
+    score: float
+    reason: str

deepeval/metrics/tool_use/template.py

@@ -0,0 +1,220 @@
+import textwrap
+import json
+
+
+class ToolUseTemplate:
+
+    @staticmethod
+    def get_tool_selection_score(
+        user_input: str,
+        assistant_messages: str,
+        tools_called: str,
+        available_tools: str,
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are an expert evaluator assessing the **Tool Selection Quality** of an AI agent.
+
+                OBJECTIVE
+                Evaluate whether the agent **selected the most appropriate tools** for completing the user's task, given a list of available tools.
+
+                This metric focuses **only** on which tools were chosen — **not** how they were used or whether they succeeded.
+
+                EVALUATION RULES
+
+                1. Relevance
+                - Each tool used must directly support the user's stated goal or a clear sub-task derived from it.
+                - Tools unrelated to the goal lower the score sharply.
+
+                2. Appropriateness
+                - The chosen tools must match their described purpose.
+                - If a more suitable tool existed and was ignored, score ≤ 0.5.
+
+                3. Necessity
+                - Every tool call must be justified by clear need.
+                - Redundant or speculative tool use (e.g., calling multiple tools that overlap) reduces the score.
+
+                4. Strictness
+                - When uncertain if a tool was required or correctly chosen, assume it was **not** appropriate.
+                - Only perfect alignment between the task and tool choice earns a high score.
+
+                SCORING GUIDE:
+
+                - **1.0** → Every tool used was necessary and perfectly matched to the task; no better alternative ignored.  
+                - **0.75** → Tool selection was mostly correct, with only minor redundancy or a small omission.  
+                - **0.5** → Mixed quality; some appropriate selections, but others questionable or missing.  
+                - **0.25** → Poor selection; major mismatches or misuse of available tools.  
+                - **0.0** → Tool selection irrelevant, random, or unjustified.
+
+                OUTPUT FORMAT:
+
+                Return a JSON object with:
+                
+                {{
+                    "score": float between 0.0 and 1.0,
+                    "reason": "1-3 factual sentences explaining which tools were appropriate or inappropriate for the task, referencing specific tool names."
+                }}
+
+                USER INPUT:
+                {user_input}
+
+                ASSISTANT MESSAGES:
+                {assistant_messages}
+
+                TOOLS CALLED:
+                {tools_called}
+
+                AVAILABLE TOOLS:
+                {available_tools}
+
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def get_argument_correctness_score(
+        user_input: str,
+        assistant_messages: str,
+        tools_called: str,
+        available_tools: str,
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are an expert evaluator assessing the **Tool Argument Quality** of an AI agent.
+
+                OBJECTIVE:
+
+                Evaluate whether the **arguments and parameters** passed to each tool were:
+                - Correctly structured and complete.
+                - Contextually appropriate for the user's goal.
+                - Compatible with each tool's intended purpose.
+
+                This metric focuses **only** on argument-level correctness and relevance — not which tools were chosen.
+
+                EVALUATION RULES
+
+                1. Relevance
+                - Each argument must align with the task and the tool's documented input fields.
+                - Unrelated, empty, or default arguments reduce the score sharply.
+
+                2. **Completeness**
+                - All required parameters must be provided.
+                - Missing or malformed arguments (e.g., wrong data types or incomplete context) lower the score.
+
+                3. **Specificity**
+                - Arguments should reflect task-specific values, not generic placeholders.
+                - Overly vague or default arguments are penalized.
+
+                4. **Justification**
+                - Each argument must make sense in context.
+                - If it doesn't clearly derive from the user's request, assume it's incorrect.
+
+                5. **Strict Bias**
+                - When uncertain whether arguments fit the tool or task, assume they were **incorrect**.
+
+                SCORING GUIDE:
+
+                - **1.0** → All arguments are accurate, specific, and fully aligned with both the task and tool requirements.  
+                - **0.75** → Mostly correct; minor omissions or small mismatches.  
+                - **0.5** → Partial correctness; some valid parameters, but key ones missing or off-target.  
+                - **0.25** → Poor argument quality; several invalid or irrelevant fields.  
+                - **0.0** → Arguments nonsensical, generic, or unrelated to task/tool intent.
+
+                OUTPUT FORMAT:
+
+                Return a JSON object with:
+                {{
+                    "score": float between 0.0 and 1.0,
+                    "reason": "1-3 sentences explaining argument alignment or issues, referencing specific parameter names or values when possible."
+                }}
+
+                ---
+
+                USER INPUT:
+                {user_input}
+
+                ASSISTANT MESSAGES:
+                {assistant_messages}
+
+                TOOLS CALLED (with arguments):
+                {tools_called}
+
+                AVAILABLE TOOLS:
+                {available_tools}
+
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def get_tool_selection_final_reason(
+        all_scores_and_reasons: str, final_score: float, threshold: float
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are an expert evaluator summarizing the outcome of a **Tool Selection** evaluation.
+
+            You are given:
+            - A list of **tool selection sub-scores and reasons**, each describing how appropriately the agent chose tools for its task.
+            - The **final aggregated score** across all sub-evaluations.
+            - A **threshold** representing the minimum passing score.
+
+            Your task is to write a **single concise explanation (1-3 sentences)** that captures:
+            - Why the agent **passed or failed** based on tool choice quality.
+            - The key patterns or trends in the sub-reasons (e.g., consistent correct choices, repeated irrelevant tool calls, missed best-fit tools).
+            - A clear statement linking the **score** and **threshold** outcome (e.g., “The agent passed because…” or “Failed because…”).
+
+            RULES:
+            - Focus on *which tools were selected* and *why that selection pattern was or wasn't appropriate*.
+            - Mention specific issues or strengths like redundancy, misuse, or perfect matching.
+            - Avoid vague or subjective language such as “pretty good” or “reasonable”.
+            - Do **not** reference argument-level details; this summary is only for tool choice quality.
+            - The result must read as a self-contained, factual justification.
+
+            FORMAT:
+            Return only a single plain-text string. Do **not** include JSON or other formatting.
+
+            All Tool Selection Sub-Scores and Reasons:
+            {all_scores_and_reasons}
+
+            Final Score: {final_score}
+            Threshold: {threshold}
+            Result: {"PASS" if final_score >= threshold else "FAIL"}
+
+            Final Reason:
+            """
+        )
+
+    @staticmethod
+    def get_tool_argument_final_reason(
+        all_scores_and_reasons: str, final_score: float, threshold: float
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are an expert evaluator summarizing the outcome of a **Tool Argument Quality** evaluation.
+
+            You are given:
+            - A list of **argument-level sub-scores and reasons**, each evaluating whether the arguments passed to tools were accurate, complete, and contextually appropriate.
+            - The **final aggregated score** across all argument evaluations.
+            - A **threshold** representing the minimum passing score.
+
+            Your task is to write a **single concise explanation (1-3 sentences)** that clearly states:
+            - Why the agent **passed or failed** in its use of tool arguments.
+            - The dominant strengths or weaknesses from the sub-reasons (e.g., correct parameterization, missing required fields, generic values, or misaligned arguments).
+            - Whether the agent met or fell short of the threshold and why.
+
+            RULES:
+            - Focus strictly on **argument correctness** and **context alignment** — not which tools were chosen.
+            - Reference specific argument-level problems or successes where helpful.
+            - Keep language objective and factual; avoid speculation or vague phrasing.
+            - The summary must stand alone as a clear explanation of the final result.
+
+            FORMAT:
+            Return only a single plain-text string. Do **not** include JSON or any extra formatting.
+
+            All Tool Argument Sub-Scores and Reasons:
+            {all_scores_and_reasons}
+
+            Final Score: {final_score}
+            Threshold: {threshold}
+            Result: {"PASS" if final_score >= threshold else "FAIL"}
+
+            Final Reason:
+            """
+        )