deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/task_completion/task_completion.py

@@ -1,11 +1,12 @@
 from typing import Optional, List, Tuple, Union, Dict
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import get_or_create_event_loop
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -15,7 +16,11 @@ from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.task_completion.template import TaskCompletionTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.task_completion.schema import *
+from deepeval.metrics.task_completion.schema import (
+    TaskAndOutcome,
+    TaskCompletionVerdict,
+)
+from deepeval.metrics.api import metric_data_manager
 
 
 class TaskCompletionMetric(BaseMetric):
@@ -23,7 +28,6 @@ class TaskCompletionMetric(BaseMetric):
     _required_params: List[LLMTestCaseParams] = [
         LLMTestCaseParams.INPUT,
         LLMTestCaseParams.ACTUAL_OUTPUT,
-        LLMTestCaseParams.TOOLS_CALLED,
     ]
 
     def __init__(
@@ -58,9 +62,15 @@ class TaskCompletionMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -91,6 +101,12 @@ class TaskCompletionMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
             return self.score
 
     async def a_measure(
@@ -100,9 +116,15 @@ class TaskCompletionMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -127,6 +149,12 @@ class TaskCompletionMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
             return self.score
 
     async def _a_generate_verdicts(self) -> Tuple:
@@ -134,44 +162,26 @@ class TaskCompletionMetric(BaseMetric):
             task=self.task,
             actual_outcome=self.outcome,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TaskCompletionVerdict
-            )
-            self.evaluation_cost += cost
-            return res.verdict, res.reason
-        else:
-            try:
-                res: TaskCompletionVerdict = await self.model.a_generate(
-                    prompt, schema=TaskCompletionVerdict
-                )
-                return res.verdict, res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["verdict"], data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskCompletionVerdict,
+            extract_schema=lambda s: (s.verdict, s.reason),
+            extract_json=lambda data: (data["verdict"], data["reason"]),
+        )
 
     def _generate_verdicts(self) -> Tuple:
         prompt = TaskCompletionTemplate.generate_verdict(
             task=self.task,
             actual_outcome=self.outcome,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=TaskCompletionVerdict
-            )
-            self.evaluation_cost += cost
-            return res.verdict, res.reason
-        else:
-            try:
-                res: TaskCompletionVerdict = self.model.generate(
-                    prompt, schema=TaskCompletionVerdict
-                )
-                return res.verdict, res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["verdict"], data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskCompletionVerdict,
+            extract_schema=lambda s: (s.verdict, s.reason),
+            extract_json=lambda data: (data["verdict"], data["reason"]),
+        )
 
     async def _a_extract_task_and_outcome(
         self,
@@ -189,22 +199,13 @@ class TaskCompletionMetric(BaseMetric):
                 actual_output=test_case.actual_output,
                 tools_called=test_case.tools_called,
             )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TaskAndOutcome
-            )
-            self.evaluation_cost += cost
-            return res.task, res.outcome
-        else:
-            try:
-                res: TaskAndOutcome = await self.model.a_generate(
-                    prompt, schema=TaskAndOutcome
-                )
-                return res.task, res.outcome
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"], data["outcome"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskAndOutcome,
+            extract_schema=lambda s: (s.task, s.outcome),
+            extract_json=lambda data: (data["task"], data["outcome"]),
+        )
 
     def _extract_task_and_outcome(
         self,
@@ -222,20 +223,13 @@ class TaskCompletionMetric(BaseMetric):
                 actual_output=test_case.actual_output,
                 tools_called=test_case.tools_called,
             )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=TaskAndOutcome)
-            self.evaluation_cost += cost
-            return res.task, res.outcome
-        else:
-            try:
-                res: TaskAndOutcome = self.model.generate(
-                    prompt, schema=TaskAndOutcome
-                )
-                return res.task, res.outcome
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"], data["outcome"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskAndOutcome,
+            extract_schema=lambda s: (s.task, s.outcome),
+            extract_json=lambda data: (data["task"], data["outcome"]),
+        )
 
     def _calculate_score(self):
         return (
@@ -250,7 +244,7 @@ class TaskCompletionMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -1,13 +1,14 @@
-from typing import List, Dict, Optional, Union
+from typing import List, Dict, Optional, Union, Tuple
 
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import get_or_create_event_loop
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
-    trimAndLoadJson,
     initialize_model,
     print_tools_called,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.test_case import (
@@ -62,7 +63,15 @@ class ToolCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
         self.test_case = test_case
         self.evaluation_cost = 0 if self.using_native_model else None
 
@@ -90,11 +99,9 @@ class ToolCorrectnessMetric(BaseMetric):
                         self.available_tools,
                     )
                 else:
-                    tool_selection_score = tool_selection_score = (
-                        ToolSelectionScore(
-                            score=1,
-                            reason="No available tools were provided to assess tool selection criteria",
-                        )
+                    tool_selection_score = ToolSelectionScore(
+                        score=1,
+                        reason="No available tools were provided to assess tool selection criteria",
                     )
                 score = min(tool_calling_score, tool_selection_score.score)
                 self.score = (
@@ -165,7 +172,15 @@ class ToolCorrectnessMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -324,18 +339,13 @@ class ToolCorrectnessMetric(BaseMetric):
         prompt = ToolCorrectnessTemplate.get_tool_selection_score(
             user_input, tools_called_formatted, available_tools_formatted
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ToolSelectionScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = self.model.generate(prompt, schema=ToolSelectionScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     async def _a_get_tool_selection_score(
         self, user_input, tools_called, available_tools
@@ -345,25 +355,16 @@ class ToolCorrectnessMetric(BaseMetric):
         prompt = ToolCorrectnessTemplate.get_tool_selection_score(
             user_input, tools_called_formatted, available_tools_formatted
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ToolSelectionScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = await self.model.a_generate(
-                    prompt, schema=ToolSelectionScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     # Calculate score
-    def _calculate_score(self):
+    def _calculate_score(self) -> float:
         if self.should_exact_match:
             score = self._calculate_exact_match_score()
         elif self.should_consider_ordering:
@@ -382,7 +383,7 @@ class ToolCorrectnessMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     # Exact matching score
-    def _calculate_exact_match_score(self):
+    def _calculate_exact_match_score(self) -> float:
         if len(self.tools_called) != len(self.expected_tools):
             return 0.0
         if (
@@ -405,7 +406,7 @@ class ToolCorrectnessMetric(BaseMetric):
         return 1.0
 
     # Non exact matching score
-    def _calculate_non_exact_match_score(self):
+    def _calculate_non_exact_match_score(self) -> float:
         total_score = 0.0
         matched_called_tools = set()
         for expected_tool in self.expected_tools:
@@ -445,7 +446,7 @@ class ToolCorrectnessMetric(BaseMetric):
         )
 
     # Consider ordering score
-    def _compute_weighted_lcs(self):
+    def _compute_weighted_lcs(self) -> Tuple[List[ToolCall], float]:
         m, n = len(self.expected_tools), len(self.tools_called)
         dp = [[0.0] * (n + 1) for _ in range(m + 1)]
         for i in range(1, m + 1):

deepeval/metrics/tool_use/schema.py

@@ -17,3 +17,7 @@ class ToolSelectionScore(BaseModel):
 class ArgumentCorrectnessScore(BaseModel):
     score: float
     reason: str
+
+
+class Reason(BaseModel):
+    reason: str

deepeval/metrics/tool_use/template.py

@@ -161,6 +161,13 @@ class ToolUseTemplate:
             - The key patterns or trends in the sub-reasons (e.g., consistent correct choices, repeated irrelevant tool calls, missed best-fit tools).
             - A clear statement linking the **score** and **threshold** outcome (e.g., “The agent passed because…” or “Failed because…”).
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <score> because <your_reason>."
+            }}
+
             RULES:
             - Focus on *which tools were selected* and *why that selection pattern was or wasn't appropriate*.
             - Mention specific issues or strengths like redundancy, misuse, or perfect matching.
@@ -178,7 +185,7 @@ class ToolUseTemplate:
             Threshold: {threshold}
             Result: {"PASS" if final_score >= threshold else "FAIL"}
 
-            Final Reason:
+            JSON:
             """
         )
 
@@ -199,6 +206,13 @@ class ToolUseTemplate:
             - The dominant strengths or weaknesses from the sub-reasons (e.g., correct parameterization, missing required fields, generic values, or misaligned arguments).
             - Whether the agent met or fell short of the threshold and why.
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <score> because <your_reason>."
+            }}
+
             RULES:
             - Focus strictly on **argument correctness** and **context alignment** — not which tools were chosen.
             - Reference specific argument-level problems or successes where helpful.
@@ -215,6 +229,6 @@ class ToolUseTemplate:
             Threshold: {threshold}
             Result: {"PASS" if final_score >= threshold else "FAIL"}
 
-            Final Reason:
+            JSON:
             """
         )

deepeval/metrics/tool_use/tool_use.py

@@ -3,11 +3,11 @@ import asyncio
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     get_unit_interactions,
-    print_tools_called,
     check_conversational_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     ConversationalTestCase,
@@ -23,6 +23,7 @@ from deepeval.metrics.tool_use.schema import (
     ToolSelectionScore,
     UserInputAndTools,
     ArgumentCorrectnessScore,
+    Reason,
 )
 from deepeval.metrics.api import metric_data_manager
 
@@ -61,7 +62,12 @@ class ToolUseMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -136,7 +142,12 @@ class ToolUseMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -206,22 +217,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ArgumentCorrectnessScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ArgumentCorrectnessScore = self.model.generate(
-                    prompt, schema=ArgumentCorrectnessScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ArgumentCorrectnessScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgumentCorrectnessScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ArgumentCorrectnessScore(**data),
+        )
 
     async def _a_get_argument_correctness_score(
         self,
@@ -233,22 +235,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ArgumentCorrectnessScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ArgumentCorrectnessScore = await self.model.a_generate(
-                    prompt, schema=ArgumentCorrectnessScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ArgumentCorrectnessScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgumentCorrectnessScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ArgumentCorrectnessScore(**data),
+        )
 
     def _get_tool_selection_score(
         self,
@@ -260,20 +253,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ToolSelectionScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ToolSelectionScore = self.model.generate(
-                    prompt, schema=ToolSelectionScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     async def _a_get_tool_selection_score(
         self,
@@ -285,22 +271,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ToolSelectionScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ToolSelectionScore = await self.model.a_generate(
-                    prompt, schema=ToolSelectionScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     def _get_user_input_and_turns(
         self,
@@ -380,13 +357,14 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason_for_argument_correctness(
         self,
@@ -400,13 +378,13 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason_for_tool_selection(
         self, tool_use_scores: List[ToolSelectionScore]
@@ -419,13 +397,13 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason_for_argument_correctness(
         self, argument_correctness_scores: List[ArgumentCorrectnessScore]
@@ -438,13 +416,13 @@ class ToolUseMetric(BaseConversationalMetric):
         prompt = ToolUseTemplate.get_tool_selection_final_reason(
             scores_and_reasons, self.score, self.threshold
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def is_successful(self) -> bool:
         try: