deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/task_completion/task_completion.py

@@ -1,11 +1,12 @@
 from typing import Optional, List, Tuple, Union, Dict
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import get_or_create_event_loop
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -15,7 +16,11 @@ from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.task_completion.template import TaskCompletionTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.task_completion.schema import *
+from deepeval.metrics.task_completion.schema import (
+    TaskAndOutcome,
+    TaskCompletionVerdict,
+)
+from deepeval.metrics.api import metric_data_manager
 
 
 class TaskCompletionMetric(BaseMetric):
@@ -23,7 +28,6 @@ class TaskCompletionMetric(BaseMetric):
     _required_params: List[LLMTestCaseParams] = [
         LLMTestCaseParams.INPUT,
         LLMTestCaseParams.ACTUAL_OUTPUT,
-        LLMTestCaseParams.TOOLS_CALLED,
     ]
 
     def __init__(
@@ -58,9 +62,15 @@ class TaskCompletionMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -91,6 +101,12 @@ class TaskCompletionMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
             return self.score
 
     async def a_measure(
@@ -100,9 +116,15 @@ class TaskCompletionMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -127,6 +149,12 @@ class TaskCompletionMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
             return self.score
 
     async def _a_generate_verdicts(self) -> Tuple:
@@ -134,44 +162,26 @@ class TaskCompletionMetric(BaseMetric):
             task=self.task,
             actual_outcome=self.outcome,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TaskCompletionVerdict
-            )
-            self.evaluation_cost += cost
-            return res.verdict, res.reason
-        else:
-            try:
-                res: TaskCompletionVerdict = await self.model.a_generate(
-                    prompt, schema=TaskCompletionVerdict
-                )
-                return res.verdict, res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["verdict"], data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskCompletionVerdict,
+            extract_schema=lambda s: (s.verdict, s.reason),
+            extract_json=lambda data: (data["verdict"], data["reason"]),
+        )
 
     def _generate_verdicts(self) -> Tuple:
         prompt = TaskCompletionTemplate.generate_verdict(
             task=self.task,
             actual_outcome=self.outcome,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=TaskCompletionVerdict
-            )
-            self.evaluation_cost += cost
-            return res.verdict, res.reason
-        else:
-            try:
-                res: TaskCompletionVerdict = self.model.generate(
-                    prompt, schema=TaskCompletionVerdict
-                )
-                return res.verdict, res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["verdict"], data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskCompletionVerdict,
+            extract_schema=lambda s: (s.verdict, s.reason),
+            extract_json=lambda data: (data["verdict"], data["reason"]),
+        )
 
     async def _a_extract_task_and_outcome(
         self,
@@ -189,22 +199,13 @@ class TaskCompletionMetric(BaseMetric):
                 actual_output=test_case.actual_output,
                 tools_called=test_case.tools_called,
             )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=TaskAndOutcome
-            )
-            self.evaluation_cost += cost
-            return res.task, res.outcome
-        else:
-            try:
-                res: TaskAndOutcome = await self.model.a_generate(
-                    prompt, schema=TaskAndOutcome
-                )
-                return res.task, res.outcome
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"], data["outcome"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskAndOutcome,
+            extract_schema=lambda s: (s.task, s.outcome),
+            extract_json=lambda data: (data["task"], data["outcome"]),
+        )
 
     def _extract_task_and_outcome(
         self,
@@ -222,20 +223,13 @@ class TaskCompletionMetric(BaseMetric):
                 actual_output=test_case.actual_output,
                 tools_called=test_case.tools_called,
             )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=TaskAndOutcome)
-            self.evaluation_cost += cost
-            return res.task, res.outcome
-        else:
-            try:
-                res: TaskAndOutcome = self.model.generate(
-                    prompt, schema=TaskAndOutcome
-                )
-                return res.task, res.outcome
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"], data["outcome"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskAndOutcome,
+            extract_schema=lambda s: (s.task, s.outcome),
+            extract_json=lambda data: (data["task"], data["outcome"]),
+        )
 
     def _calculate_score(self):
         return (
@@ -250,7 +244,7 @@ class TaskCompletionMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -1,13 +1,14 @@
-from typing import List, Dict, Optional, Union
+from typing import List, Dict, Optional, Union, Tuple
 
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import get_or_create_event_loop
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
-    trimAndLoadJson,
     initialize_model,
     print_tools_called,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.test_case import (
@@ -62,7 +63,15 @@ class ToolCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
         self.test_case = test_case
         self.evaluation_cost = 0 if self.using_native_model else None
 
@@ -83,18 +92,16 @@ class ToolCorrectnessMetric(BaseMetric):
                 self.tools_called: List[ToolCall] = test_case.tools_called
                 self.expected_tools: List[ToolCall] = test_case.expected_tools
                 tool_calling_score = self._calculate_score()
-                if self.available_tools:
+                if self.available_tools and not test_case.multimodal:
                     tool_selection_score = self._get_tool_selection_score(
                         test_case.input,
                         test_case.tools_called,
                         self.available_tools,
                     )
                 else:
-                    tool_selection_score = tool_selection_score = (
-                        ToolSelectionScore(
-                            score=1,
-                            reason="No available tools were provided to assess tool selection criteria",
-                        )
+                    tool_selection_score = ToolSelectionScore(
+                        score=1,
+                        reason="No available tools were provided to assess tool selection criteria",
                     )
                 score = min(tool_calling_score, tool_selection_score.score)
                 self.score = (
@@ -165,7 +172,15 @@ class ToolCorrectnessMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -177,7 +192,7 @@ class ToolCorrectnessMetric(BaseMetric):
             self.tools_called: List[ToolCall] = test_case.tools_called
             self.expected_tools: List[ToolCall] = test_case.expected_tools
             tool_calling_score = self._calculate_score()
-            if self.available_tools:
+            if self.available_tools and not test_case.multimodal:
                 tool_selection_score = await self._a_get_tool_selection_score(
                     test_case.input,
                     test_case.tools_called,
@@ -324,18 +339,13 @@ class ToolCorrectnessMetric(BaseMetric):
         prompt = ToolCorrectnessTemplate.get_tool_selection_score(
             user_input, tools_called_formatted, available_tools_formatted
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ToolSelectionScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = self.model.generate(prompt, schema=ToolSelectionScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     async def _a_get_tool_selection_score(
         self, user_input, tools_called, available_tools
@@ -345,25 +355,16 @@ class ToolCorrectnessMetric(BaseMetric):
         prompt = ToolCorrectnessTemplate.get_tool_selection_score(
             user_input, tools_called_formatted, available_tools_formatted
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ToolSelectionScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = await self.model.a_generate(
-                    prompt, schema=ToolSelectionScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     # Calculate score
-    def _calculate_score(self):
+    def _calculate_score(self) -> float:
         if self.should_exact_match:
             score = self._calculate_exact_match_score()
         elif self.should_consider_ordering:
@@ -382,7 +383,7 @@ class ToolCorrectnessMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     # Exact matching score
-    def _calculate_exact_match_score(self):
+    def _calculate_exact_match_score(self) -> float:
         if len(self.tools_called) != len(self.expected_tools):
             return 0.0
         if (
@@ -405,7 +406,7 @@ class ToolCorrectnessMetric(BaseMetric):
         return 1.0
 
     # Non exact matching score
-    def _calculate_non_exact_match_score(self):
+    def _calculate_non_exact_match_score(self) -> float:
         total_score = 0.0
         matched_called_tools = set()
         for expected_tool in self.expected_tools:
@@ -445,7 +446,7 @@ class ToolCorrectnessMetric(BaseMetric):
         )
 
     # Consider ordering score
-    def _compute_weighted_lcs(self):
+    def _compute_weighted_lcs(self) -> Tuple[List[ToolCall], float]:
         m, n = len(self.expected_tools), len(self.tools_called)
         dp = [[0.0] * (n + 1) for _ in range(m + 1)]
         for i in range(1, m + 1):

deepeval/metrics/tool_use/tool_use.py

@@ -3,11 +3,11 @@ import asyncio
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     get_unit_interactions,
-    print_tools_called,
     check_conversational_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     ConversationalTestCase,
@@ -61,7 +61,12 @@ class ToolUseMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -136,7 +141,12 @@ class ToolUseMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -206,22 +216,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ArgumentCorrectnessScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ArgumentCorrectnessScore = self.model.generate(
-                    prompt, schema=ArgumentCorrectnessScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ArgumentCorrectnessScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgumentCorrectnessScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ArgumentCorrectnessScore(**data),
+        )
 
     async def _a_get_argument_correctness_score(
         self,
@@ -233,22 +234,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ArgumentCorrectnessScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ArgumentCorrectnessScore = await self.model.a_generate(
-                    prompt, schema=ArgumentCorrectnessScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ArgumentCorrectnessScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgumentCorrectnessScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ArgumentCorrectnessScore(**data),
+        )
 
     def _get_tool_selection_score(
         self,
@@ -260,20 +252,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ToolSelectionScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ToolSelectionScore = self.model.generate(
-                    prompt, schema=ToolSelectionScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     async def _a_get_tool_selection_score(
         self,
@@ -285,22 +270,13 @@ class ToolUseMetric(BaseConversationalMetric):
             user_and_tools.tools_called,
             user_and_tools.available_tools,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ToolSelectionScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ToolSelectionScore = await self.model.a_generate(
-                    prompt, schema=ToolSelectionScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolSelectionScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolSelectionScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolSelectionScore(**data),
+        )
 
     def _get_user_input_and_turns(
         self,

deepeval/metrics/topic_adherence/template.py

@@ -3,6 +3,13 @@ import textwrap
 
 
 class TopicAdherenceTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
 
     @staticmethod
     def get_qa_pairs(
@@ -19,6 +26,8 @@ class TopicAdherenceTemplate:
                 Do not infer information beyond what is stated. Ignore irrelevant or conversational turns (e.g. greetings, affirmations) that do not constitute clear QA pairs.
                 If there are multiple questions and multiple answers in a single sentence, break them into separate pairs. Each pair must be standalone, and should not contain more than one question or response.
 
+                {TopicAdherenceTemplate.multimodal_rules}
+
                 OUTPUT Format:
                 Return a **JSON object** with a single 2 keys:
                 - `"question"`: the user's question
@@ -82,6 +91,8 @@ class TopicAdherenceTemplate:
                 3. Based on both relevance and correctness, assign one of four possible verdicts.
                 4. Give a simple, comprehensive reason explaining why this question-answer pair was assigned this verdict
 
+                {TopicAdherenceTemplate.multimodal_rules}
+
                 VERDICTS:
                 - `"TP"` (True Positive): Question is relevant and the response correctly answers it.
                 - `"FN"` (False Negative): Question is relevant, but the assistant refused to answer or gave an irrelevant response.
@@ -138,6 +149,8 @@ class TopicAdherenceTemplate:
                 
                 Your task is to go through these reasons and give a single final explaination that clearly explains why this metric has failed or passed.
 
+                {TopicAdherenceTemplate.multimodal_rules}
+
                 Pass: {success}
                 Score: {score}
                 Threshold: {threshold}