deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/indicator.py

@@ -10,10 +10,9 @@ from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
     BaseArenaMetric,
 )
-from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.test_run.cache import CachedTestCase, Cache
 from deepeval.telemetry import capture_metric_type
 from deepeval.utils import update_pbar
@@ -74,8 +73,8 @@ def metric_progress_indicator(
 async def measure_metric_task(
     task_id,
     progress,
-    metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
-    test_case: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    metric: Union[BaseMetric, BaseConversationalMetric],
+    test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -156,10 +155,8 @@ async def measure_metric_task(
 
 
 async def measure_metrics_with_indicator(
-    metrics: List[
-        Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric]
-    ],
-    test_case: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    metrics: List[Union[BaseMetric, BaseConversationalMetric]],
+    test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -238,8 +235,8 @@ async def measure_metrics_with_indicator(
 
 
 async def safe_a_measure(
-    metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
-    tc: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    metric: Union[BaseMetric, BaseConversationalMetric],
+    tc: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     ignore_errors: bool,
     skip_on_missing_params: bool,
     progress: Optional[Progress] = None,

deepeval/metrics/json_correctness/json_correctness.py

@@ -11,7 +11,8 @@ from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
     initialize_model,
-    trimAndLoadJson,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
@@ -46,6 +47,7 @@ class JsonCorrectnessMetric(BaseMetric):
         self.async_mode = async_mode
         self.verbose_mode = verbose_mode
         self.expected_schema = expected_schema
+        self.evaluation_model = self.model.get_model_name()
 
     def measure(
         self,
@@ -55,7 +57,16 @@ class JsonCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -77,7 +88,7 @@ class JsonCorrectnessMetric(BaseMetric):
                     self.expected_schema.model_validate_json(
                         test_case.actual_output
                     )
-                except ValidationError as e:
+                except ValidationError:
                     valid_json = False
 
                 self.score = 1 if valid_json else 0
@@ -106,7 +117,16 @@ class JsonCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -120,7 +140,7 @@ class JsonCorrectnessMetric(BaseMetric):
                 self.expected_schema.model_validate_json(
                     test_case.actual_output
                 )
-            except ValidationError as e:
+            except ValidationError:
                 valid_json = False
 
             self.score = 1 if valid_json else 0
@@ -156,22 +176,13 @@ class JsonCorrectnessMetric(BaseMetric):
             is_valid_json=is_valid_json,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=JsonCorrectnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: JsonCorrectnessScoreReason = await self.model.a_generate(
-                    prompt, schema=JsonCorrectnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=JsonCorrectnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def generate_reason(self, actual_output: str) -> str:
         if self.include_reason is False:
@@ -189,22 +200,13 @@ class JsonCorrectnessMetric(BaseMetric):
             is_valid_json=is_valid_json,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=JsonCorrectnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: JsonCorrectnessScoreReason = self.model.generate(
-                    prompt, schema=JsonCorrectnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=JsonCorrectnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -212,7 +214,7 @@ class JsonCorrectnessMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/json_correctness/template.py

@@ -2,12 +2,22 @@ from typing import Optional
 
 
 class JsonCorrectnessTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_reason(
         actual_output: str, expected_schema: str, is_valid_json: bool
     ):
         return f"""Based on the given generated json, generated by an LLM, and a boolean stating whether it is a valid JSON based on the expected json schema, give a reason why it is OR is not a valid Json.
 
+{JsonCorrectnessTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON:

deepeval/metrics/knowledge_retention/knowledge_retention.py

@@ -5,9 +5,10 @@ from deepeval.metrics import BaseConversationalMetric
 from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
-    trimAndLoadJson,
     initialize_model,
     convert_turn_to_dict,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.knowledge_retention.template import (
@@ -51,7 +52,12 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -101,7 +107,12 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -147,23 +158,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             attritions=attritions,
             score=format(self.score, ".2f"),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: KnowledgeRetentionScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=KnowledgeRetentionScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=KnowledgeRetentionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self) -> str:
         if self.include_reason is False:
@@ -178,21 +179,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             attritions=attritions,
             score=format(self.score, ".2f"),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: KnowledgeRetentionScoreReason = self.model.generate(
-                    prompt, schema=KnowledgeRetentionScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=KnowledgeRetentionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, turns: List[Turn]
@@ -205,7 +198,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             accumulated_knowledge = [
                 knowledge.data
                 for knowledge in self.knowledges[:i]
-                if knowledge is not None
+                if knowledge is not None and knowledge.data
             ]
             if len(accumulated_knowledge) == 0:
                 continue
@@ -214,22 +207,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                 llm_message=turns[i].content,
                 accumulated_knowledge=accumulated_knowledge,
             )
-            if self.using_native_model:
-                res, cost = await self.model.a_generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                verdict = KnowledgeRetentionVerdict(**data)
-            else:
-                try:
-                    verdict: KnowledgeRetentionVerdict = (
-                        await self.model.a_generate(
-                            prompt, schema=KnowledgeRetentionVerdict
-                        )
-                    )
-                except TypeError:
-                    res = await self.model.a_generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    verdict = KnowledgeRetentionVerdict(**data)
+            verdict = await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=KnowledgeRetentionVerdict,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: KnowledgeRetentionVerdict(**data),
+            )
             verdicts.append(verdict)
         return verdicts
 
@@ -244,7 +228,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             accumulated_knowledge = [
                 knowledge.data
                 for knowledge in self.knowledges[:i]
-                if knowledge is not None
+                if knowledge is not None and knowledge.data
             ]
             if len(accumulated_knowledge) == 0:
                 continue
@@ -254,20 +238,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                 accumulated_knowledge=accumulated_knowledge,
             )
 
-            if self.using_native_model:
-                res, cost = self.model.generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                verdict = KnowledgeRetentionVerdict(**data)
-            else:
-                try:
-                    verdict: KnowledgeRetentionVerdict = self.model.generate(
-                        prompt, schema=KnowledgeRetentionVerdict
-                    )
-                except TypeError:
-                    res = self.model.generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    verdict = KnowledgeRetentionVerdict(**data)
+            verdict = generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=KnowledgeRetentionVerdict,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: KnowledgeRetentionVerdict(**data),
+            )
             verdicts.append(verdict)
         return verdicts
 
@@ -289,20 +266,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                     convert_turn_to_dict(turn) for turn in previous_turns
                 ],
             )
-            if self.using_native_model:
-                res, cost = await self.model.a_generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                knowledges[i] = Knowledge(data=data)
-            else:
-                try:
-                    knowledges[i] = await self.model.a_generate(
-                        prompt, schema=Knowledge
-                    )
-                except TypeError:
-                    res = await self.model.a_generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    knowledges[i] = Knowledge(data=data)
+            knowledges[i] = await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=Knowledge,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: Knowledge(data=data),
+            )
 
         return knowledges
 
@@ -325,20 +295,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                 ],
             )
 
-            if self.using_native_model:
-                res, cost = self.model.generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                knowledges[i] = Knowledge(data=data)
-            else:
-                try:
-                    knowledges[i] = self.model.generate(
-                        prompt, schema=Knowledge
-                    )
-                except TypeError:
-                    res = self.model.generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    knowledges[i] = Knowledge(data=data)
+            knowledges[i] = generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=Knowledge,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: Knowledge(data=data),
+            )
 
         return knowledges
 
@@ -361,8 +324,8 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/knowledge_retention/schema.py

@@ -1,15 +1,21 @@
-from typing import Dict, Optional, Any
-from pydantic import BaseModel
+from typing import Dict, Optional, Union, List
+from pydantic import BaseModel, ConfigDict
 
 
 class Knowledge(BaseModel):
-    data: Dict[str, Any]
+    # Each fact’s value is either a string or a list of strings
+    # data: Dict[str, Union[str, List[str]]]
+    data: Dict[str, Union[str, List[str]]] | None = None
+    # Forbid extra top-level fields to satisfy OpenAI’s schema requirements
+    model_config = ConfigDict(extra="forbid")
 
 
 class KnowledgeRetentionVerdict(BaseModel):
     verdict: str
     reason: Optional[str] = None
+    model_config = ConfigDict(extra="forbid")
 
 
 class KnowledgeRetentionScoreReason(BaseModel):
     reason: str
+    model_config = ConfigDict(extra="forbid")

deepeval/metrics/knowledge_retention/template.py

@@ -2,10 +2,20 @@ from typing import List, Dict, Any
 
 
 class KnowledgeRetentionTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_reason(attritions, score):
         return f"""Given a list of attritions, which highlights forgetfulness in the LLM response and knowledge established previously in the conversation, use it to CONCISELY provide a reason for the knowledge retention score. Note that The knowledge retention score ranges from 0 - 1, and the higher the better.
 
+{KnowledgeRetentionTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON:
@@ -33,6 +43,8 @@ JSON:
 
 Your task is to determine whether the LLM message **contradicts** or **forgets** any of the known facts.
 
+{KnowledgeRetentionTemplate.multimodal_rules}
+
 ---
 **Output format:**
 

deepeval/metrics/mcp/mcp_task_completion.py

@@ -7,8 +7,9 @@ from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
     get_unit_interactions,
-    trimAndLoadJson,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, TurnParams
@@ -50,7 +51,12 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -107,7 +113,12 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -149,48 +160,67 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
 
         return self.score
 
-    def _generate_reason(self, task_scores: List[TaskScore]) -> str:
-        reason = "["
+    def _generate_reason(self, task_scores: List[TaskScore]) -> Optional[str]:
+        if not self.include_reason:
+            return None
+
+        reasons = []
         for task_score in task_scores:
-            if task_score.score < self.threshold:
-                reason += (
-                    f"\nScore: {task_score.score}\n"
-                    f"Reason: {task_score.reason}\n"
-                )
-        reason += "]"
-        return reason
+            reasons.append(task_score.reason)
+
+        prompt = MCPTaskCompletionTemplate.generate_final_reason(
+            self.score, self.success, reasons
+        )
 
-    def _get_task_score(self, task: Task) -> TaskScore:
-        prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=TaskScore)
+            res, cost = self.model.generate(prompt)
             self.evaluation_cost += cost
             return res
         else:
-            try:
-                res: TaskScore = self.model.generate(prompt, schema=TaskScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TaskScore(**data)
+            res = self.model.generate(prompt)
+            return res
+
+    async def _a_generate_reason(
+        self, task_scores: List[TaskScore]
+    ) -> Optional[str]:
+        if not self.include_reason:
+            return None
+
+        reasons = []
+        for task_score in task_scores:
+            reasons.append(task_score.reason)
+
+        prompt = MCPTaskCompletionTemplate.generate_final_reason(
+            self.score, self.success, reasons
+        )
 
-    async def _a_get_task_score(self, task: Task) -> TaskScore:
-        prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=TaskScore)
+            res, cost = await self.model.a_generate(prompt)
             self.evaluation_cost += cost
             return res
         else:
-            try:
-                res: TaskScore = await self.model.a_generate(
-                    prompt, schema=TaskScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TaskScore(**data)
+            res = await self.model.a_generate(prompt)
+            return res
+
+    def _get_task_score(self, task: Task) -> TaskScore:
+        prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TaskScore(**data),
+        )
+
+    async def _a_get_task_score(self, task: Task) -> TaskScore:
+        prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TaskScore(**data),
+        )
 
     def _get_tasks(self, unit_interactions: List) -> List[Task]:
         tasks = []
@@ -244,9 +274,9 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         return tasks
 
     def _calculate_score(self, scores: List[TaskScore]) -> float:
-        score_divsor = len(scores) if len(scores) > 0 else 1
+        score_divisor = len(scores) if len(scores) > 0 else 1
         total_score = sum(score.score for score in scores)
-        score = total_score / score_divsor
+        score = total_score / score_divisor
         return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
@@ -254,8 +284,8 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success