deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/knowledge_retention/knowledge_retention.py

@@ -5,9 +5,10 @@ from deepeval.metrics import BaseConversationalMetric
 from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
-    trimAndLoadJson,
     initialize_model,
     convert_turn_to_dict,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.knowledge_retention.template import (
@@ -51,7 +52,12 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -101,7 +107,12 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -147,23 +158,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             attritions=attritions,
             score=format(self.score, ".2f"),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: KnowledgeRetentionScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=KnowledgeRetentionScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=KnowledgeRetentionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self) -> str:
         if self.include_reason is False:
@@ -178,21 +179,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             attritions=attritions,
             score=format(self.score, ".2f"),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
-        else:
-            try:
-                res: KnowledgeRetentionScoreReason = self.model.generate(
-                    prompt, schema=KnowledgeRetentionScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=KnowledgeRetentionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, turns: List[Turn]
@@ -205,7 +198,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             accumulated_knowledge = [
                 knowledge.data
                 for knowledge in self.knowledges[:i]
-                if knowledge is not None
+                if knowledge is not None and knowledge.data
             ]
             if len(accumulated_knowledge) == 0:
                 continue
@@ -214,22 +207,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                 llm_message=turns[i].content,
                 accumulated_knowledge=accumulated_knowledge,
             )
-            if self.using_native_model:
-                res, cost = await self.model.a_generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                verdict = KnowledgeRetentionVerdict(**data)
-            else:
-                try:
-                    verdict: KnowledgeRetentionVerdict = (
-                        await self.model.a_generate(
-                            prompt, schema=KnowledgeRetentionVerdict
-                        )
-                    )
-                except TypeError:
-                    res = await self.model.a_generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    verdict = KnowledgeRetentionVerdict(**data)
+            verdict = await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=KnowledgeRetentionVerdict,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: KnowledgeRetentionVerdict(**data),
+            )
             verdicts.append(verdict)
         return verdicts
 
@@ -244,7 +228,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             accumulated_knowledge = [
                 knowledge.data
                 for knowledge in self.knowledges[:i]
-                if knowledge is not None
+                if knowledge is not None and knowledge.data
             ]
             if len(accumulated_knowledge) == 0:
                 continue
@@ -254,20 +238,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                 accumulated_knowledge=accumulated_knowledge,
             )
 
-            if self.using_native_model:
-                res, cost = self.model.generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                verdict = KnowledgeRetentionVerdict(**data)
-            else:
-                try:
-                    verdict: KnowledgeRetentionVerdict = self.model.generate(
-                        prompt, schema=KnowledgeRetentionVerdict
-                    )
-                except TypeError:
-                    res = self.model.generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    verdict = KnowledgeRetentionVerdict(**data)
+            verdict = generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=KnowledgeRetentionVerdict,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: KnowledgeRetentionVerdict(**data),
+            )
             verdicts.append(verdict)
         return verdicts
 
@@ -289,20 +266,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                     convert_turn_to_dict(turn) for turn in previous_turns
                 ],
             )
-            if self.using_native_model:
-                res, cost = await self.model.a_generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                knowledges[i] = Knowledge(data=data)
-            else:
-                try:
-                    knowledges[i] = await self.model.a_generate(
-                        prompt, schema=Knowledge
-                    )
-                except TypeError:
-                    res = await self.model.a_generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    knowledges[i] = Knowledge(data=data)
+            knowledges[i] = await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=Knowledge,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: Knowledge(data=data),
+            )
 
         return knowledges
 
@@ -325,20 +295,13 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                 ],
             )
 
-            if self.using_native_model:
-                res, cost = self.model.generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                knowledges[i] = Knowledge(data=data)
-            else:
-                try:
-                    knowledges[i] = self.model.generate(
-                        prompt, schema=Knowledge
-                    )
-                except TypeError:
-                    res = self.model.generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    knowledges[i] = Knowledge(data=data)
+            knowledges[i] = generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=Knowledge,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: Knowledge(data=data),
+            )
 
         return knowledges
 
@@ -361,8 +324,8 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/knowledge_retention/schema.py

@@ -1,15 +1,21 @@
-from typing import Dict, Optional, Any
-from pydantic import BaseModel
+from typing import Dict, Optional, Union, List
+from pydantic import BaseModel, ConfigDict
 
 
 class Knowledge(BaseModel):
-    data: Dict[str, Any]
+    # Each fact’s value is either a string or a list of strings
+    # data: Dict[str, Union[str, List[str]]]
+    data: Dict[str, Union[str, List[str]]] | None = None
+    # Forbid extra top-level fields to satisfy OpenAI’s schema requirements
+    model_config = ConfigDict(extra="forbid")
 
 
 class KnowledgeRetentionVerdict(BaseModel):
     verdict: str
     reason: Optional[str] = None
+    model_config = ConfigDict(extra="forbid")
 
 
 class KnowledgeRetentionScoreReason(BaseModel):
     reason: str
+    model_config = ConfigDict(extra="forbid")

deepeval/metrics/knowledge_retention/template.py

@@ -2,10 +2,20 @@ from typing import List, Dict, Any
 
 
 class KnowledgeRetentionTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_reason(attritions, score):
         return f"""Given a list of attritions, which highlights forgetfulness in the LLM response and knowledge established previously in the conversation, use it to CONCISELY provide a reason for the knowledge retention score. Note that The knowledge retention score ranges from 0 - 1, and the higher the better.
 
+{KnowledgeRetentionTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON:
@@ -33,6 +43,8 @@ JSON:
 
 Your task is to determine whether the LLM message **contradicts** or **forgets** any of the known facts.
 
+{KnowledgeRetentionTemplate.multimodal_rules}
+
 ---
 **Output format:**
 

deepeval/metrics/mcp/mcp_task_completion.py

@@ -7,13 +7,14 @@ from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
     get_unit_interactions,
-    trimAndLoadJson,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.mcp.schema import Task, TaskScore
+from deepeval.metrics.mcp.schema import Task, TaskScore, Reason
 from deepeval.metrics.mcp.template import MCPTaskCompletionTemplate
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics.api import metric_data_manager
@@ -50,7 +51,12 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -107,7 +113,12 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -149,48 +160,66 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
 
         return self.score
 
-    def _generate_reason(self, task_scores: List[TaskScore]) -> str:
-        reason = "["
+    def _generate_reason(self, task_scores: List[TaskScore]) -> Optional[str]:
+        if not self.include_reason:
+            return None
+
+        reasons = []
         for task_score in task_scores:
-            if task_score.score < self.threshold:
-                reason += (
-                    f"\nScore: {task_score.score}\n"
-                    f"Reason: {task_score.reason}\n"
-                )
-        reason += "]"
-        return reason
+            reasons.append(task_score.reason)
+
+        prompt = MCPTaskCompletionTemplate.generate_final_reason(
+            self.score, self.success, reasons
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
+
+    async def _a_generate_reason(
+        self, task_scores: List[TaskScore]
+    ) -> Optional[str]:
+        if not self.include_reason:
+            return None
+
+        reasons = []
+        for task_score in task_scores:
+            reasons.append(task_score.reason)
+
+        prompt = MCPTaskCompletionTemplate.generate_final_reason(
+            self.score, self.success, reasons
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Reason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_task_score(self, task: Task) -> TaskScore:
         prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=TaskScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: TaskScore = self.model.generate(prompt, schema=TaskScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TaskScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TaskScore(**data),
+        )
 
     async def _a_get_task_score(self, task: Task) -> TaskScore:
         prompt = MCPTaskCompletionTemplate.get_task_completion_score(task)
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=TaskScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: TaskScore = await self.model.a_generate(
-                    prompt, schema=TaskScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return TaskScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=TaskScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: TaskScore(**data),
+        )
 
     def _get_tasks(self, unit_interactions: List) -> List[Task]:
         tasks = []
@@ -244,9 +273,9 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         return tasks
 
     def _calculate_score(self, scores: List[TaskScore]) -> float:
-        score_divsor = len(scores) if len(scores) > 0 else 1
+        score_divisor = len(scores) if len(scores) > 0 else 1
         total_score = sum(score.score for score in scores)
-        score = total_score / score_divsor
+        score = total_score / score_divisor
         return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
@@ -254,8 +283,8 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success