deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/summarization/summarization.py

@@ -5,16 +5,16 @@ from deepeval.metrics.api import metric_data_manager
 from deepeval.test_case import (
     LLMTestCase,
     LLMTestCaseParams,
-    ConversationalTestCase,
 )
 from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.summarization.template import SummarizationTemplate
 from deepeval.metrics.faithfulness.template import FaithfulnessTemplate
@@ -77,7 +77,15 @@ class SummarizationMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -138,7 +146,15 @@ class SummarizationMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -185,7 +201,7 @@ class SummarizationMetric(BaseMetric):
 
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -221,24 +237,15 @@ class SummarizationMetric(BaseMetric):
         prompt += """JSON:
 """
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=SummarizationScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: SummarizationScoreReason = await self.model.a_generate(
-                    prompt, schema=SummarizationScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=SummarizationScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
-    def _generate_reason(self) -> str:
+    def _generate_reason(self) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -274,22 +281,13 @@ class SummarizationMetric(BaseMetric):
         prompt += """JSON:
 """
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=SummarizationScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: SummarizationScoreReason = self.model.generate(
-                    prompt, schema=SummarizationScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=SummarizationScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(self, score_type: ScoreType) -> float:
         if score_type == ScoreType.ALIGNMENT:
@@ -327,69 +325,45 @@ class SummarizationMetric(BaseMetric):
         prompt = SummarizationTemplate.generate_answers(
             questions=self.assessment_questions, text=text
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Answers)
-            self.evaluation_cost += cost
-            return res.answers
-        else:
-            try:
-                res: Answers = await self.model.a_generate(
-                    prompt, schema=Answers
-                )
-                return res.answers
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["answers"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Answers,
+            extract_schema=lambda s: s.answers,
+            extract_json=lambda data: data["answers"],
+        )
 
     def _generate_answers(self, text: str) -> List[str]:
         prompt = SummarizationTemplate.generate_answers(
             questions=self.assessment_questions, text=text
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Answers)
-            self.evaluation_cost += cost
-            return res.answers
-        else:
-            try:
-                res: Answers = self.model.generate(prompt, schema=Answers)
-                return res.answers
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["answers"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Answers,
+            extract_schema=lambda s: s.answers,
+            extract_json=lambda data: data["answers"],
+        )
 
-    async def _a_generate_assessment_questions(self, text: str):
+    async def _a_generate_assessment_questions(self, text: str) -> List[str]:
         prompt = SummarizationTemplate.generate_questions(text=text, n=self.n)
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Questions)
-            self.evaluation_cost += cost
-            return res.questions
-        else:
-            try:
-                res: Questions = await self.model.a_generate(
-                    prompt, schema=Questions
-                )
-                return res.questions
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["questions"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Questions,
+            extract_schema=lambda s: s.questions,
+            extract_json=lambda data: data["questions"],
+        )
 
-    def _generate_assessment_questions(self, text: str):
+    def _generate_assessment_questions(self, text: str) -> List[str]:
         prompt = SummarizationTemplate.generate_questions(text=text, n=self.n)
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Questions)
-            self.evaluation_cost += cost
-            return res.questions
-        else:
-            try:
-                res: Questions = self.model.generate(prompt, schema=Questions)
-                return res.questions
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["questions"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Questions,
+            extract_schema=lambda s: s.questions,
+            extract_json=lambda data: data["questions"],
+        )
 
     async def _a_generate_coverage_verdicts(
         self, test_case: LLMTestCase
@@ -453,30 +427,19 @@ class SummarizationMetric(BaseMetric):
         if len(self.claims) == 0:
             return []
 
-        verdicts: List[SummarizationAlignmentVerdict] = []
         prompt = SummarizationTemplate.generate_alignment_verdicts(
             summary_claims=self.claims, original_text="\n\n".join(self.truths)
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    SummarizationAlignmentVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                SummarizationAlignmentVerdict(**item)
+                for item in data["verdicts"]
+            ],
+        )
 
     def _generate_alignment_verdicts(
         self,
@@ -484,30 +447,19 @@ class SummarizationMetric(BaseMetric):
         if len(self.claims) == 0:
             return []
 
-        verdicts: List[SummarizationAlignmentVerdict] = []
         prompt = SummarizationTemplate.generate_alignment_verdicts(
             summary_claims=self.claims, original_text="\n\n".join(self.truths)
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=SummarizationAlignmentVerdict
-            )
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    SummarizationAlignmentVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                SummarizationAlignmentVerdict(**item)
+                for item in data["verdicts"]
+            ],
+        )
 
     async def _a_generate_truths(self, text: str) -> List[str]:
         # Borrow faithfulness template
@@ -515,34 +467,24 @@ class SummarizationMetric(BaseMetric):
             retrieval_context=text,
             extraction_limit=self.truths_extraction_limit,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = await self.model.a_generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
+        )
 
     async def _a_generate_claims(self, text: str) -> List[str]:
         # Borrow faithfulness template
         prompt = FaithfulnessTemplate.generate_claims(actual_output=text)
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = await self.model.a_generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
+        )
 
     def _generate_truths(self, text: str) -> List[str]:
         # Borrow faithfulness template
@@ -550,34 +492,24 @@ class SummarizationMetric(BaseMetric):
             retrieval_context=text,
             extraction_limit=self.truths_extraction_limit,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = self.model.generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
+        )
 
     def _generate_claims(self, text: str) -> List[str]:
         # Borrow faithfulness template
         prompt = FaithfulnessTemplate.generate_claims(actual_output=text)
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = self.model.generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -585,7 +517,7 @@ class SummarizationMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/summarization/template.py

@@ -1,9 +1,20 @@
+multimodal_rules = """
+    --- MULTIMODAL INPUT RULES ---
+    - Treat image content as factual evidence.
+    - Only reference visual details that are explicitly and clearly visible.
+    - Do not infer or guess objects, text, or details not visibly present.
+    - If an image is unclear or ambiguous, mark uncertainty explicitly.
+"""
+
+
 class SummarizationTemplate:
     @staticmethod
     def generate_reason(contradictions, redundancies, questions, score):
         return f"""You will be given the following: 1) information in the summary contradicting the original text, 2) extra information in the summary not mentioned in the original text, 3) [Optional] questions cannot be answered by the summary. Your task is to explain the quality of this summarization task.
 Given the summarization score, which is a 0-1 score indicating how good the summary is to the original text (higher the better), CONCISELY summarize the provided information to justify the score.  
 
+{multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON:
@@ -28,6 +39,9 @@ Extra Information not mentioned in the original text:
     @staticmethod
     def generate_answers(questions, text):
         return f"""Based on the list of close-ended 'yes' or 'no' questions, generate a JSON with key 'answers', which is a list of strings that determines whether the provided text contains sufficient information to answer EACH question.
+
+{multimodal_rules}
+
 Answers should STRICTLY be either 'yes' or 'no'.
 Answer 'no' if the provided text does not contain enough information to answer the question.
 **
@@ -57,6 +71,8 @@ JSON:
     def generate_questions(text, n):
         return f"""Based on the given text, generate {n} closed-ended questions that can be answered with either a 'yes' or 'no'. 
 The questions generated should ALWAYS result in a 'yes' based on the given text. 
+
+{multimodal_rules}
         
 ** IMPORTANT
 Only return a JSON with a 'questions' key, which is a list of strings. 
@@ -72,6 +88,9 @@ JSON:
     @staticmethod
     def generate_alignment_verdicts(original_text, summary_claims):
         return f"""Based on the given summary claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH piece of info contradicts any facts in the original text. The JSON will have 2 fields: 'verdict' and 'reason'.
+
+{multimodal_rules}
+
 The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given summary claim agrees with the original text. 
 Provide a 'reason' ONLY if the answer is 'no' OR 'idk'. 
 The provided summary claims is drawn from the summary. Try to provide a correction in the reason using the facts in the original text.