deepeval 3.7.5__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -6,17 +6,22 @@ from deepeval.utils import (
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
-    check_mllm_test_case_params,
     initialize_model,
+    generate_with_schema_and_extract,
+    a_generate_with_schema_and_extract,
 )
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
 from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.answer_relevancy.template import AnswerRelevancyTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.answer_relevancy.schema import *
+from deepeval.metrics.answer_relevancy.schema import (
+    Statements,
+    AnswerRelevancyVerdict,
+    Verdicts,
+    AnswerRelevancyScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -55,13 +60,15 @@ class AnswerRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        multimodal = test_case.multimodal
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -82,13 +89,13 @@ class AnswerRelevancyMetric(BaseMetric):
                 actual_output = test_case.actual_output
 
                 self.statements: List[str] = self._generate_statements(
-                    actual_output, multimodal
+                    actual_output, test_case.multimodal
                 )
                 self.verdicts: List[AnswerRelevancyVerdict] = (
-                    self._generate_verdicts(input, multimodal)
+                    self._generate_verdicts(input, test_case.multimodal)
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(input, multimodal)
+                self.reason = self._generate_reason(input, test_case.multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -113,13 +120,15 @@ class AnswerRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        multimodal = test_case.multimodal
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -132,13 +141,15 @@ class AnswerRelevancyMetric(BaseMetric):
             actual_output = test_case.actual_output
 
             self.statements: List[str] = await self._a_generate_statements(
-                actual_output, multimodal
+                actual_output, test_case.multimodal
             )
             self.verdicts: List[AnswerRelevancyVerdict] = (
-                await self._a_generate_verdicts(input, multimodal)
+                await self._a_generate_verdicts(input, test_case.multimodal)
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(input, multimodal)
+            self.reason = await self._a_generate_reason(
+                input, test_case.multimodal
+            )
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -170,22 +181,13 @@ class AnswerRelevancyMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=AnswerRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: AnswerRelevancyScoreReason = await self.model.a_generate(
-                    prompt=prompt, schema=AnswerRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AnswerRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
@@ -203,22 +205,13 @@ class AnswerRelevancyMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=AnswerRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: AnswerRelevancyScoreReason = self.model.generate(
-                    prompt=prompt, schema=AnswerRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AnswerRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, input: str, multimodal: bool
@@ -230,22 +223,15 @@ class AnswerRelevancyMetric(BaseMetric):
             input=input, statements=self.statements, multimodal=multimodal
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    AnswerRelevancyVerdict(**item) for item in data["verdicts"]
-                ]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                AnswerRelevancyVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(
         self, input: str, multimodal: bool
@@ -257,22 +243,17 @@ class AnswerRelevancyMetric(BaseMetric):
             input=input, statements=self.statements, multimodal=multimodal
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    AnswerRelevancyVerdict(**item) for item in data["verdicts"]
-                ]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                AnswerRelevancyVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
-    async def _a_generate_statements(
+    def _generate_statements(
         self,
         actual_output: str,
         multimodal: bool,
@@ -280,31 +261,18 @@ class AnswerRelevancyMetric(BaseMetric):
         prompt = self.evaluation_template.generate_statements(
             actual_output=actual_output, multimodal=multimodal
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Statements)
-            self.evaluation_cost += cost
-            statements: List[str] = res.statements + [
-                ele for ele in actual_output if isinstance(ele, MLLMImage)
-            ]
-            return statements
-        else:
-            try:
-                res: Statements = await self.model.a_generate(
-                    prompt, schema=Statements
-                )
-                statements: List[str] = res.statements + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                statements = data["statements"] + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
 
-    def _generate_statements(
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Statements,
+            extract_schema=lambda s: s.statements
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+            extract_json=lambda d: d["statements"]
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+        )
+
+    async def _a_generate_statements(
         self,
         actual_output: str,
         multimodal: bool,
@@ -312,27 +280,16 @@ class AnswerRelevancyMetric(BaseMetric):
         prompt = self.evaluation_template.generate_statements(
             actual_output=actual_output, multimodal=multimodal
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Statements)
-            self.evaluation_cost += cost
-            statements = res.statements + [
-                ele for ele in actual_output if isinstance(ele, MLLMImage)
-            ]
-            return statements
-        else:
-            try:
-                res: Statements = self.model.generate(prompt, schema=Statements)
-                statements = res.statements + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                statements = data["statements"] + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Statements,
+            extract_schema=lambda s: s.statements
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+            extract_json=lambda d: d["statements"]
+            + [ele for ele in actual_output if isinstance(ele, MLLMImage)],
+        )
 
     def _calculate_score(self):
         number_of_verdicts = len(self.verdicts)
@@ -353,7 +310,7 @@ class AnswerRelevancyMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/answer_relevancy/template.py

@@ -3,170 +3,93 @@ import textwrap
 
 
 class AnswerRelevancyTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_statements(actual_output: str, multimodal: bool = False):
-        multimodal_instruction = ""
-        example_text = ""
-        example_json = ""
-
-        if multimodal:
-            multimodal_instruction = " The text may contain images as well."
-            example_text = "Shoes. The shoes can be refunded at no extra cost. Thanks for asking the question!"
-            example_json = textwrap.dedent(
-                """
-                {{
-                    "statements": ["Shoes.", "Shoes can be refunded at no extra cost", "Thanks for asking the question!"]
-                }}
-                """
-            )
-        else:
-            example_text = "Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we've added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support."
-            example_json = textwrap.dedent(
-                """
-                {{
-                    "statements": [
-                        "The new laptop model has a high-resolution Retina display.",
-                        "It includes a fast-charging battery with up to 12 hours of usage.",
-                        "Security features include fingerprint authentication and an encrypted SSD.",
-                        "Every purchase comes with a one-year warranty.",
-                        "24/7 customer support is included."
-                    ]
-                }}
-                """
-            )
-
-        coherence_note = (
-            ""
-            if multimodal
-            else " Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement."
-        )
-
-        return textwrap.dedent(
-            f"""Given the text, breakdown and generate a list of statements presented.{coherence_note}{multimodal_instruction}
-
-            Example:
-            Example text: 
-            {example_text}
-
-            {example_json}
-            ===== END OF EXAMPLE ======
-                    
-            **
-            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-            **
-
-            Text:
-            {actual_output}
-
-            JSON:
-            """
-        )
+        return f"""Given the text, breakdown and generate a list of statements presented. Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement.
+
+Example:
+Example text: 
+Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we’ve added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support.
+
+{AnswerRelevancyTemplate.multimodal_rules if multimodal else ""}
+
+{{
+    "statements": [
+        "The new laptop model has a high-resolution Retina display.",
+        "It includes a fast-charging battery with up to 12 hours of usage.",
+        "Security features include fingerprint authentication and an encrypted SSD.",
+        "Every purchase comes with a one-year warranty.",
+        "24/7 customer support is included."
+    ]
+}}
+===== END OF EXAMPLE ======
+        
+**
+IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+**
+
+Text:
+{actual_output}
+
+JSON:
+"""
 
     @staticmethod
     def generate_verdicts(
         input: str, statements: str, multimodal: bool = False
     ):
-        content_type = (
-            "statements (which can contain images)"
-            if multimodal
-            else "list of statements"
-        )
-        statement_or_image = "statement or image" if multimodal else "statement"
-
-        format_instruction = textwrap.dedent(
-            """
-            Expected JSON format:
-            {{
-                "verdicts": [
-                    {{
-                        "verdict": "yes"
-                    }},
-                    {{
-                        "reason": <explanation_for_irrelevance>,
-                        "verdict": "no"
-                    }},
-                    {{
-                        "reason": <explanation_for_ambiguity>,
-                        "verdict": "idk"
-                    }}
-                ]  
-            }}
-            """
-        )
-
-        example_section = ""
-        if multimodal:
-            example_section = textwrap.dedent(
-                """
-                Example input: What should I do if there is an earthquake?
-                Example statements: ["Shoes.", "Thanks for asking the question!", "Is there anything else I can help you with?", "Duck and hide"]
-                Example JSON:
-                {{
-                    "verdicts": [
-                        {{
-                            "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake.",
-                            "verdict": "no"
-                        }},
-                        {{
-                            "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant.",
-                            "verdict": "idk"
-                        }},
-                        {{
-                            "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant.",
-                            "verdict": "idk"
-                        }},
-                        {{
-                            "verdict": "yes"
-                        }}
-                    ]  
-                }}
-                """
-            )
-
-        guidelines = ""
-        if multimodal:
-            guidelines = textwrap.dedent(
-                f"""
-                Since you are going to generate a verdict for each statement and image, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of `statements`.
-                """
-            )
-        else:
-            guidelines = textwrap.dedent(
-                f"""
-                Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
-                'verdict' must be STRICTLY 'yes', 'no', or 'idk':
-                - 'yes': statement is relevant to addressing the input
-                - 'no': statement is irrelevant to the input  
-                - 'idk': statement is ambiguous (not directly relevant but could be supporting information)
-                Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-                """
-            )
-
-        return textwrap.dedent(
-            f"""For the provided {content_type}, determine whether each {statement_or_image} is relevant to address the input.
-            {"Please generate a list of JSON with two keys: `verdict` and `reason`." if multimodal else "Generate JSON objects with 'verdict' and 'reason' fields."}
-            The 'verdict' {"key " if multimodal else ''}should {"STRICTLY be either a 'yes', 'idk' or 'no'" if multimodal else "be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information)"}. {"Answer 'yes' if the " + statement_or_image + ' is relevant to addressing the original input, no if the ' + statement_or_image + ' is irrelevant, and "idk" if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).' if multimodal else ""}
-            {"The 'reason' is the reason for the verdict.' if multimodal else '"}
-            Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-            The {"provided statements are statements and images' if multimodal else 'statements are from an AI's actual output"} generated in the actual output.
-
-            **
-            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-
-            {format_instruction if not multimodal else ''}
-            {example_section}
-            {guidelines}
-            **          
-
-            Input:
-            {input}
-
-            Statements:
-            {statements}
-
-            JSON:
-            """
-        )
+        return f"""For the provided list of statements, determine whether each statement is relevant to address the input.
+Generate JSON objects with 'verdict' and 'reason' fields.
+The 'verdict' should be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information).
+Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+The statements are from an AI's actual output.
+
+{AnswerRelevancyTemplate.multimodal_rules if multimodal else ""}
+
+**
+IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+
+Expected JSON format:
+{{
+    "verdicts": [
+        {{
+            "verdict": "yes"
+        }},
+        {{
+            "reason": <explanation_for_irrelevance>,
+            "verdict": "no"
+        }},
+        {{
+            "reason": <explanation_for_ambiguity>,
+            "verdict": "idk"
+        }}
+    ]  
+}}
+
+Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
+'verdict' must be STRICTLY 'yes', 'no', or 'idk':
+- 'yes': statement is relevant to addressing the input
+- 'no': statement is irrelevant to the input  
+- 'idk': statement is ambiguous (not directly relevant but could be supporting information)
+Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+**          
+
+Input:
+{input}
+
+Statements:
+{statements}
+
+JSON:
+"""
 
     @staticmethod
     def generate_reason(
@@ -175,32 +98,32 @@ class AnswerRelevancyTemplate:
         score: float,
         multimodal: bool = False,
     ):
-        return textwrap.dedent(
-            f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
-            The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
-            If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+        return f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
+The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
+If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+
+{AnswerRelevancyTemplate.multimodal_rules if multimodal else ""}
 
+**
+IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
 
-            **
-            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+Example:
+Example JSON:
+{{
+    "reason": "The score is <answer_relevancy_score> because <your_reason>."
+}}
+===== END OF EXAMPLE ======
+**
 
-            {"Example:' if not multimodal else '"}
-            Example JSON:
-            {{
-                "reason": "The score is <answer_relevancy_score> because <your_reason>."
-            }}
-            {"===== END OF EXAMPLE ======' if not multimodal else '"}
-            **
 
-            Answer Relevancy Score:
-            {score}
+Answer Relevancy Score:
+{score}
 
-            Reasons why the score can't be higher based on irrelevant statements in the actual output:
-            {irrelevant_statements}
+Reasons why the score can't be higher based on irrelevant statements in the actual output:
+{irrelevant_statements}
 
-            Input:
-            {input}
+Input:
+{input}
 
-            JSON:
-            """
-        )
+JSON:
+"""