deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/answer_relevancy/template.py

@@ -3,170 +3,93 @@ import textwrap
 
 
 class AnswerRelevancyTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_statements(actual_output: str, multimodal: bool = False):
-        multimodal_instruction = ""
-        example_text = ""
-        example_json = ""
-
-        if multimodal:
-            multimodal_instruction = " The text may contain images as well."
-            example_text = "Shoes. The shoes can be refunded at no extra cost. Thanks for asking the question!"
-            example_json = textwrap.dedent(
-                """
-                {{
-                    "statements": ["Shoes.", "Shoes can be refunded at no extra cost", "Thanks for asking the question!"]
-                }}
-                """
-            )
-        else:
-            example_text = "Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we've added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support."
-            example_json = textwrap.dedent(
-                """
-                {{
-                    "statements": [
-                        "The new laptop model has a high-resolution Retina display.",
-                        "It includes a fast-charging battery with up to 12 hours of usage.",
-                        "Security features include fingerprint authentication and an encrypted SSD.",
-                        "Every purchase comes with a one-year warranty.",
-                        "24/7 customer support is included."
-                    ]
-                }}
-                """
-            )
-
-        coherence_note = (
-            ""
-            if multimodal
-            else " Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement."
-        )
-
-        return textwrap.dedent(
-            f"""Given the text, breakdown and generate a list of statements presented.{coherence_note}{multimodal_instruction}
-
-            Example:
-            Example text: 
-            {example_text}
-
-            {example_json}
-            ===== END OF EXAMPLE ======
-                    
-            **
-            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-            **
-
-            Text:
-            {actual_output}
-
-            JSON:
-            """
-        )
+        return f"""Given the text, breakdown and generate a list of statements presented. Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement.
+
+Example:
+Example text: 
+Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we’ve added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support.
+
+{AnswerRelevancyTemplate.multimodal_rules if multimodal else ""}
+
+{{
+    "statements": [
+        "The new laptop model has a high-resolution Retina display.",
+        "It includes a fast-charging battery with up to 12 hours of usage.",
+        "Security features include fingerprint authentication and an encrypted SSD.",
+        "Every purchase comes with a one-year warranty.",
+        "24/7 customer support is included."
+    ]
+}}
+===== END OF EXAMPLE ======
+        
+**
+IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+**
+
+Text:
+{actual_output}
+
+JSON:
+"""
 
     @staticmethod
     def generate_verdicts(
         input: str, statements: str, multimodal: bool = False
     ):
-        content_type = (
-            "statements (which can contain images)"
-            if multimodal
-            else "list of statements"
-        )
-        statement_or_image = "statement or image" if multimodal else "statement"
-
-        format_instruction = textwrap.dedent(
-            """
-            Expected JSON format:
-            {{
-                "verdicts": [
-                    {{
-                        "verdict": "yes"
-                    }},
-                    {{
-                        "reason": <explanation_for_irrelevance>,
-                        "verdict": "no"
-                    }},
-                    {{
-                        "reason": <explanation_for_ambiguity>,
-                        "verdict": "idk"
-                    }}
-                ]  
-            }}
-            """
-        )
-
-        example_section = ""
-        if multimodal:
-            example_section = textwrap.dedent(
-                """
-                Example input: What should I do if there is an earthquake?
-                Example statements: ["Shoes.", "Thanks for asking the question!", "Is there anything else I can help you with?", "Duck and hide"]
-                Example JSON:
-                {{
-                    "verdicts": [
-                        {{
-                            "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake.",
-                            "verdict": "no"
-                        }},
-                        {{
-                            "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant.",
-                            "verdict": "idk"
-                        }},
-                        {{
-                            "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant.",
-                            "verdict": "idk"
-                        }},
-                        {{
-                            "verdict": "yes"
-                        }}
-                    ]  
-                }}
-                """
-            )
-
-        guidelines = ""
-        if multimodal:
-            guidelines = textwrap.dedent(
-                f"""
-                Since you are going to generate a verdict for each statement and image, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of `statements`.
-                """
-            )
-        else:
-            guidelines = textwrap.dedent(
-                f"""
-                Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
-                'verdict' must be STRICTLY 'yes', 'no', or 'idk':
-                - 'yes': statement is relevant to addressing the input
-                - 'no': statement is irrelevant to the input  
-                - 'idk': statement is ambiguous (not directly relevant but could be supporting information)
-                Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-                """
-            )
-
-        return textwrap.dedent(
-            f"""For the provided {content_type}, determine whether each {statement_or_image} is relevant to address the input.
-            {"Please generate a list of JSON with two keys: `verdict` and `reason`." if multimodal else "Generate JSON objects with 'verdict' and 'reason' fields."}
-            The 'verdict' {"key " if multimodal else ''}should {"STRICTLY be either a 'yes', 'idk' or 'no'" if multimodal else "be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information)"}. {"Answer 'yes' if the " + statement_or_image + ' is relevant to addressing the original input, no if the ' + statement_or_image + ' is irrelevant, and "idk" if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).' if multimodal else ""}
-            {"The 'reason' is the reason for the verdict.' if multimodal else '"}
-            Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-            The {"provided statements are statements and images' if multimodal else 'statements are from an AI's actual output"} generated in the actual output.
-
-            **
-            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-
-            {format_instruction if not multimodal else ''}
-            {example_section}
-            {guidelines}
-            **          
-
-            Input:
-            {input}
-
-            Statements:
-            {statements}
-
-            JSON:
-            """
-        )
+        return f"""For the provided list of statements, determine whether each statement is relevant to address the input.
+Generate JSON objects with 'verdict' and 'reason' fields.
+The 'verdict' should be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information).
+Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+The statements are from an AI's actual output.
+
+{AnswerRelevancyTemplate.multimodal_rules if multimodal else ""}
+
+**
+IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+
+Expected JSON format:
+{{
+    "verdicts": [
+        {{
+            "verdict": "yes"
+        }},
+        {{
+            "reason": <explanation_for_irrelevance>,
+            "verdict": "no"
+        }},
+        {{
+            "reason": <explanation_for_ambiguity>,
+            "verdict": "idk"
+        }}
+    ]  
+}}
+
+Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
+'verdict' must be STRICTLY 'yes', 'no', or 'idk':
+- 'yes': statement is relevant to addressing the input
+- 'no': statement is irrelevant to the input  
+- 'idk': statement is ambiguous (not directly relevant but could be supporting information)
+Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+**          
+
+Input:
+{input}
+
+Statements:
+{statements}
+
+JSON:
+"""
 
     @staticmethod
     def generate_reason(
@@ -175,32 +98,32 @@ class AnswerRelevancyTemplate:
         score: float,
         multimodal: bool = False,
     ):
-        return textwrap.dedent(
-            f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
-            The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
-            If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+        return f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
+The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
+If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+
+{AnswerRelevancyTemplate.multimodal_rules if multimodal else ""}
 
+**
+IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
 
-            **
-            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+Example:
+Example JSON:
+{{
+    "reason": "The score is <answer_relevancy_score> because <your_reason>."
+}}
+===== END OF EXAMPLE ======
+**
 
-            {"Example:' if not multimodal else '"}
-            Example JSON:
-            {{
-                "reason": "The score is <answer_relevancy_score> because <your_reason>."
-            }}
-            {"===== END OF EXAMPLE ======' if not multimodal else '"}
-            **
 
-            Answer Relevancy Score:
-            {score}
+Answer Relevancy Score:
+{score}
 
-            Reasons why the score can't be higher based on irrelevant statements in the actual output:
-            {irrelevant_statements}
+Reasons why the score can't be higher based on irrelevant statements in the actual output:
+{irrelevant_statements}
 
-            Input:
-            {input}
+Input:
+{input}
 
-            JSON:
-            """
-        )
+JSON:
+"""

deepeval/metrics/arena_g_eval/arena_g_eval.py

@@ -14,12 +14,17 @@ from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     check_arena_test_case_params,
     construct_verbose_logs,
-    trimAndLoadJson,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.arena_g_eval.schema import *
+from deepeval.metrics.arena_g_eval.schema import (
+    RewrittenReason,
+    Winner,
+    Steps,
+)
 from deepeval.metrics.g_eval.utils import (
     construct_g_eval_params_string,
     validate_criteria_and_evaluation_steps,
@@ -62,7 +67,13 @@ class ArenaGEval(BaseArenaMetric):
         _progress: Optional[Progress] = None,
         _pbar_id: Optional[int] = None,
     ) -> str:
-        check_arena_test_case_params(test_case, self.evaluation_params, self)
+        check_arena_test_case_params(
+            test_case,
+            self.evaluation_params,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
         self.evaluation_cost = 0 if self.using_native_model else None
 
         with metric_progress_indicator(self, _show_indicator=_show_indicator):
@@ -76,12 +87,12 @@ class ArenaGEval(BaseArenaMetric):
                 )
             else:
                 self.evaluation_steps: List[str] = (
-                    self._generate_evaluation_steps()
+                    self._generate_evaluation_steps(test_case.multimodal)
                 )
                 if _progress:
                     update_pbar(_progress, _pbar_id)
                 masked_winner, masked_reason, dummy_to_real_names = (
-                    self._compare(test_case)
+                    self._compare(test_case, test_case.multimodal)
                 )
                 if _progress:
                     update_pbar(_progress, _pbar_id)
@@ -111,7 +122,13 @@ class ArenaGEval(BaseArenaMetric):
         _progress: Optional[Progress] = None,
         _pbar_id: Optional[int] = None,
     ) -> str:
-        check_arena_test_case_params(test_case, self.evaluation_params, self)
+        check_arena_test_case_params(
+            test_case,
+            self.evaluation_params,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
         self.evaluation_cost = 0 if self.using_native_model else None
 
         with metric_progress_indicator(
@@ -120,12 +137,12 @@ class ArenaGEval(BaseArenaMetric):
             _show_indicator=_show_indicator,
         ):
             self.evaluation_steps: List[str] = (
-                await self._a_generate_evaluation_steps()
+                await self._a_generate_evaluation_steps(test_case.multimodal)
             )
             if _progress:
                 update_pbar(_progress, _pbar_id)
             masked_winner, masked_reason, dummy_to_real_names = (
-                await self._a_compare(test_case)
+                await self._a_compare(test_case, test_case.multimodal)
             )
             if _progress:
                 update_pbar(_progress, _pbar_id)
@@ -147,7 +164,7 @@ class ArenaGEval(BaseArenaMetric):
             )
             return self.winner
 
-    async def _a_generate_evaluation_steps(self) -> List[str]:
+    async def _a_generate_evaluation_steps(self, multimodal: bool) -> List[str]:
         if self.evaluation_steps:
             return self.evaluation_steps
 
@@ -155,23 +172,20 @@ class ArenaGEval(BaseArenaMetric):
             self.evaluation_params
         )
         prompt = ArenaGEvalTemplate.generate_evaluation_steps(
-            criteria=self.criteria, parameters=g_eval_params_str
+            criteria=self.criteria,
+            parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            self,
+            prompt,
+            Steps,
+            extract_schema=lambda s: s.steps,
+            extract_json=lambda data: data["steps"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["steps"]
-        else:
-            try:
-                res: Steps = await self.model.a_generate(prompt, schema=Steps)
-                return res.steps
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["steps"]
 
-    def _generate_evaluation_steps(self) -> List[str]:
+    def _generate_evaluation_steps(self, multimodal: bool) -> List[str]:
         if self.evaluation_steps:
             return self.evaluation_steps
 
@@ -179,25 +193,20 @@ class ArenaGEval(BaseArenaMetric):
             self.evaluation_params
         )
         prompt = ArenaGEvalTemplate.generate_evaluation_steps(
-            criteria=self.criteria, parameters=g_eval_params_str
+            criteria=self.criteria,
+            parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            self,
+            prompt,
+            Steps,
+            extract_schema=lambda s: s.steps,
+            extract_json=lambda data: data["steps"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["steps"]
-        else:
-            try:
-                res: Steps = self.model.generate(prompt, schema=Steps)
-                return res.steps
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["steps"]
 
     async def _a_compare(
-        self,
-        test_case: ArenaTestCase,
+        self, test_case: ArenaTestCase, multimodal: bool
     ) -> Tuple[str, str, Dict[str, str]]:
         formatted_test_case, dummy_to_real_names = format_arena_test_case(
             self.evaluation_params, test_case
@@ -209,23 +218,27 @@ class ArenaGEval(BaseArenaMetric):
             evaluation_steps=number_evaluation_steps(self.evaluation_steps),
             test_case_contents=formatted_test_case,
             parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            self,
+            prompt,
+            Winner,
+            extract_schema=lambda s: (
+                s.winner,
+                s.reason,
+                dummy_to_real_names,
+            ),
+            extract_json=lambda data: (
+                data["winner"],
+                data["reason"],
+                dummy_to_real_names,
+            ),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Winner)
-            self.evaluation_cost += cost
-            return res.winner, res.reason, dummy_to_real_names
-        else:
-            try:
-                res: Winner = await self.model.a_generate(prompt, schema=Winner)
-                return res.winner, res.reason, dummy_to_real_names
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["winner"], data["reason"], dummy_to_real_names
 
     def _compare(
-        self,
-        test_case: ArenaTestCase,
+        self, test_case: ArenaTestCase, multimodal: bool
     ) -> Tuple[str, str, Dict[str, str]]:
         formatted_test_case, dummy_to_real_names = format_arena_test_case(
             self.evaluation_params, test_case
@@ -237,19 +250,23 @@ class ArenaGEval(BaseArenaMetric):
             evaluation_steps=number_evaluation_steps(self.evaluation_steps),
             test_case_contents=formatted_test_case,
             parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            self,
+            prompt,
+            Winner,
+            extract_schema=lambda s: (
+                s.winner,
+                s.reason,
+                dummy_to_real_names,
+            ),
+            extract_json=lambda data: (
+                data["winner"],
+                data["reason"],
+                dummy_to_real_names,
+            ),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Winner)
-            self.evaluation_cost += cost
-            return res.winner, res.reason, dummy_to_real_names
-        else:
-            try:
-                res: Winner = self.model.generate(prompt, schema=Winner)
-                return res.winner, res.reason, dummy_to_real_names
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["winner"], data["reason"], dummy_to_real_names
 
     async def _a_generate_rewritten_reason(
         self,
@@ -260,22 +277,14 @@ class ArenaGEval(BaseArenaMetric):
             reason=reason,
             dummy_to_real_names=dummy_to_real_names,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=RewrittenReason
-            )
-            self.evaluation_cost += cost
-            return res.rewritten_reason
-        else:
-            try:
-                res: RewrittenReason = await self.model.a_generate(
-                    prompt, schema=RewrittenReason
-                )
-                return res.rewritten_reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["rewritten_reason"]
+
+        return await a_generate_with_schema_and_extract(
+            self,
+            prompt,
+            RewrittenReason,
+            extract_schema=lambda s: s.rewritten_reason,
+            extract_json=lambda data: data["rewritten_reason"],
+        )
 
     def _generate_rewritten_reason(
         self,
@@ -286,20 +295,13 @@ class ArenaGEval(BaseArenaMetric):
             reason=reason,
             dummy_to_real_names=dummy_to_real_names,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=RewrittenReason)
-            self.evaluation_cost += cost
-            return res.rewritten_reason
-        else:
-            try:
-                res: RewrittenReason = self.model.generate(
-                    prompt, schema=RewrittenReason
-                )
-                return res.rewritten_reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["rewritten_reason"]
+        return generate_with_schema_and_extract(
+            self,
+            prompt,
+            RewrittenReason,
+            extract_schema=lambda s: s.rewritten_reason,
+            extract_json=lambda data: data["rewritten_reason"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/arena_g_eval/template.py

@@ -3,11 +3,23 @@ import textwrap
 
 
 class ArenaGEvalTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def generate_evaluation_steps(parameters: str, criteria: str):
+    def generate_evaluation_steps(
+        parameters: str, criteria: str, multimodal: Optional[bool]
+    ):
         return textwrap.dedent(
             f"""Given an evaluation criteria which outlines how you should choose the winner out of all contestants based on the {parameters}, generate 3-4 concise evaluation steps based on the criteria below. You MUST make it clear how to evaluate {parameters} in relation to one another.
 
+            {ArenaGEvalTemplate.multimodal_rules if multimodal else ""}
+
             Evaluation Criteria:
             {criteria}
 
@@ -28,6 +40,7 @@ class ArenaGEvalTemplate:
         evaluation_steps: str,
         test_case_contents: List[str],
         parameters: str,
+        multimodal: Optional[bool],
     ):
         reasoning_expectation = (
             "Be specific and grounded in the evaluation steps."
@@ -36,6 +49,9 @@ class ArenaGEvalTemplate:
         return textwrap.dedent(
             f"""
             You are a judge. Given the following evaluation steps, select the single contestant that best aligns with the evaluation steps.
+
+            {ArenaGEvalTemplate.multimodal_rules if multimodal else ""}
+
             Return a JSON object with three fields:
 
             - `"winner"`: the contestant that is best aligned with the evaluation steps.