deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/answer_relevancy/template.py

@@ -1,110 +1,206 @@
 from typing import List
+import textwrap
 
 
 class AnswerRelevancyTemplate:
     @staticmethod
-    def generate_statements(actual_output: str):
-        return f"""Given the text, breakdown and generate a list of statements presented. Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement.
-
-Example:
-Example text: 
-Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we’ve added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support.
-
-{{
-    "statements": [
-        "The new laptop model has a high-resolution Retina display.",
-        "It includes a fast-charging battery with up to 12 hours of usage.",
-        "Security features include fingerprint authentication and an encrypted SSD.",
-        "Every purchase comes with a one-year warranty.",
-        "24/7 customer support is included."
-    ]
-}}
-===== END OF EXAMPLE ======
-        
-**
-IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-**
-
-Text:
-{actual_output}
-
-JSON:
-"""
+    def generate_statements(actual_output: str, multimodal: bool = False):
+        multimodal_instruction = ""
+        example_text = ""
+        example_json = ""
+
+        if multimodal:
+            multimodal_instruction = " The text may contain images as well."
+            example_text = "Shoes. The shoes can be refunded at no extra cost. Thanks for asking the question!"
+            example_json = textwrap.dedent(
+                """
+                {{
+                    "statements": ["Shoes.", "Shoes can be refunded at no extra cost", "Thanks for asking the question!"]
+                }}
+                """
+            )
+        else:
+            example_text = "Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we've added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support."
+            example_json = textwrap.dedent(
+                """
+                {{
+                    "statements": [
+                        "The new laptop model has a high-resolution Retina display.",
+                        "It includes a fast-charging battery with up to 12 hours of usage.",
+                        "Security features include fingerprint authentication and an encrypted SSD.",
+                        "Every purchase comes with a one-year warranty.",
+                        "24/7 customer support is included."
+                    ]
+                }}
+                """
+            )
+
+        coherence_note = (
+            ""
+            if multimodal
+            else " Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement."
+        )
+
+        return textwrap.dedent(
+            f"""Given the text, breakdown and generate a list of statements presented.{coherence_note}{multimodal_instruction}
+
+            Example:
+            Example text: 
+            {example_text}
+
+            {example_json}
+            ===== END OF EXAMPLE ======
+                    
+            **
+            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+            **
+
+            Text:
+            {actual_output}
+
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(input: str, statements: str):
-        return f"""For the provided list of statements, determine whether each statement is relevant to address the input.
-Generate JSON objects with 'verdict' and 'reason' fields.
-The 'verdict' should be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information).
-Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-The statements are from an AI's actual output.
-
-**
-IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-
-Expected JSON format:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes"
-        }},
-        {{
-            "verdict": "no",
-            "reason": <explanation_for_irrelevance>
-        }},
-        {{
-            "verdict": "idk",
-            "reason": <explanation_for_ambiguity>
-        }}
-    ]  
-}}
-
-Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
-'verdict' must be STRICTLY 'yes', 'no', or 'idk':
-- 'yes': statement is relevant to addressing the input
-- 'no': statement is irrelevant to the input  
-- 'idk': statement is ambiguous (not directly relevant but could be supporting information)
-Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-**          
-
-Input:
-{input}
-
-Statements:
-{statements}
-
-JSON:
-"""
+    def generate_verdicts(
+        input: str, statements: str, multimodal: bool = False
+    ):
+        content_type = (
+            "statements (which can contain images)"
+            if multimodal
+            else "list of statements"
+        )
+        statement_or_image = "statement or image" if multimodal else "statement"
+
+        format_instruction = textwrap.dedent(
+            """
+            Expected JSON format:
+            {{
+                "verdicts": [
+                    {{
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": <explanation_for_irrelevance>,
+                        "verdict": "no"
+                    }},
+                    {{
+                        "reason": <explanation_for_ambiguity>,
+                        "verdict": "idk"
+                    }}
+                ]  
+            }}
+            """
+        )
+
+        example_section = ""
+        if multimodal:
+            example_section = textwrap.dedent(
+                """
+                Example input: What should I do if there is an earthquake?
+                Example statements: ["Shoes.", "Thanks for asking the question!", "Is there anything else I can help you with?", "Duck and hide"]
+                Example JSON:
+                {{
+                    "verdicts": [
+                        {{
+                            "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake.",
+                            "verdict": "no"
+                        }},
+                        {{
+                            "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "verdict": "yes"
+                        }}
+                    ]  
+                }}
+                """
+            )
+
+        guidelines = ""
+        if multimodal:
+            guidelines = textwrap.dedent(
+                f"""
+                Since you are going to generate a verdict for each statement and image, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of `statements`.
+                """
+            )
+        else:
+            guidelines = textwrap.dedent(
+                f"""
+                Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
+                'verdict' must be STRICTLY 'yes', 'no', or 'idk':
+                - 'yes': statement is relevant to addressing the input
+                - 'no': statement is irrelevant to the input  
+                - 'idk': statement is ambiguous (not directly relevant but could be supporting information)
+                Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+                """
+            )
+
+        return textwrap.dedent(
+            f"""For the provided {content_type}, determine whether each {statement_or_image} is relevant to address the input.
+            {"Please generate a list of JSON with two keys: `verdict` and `reason`." if multimodal else "Generate JSON objects with 'verdict' and 'reason' fields."}
+            The 'verdict' {"key " if multimodal else ''}should {"STRICTLY be either a 'yes', 'idk' or 'no'" if multimodal else "be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information)"}. {"Answer 'yes' if the " + statement_or_image + ' is relevant to addressing the original input, no if the ' + statement_or_image + ' is irrelevant, and "idk" if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).' if multimodal else ""}
+            {"The 'reason' is the reason for the verdict.' if multimodal else '"}
+            Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+            The {"provided statements are statements and images' if multimodal else 'statements are from an AI's actual output"} generated in the actual output.
+
+            **
+            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+
+            {format_instruction if not multimodal else ''}
+            {example_section}
+            {guidelines}
+            **          
+
+            Input:
+            {input}
+
+            Statements:
+            {statements}
+
+            JSON:
+            """
+        )
 
     @staticmethod
     def generate_reason(
-        irrelevant_statements: List[str], input: str, score: float
+        irrelevant_statements: List[str],
+        input: str,
+        score: float,
+        multimodal: bool = False,
     ):
-        return f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
-The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
-If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-
+        return textwrap.dedent(
+            f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
+            The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
+            If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
 
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
 
-Example:
-Example JSON:
-{{
-    "reason": "The score is <answer_relevancy_score> because <your_reason>."
-}}
-===== END OF EXAMPLE ======
-**
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
 
+            {"Example:' if not multimodal else '"}
+            Example JSON:
+            {{
+                "reason": "The score is <answer_relevancy_score> because <your_reason>."
+            }}
+            {"===== END OF EXAMPLE ======' if not multimodal else '"}
+            **
 
-Answer Relevancy Score:
-{score}
+            Answer Relevancy Score:
+            {score}
 
-Reasons why the score can't be higher based on irrelevant statements in the actual output:
-{irrelevant_statements}
+            Reasons why the score can't be higher based on irrelevant statements in the actual output:
+            {irrelevant_statements}
 
-Input:
-{input}
+            Input:
+            {input}
 
-JSON:
-"""
+            JSON:
+            """
+        )

deepeval/metrics/argument_correctness/template.py

@@ -70,8 +70,8 @@ class ArgumentCorrectnessTemplate:
                         "verdict": "yes"
                     }},
                     {{
-                        "verdict": "no",
-                        "reason": "Recommending romantic Parisian comedies does not help find the highest temperature in 2023."
+                        "reason": "Recommending romantic Parisian comedies does not help find the highest temperature in 2023.",
+                        "verdict": "no"
                     }}
                 ]  
             }}

deepeval/metrics/base_metric.py

@@ -4,7 +4,6 @@ from typing import Optional, Dict, List
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
     LLMTestCaseParams,
     ArenaTestCase,
 )
@@ -113,13 +112,11 @@ class BaseMultimodalMetric:
         self._threshold = value
 
     @abstractmethod
-    def measure(self, test_case: MLLMTestCase, *args, **kwargs) -> float:
+    def measure(self, test_case: LLMTestCase, *args, **kwargs) -> float:
         raise NotImplementedError
 
     @abstractmethod
-    async def a_measure(
-        self, test_case: MLLMTestCase, *args, **kwargs
-    ) -> float:
+    async def a_measure(self, test_case: LLMTestCase, *args, **kwargs) -> float:
         raise NotImplementedError(
             f"Async execution for {self.__class__.__name__} not supported yet. Please set 'async_mode' to 'False'."
         )

deepeval/metrics/bias/template.py

@@ -64,15 +64,15 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "The opinion "Government meddling in healthcare bloats costs and quashes innovation" reveals a political bias, emphasizing negative views on government involvement."
+            "reason": "The opinion "Government meddling in healthcare bloats costs and quashes innovation" reveals a political bias, emphasizing negative views on government involvement.",
+            "verdict": "yes"
         }},
         {{
             "verdict": "no"
         }},
         {{
             "verdict": "no"
-        }},
+        }}
     ]  
 }}
 

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -1,10 +1,14 @@
 from typing import Optional, List, Type, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.test_case import (
@@ -56,7 +60,15 @@ class ContextualPrecisionMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -73,15 +85,20 @@ class ContextualPrecisionMetric(BaseMetric):
                     )
                 )
             else:
+                input = test_case.input
+                expected_output = test_case.expected_output
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                     self._generate_verdicts(
-                        test_case.input,
-                        test_case.expected_output,
-                        test_case.retrieval_context,
+                        input,
+                        expected_output,
+                        retrieval_context,
+                        multimodal,
                     )
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -104,7 +121,14 @@ class ContextualPrecisionMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -113,15 +137,17 @@ class ContextualPrecisionMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            expected_output = test_case.expected_output
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                 await self._a_generate_verdicts(
-                    test_case.input,
-                    test_case.expected_output,
-                    test_case.retrieval_context,
+                    input, expected_output, retrieval_context, multimodal
                 )
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -136,7 +162,7 @@ class ContextualPrecisionMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str):
+    async def _a_generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -148,6 +174,7 @@ class ContextualPrecisionMetric(BaseMetric):
             input=input,
             verdicts=retrieval_contexts_verdicts,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -169,7 +196,7 @@ class ContextualPrecisionMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str):
+    def _generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -181,6 +208,7 @@ class ContextualPrecisionMetric(BaseMetric):
             input=input,
             verdicts=retrieval_contexts_verdicts,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -203,12 +231,17 @@ class ContextualPrecisionMetric(BaseMetric):
                 return data["reason"]
 
     async def _a_generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
+        self,
+        input: str,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(
@@ -234,12 +267,17 @@ class ContextualPrecisionMetric(BaseMetric):
                 return verdicts
 
     def _generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
+        self,
+        input: str,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=cpschema.Verdicts)