deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -1,10 +1,14 @@
 from typing import Optional, List, Type, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.test_case import (
@@ -56,7 +60,15 @@ class ContextualPrecisionMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -73,15 +85,20 @@ class ContextualPrecisionMetric(BaseMetric):
                     )
                 )
             else:
+                input = test_case.input
+                expected_output = test_case.expected_output
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                     self._generate_verdicts(
-                        test_case.input,
-                        test_case.expected_output,
-                        test_case.retrieval_context,
+                        input,
+                        expected_output,
+                        retrieval_context,
+                        multimodal,
                     )
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -104,7 +121,14 @@ class ContextualPrecisionMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -113,15 +137,17 @@ class ContextualPrecisionMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            expected_output = test_case.expected_output
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                 await self._a_generate_verdicts(
-                    test_case.input,
-                    test_case.expected_output,
-                    test_case.retrieval_context,
+                    input, expected_output, retrieval_context, multimodal
                 )
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -136,7 +162,7 @@ class ContextualPrecisionMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str):
+    async def _a_generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -148,6 +174,7 @@ class ContextualPrecisionMetric(BaseMetric):
             input=input,
             verdicts=retrieval_contexts_verdicts,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -169,7 +196,7 @@ class ContextualPrecisionMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str):
+    def _generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -181,6 +208,7 @@ class ContextualPrecisionMetric(BaseMetric):
             input=input,
             verdicts=retrieval_contexts_verdicts,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -203,12 +231,17 @@ class ContextualPrecisionMetric(BaseMetric):
                 return data["reason"]
 
     async def _a_generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
+        self,
+        input: str,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(
@@ -234,12 +267,17 @@ class ContextualPrecisionMetric(BaseMetric):
                 return verdicts
 
     def _generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
+        self,
+        input: str,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=cpschema.Verdicts)

deepeval/metrics/contextual_precision/template.py

@@ -1,84 +1,133 @@
-from typing import List, Dict
+from typing import List, Dict, Union
+import textwrap
+from deepeval.test_case import MLLMImage
+from deepeval.utils import convert_to_multi_modal_array
 
 
 class ContextualPrecisionTemplate:
     @staticmethod
     def generate_verdicts(
-        input: str, expected_output: str, retrieval_context: List[str]
+        input: str,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool = False,
     ):
         document_count_str = f" ({len(retrieval_context)} document{'s' if len(retrieval_context) > 1 else ''})"
-        return f"""Given the input, expected output, and retrieval context, please generate a list of JSON objects to determine whether each node in the retrieval context was remotely useful in arriving at the expected output.
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, you should aim to quote parts of the context.
-Example Retrieval Context: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect", "He won the Nobel Prize in 1968.", "There was a cat."]
-Example Input: "Who won the Nobel Prize in 1968 and for what?"
-Example Expected Output: "Einstein won the Nobel Prize in 1968 for his discovery of the photoelectric effect."
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'",
-            "verdict": "yes"
-        }},
-        {{
-            "reason": "The text verifies that the prize was indeed won in 1968.",
-            "verdict": "yes"
-        }},
-        {{
-            "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize.",
-            "verdict": "no"
-        }}
-    ]  
-}}
-Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of the contexts.
-**
-        
-Input:
-{input}
-
-Expected output:
-{expected_output}
-
-Retrieval Context{document_count_str}:
-{retrieval_context}
-
-JSON:
-"""
+
+        # For multimodal, we need to annotate the retrieval context with node IDs
+        context_to_display = (
+            ContextualPrecisionTemplate.id_retrieval_context(retrieval_context)
+            if multimodal
+            else retrieval_context
+        )
+
+        multimodal_note = (
+            " (which can be text or an image)" if multimodal else ""
+        )
+
+        prompt_template = textwrap.dedent(
+            f"""Given the input, expected output, and retrieval context, please generate a list of JSON objects to determine whether each node in the retrieval context was remotely useful in arriving at the expected output.
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, you should aim to quote parts of the context {multimodal_note}.
+            Example Retrieval Context: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect", "He won the Nobel Prize in 1968.", "There was a cat."]
+            Example Input: "Who won the Nobel Prize in 1968 and for what?"
+            Example Expected Output: "Einstein won the Nobel Prize in 1968 for his discovery of the photoelectric effect."
+
+            Example:
+            {{
+                "verdicts": [
+                    {{
+                        "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": "The text verifies that the prize was indeed won in 1968.",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize.",
+                        "verdict": "no"
+                    }}
+                ]  
+            }}
+            Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of the contexts.
+            **
+
+            Input:
+            {input}
+
+            Expected output:
+            {expected_output}
+
+            Retrieval Context {document_count_str}:
+            {context_to_display}
+
+            JSON:
+            """
+        )
+
+        return prompt_template
 
     @staticmethod
     def generate_reason(
-        input: str, score: float, verdicts: List[Dict[str, str]]
+        input: str,
+        score: float,
+        verdicts: List[Dict[str, str]],
+        multimodal: bool = False,
     ):
-        # given the input and retrieval context for this input, where the verdict is whether ... and the node is the ..., give a reason for the score
-        return f"""Given the input, retrieval contexts, and contextual precision score, provide a CONCISE summary for the score. Explain why it is not higher, but also why it is at its current score.
-The retrieval contexts is a list of JSON with three keys: `verdict`, `reason` (reason for the verdict) and `node`. `verdict` will be either 'yes' or 'no', which represents whether the corresponding 'node' in the retrieval context is relevant to the input. 
-Contextual precision represents if the relevant nodes are ranked higher than irrelevant nodes. Also note that retrieval contexts is given IN THE ORDER OF THEIR RANKINGS.
+        return textwrap.dedent(
+            f"""Given the input, retrieval contexts, and contextual precision score, provide a CONCISE {'summarize' if multimodal else 'summary'} for the score. Explain why it is not higher, but also why it is at its current score.
+            The retrieval contexts is a list of JSON with three keys: `verdict`, `reason` (reason for the verdict) and `node`. `verdict` will be either 'yes' or 'no', which represents whether the corresponding 'node' in the retrieval context is relevant to the input. 
+            Contextual precision represents if the relevant nodes are ranked higher than irrelevant nodes. Also note that retrieval contexts is given IN THE ORDER OF THEIR RANKINGS.
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_precision_score> because <your_reason>."
+            }}
 
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_precision_score> because <your_reason>."
-}}
 
+            DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' {'are' if multimodal else 'is'} just here for you to understand the broader scope of things.
+            Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
+            In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
+            When addressing nodes, make it explicit that {'it is' if multimodal else 'they are'} nodes in {'retrieval context' if multimodal else 'retrieval contexts'}.
+            If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it{',' if multimodal else ''} otherwise it gets annoying).
+            **
 
-DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' is just here for you to understand the broader scope of things.
-Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
-In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
-When addressing nodes, make it explicit that they are nodes in retrieval contexts.
-If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it, otherwise it gets annoying).
-**
+            Contextual Precision Score:
+            {score}
 
-Contextual Precision Score:
-{score}
+            Input:
+            {input}
 
-Input:
-{input}
+            Retrieval Contexts:
+            {verdicts}
+
+            JSON:
+            """
+        )
+
+    @staticmethod
+    def id_retrieval_context(
+        retrieval_context: List[str],
+    ) -> List[str]:
+        """
+        Annotates retrieval context with node IDs for multimodal processing.
 
-Retrieval Contexts:
-{verdicts}
+        Args:
+            retrieval_context: List of contexts (can be strings or MLLMImages)
 
-JSON:
-"""
+        Returns:
+            Annotated list with "Node X:" prefixes
+        """
+        annotated_retrieval_context = []
+        retrieval_context = convert_to_multi_modal_array(retrieval_context)
+        for i, context in enumerate(retrieval_context):
+            if isinstance(context, str):
+                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
+            elif isinstance(context, MLLMImage):
+                annotated_retrieval_context.append(f"Node {i + 1}:")
+                annotated_retrieval_context.append(context)
+        return annotated_retrieval_context

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -1,10 +1,15 @@
 from typing import Optional, List, Type, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+    convert_to_multi_modal_array,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.test_case import (
@@ -55,7 +60,14 @@ class ContextualRecallMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -72,13 +84,16 @@ class ContextualRecallMetric(BaseMetric):
                     )
                 )
             else:
+                expected_output = test_case.expected_output
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts: List[ContextualRecallVerdict] = (
                     self._generate_verdicts(
-                        test_case.expected_output, test_case.retrieval_context
+                        expected_output, retrieval_context, multimodal
                     )
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.expected_output)
+                self.reason = self._generate_reason(expected_output, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -101,7 +116,14 @@ class ContextualRecallMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -110,14 +132,17 @@ class ContextualRecallMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            expected_output = test_case.expected_output
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts: List[ContextualRecallVerdict] = (
                 await self._a_generate_verdicts(
-                    test_case.expected_output, test_case.retrieval_context
+                    expected_output, retrieval_context, multimodal
                 )
             )
             self.score = self._calculate_score()
             self.reason = await self._a_generate_reason(
-                test_case.expected_output
+                expected_output, multimodal
             )
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
@@ -133,7 +158,7 @@ class ContextualRecallMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, expected_output: str):
+    async def _a_generate_reason(self, expected_output: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -150,6 +175,7 @@ class ContextualRecallMetric(BaseMetric):
             supportive_reasons=supportive_reasons,
             unsupportive_reasons=unsupportive_reasons,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -169,7 +195,7 @@ class ContextualRecallMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, expected_output: str):
+    def _generate_reason(self, expected_output: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -186,6 +212,7 @@ class ContextualRecallMetric(BaseMetric):
             supportive_reasons=supportive_reasons,
             unsupportive_reasons=unsupportive_reasons,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -219,10 +246,15 @@ class ContextualRecallMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     async def _a_generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
+        self,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[ContextualRecallVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
+            expected_output=expected_output,
+            retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Verdicts)
@@ -245,10 +277,15 @@ class ContextualRecallMetric(BaseMetric):
                 return verdicts
 
     def _generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
+        self,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[ContextualRecallVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
+            expected_output=expected_output,
+            retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Verdicts)