deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/contextual_precision/template.py

@@ -1,84 +1,133 @@
-from typing import List, Dict
+from typing import List, Dict, Union
+import textwrap
+from deepeval.test_case import MLLMImage
+from deepeval.utils import convert_to_multi_modal_array
 
 
 class ContextualPrecisionTemplate:
     @staticmethod
     def generate_verdicts(
-        input: str, expected_output: str, retrieval_context: List[str]
+        input: str,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool = False,
     ):
         document_count_str = f" ({len(retrieval_context)} document{'s' if len(retrieval_context) > 1 else ''})"
-        return f"""Given the input, expected output, and retrieval context, please generate a list of JSON objects to determine whether each node in the retrieval context was remotely useful in arriving at the expected output.
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, you should aim to quote parts of the context.
-Example Retrieval Context: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect", "He won the Nobel Prize in 1968.", "There was a cat."]
-Example Input: "Who won the Nobel Prize in 1968 and for what?"
-Example Expected Output: "Einstein won the Nobel Prize in 1968 for his discovery of the photoelectric effect."
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'"
-        }},
-        {{
-            "verdict": "yes",
-            "reason": "The text verifies that the prize was indeed won in 1968."
-        }},
-        {{
-            "verdict": "no",
-            "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize."
-        }}
-    ]  
-}}
-Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of the contexts.
-**
-        
-Input:
-{input}
-
-Expected output:
-{expected_output}
-
-Retrieval Context{document_count_str}:
-{retrieval_context}
-
-JSON:
-"""
+
+        # For multimodal, we need to annotate the retrieval context with node IDs
+        context_to_display = (
+            ContextualPrecisionTemplate.id_retrieval_context(retrieval_context)
+            if multimodal
+            else retrieval_context
+        )
+
+        multimodal_note = (
+            " (which can be text or an image)" if multimodal else ""
+        )
+
+        prompt_template = textwrap.dedent(
+            f"""Given the input, expected output, and retrieval context, please generate a list of JSON objects to determine whether each node in the retrieval context was remotely useful in arriving at the expected output.
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, you should aim to quote parts of the context {multimodal_note}.
+            Example Retrieval Context: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect", "He won the Nobel Prize in 1968.", "There was a cat."]
+            Example Input: "Who won the Nobel Prize in 1968 and for what?"
+            Example Expected Output: "Einstein won the Nobel Prize in 1968 for his discovery of the photoelectric effect."
+
+            Example:
+            {{
+                "verdicts": [
+                    {{
+                        "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": "The text verifies that the prize was indeed won in 1968.",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize.",
+                        "verdict": "no"
+                    }}
+                ]  
+            }}
+            Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of the contexts.
+            **
+
+            Input:
+            {input}
+
+            Expected output:
+            {expected_output}
+
+            Retrieval Context {document_count_str}:
+            {context_to_display}
+
+            JSON:
+            """
+        )
+
+        return prompt_template
 
     @staticmethod
     def generate_reason(
-        input: str, score: float, verdicts: List[Dict[str, str]]
+        input: str,
+        score: float,
+        verdicts: List[Dict[str, str]],
+        multimodal: bool = False,
     ):
-        # given the input and retrieval context for this input, where the verdict is whether ... and the node is the ..., give a reason for the score
-        return f"""Given the input, retrieval contexts, and contextual precision score, provide a CONCISE summary for the score. Explain why it is not higher, but also why it is at its current score.
-The retrieval contexts is a list of JSON with three keys: `verdict`, `reason` (reason for the verdict) and `node`. `verdict` will be either 'yes' or 'no', which represents whether the corresponding 'node' in the retrieval context is relevant to the input. 
-Contextual precision represents if the relevant nodes are ranked higher than irrelevant nodes. Also note that retrieval contexts is given IN THE ORDER OF THEIR RANKINGS.
+        return textwrap.dedent(
+            f"""Given the input, retrieval contexts, and contextual precision score, provide a CONCISE {'summarize' if multimodal else 'summary'} for the score. Explain why it is not higher, but also why it is at its current score.
+            The retrieval contexts is a list of JSON with three keys: `verdict`, `reason` (reason for the verdict) and `node`. `verdict` will be either 'yes' or 'no', which represents whether the corresponding 'node' in the retrieval context is relevant to the input. 
+            Contextual precision represents if the relevant nodes are ranked higher than irrelevant nodes. Also note that retrieval contexts is given IN THE ORDER OF THEIR RANKINGS.
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_precision_score> because <your_reason>."
+            }}
 
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_precision_score> because <your_reason>."
-}}
 
+            DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' {'are' if multimodal else 'is'} just here for you to understand the broader scope of things.
+            Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
+            In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
+            When addressing nodes, make it explicit that {'it is' if multimodal else 'they are'} nodes in {'retrieval context' if multimodal else 'retrieval contexts'}.
+            If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it{',' if multimodal else ''} otherwise it gets annoying).
+            **
 
-DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' is just here for you to understand the broader scope of things.
-Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
-In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
-When addressing nodes, make it explicit that they are nodes in retrieval contexts.
-If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it, otherwise it gets annoying).
-**
+            Contextual Precision Score:
+            {score}
 
-Contextual Precision Score:
-{score}
+            Input:
+            {input}
 
-Input:
-{input}
+            Retrieval Contexts:
+            {verdicts}
+
+            JSON:
+            """
+        )
+
+    @staticmethod
+    def id_retrieval_context(
+        retrieval_context: List[str],
+    ) -> List[str]:
+        """
+        Annotates retrieval context with node IDs for multimodal processing.
 
-Retrieval Contexts:
-{verdicts}
+        Args:
+            retrieval_context: List of contexts (can be strings or MLLMImages)
 
-JSON:
-"""
+        Returns:
+            Annotated list with "Node X:" prefixes
+        """
+        annotated_retrieval_context = []
+        retrieval_context = convert_to_multi_modal_array(retrieval_context)
+        for i, context in enumerate(retrieval_context):
+            if isinstance(context, str):
+                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
+            elif isinstance(context, MLLMImage):
+                annotated_retrieval_context.append(f"Node {i + 1}:")
+                annotated_retrieval_context.append(context)
+        return annotated_retrieval_context

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -1,10 +1,15 @@
 from typing import Optional, List, Type, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+    convert_to_multi_modal_array,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.test_case import (
@@ -55,7 +60,14 @@ class ContextualRecallMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -72,13 +84,16 @@ class ContextualRecallMetric(BaseMetric):
                     )
                 )
             else:
+                expected_output = test_case.expected_output
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts: List[ContextualRecallVerdict] = (
                     self._generate_verdicts(
-                        test_case.expected_output, test_case.retrieval_context
+                        expected_output, retrieval_context, multimodal
                     )
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.expected_output)
+                self.reason = self._generate_reason(expected_output, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -101,7 +116,14 @@ class ContextualRecallMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -110,14 +132,17 @@ class ContextualRecallMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            expected_output = test_case.expected_output
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts: List[ContextualRecallVerdict] = (
                 await self._a_generate_verdicts(
-                    test_case.expected_output, test_case.retrieval_context
+                    expected_output, retrieval_context, multimodal
                 )
             )
             self.score = self._calculate_score()
             self.reason = await self._a_generate_reason(
-                test_case.expected_output
+                expected_output, multimodal
             )
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
@@ -133,7 +158,7 @@ class ContextualRecallMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, expected_output: str):
+    async def _a_generate_reason(self, expected_output: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -150,6 +175,7 @@ class ContextualRecallMetric(BaseMetric):
             supportive_reasons=supportive_reasons,
             unsupportive_reasons=unsupportive_reasons,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -169,7 +195,7 @@ class ContextualRecallMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, expected_output: str):
+    def _generate_reason(self, expected_output: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -186,6 +212,7 @@ class ContextualRecallMetric(BaseMetric):
             supportive_reasons=supportive_reasons,
             unsupportive_reasons=unsupportive_reasons,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -219,10 +246,15 @@ class ContextualRecallMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     async def _a_generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
+        self,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[ContextualRecallVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
+            expected_output=expected_output,
+            retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Verdicts)
@@ -245,10 +277,15 @@ class ContextualRecallMetric(BaseMetric):
                 return verdicts
 
     def _generate_verdicts(
-        self, expected_output: str, retrieval_context: List[str]
+        self,
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool,
     ) -> List[ContextualRecallVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
+            expected_output=expected_output,
+            retrieval_context=retrieval_context,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Verdicts)

deepeval/metrics/contextual_recall/template.py

@@ -1,4 +1,7 @@
-from typing import List
+from typing import List, Union
+import textwrap
+from deepeval.test_case import MLLMImage
+from deepeval.utils import convert_to_multi_modal_array
 
 
 class ContextualRecallTemplate:
@@ -8,68 +11,116 @@ class ContextualRecallTemplate:
         supportive_reasons: str,
         unsupportive_reasons: str,
         score: float,
+        multimodal: bool = False,
     ):
-        return f"""
-Given the original expected output, a list of supportive reasons, and a list of unsupportive reasons (which are deduced directly from the 'expected output'), and a contextual recall score (closer to 1 the better), summarize a CONCISE reason for the score.
-A supportive reason is the reason why a certain sentence in the original expected output can be attributed to the node in the retrieval context.
-An unsupportive reason is the reason why a certain sentence in the original expected output cannot be attributed to anything in the retrieval context.
-In your reason, you should relate supportive/unsupportive reasons to the sentence number in expected output, and include info regarding the node number in retrieval context to support your final reason. The first mention of "node(s)" should specify "node(s) in retrieval context".
+        content_type = "sentence or image" if multimodal else "sentence"
 
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_recall_score> because <your_reason>."
-}}
+        return textwrap.dedent(
+            f"""Given the original expected output, a list of supportive reasons, and a list of unsupportive reasons ({'which is' if multimodal else 'which are'} deduced directly from the {'"expected output"' if multimodal else 'original expected output'}), and a contextual recall score (closer to 1 the better), summarize a CONCISE reason for the score.
+            A supportive reason is the reason why a certain {content_type} in the original expected output can be attributed to the node in the retrieval context.
+            An unsupportive reason is the reason why a certain {content_type} in the original expected output cannot be attributed to anything in the retrieval context.
+            In your reason, you should {'related' if multimodal else 'relate'} supportive/unsupportive reasons to the {content_type} number in expected output, and {'info' if multimodal else 'include info'} regarding the node number in retrieval context to support your final reason. The first mention of "node(s)" should specify "node(s) in retrieval context{')' if multimodal else ''}.
 
-DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it, otherwise it gets annoying).
-**
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_recall_score> because <your_reason>."
+            }}
 
-Contextual Recall Score:
-{score}
+            DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
+            If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it{',' if multimodal else ''} otherwise it gets annoying).
+            **
 
-Expected Output:
-{expected_output}
+            Contextual Recall Score:
+            {score}
 
-Supportive Reasons:
-{supportive_reasons}
+            Expected Output:
+            {expected_output}
 
-Unsupportive Reasons:
-{unsupportive_reasons}
+            Supportive Reasons:
+            {supportive_reasons}
 
-JSON:
-"""
+            Unsupportive Reasons:
+            {unsupportive_reasons}
 
-    @staticmethod
-    def generate_verdicts(expected_output: str, retrieval_context: List[str]):
-        return f"""
-For EACH sentence in the given expected output below, determine whether the sentence can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: `verdict` and `reason`.
-The `verdict` key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the sentence can be attributed to any parts of the retrieval context, else answer 'no'.
-The `reason` key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said sentence. You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
-
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: `verdict` and `reason`.
-
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "reason": "..."
-        }},
-        ...
-    ]  
-}}
-
-Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of sentences in `expected output`.
-**
+            JSON:
+            """
+        )
 
-Expected Output:
-{expected_output}
-
-Retrieval Context:
-{retrieval_context}
+    @staticmethod
+    def generate_verdicts(
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool = False,
+    ):
+        content_type = "sentence and image" if multimodal else "sentence"
+        content_type_plural = (
+            "sentences and images" if multimodal else "sentences"
+        )
+        content_or = "sentence or image" if multimodal else "sentence"
+
+        # For multimodal, we need to annotate the retrieval context with node IDs
+        context_to_display = (
+            ContextualRecallTemplate.id_retrieval_context(retrieval_context)
+            if multimodal
+            else retrieval_context
+        )
+
+        node_instruction = ""
+        if multimodal:
+            node_instruction = " A node is either a string or image, but not both (so do not group images and texts in the same nodes)."
+
+        return textwrap.dedent(
+            f"""For EACH {content_type} in the given expected output below, determine whether the {content_or} can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: `verdict` and `reason`.
+            The `verdict` key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the {content_or} can be attributed to any parts of the retrieval context, else answer 'no'.
+            The `reason` key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said {content_or}.{node_instruction} You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: `verdict` and `reason`.
+
+            {{
+                "verdicts": [
+                    {{
+                        "reason": "...",
+                        "verdict": "yes"
+                    }},
+                    ...
+                ]  
+            }}
+
+            Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of {content_type_plural} in {'the' if multimodal else '`expected output`'}{' `expected output`' if multimodal else ''}.
+            **
+
+            Expected Output:
+            {expected_output}
+
+            Retrieval Context:
+            {context_to_display}
+
+            JSON:
+            """
+        )
 
-JSON:
-"""
+    @staticmethod
+    def id_retrieval_context(
+        retrieval_context: List[str],
+    ) -> List[str]:
+        """
+        Annotates retrieval context with node IDs for multimodal processing.
+
+        Args:
+            retrieval_context: List of contexts (can be strings or MLLMImages)
+
+        Returns:
+            Annotated list with "Node X:" prefixes
+        """
+        annotated_retrieval_context = []
+        retrieval_context = convert_to_multi_modal_array(retrieval_context)
+        for i, context in enumerate(retrieval_context):
+            if isinstance(context, str):
+                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
+            elif isinstance(context, MLLMImage):
+                annotated_retrieval_context.append(f"Node {i + 1}:")
+                annotated_retrieval_context.append(context)
+        return annotated_retrieval_context