deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/contextual_recall/template.py

@@ -1,4 +1,7 @@
-from typing import List
+from typing import List, Union
+import textwrap
+from deepeval.test_case import MLLMImage
+from deepeval.utils import convert_to_multi_modal_array
 
 
 class ContextualRecallTemplate:
@@ -8,68 +11,116 @@ class ContextualRecallTemplate:
         supportive_reasons: str,
         unsupportive_reasons: str,
         score: float,
+        multimodal: bool = False,
     ):
-        return f"""
-Given the original expected output, a list of supportive reasons, and a list of unsupportive reasons (which are deduced directly from the 'expected output'), and a contextual recall score (closer to 1 the better), summarize a CONCISE reason for the score.
-A supportive reason is the reason why a certain sentence in the original expected output can be attributed to the node in the retrieval context.
-An unsupportive reason is the reason why a certain sentence in the original expected output cannot be attributed to anything in the retrieval context.
-In your reason, you should relate supportive/unsupportive reasons to the sentence number in expected output, and include info regarding the node number in retrieval context to support your final reason. The first mention of "node(s)" should specify "node(s) in retrieval context".
+        content_type = "sentence or image" if multimodal else "sentence"
 
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_recall_score> because <your_reason>."
-}}
+        return textwrap.dedent(
+            f"""Given the original expected output, a list of supportive reasons, and a list of unsupportive reasons ({'which is' if multimodal else 'which are'} deduced directly from the {'"expected output"' if multimodal else 'original expected output'}), and a contextual recall score (closer to 1 the better), summarize a CONCISE reason for the score.
+            A supportive reason is the reason why a certain {content_type} in the original expected output can be attributed to the node in the retrieval context.
+            An unsupportive reason is the reason why a certain {content_type} in the original expected output cannot be attributed to anything in the retrieval context.
+            In your reason, you should {'related' if multimodal else 'relate'} supportive/unsupportive reasons to the {content_type} number in expected output, and {'info' if multimodal else 'include info'} regarding the node number in retrieval context to support your final reason. The first mention of "node(s)" should specify "node(s) in retrieval context{')' if multimodal else ''}.
 
-DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it, otherwise it gets annoying).
-**
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_recall_score> because <your_reason>."
+            }}
 
-Contextual Recall Score:
-{score}
+            DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
+            If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it{',' if multimodal else ''} otherwise it gets annoying).
+            **
 
-Expected Output:
-{expected_output}
+            Contextual Recall Score:
+            {score}
 
-Supportive Reasons:
-{supportive_reasons}
+            Expected Output:
+            {expected_output}
 
-Unsupportive Reasons:
-{unsupportive_reasons}
+            Supportive Reasons:
+            {supportive_reasons}
 
-JSON:
-"""
+            Unsupportive Reasons:
+            {unsupportive_reasons}
 
-    @staticmethod
-    def generate_verdicts(expected_output: str, retrieval_context: List[str]):
-        return f"""
-For EACH sentence in the given expected output below, determine whether the sentence can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: `verdict` and `reason`.
-The `verdict` key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the sentence can be attributed to any parts of the retrieval context, else answer 'no'.
-The `reason` key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said sentence. You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
-
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: `verdict` and `reason`.
-
-{{
-    "verdicts": [
-        {{
-            "reason": "...",
-            "verdict": "yes"
-        }},
-        ...
-    ]  
-}}
-
-Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of sentences in `expected output`.
-**
+            JSON:
+            """
+        )
 
-Expected Output:
-{expected_output}
-
-Retrieval Context:
-{retrieval_context}
+    @staticmethod
+    def generate_verdicts(
+        expected_output: str,
+        retrieval_context: List[str],
+        multimodal: bool = False,
+    ):
+        content_type = "sentence and image" if multimodal else "sentence"
+        content_type_plural = (
+            "sentences and images" if multimodal else "sentences"
+        )
+        content_or = "sentence or image" if multimodal else "sentence"
+
+        # For multimodal, we need to annotate the retrieval context with node IDs
+        context_to_display = (
+            ContextualRecallTemplate.id_retrieval_context(retrieval_context)
+            if multimodal
+            else retrieval_context
+        )
+
+        node_instruction = ""
+        if multimodal:
+            node_instruction = " A node is either a string or image, but not both (so do not group images and texts in the same nodes)."
+
+        return textwrap.dedent(
+            f"""For EACH {content_type} in the given expected output below, determine whether the {content_or} can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: `verdict` and `reason`.
+            The `verdict` key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the {content_or} can be attributed to any parts of the retrieval context, else answer 'no'.
+            The `reason` key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said {content_or}.{node_instruction} You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: `verdict` and `reason`.
+
+            {{
+                "verdicts": [
+                    {{
+                        "reason": "...",
+                        "verdict": "yes"
+                    }},
+                    ...
+                ]  
+            }}
+
+            Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of {content_type_plural} in {'the' if multimodal else '`expected output`'}{' `expected output`' if multimodal else ''}.
+            **
+
+            Expected Output:
+            {expected_output}
+
+            Retrieval Context:
+            {context_to_display}
+
+            JSON:
+            """
+        )
 
-JSON:
-"""
+    @staticmethod
+    def id_retrieval_context(
+        retrieval_context: List[str],
+    ) -> List[str]:
+        """
+        Annotates retrieval context with node IDs for multimodal processing.
+
+        Args:
+            retrieval_context: List of contexts (can be strings or MLLMImages)
+
+        Returns:
+            Annotated list with "Node X:" prefixes
+        """
+        annotated_retrieval_context = []
+        retrieval_context = convert_to_multi_modal_array(retrieval_context)
+        for i, context in enumerate(retrieval_context):
+            if isinstance(context, str):
+                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
+            elif isinstance(context, MLLMImage):
+                annotated_retrieval_context.append(f"Node {i + 1}:")
+                annotated_retrieval_context.append(context)
+        return annotated_retrieval_context

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -1,11 +1,16 @@
 from typing import Optional, List, Type, Union
 import asyncio
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+    convert_to_multi_modal_array,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.test_case import (
@@ -57,7 +62,14 @@ class ContextualRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -74,12 +86,16 @@ class ContextualRelevancyMetric(BaseMetric):
                     )
                 )
             else:
+
+                input = test_case.input
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts_list: List[ContextualRelevancyVerdicts] = [
-                    (self._generate_verdicts(test_case.input, context))
-                    for context in test_case.retrieval_context
+                    (self._generate_verdicts(input, context, multimodal))
+                    for context in retrieval_context
                 ]
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -103,7 +119,14 @@ class ContextualRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -112,16 +135,19 @@ class ContextualRelevancyMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts_list: List[ContextualRelevancyVerdicts] = (
                 await asyncio.gather(
                     *[
-                        self._a_generate_verdicts(test_case.input, context)
-                        for context in test_case.retrieval_context
+                        self._a_generate_verdicts(input, context, multimodal)
+                        for context in retrieval_context
                     ]
                 )
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -136,7 +162,7 @@ class ContextualRelevancyMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str):
+    async def _a_generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -154,7 +180,9 @@ class ContextualRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             relevant_statements=relevant_statements,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(
                 prompt, schema=ContextualRelevancyScoreReason
@@ -174,7 +202,7 @@ class ContextualRelevancyMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str):
+    def _generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -192,7 +220,9 @@ class ContextualRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             relevant_statements=relevant_statements,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(
                 prompt, schema=ContextualRelevancyScoreReason
@@ -226,11 +256,12 @@ class ContextualRelevancyMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     async def _a_generate_verdicts(
-        self, input: str, context: List[str]
+        self, input: str, context: List[str], multimodal: bool
     ) -> ContextualRelevancyVerdicts:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input, context=context
+            input=input, context=context, multimodal=multimodal
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(
                 prompt, schema=ContextualRelevancyVerdicts
@@ -249,11 +280,12 @@ class ContextualRelevancyMetric(BaseMetric):
                 return ContextualRelevancyVerdicts(**data)
 
     def _generate_verdicts(
-        self, input: str, context: str
+        self, input: str, context: str, multimodal: bool
     ) -> ContextualRelevancyVerdicts:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input, context=context
+            input=input, context=context, multimodal=multimodal
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(
                 prompt, schema=ContextualRelevancyVerdicts

deepeval/metrics/contextual_relevancy/template.py

@@ -1,4 +1,5 @@
-from typing import List
+from typing import List, Union
+import textwrap
 
 
 class ContextualRelevancyTemplate:
@@ -8,70 +9,98 @@ class ContextualRelevancyTemplate:
         irrelevant_statements: List[str],
         relevant_statements: List[str],
         score: float,
+        multimodal: bool = False,
     ):
-        return f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
-In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
+        # Note: irrelevancies parameter name in multimodal version is kept as irrelevant_statements for consistency
+        return textwrap.dedent(
+            f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
+            In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
 
-** 
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_relevancy_score> because <your_reason>."
-}}
+            ** 
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_relevancy_score> because <your_reason>."
+            }}
 
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-**
+            If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+            **
 
 
-Contextual Relevancy Score:
-{score}
+            Contextual Relevancy Score:
+            {score}
 
-Input:
-{input}
+            Input:
+            {input}
+            
+            Reasons for why the retrieval context is irrelevant to the input:
+            {irrelevant_statements}
 
-Reasons for why the retrieval context is irrelevant to the input:
-{irrelevant_statements}
+            Statement in the retrieval context that is relevant to the input:
+            {relevant_statements}
 
-Statement in the retrieval context that is relevant to the input:
-{relevant_statements}
-
-JSON:
-"""
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(input: str, context: str):
-        return f"""Based on the input and context, please generate a JSON object to indicate whether each statement found in the context is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
-You should first extract statements found in the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
-The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the statement is relevant to the input.
-Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the statement to back up your reason.
-If provided context contains no actual content or statements then: give \"no\" as a \"verdict\",
-put context into \"statement\", and \"No statements found in provided context.\" into \"reason\".
-**
-IMPORTANT: Please make sure to only return in JSON format.
-Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
-Example Input: "What were some of Einstein's achievements?"
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
-            "verdict": "yes"
-        }},
-        {{
-            "statement": "There was a cat.",
-            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
-            "verdict": "no"
-        }}
-    ]
-}}
-**
-
-Input:
-{input}
-
-Context:
-{context}
-
-JSON:
-"""
+    def generate_verdicts(
+        input: str,
+        context: str,
+        multimodal: bool = False,
+    ):
+        context_type = "context (image or string)" if multimodal else "context"
+        statement_or_image = "statement or image" if multimodal else "statement"
+
+        # Conditional instructions based on mode
+        extraction_instructions = ""
+        if multimodal:
+            extraction_instructions = textwrap.dedent(
+                """
+                If the context is textual, you should first extract the statements found in the context if the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
+                If the context is an image, `statement` should be a description of the image. Do not assume any information not visibly available.
+                """
+            ).strip()
+        else:
+            extraction_instructions = "You should first extract statements found in the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement."
+
+        # Additional instruction for empty context (only in non-multimodal)
+        empty_context_instruction = ""
+        if not multimodal:
+            empty_context_instruction = '\nIf provided context contains no actual content or statements then: give "no" as a "verdict",\nput context into "statement", and "No statements found in provided context." into "reason".'
+
+        return textwrap.dedent(
+            f"""Based on the input and {context_type}, please generate a JSON object to indicate whether {'the context' if multimodal else 'each statement found in the context'} is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
+            {extraction_instructions}
+            The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the {statement_or_image} is relevant to the input.
+            Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the {statement_or_image} to back up your reason.{empty_context_instruction}
+            **
+            IMPORTANT: Please make sure to only return in JSON format.
+            Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
+            Example Input: "What were some of Einstein's achievements?"
+
+            Example:
+            {{
+                "verdicts": [
+                    {{
+                        "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "statement": "There was a cat.",
+                        "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
+                        "verdict": "no"
+                    }}
+                ]
+            }}
+            **
+
+            Input:
+            {input}
+
+            Context:
+            {context}
+
+            JSON:
+            """
+        )

deepeval/metrics/dag/templates.py

@@ -60,11 +60,11 @@ class BinaryJudgementTemplate:
 {text}
 
 **
-IMPORTANT: Please make sure to only return a json with two keys: `verdict` (True or False), and the 'reason' key providing the reason. The verdict must be a boolean only, either True or False.
+IMPORTANT: Please make sure to only return a json with two keys: `verdict` (true or false), and the 'reason' key providing the reason. The verdict must be a boolean only, either true or false.
 Example JSON:
 {{
     "reason": "...",
-    "verdict": True
+    "verdict": true
 }}
 **