deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/__init__.py

@@ -42,6 +42,16 @@ from .mcp_use_metric.mcp_use_metric import MCPUseMetric
 from .turn_relevancy.turn_relevancy import (
     TurnRelevancyMetric,
 )
+from .turn_faithfulness.turn_faithfulness import TurnFaithfulnessMetric
+from .turn_contextual_precision.turn_contextual_precision import (
+    TurnContextualPrecisionMetric,
+)
+from .turn_contextual_recall.turn_contextual_recall import (
+    TurnContextualRecallMetric,
+)
+from .turn_contextual_relevancy.turn_contextual_relevancy import (
+    TurnContextualRelevancyMetric,
+)
 from .conversation_completeness.conversation_completeness import (
     ConversationCompletenessMetric,
 )
@@ -55,12 +65,6 @@ from .multimodal_metrics import (
     ImageCoherenceMetric,
     ImageHelpfulnessMetric,
     ImageReferenceMetric,
-    MultimodalContextualRecallMetric,
-    MultimodalContextualRelevancyMetric,
-    MultimodalContextualPrecisionMetric,
-    MultimodalAnswerRelevancyMetric,
-    MultimodalFaithfulnessMetric,
-    MultimodalToolCorrectnessMetric,
     MultimodalGEval,
 )
 
@@ -119,17 +123,15 @@ __all__ = [
     # Conversational metrics
     "TurnRelevancyMetric",
     "ConversationCompletenessMetric",
+    "TurnFaithfulnessMetric",
+    "TurnContextualPrecisionMetric",
+    "TurnContextualRecallMetric",
+    "TurnContextualRelevancyMetric",
     # Multimodal metrics
     "TextToImageMetric",
     "ImageEditingMetric",
     "ImageCoherenceMetric",
     "ImageHelpfulnessMetric",
     "ImageReferenceMetric",
-    "MultimodalContextualRecallMetric",
-    "MultimodalContextualRelevancyMetric",
-    "MultimodalContextualPrecisionMetric",
-    "MultimodalAnswerRelevancyMetric",
-    "MultimodalFaithfulnessMetric",
-    "MultimodalToolCorrectnessMetric",
     "MultimodalGEval",
 ]

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -1,16 +1,17 @@
 from typing import Optional, List, Type, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
-from deepeval.test_case import (
-    LLMTestCase,
-    LLMTestCaseParams,
-)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
 from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.answer_relevancy.template import AnswerRelevancyTemplate
@@ -53,7 +54,14 @@ class AnswerRelevancyMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -70,14 +78,17 @@ class AnswerRelevancyMetric(BaseMetric):
                     )
                 )
             else:
+                input = test_case.input
+                actual_output = test_case.actual_output
+
                 self.statements: List[str] = self._generate_statements(
-                    test_case.actual_output
+                    actual_output, multimodal
                 )
                 self.verdicts: List[AnswerRelevancyVerdict] = (
-                    self._generate_verdicts(test_case.input)
+                    self._generate_verdicts(input, multimodal)
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -101,7 +112,14 @@ class AnswerRelevancyMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -110,14 +128,17 @@ class AnswerRelevancyMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            actual_output = test_case.actual_output
+
             self.statements: List[str] = await self._a_generate_statements(
-                test_case.actual_output
+                actual_output, multimodal
             )
             self.verdicts: List[AnswerRelevancyVerdict] = (
-                await self._a_generate_verdicts(test_case.input)
+                await self._a_generate_verdicts(input, multimodal)
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -133,7 +154,7 @@ class AnswerRelevancyMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str) -> str:
+    async def _a_generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -146,7 +167,9 @@ class AnswerRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             input=input,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(
                 prompt, schema=AnswerRelevancyScoreReason
@@ -164,7 +187,7 @@ class AnswerRelevancyMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str) -> str:
+    def _generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -177,6 +200,7 @@ class AnswerRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             input=input,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -197,14 +221,13 @@ class AnswerRelevancyMetric(BaseMetric):
                 return data["reason"]
 
     async def _a_generate_verdicts(
-        self, input: str
+        self, input: str, multimodal: bool
     ) -> List[AnswerRelevancyVerdict]:
         if len(self.statements) == 0:
             return []
 
         prompt = self.evaluation_template.generate_verdicts(
-            input=input,
-            statements=self.statements,
+            input=input, statements=self.statements, multimodal=multimodal
         )
 
         if self.using_native_model:
@@ -224,14 +247,16 @@ class AnswerRelevancyMetric(BaseMetric):
                     AnswerRelevancyVerdict(**item) for item in data["verdicts"]
                 ]
 
-    def _generate_verdicts(self, input: str) -> List[AnswerRelevancyVerdict]:
+    def _generate_verdicts(
+        self, input: str, multimodal: bool
+    ) -> List[AnswerRelevancyVerdict]:
         if len(self.statements) == 0:
             return []
 
         prompt = self.evaluation_template.generate_verdicts(
-            input=input,
-            statements=self.statements,
+            input=input, statements=self.statements, multimodal=multimodal
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
@@ -250,44 +275,64 @@ class AnswerRelevancyMetric(BaseMetric):
     async def _a_generate_statements(
         self,
         actual_output: str,
+        multimodal: bool,
     ) -> List[str]:
         prompt = self.evaluation_template.generate_statements(
-            actual_output=actual_output,
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Statements)
             self.evaluation_cost += cost
-            return res.statements
+            statements: List[str] = res.statements + [
+                ele for ele in actual_output if isinstance(ele, MLLMImage)
+            ]
+            return statements
         else:
             try:
                 res: Statements = await self.model.a_generate(
                     prompt, schema=Statements
                 )
-                return res.statements
+                statements: List[str] = res.statements + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
             except TypeError:
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
-                return data["statements"]
+                statements = data["statements"] + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
 
     def _generate_statements(
         self,
         actual_output: str,
+        multimodal: bool,
     ) -> List[str]:
         prompt = self.evaluation_template.generate_statements(
-            actual_output=actual_output,
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Statements)
             self.evaluation_cost += cost
-            return res.statements
+            statements = res.statements + [
+                ele for ele in actual_output if isinstance(ele, MLLMImage)
+            ]
+            return statements
         else:
             try:
                 res: Statements = self.model.generate(prompt, schema=Statements)
-                return res.statements
+                statements = res.statements + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
-                return data["statements"]
+                statements = data["statements"] + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
 
     def _calculate_score(self):
         number_of_verdicts = len(self.verdicts)

deepeval/metrics/answer_relevancy/template.py

@@ -1,110 +1,206 @@
 from typing import List
+import textwrap
 
 
 class AnswerRelevancyTemplate:
     @staticmethod
-    def generate_statements(actual_output: str):
-        return f"""Given the text, breakdown and generate a list of statements presented. Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement.
-
-Example:
-Example text: 
-Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we’ve added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support.
-
-{{
-    "statements": [
-        "The new laptop model has a high-resolution Retina display.",
-        "It includes a fast-charging battery with up to 12 hours of usage.",
-        "Security features include fingerprint authentication and an encrypted SSD.",
-        "Every purchase comes with a one-year warranty.",
-        "24/7 customer support is included."
-    ]
-}}
-===== END OF EXAMPLE ======
-        
-**
-IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-**
-
-Text:
-{actual_output}
-
-JSON:
-"""
+    def generate_statements(actual_output: str, multimodal: bool = False):
+        multimodal_instruction = ""
+        example_text = ""
+        example_json = ""
+
+        if multimodal:
+            multimodal_instruction = " The text may contain images as well."
+            example_text = "Shoes. The shoes can be refunded at no extra cost. Thanks for asking the question!"
+            example_json = textwrap.dedent(
+                """
+                {{
+                    "statements": ["Shoes.", "Shoes can be refunded at no extra cost", "Thanks for asking the question!"]
+                }}
+                """
+            )
+        else:
+            example_text = "Our new laptop model features a high-resolution Retina display for crystal-clear visuals. It also includes a fast-charging battery, giving you up to 12 hours of usage on a single charge. For security, we've added fingerprint authentication and an encrypted SSD. Plus, every purchase comes with a one-year warranty and 24/7 customer support."
+            example_json = textwrap.dedent(
+                """
+                {{
+                    "statements": [
+                        "The new laptop model has a high-resolution Retina display.",
+                        "It includes a fast-charging battery with up to 12 hours of usage.",
+                        "Security features include fingerprint authentication and an encrypted SSD.",
+                        "Every purchase comes with a one-year warranty.",
+                        "24/7 customer support is included."
+                    ]
+                }}
+                """
+            )
+
+        coherence_note = (
+            ""
+            if multimodal
+            else " Ambiguous statements and single words can be considered as statements, but only if outside of a coherent statement."
+        )
+
+        return textwrap.dedent(
+            f"""Given the text, breakdown and generate a list of statements presented.{coherence_note}{multimodal_instruction}
+
+            Example:
+            Example text: 
+            {example_text}
+
+            {example_json}
+            ===== END OF EXAMPLE ======
+                    
+            **
+            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the "statements" key mapping to a list of strings. No words or explanation are needed. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+            **
+
+            Text:
+            {actual_output}
+
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(input: str, statements: str):
-        return f"""For the provided list of statements, determine whether each statement is relevant to address the input.
-Generate JSON objects with 'verdict' and 'reason' fields.
-The 'verdict' should be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information).
-Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-The statements are from an AI's actual output.
-
-**
-IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
-
-Expected JSON format:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes"
-        }},
-        {{
-            "reason": <explanation_for_irrelevance>,
-            "verdict": "no"
-        }},
-        {{
-            "reason": <explanation_for_ambiguity>,
-            "verdict": "idk"
-        }}
-    ]  
-}}
-
-Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
-'verdict' must be STRICTLY 'yes', 'no', or 'idk':
-- 'yes': statement is relevant to addressing the input
-- 'no': statement is irrelevant to the input  
-- 'idk': statement is ambiguous (not directly relevant but could be supporting information)
-Provide 'reason' ONLY for 'no' or 'idk' verdicts.
-**          
-
-Input:
-{input}
-
-Statements:
-{statements}
-
-JSON:
-"""
+    def generate_verdicts(
+        input: str, statements: str, multimodal: bool = False
+    ):
+        content_type = (
+            "statements (which can contain images)"
+            if multimodal
+            else "list of statements"
+        )
+        statement_or_image = "statement or image" if multimodal else "statement"
+
+        format_instruction = textwrap.dedent(
+            """
+            Expected JSON format:
+            {{
+                "verdicts": [
+                    {{
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": <explanation_for_irrelevance>,
+                        "verdict": "no"
+                    }},
+                    {{
+                        "reason": <explanation_for_ambiguity>,
+                        "verdict": "idk"
+                    }}
+                ]  
+            }}
+            """
+        )
+
+        example_section = ""
+        if multimodal:
+            example_section = textwrap.dedent(
+                """
+                Example input: What should I do if there is an earthquake?
+                Example statements: ["Shoes.", "Thanks for asking the question!", "Is there anything else I can help you with?", "Duck and hide"]
+                Example JSON:
+                {{
+                    "verdicts": [
+                        {{
+                            "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake.",
+                            "verdict": "no"
+                        }},
+                        {{
+                            "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "verdict": "yes"
+                        }}
+                    ]  
+                }}
+                """
+            )
+
+        guidelines = ""
+        if multimodal:
+            guidelines = textwrap.dedent(
+                f"""
+                Since you are going to generate a verdict for each statement and image, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of `statements`.
+                """
+            )
+        else:
+            guidelines = textwrap.dedent(
+                f"""
+                Generate ONE verdict per statement - number of 'verdicts' MUST equal number of statements.
+                'verdict' must be STRICTLY 'yes', 'no', or 'idk':
+                - 'yes': statement is relevant to addressing the input
+                - 'no': statement is irrelevant to the input  
+                - 'idk': statement is ambiguous (not directly relevant but could be supporting information)
+                Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+                """
+            )
+
+        return textwrap.dedent(
+            f"""For the provided {content_type}, determine whether each {statement_or_image} is relevant to address the input.
+            {"Please generate a list of JSON with two keys: `verdict` and `reason`." if multimodal else "Generate JSON objects with 'verdict' and 'reason' fields."}
+            The 'verdict' {"key " if multimodal else ''}should {"STRICTLY be either a 'yes', 'idk' or 'no'" if multimodal else "be 'yes' (relevant), 'no' (irrelevant), or 'idk' (ambiguous/supporting information)"}. {"Answer 'yes' if the " + statement_or_image + ' is relevant to addressing the original input, no if the ' + statement_or_image + ' is irrelevant, and "idk" if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).' if multimodal else ""}
+            {"The 'reason' is the reason for the verdict.' if multimodal else '"}
+            Provide 'reason' ONLY for 'no' or 'idk' verdicts.
+            The {"provided statements are statements and images' if multimodal else 'statements are from an AI's actual output"} generated in the actual output.
+
+            **
+            IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
+
+            {format_instruction if not multimodal else ''}
+            {example_section}
+            {guidelines}
+            **          
+
+            Input:
+            {input}
+
+            Statements:
+            {statements}
+
+            JSON:
+            """
+        )
 
     @staticmethod
     def generate_reason(
-        irrelevant_statements: List[str], input: str, score: float
+        irrelevant_statements: List[str],
+        input: str,
+        score: float,
+        multimodal: bool = False,
     ):
-        return f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
-The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
-If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-
+        return textwrap.dedent(
+            f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
+            The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
+            If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
 
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
 
-Example:
-Example JSON:
-{{
-    "reason": "The score is <answer_relevancy_score> because <your_reason>."
-}}
-===== END OF EXAMPLE ======
-**
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
 
+            {"Example:' if not multimodal else '"}
+            Example JSON:
+            {{
+                "reason": "The score is <answer_relevancy_score> because <your_reason>."
+            }}
+            {"===== END OF EXAMPLE ======' if not multimodal else '"}
+            **
 
-Answer Relevancy Score:
-{score}
+            Answer Relevancy Score:
+            {score}
 
-Reasons why the score can't be higher based on irrelevant statements in the actual output:
-{irrelevant_statements}
+            Reasons why the score can't be higher based on irrelevant statements in the actual output:
+            {irrelevant_statements}
 
-Input:
-{input}
+            Input:
+            {input}
 
-JSON:
-"""
+            JSON:
+            """
+        )

deepeval/metrics/base_metric.py

@@ -4,7 +4,6 @@ from typing import Optional, Dict, List
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
     LLMTestCaseParams,
     ArenaTestCase,
 )
@@ -113,13 +112,11 @@ class BaseMultimodalMetric:
         self._threshold = value
 
     @abstractmethod
-    def measure(self, test_case: MLLMTestCase, *args, **kwargs) -> float:
+    def measure(self, test_case: LLMTestCase, *args, **kwargs) -> float:
         raise NotImplementedError
 
     @abstractmethod
-    async def a_measure(
-        self, test_case: MLLMTestCase, *args, **kwargs
-    ) -> float:
+    async def a_measure(self, test_case: LLMTestCase, *args, **kwargs) -> float:
         raise NotImplementedError(
             f"Async execution for {self.__class__.__name__} not supported yet. Please set 'async_mode' to 'False'."
         )