deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/faithfulness/faithfulness.py

@@ -1,16 +1,17 @@
 from typing import List, Optional, Union, Type
 import asyncio
 
-from deepeval.test_case import (
-    LLMTestCase,
-    LLMTestCaseParams,
-)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
 from deepeval.metrics import BaseMetric
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.models import DeepEvalBaseLLM
@@ -67,7 +68,13 @@ class FaithfulnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -84,11 +91,16 @@ class FaithfulnessMetric(BaseMetric):
                     )
                 )
             else:
-                self.truths = self._generate_truths(test_case.retrieval_context)
-                self.claims = self._generate_claims(test_case.actual_output)
-                self.verdicts = self._generate_verdicts()
+                retrieval_context = test_case.retrieval_context
+                actual_output = test_case.actual_output
+
+                self.truths = self._generate_truths(
+                    retrieval_context, multimodal
+                )
+                self.claims = self._generate_claims(actual_output, multimodal)
+                self.verdicts = self._generate_verdicts(multimodal)
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason()
+                self.reason = self._generate_reason(multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -114,7 +126,13 @@ class FaithfulnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -123,13 +141,16 @@ class FaithfulnessMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            retrieval_context = test_case.retrieval_context
+            actual_output = test_case.actual_output
+
             self.truths, self.claims = await asyncio.gather(
-                self._a_generate_truths(test_case.retrieval_context),
-                self._a_generate_claims(test_case.actual_output),
+                self._a_generate_truths(retrieval_context, multimodal),
+                self._a_generate_claims(actual_output, multimodal),
             )
-            self.verdicts = await self._a_generate_verdicts()
+            self.verdicts = await self._a_generate_verdicts(multimodal)
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
+            self.reason = await self._a_generate_reason(multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -146,7 +167,7 @@ class FaithfulnessMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -158,6 +179,7 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -177,7 +199,7 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self) -> str:
+    def _generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -189,6 +211,7 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -208,14 +231,20 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    async def _a_generate_verdicts(self) -> List[FaithfulnessVerdict]:
+    async def _a_generate_verdicts(
+        self, multimodal: bool
+    ) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
         verdicts: List[FaithfulnessVerdict] = []
+
         prompt = self.evaluation_template.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
+            claims=self.claims,
+            retrieval_context="\n\n".join(self.truths),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
@@ -236,14 +265,18 @@ class FaithfulnessMetric(BaseMetric):
                 ]
                 return verdicts
 
-    def _generate_verdicts(self) -> List[FaithfulnessVerdict]:
+    def _generate_verdicts(self, multimodal: bool) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
         verdicts: List[FaithfulnessVerdict] = []
+
         prompt = self.evaluation_template.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
+            claims=self.claims,
+            retrieval_context="\n\n".join(self.truths),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
@@ -262,10 +295,13 @@ class FaithfulnessMetric(BaseMetric):
                 ]
                 return verdicts
 
-    async def _a_generate_truths(self, retrieval_context: str) -> List[str]:
+    async def _a_generate_truths(
+        self, retrieval_context: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_truths(
             retrieval_context="\n\n".join(retrieval_context),
             extraction_limit=self.truths_extraction_limit,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Truths)
@@ -280,10 +316,13 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["truths"]
 
-    def _generate_truths(self, retrieval_context: str) -> List[str]:
+    def _generate_truths(
+        self, retrieval_context: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_truths(
             retrieval_context="\n\n".join(retrieval_context),
             extraction_limit=self.truths_extraction_limit,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Truths)
@@ -298,9 +337,11 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["truths"]
 
-    async def _a_generate_claims(self, actual_output: str) -> List[str]:
+    async def _a_generate_claims(
+        self, actual_output: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_claims(
-            actual_output=actual_output
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Claims)
@@ -315,9 +356,11 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["claims"]
 
-    def _generate_claims(self, actual_output: str) -> List[str]:
+    def _generate_claims(
+        self, actual_output: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_claims(
-            actual_output=actual_output
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Claims)

deepeval/metrics/faithfulness/schema.py

@@ -3,7 +3,7 @@ from pydantic import BaseModel, Field
 
 
 class FaithfulnessVerdict(BaseModel):
-    verdict: Literal["yes", "idk", "no"]
+    verdict: Literal["yes", "no", "idk"]
     reason: Optional[str] = Field(default=None)
 
 

deepeval/metrics/faithfulness/template.py

@@ -1,41 +1,50 @@
 from typing import Optional, List
+import textwrap
 
 
 class FaithfulnessTemplate:
     @staticmethod
-    def generate_claims(actual_output: str):
-        return f"""Based on the given text, please extract a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided actual AI output.
-These truths, MUST BE COHERENT, and CANNOT be taken out of context.
-    
-Example:
-Example Text: 
-"Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
-
-Example JSON: 
-{{
-    "claims": [
-        "Einstein won the noble prize for his discovery of the photoelectric effect in 1968."
-        "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
-    ]  
-}}
-===== END OF EXAMPLE ======
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
-Only include claims that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT. The claims you extract should include the full context it was presented in, NOT cherry picked facts.
-You should NOT include any prior knowledge, and take the text at face value when extracting claims.
-You should be aware that it is an AI that is outputting these claims.
-**
-
-AI Output:
-{actual_output}
-
-JSON:
-"""
+    def generate_claims(actual_output: str, multimodal: bool = False):
+        multimodal_instruction = ""
+        if multimodal:
+            multimodal_instruction = " The excerpt may contain both text and images, so extract claims from all provided content."
+
+        return textwrap.dedent(
+            f"""Based on the given {'excerpt' if multimodal else 'text'}, please extract a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided actual AI output. {multimodal_instruction}
+            These truths, MUST BE COHERENT, and CANNOT be taken out of context.
+                
+            Example:
+            Example Text: 
+            "Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
+
+            Example JSON: 
+            {{
+                "claims": [
+                    "Einstein won the noble prize for his discovery of the photoelectric effect in 1968.",
+                    "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
+                ]  
+            }}
+            ===== END OF EXAMPLE ======
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
+            Only include claims that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT. The claims you extract should include the full context it was presented in, NOT cherry picked facts.
+            You should NOT include any prior knowledge, and take the text at face value when extracting claims.
+            You should be aware that it is an AI that is outputting these claims.
+            **
+
+            {'Excerpt' if multimodal else 'AI Output'}:
+            {actual_output}
+
+            JSON:
+            """
+        )
 
     @staticmethod
     def generate_truths(
-        retrieval_context: str, extraction_limit: Optional[int] = None
+        retrieval_context: str,
+        extraction_limit: Optional[int] = None,
+        multimodal: bool = False,
     ):
         if extraction_limit is None:
             limit = " FACTUAL, undisputed truths"
@@ -43,98 +52,174 @@ JSON:
             limit = " the single most important FACTUAL, undisputed truth"
         else:
             limit = f" the {extraction_limit} most important FACTUAL, undisputed truths per document"
-        return f"""Based on the given text, please generate a comprehensive list of{limit}, that can inferred from the provided text.
-These truths, MUST BE COHERENT. They must NOT be taken out of context.
-        
-Example:
-Example Text: 
-"Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
-
-Example JSON: 
-{{
-    "truths": [
-        "Einstein won the noble prize for his discovery of the photoelectric effect in 1968."
-        "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
-    ]  
-}}
-===== END OF EXAMPLE ======
-**
-IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
-Only include truths that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT.
-**
-
-Text:
-{retrieval_context}
-
-JSON:
-"""
+
+        multimodal_instruction = ""
+        if multimodal:
+            multimodal_instruction = (
+                " The excerpt may contain both text and images."
+            )
+
+        return textwrap.dedent(
+            f"""Based on the given {'excerpt (text and images)' if multimodal else 'text'}, please generate a comprehensive list of{limit}, that can inferred from the provided {'excerpt' if multimodal else 'text'}.{multimodal_instruction}
+            These truths, MUST BE COHERENT. They must NOT be taken out of context.
+                    
+            Example:
+            Example Text: 
+            "Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
+
+            Example JSON: 
+            {{
+                "truths": [
+                    "Einstein won the noble prize for his discovery of the photoelectric effect in 1968.",
+                    "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
+                ]  
+            }}
+            ===== END OF EXAMPLE ======
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
+            Only include truths that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT.
+            **
+
+            {'Excerpt' if multimodal else 'Text'}:
+            {retrieval_context}
+
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(claims: List[str], retrieval_context: str):
-        return f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
-The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
-Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
-The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
-
-Expected JSON format:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes"
-        }},
-        {{
-            "reason": <explanation_for_contradiction>,
-            "verdict": "no"
-        }},
-        {{
-            "reason": <explanation_for_uncertainty>,
-            "verdict": "idk"
-        }}
-    ]  
-}}
-
-Generate ONE verdict per claim - length of 'verdicts' MUST equal number of claims.
-No 'reason' needed for 'yes' verdicts.
-Only use 'no' if retrieval context DIRECTLY CONTRADICTS the claim - never use prior knowledge.
-Use 'idk' for claims not backed up by context OR factually incorrect but non-contradictory - do not assume your knowledge.
-Vague/speculative language in claims (e.g. 'may have', 'possibility') does NOT count as contradiction.
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
-**
-
-Retrieval Contexts:
-{retrieval_context}
-
-Claims:
-{claims}
-
-JSON:
-"""
+    def generate_verdicts(
+        claims: List[str], retrieval_context: str, multimodal: bool = False
+    ):
+        example_section = ""
+        if multimodal:
+            example_section = textwrap.dedent(
+                """
+                Example retrieval contexts: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. Einstein won the Nobel Prize in 1968. Einstein is a German Scientist."
+                Example claims: ["Barack Obama is a caucasian male.", "Zurich is a city in London", "Einstein won the Nobel Prize for the discovery of the photoelectric effect which may have contributed to his fame.", "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect.", "Einstein was a German chef."]
+
+                Example:
+                {{
+                    "verdicts": [
+                        {{
+                            "reason": "The claim about Barack Obama is not directly addressed in the retrieval context, and so poses no contradiction.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "verdict": "yes"
+                        }},
+                        {{
+                            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead.",
+                            "verdict": "no"
+                        }},
+                        {{
+                            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead.",
+                            "verdict": "no"
+                        }}
+                    ]  
+                }}
+                ===== END OF EXAMPLE ======
+                """
+            )
+
+        format_instruction = textwrap.dedent(
+            """
+            Expected JSON format:
+            {{
+                "verdicts": [
+                    {{
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": <explanation_for_contradiction>,
+                        "verdict": "no"
+                    }},
+                    {{
+                        "reason": <explanation_for_uncertainty>,
+                        "verdict": "idk"
+                    }}
+                ]  
+            }}
+            """
+        )
+
+        guidelines = ""
+        if multimodal:
+            guidelines = textwrap.dedent(
+                """
+                The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
+                You DON'T have to provide a reason if the answer is 'yes'.
+                ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
+                Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
+                Claims that is not backed up due to a lack of information/is not mentioned in the retrieval contexts MUST be answered 'idk', otherwise I WILL DIE.
+                If there are clear contradictions or any data or images that's not mentioned in the retrieval context, just provide 'no'.
+                """
+            )
+        else:
+            guidelines = textwrap.dedent(
+                """
+                Generate ONE verdict per claim - length of 'verdicts' MUST equal number of claims.
+                No 'reason' needed for 'yes' verdicts.
+                Only use 'no' if retrieval context DIRECTLY CONTRADICTS the claim - never use prior knowledge.
+                Use 'idk' for claims not backed up by context OR factually incorrect but non-contradictory - do not assume your knowledge.
+                Vague/speculative language in claims (e.g. 'may have', 'possibility') does NOT count as contradiction.
+                """
+            )
+
+        return textwrap.dedent(
+            f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
+            The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
+            Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
+            The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
+
+            {format_instruction}
+            {example_section}
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
+            {guidelines}
+            **
+
+            Retrieval Contexts:
+            {retrieval_context}
+
+            Claims:
+            {claims}
+
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_reason(score: float, contradictions: List[str]):
-        return f"""Below is a list of Contradictions. It is a list of strings explaining why the 'actual output' does not align with the information presented in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'.
-Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. 
+    def generate_reason(
+        score: float, contradictions: List[str], multimodal: bool = False
+    ):
+        return textwrap.dedent(
+            f"""Below is a list of Contradictions. It is a list of strings explaining why the 'actual output' does not align with the information presented in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'.
+            Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. 
 
-Expected JSON format:
-{{
-    "reason": "The score is <faithfulness_score> because <your_reason>."
-}}
+            Expected JSON format:
+            {{
+                "reason": "The score is <faithfulness_score> because <your_reason>."
+            }}
 
-** 
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            ** 
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 
-If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-Your reason MUST use information in `contradiction` in your reason.
-Be sure in your reason, as if you know what the actual output is from the contradictions.
-**
+            If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+            Your reason MUST use information in `contradiction` in your reason.
+            Be sure in your reason, as if you know what the actual output is from the contradictions.
+            **
 
-Faithfulness Score:
-{score}
+            Faithfulness Score:
+            {score}
 
-Contradictions:
-{contradictions}
+            Contradictions:
+            {contradictions}
 
-JSON:
-"""
+            JSON:
+            """
+        )

deepeval/metrics/g_eval/utils.py

@@ -118,12 +118,12 @@ def no_log_prob_support(model: Union[str, DeepEvalBaseLLM]):
         return True
     elif (
         isinstance(model, GPTModel)
-        and model.model_name in unsupported_log_probs_gpt_models
+        and model.get_model_name() in unsupported_log_probs_gpt_models
     ):
         return True
     elif (
         isinstance(model, AzureOpenAIModel)
-        and model.model_name in unsupported_log_probs_gpt_models
+        and model.get_model_name() in unsupported_log_probs_gpt_models
     ):
         return True
 

deepeval/metrics/indicator.py

@@ -13,7 +13,7 @@ from deepeval.metrics import (
     BaseMultimodalMetric,
     BaseArenaMetric,
 )
-from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.test_run.cache import CachedTestCase, Cache
 from deepeval.telemetry import capture_metric_type
 from deepeval.utils import update_pbar
@@ -75,7 +75,7 @@ async def measure_metric_task(
     task_id,
     progress,
     metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
-    test_case: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -159,7 +159,7 @@ async def measure_metrics_with_indicator(
     metrics: List[
         Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric]
     ],
-    test_case: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -239,7 +239,7 @@ async def measure_metrics_with_indicator(
 
 async def safe_a_measure(
     metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
-    tc: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    tc: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     ignore_errors: bool,
     skip_on_missing_params: bool,
     progress: Optional[Progress] = None,

deepeval/metrics/multimodal_metrics/__init__.py

@@ -3,22 +3,4 @@ from .image_editing.image_editing import ImageEditingMetric
 from .image_coherence.image_coherence import ImageCoherenceMetric
 from .image_helpfulness.image_helpfulness import ImageHelpfulnessMetric
 from .image_reference.image_reference import ImageReferenceMetric
-from .multimodal_contextual_recall.multimodal_contextual_recall import (
-    MultimodalContextualRecallMetric,
-)
-from .multimodal_contextual_relevancy.multimodal_contextual_relevancy import (
-    MultimodalContextualRelevancyMetric,
-)
-from .multimodal_contextual_precision.multimodal_contextual_precision import (
-    MultimodalContextualPrecisionMetric,
-)
-from .multimodal_answer_relevancy.multimodal_answer_relevancy import (
-    MultimodalAnswerRelevancyMetric,
-)
-from .multimodal_faithfulness.multimodal_faithfulness import (
-    MultimodalFaithfulnessMetric,
-)
-from .multimodal_tool_correctness.multimodal_tool_correctness import (
-    MultimodalToolCorrectnessMetric,
-)
 from .multimodal_g_eval.multimodal_g_eval import MultimodalGEval