deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/faithfulness/template.py

@@ -1,41 +1,50 @@
 from typing import Optional, List
+import textwrap
 
 
 class FaithfulnessTemplate:
     @staticmethod
-    def generate_claims(actual_output: str):
-        return f"""Based on the given text, please extract a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided actual AI output.
-These truths, MUST BE COHERENT, and CANNOT be taken out of context.
-    
-Example:
-Example Text: 
-"Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
-
-Example JSON: 
-{{
-    "claims": [
-        "Einstein won the noble prize for his discovery of the photoelectric effect in 1968."
-        "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
-    ]  
-}}
-===== END OF EXAMPLE ======
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
-Only include claims that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT. The claims you extract should include the full context it was presented in, NOT cherry picked facts.
-You should NOT include any prior knowledge, and take the text at face value when extracting claims.
-You should be aware that it is an AI that is outputting these claims.
-**
-
-AI Output:
-{actual_output}
-
-JSON:
-"""
+    def generate_claims(actual_output: str, multimodal: bool = False):
+        multimodal_instruction = ""
+        if multimodal:
+            multimodal_instruction = " The excerpt may contain both text and images, so extract claims from all provided content."
+
+        return textwrap.dedent(
+            f"""Based on the given {'excerpt' if multimodal else 'text'}, please extract a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided actual AI output. {multimodal_instruction}
+            These truths, MUST BE COHERENT, and CANNOT be taken out of context.
+                
+            Example:
+            Example Text: 
+            "Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
+
+            Example JSON: 
+            {{
+                "claims": [
+                    "Einstein won the noble prize for his discovery of the photoelectric effect in 1968.",
+                    "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
+                ]  
+            }}
+            ===== END OF EXAMPLE ======
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
+            Only include claims that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT. The claims you extract should include the full context it was presented in, NOT cherry picked facts.
+            You should NOT include any prior knowledge, and take the text at face value when extracting claims.
+            You should be aware that it is an AI that is outputting these claims.
+            **
+
+            {'Excerpt' if multimodal else 'AI Output'}:
+            {actual_output}
+
+            JSON:
+            """
+        )
 
     @staticmethod
     def generate_truths(
-        retrieval_context: str, extraction_limit: Optional[int] = None
+        retrieval_context: str,
+        extraction_limit: Optional[int] = None,
+        multimodal: bool = False,
     ):
         if extraction_limit is None:
             limit = " FACTUAL, undisputed truths"
@@ -43,98 +52,174 @@ JSON:
             limit = " the single most important FACTUAL, undisputed truth"
         else:
             limit = f" the {extraction_limit} most important FACTUAL, undisputed truths per document"
-        return f"""Based on the given text, please generate a comprehensive list of{limit}, that can inferred from the provided text.
-These truths, MUST BE COHERENT. They must NOT be taken out of context.
-        
-Example:
-Example Text: 
-"Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
-
-Example JSON: 
-{{
-    "truths": [
-        "Einstein won the noble prize for his discovery of the photoelectric effect in 1968."
-        "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
-    ]  
-}}
-===== END OF EXAMPLE ======
-**
-IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
-Only include truths that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT.
-**
-
-Text:
-{retrieval_context}
-
-JSON:
-"""
+
+        multimodal_instruction = ""
+        if multimodal:
+            multimodal_instruction = (
+                " The excerpt may contain both text and images."
+            )
+
+        return textwrap.dedent(
+            f"""Based on the given {'excerpt (text and images)' if multimodal else 'text'}, please generate a comprehensive list of{limit}, that can inferred from the provided {'excerpt' if multimodal else 'text'}.{multimodal_instruction}
+            These truths, MUST BE COHERENT. They must NOT be taken out of context.
+                    
+            Example:
+            Example Text: 
+            "Albert Einstein, the genius often associated with wild hair and mind-bending theories, famously won the Nobel Prize in Physics—though not for his groundbreaking work on relativity, as many assume. Instead, in 1968, he was honored for his discovery of the photoelectric effect, a phenomenon that laid the foundation for quantum mechanics."
+
+            Example JSON: 
+            {{
+                "truths": [
+                    "Einstein won the noble prize for his discovery of the photoelectric effect in 1968.",
+                    "The photoelectric effect is a phenomenon that laid the foundation for quantum mechanics."
+                ]  
+            }}
+            ===== END OF EXAMPLE ======
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
+            Only include truths that are factual, BUT IT DOESN'T MATTER IF THEY ARE FACTUALLY CORRECT.
+            **
+
+            {'Excerpt' if multimodal else 'Text'}:
+            {retrieval_context}
+
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(claims: List[str], retrieval_context: str):
-        return f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
-The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
-Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
-The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
-
-Expected JSON format:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes"
-        }},
-        {{
-            "verdict": "no",
-            "reason": <explanation_for_contradiction>
-        }},
-        {{
-            "verdict": "idk",
-            "reason": <explanation_for_uncertainty>
-        }}
-    ]  
-}}
-
-Generate ONE verdict per claim - length of 'verdicts' MUST equal number of claims.
-No 'reason' needed for 'yes' verdicts.
-Only use 'no' if retrieval context DIRECTLY CONTRADICTS the claim - never use prior knowledge.
-Use 'idk' for claims not backed up by context OR factually incorrect but non-contradictory - do not assume your knowledge.
-Vague/speculative language in claims (e.g. 'may have', 'possibility') does NOT count as contradiction.
-
-**
-IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
-**
-
-Retrieval Contexts:
-{retrieval_context}
-
-Claims:
-{claims}
-
-JSON:
-"""
+    def generate_verdicts(
+        claims: List[str], retrieval_context: str, multimodal: bool = False
+    ):
+        example_section = ""
+        if multimodal:
+            example_section = textwrap.dedent(
+                """
+                Example retrieval contexts: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. Einstein won the Nobel Prize in 1968. Einstein is a German Scientist."
+                Example claims: ["Barack Obama is a caucasian male.", "Zurich is a city in London", "Einstein won the Nobel Prize for the discovery of the photoelectric effect which may have contributed to his fame.", "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect.", "Einstein was a German chef."]
+
+                Example:
+                {{
+                    "verdicts": [
+                        {{
+                            "reason": "The claim about Barack Obama is not directly addressed in the retrieval context, and so poses no contradiction.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context.",
+                            "verdict": "idk"
+                        }},
+                        {{
+                            "verdict": "yes"
+                        }},
+                        {{
+                            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead.",
+                            "verdict": "no"
+                        }},
+                        {{
+                            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead.",
+                            "verdict": "no"
+                        }}
+                    ]  
+                }}
+                ===== END OF EXAMPLE ======
+                """
+            )
+
+        format_instruction = textwrap.dedent(
+            """
+            Expected JSON format:
+            {{
+                "verdicts": [
+                    {{
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "reason": <explanation_for_contradiction>,
+                        "verdict": "no"
+                    }},
+                    {{
+                        "reason": <explanation_for_uncertainty>,
+                        "verdict": "idk"
+                    }}
+                ]  
+            }}
+            """
+        )
+
+        guidelines = ""
+        if multimodal:
+            guidelines = textwrap.dedent(
+                """
+                The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
+                You DON'T have to provide a reason if the answer is 'yes'.
+                ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
+                Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
+                Claims that is not backed up due to a lack of information/is not mentioned in the retrieval contexts MUST be answered 'idk', otherwise I WILL DIE.
+                If there are clear contradictions or any data or images that's not mentioned in the retrieval context, just provide 'no'.
+                """
+            )
+        else:
+            guidelines = textwrap.dedent(
+                """
+                Generate ONE verdict per claim - length of 'verdicts' MUST equal number of claims.
+                No 'reason' needed for 'yes' verdicts.
+                Only use 'no' if retrieval context DIRECTLY CONTRADICTS the claim - never use prior knowledge.
+                Use 'idk' for claims not backed up by context OR factually incorrect but non-contradictory - do not assume your knowledge.
+                Vague/speculative language in claims (e.g. 'may have', 'possibility') does NOT count as contradiction.
+                """
+            )
+
+        return textwrap.dedent(
+            f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
+            The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
+            Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
+            The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
+
+            {format_instruction}
+            {example_section}
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
+            {guidelines}
+            **
+
+            Retrieval Contexts:
+            {retrieval_context}
+
+            Claims:
+            {claims}
+
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_reason(score: float, contradictions: List[str]):
-        return f"""Below is a list of Contradictions. It is a list of strings explaining why the 'actual output' does not align with the information presented in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'.
-Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. 
+    def generate_reason(
+        score: float, contradictions: List[str], multimodal: bool = False
+    ):
+        return textwrap.dedent(
+            f"""Below is a list of Contradictions. It is a list of strings explaining why the 'actual output' does not align with the information presented in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'.
+            Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. 
 
-Expected JSON format:
-{{
-    "reason": "The score is <faithfulness_score> because <your_reason>."
-}}
+            Expected JSON format:
+            {{
+                "reason": "The score is <faithfulness_score> because <your_reason>."
+            }}
 
-** 
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            ** 
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 
-If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-Your reason MUST use information in `contradiction` in your reason.
-Be sure in your reason, as if you know what the actual output is from the contradictions.
-**
+            If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+            Your reason MUST use information in `contradiction` in your reason.
+            Be sure in your reason, as if you know what the actual output is from the contradictions.
+            **
 
-Faithfulness Score:
-{score}
+            Faithfulness Score:
+            {score}
 
-Contradictions:
-{contradictions}
+            Contradictions:
+            {contradictions}
 
-JSON:
-"""
+            JSON:
+            """
+        )

deepeval/metrics/g_eval/utils.py

@@ -118,12 +118,12 @@ def no_log_prob_support(model: Union[str, DeepEvalBaseLLM]):
         return True
     elif (
         isinstance(model, GPTModel)
-        and model.model_name in unsupported_log_probs_gpt_models
+        and model.get_model_name() in unsupported_log_probs_gpt_models
     ):
         return True
     elif (
         isinstance(model, AzureOpenAIModel)
-        and model.model_name in unsupported_log_probs_gpt_models
+        and model.get_model_name() in unsupported_log_probs_gpt_models
     ):
         return True
 

deepeval/metrics/hallucination/template.py

@@ -17,12 +17,12 @@ Example:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect."
+            "reason": "The actual output agrees with the provided context which states that Einstein won the Nobel Prize for his discovery of the photoelectric effect.",
+            "verdict": "yes"
         }},
         {{
-            "verdict": "no",
-            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969."
+            "reason": "The actual output contradicts the provided context which states that Einstein won the Nobel Prize in 1968, not 1969.",
+            "verdict": "no"
         }}
     ]  
 }}

deepeval/metrics/indicator.py

@@ -13,7 +13,7 @@ from deepeval.metrics import (
     BaseMultimodalMetric,
     BaseArenaMetric,
 )
-from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.test_run.cache import CachedTestCase, Cache
 from deepeval.telemetry import capture_metric_type
 from deepeval.utils import update_pbar
@@ -75,7 +75,7 @@ async def measure_metric_task(
     task_id,
     progress,
     metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
-    test_case: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -159,7 +159,7 @@ async def measure_metrics_with_indicator(
     metrics: List[
         Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric]
     ],
-    test_case: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -239,7 +239,7 @@ async def measure_metrics_with_indicator(
 
 async def safe_a_measure(
     metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
-    tc: Union[LLMTestCase, MLLMTestCase, ConversationalTestCase],
+    tc: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     ignore_errors: bool,
     skip_on_missing_params: bool,
     progress: Optional[Progress] = None,

deepeval/metrics/misuse/template.py

@@ -40,8 +40,8 @@ Example JSON:
 {{
     "verdicts": [
         {{
-            "verdict": "yes",
-            "reason": "This request falls outside the {domain} domain and should be handled by a different specialist."
+            "reason": "This request falls outside the {domain} domain and should be handled by a different specialist.",
+            "verdict": "yes"
         }},
         {{
             "verdict": "no"

deepeval/metrics/multimodal_metrics/__init__.py

@@ -3,22 +3,4 @@ from .image_editing.image_editing import ImageEditingMetric
 from .image_coherence.image_coherence import ImageCoherenceMetric
 from .image_helpfulness.image_helpfulness import ImageHelpfulnessMetric
 from .image_reference.image_reference import ImageReferenceMetric
-from .multimodal_contextual_recall.multimodal_contextual_recall import (
-    MultimodalContextualRecallMetric,
-)
-from .multimodal_contextual_relevancy.multimodal_contextual_relevancy import (
-    MultimodalContextualRelevancyMetric,
-)
-from .multimodal_contextual_precision.multimodal_contextual_precision import (
-    MultimodalContextualPrecisionMetric,
-)
-from .multimodal_answer_relevancy.multimodal_answer_relevancy import (
-    MultimodalAnswerRelevancyMetric,
-)
-from .multimodal_faithfulness.multimodal_faithfulness import (
-    MultimodalFaithfulnessMetric,
-)
-from .multimodal_tool_correctness.multimodal_tool_correctness import (
-    MultimodalToolCorrectnessMetric,
-)
 from .multimodal_g_eval.multimodal_g_eval import MultimodalGEval

deepeval/metrics/multimodal_metrics/image_coherence/image_coherence.py

@@ -2,7 +2,7 @@ import asyncio
 from typing import Optional, List, Tuple, Union
 
 from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
+from deepeval.test_case import LLMTestCaseParams, LLMTestCase, MLLMImage
 from deepeval.metrics.multimodal_metrics.image_coherence.template import (
     ImageCoherenceTemplate,
 )
@@ -10,32 +10,35 @@ from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_mllm_test_case_params,
-    initialize_multimodal_model,
+    initialize_model,
 )
-from deepeval.models import DeepEvalBaseMLLM
+from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.multimodal_metrics.image_coherence.schema import (
     ReasonScore,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.utils import get_or_create_event_loop
+from deepeval.utils import (
+    get_or_create_event_loop,
+    convert_to_multi_modal_array,
+)
 
 
 class ImageCoherenceMetric(BaseMultimodalMetric):
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
     ]
 
     def __init__(
         self,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         threshold: float = 0.5,
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
         max_context_size: Optional[int] = None,
     ):
-        self.model, self.using_native_model = initialize_multimodal_model(model)
+        self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.threshold = 1 if strict_mode else threshold
         self.strict_mode = strict_mode
@@ -45,13 +48,13 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
 
     def measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
+            test_case, self._required_params, None, None, self, self.model
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -68,7 +71,9 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
                     )
                 )
             else:
-                actual_output = test_case.actual_output
+                actual_output = convert_to_multi_modal_array(
+                    test_case.actual_output
+                )
                 self.contexts_above = []
                 self.contexts_below = []
                 self.scores = []
@@ -145,13 +150,13 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
 
     async def a_measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
+            test_case, self._required_params, None, None, self, self.model
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -160,7 +165,9 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            actual_output = test_case.actual_output
+            actual_output = convert_to_multi_modal_array(
+                test_case.actual_output
+            )
             self.contexts_above = []
             self.contexts_below = []
             self.scores = []
@@ -253,7 +260,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         instructions = ImageCoherenceTemplate.evaluate_image_coherence(
             context_above, context_below
         )
-        prompt = [instructions] + [image]
+        prompt = f"{instructions} \nImages: {image}"
         if self.using_native_model:
             res, cost = self.model.generate(prompt, ReasonScore)
             self.evaluation_cost += cost
@@ -278,7 +285,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         instructions = ImageCoherenceTemplate.evaluate_image_coherence(
             context_above, context_below
         )
-        prompt = [instructions] + [image]
+        prompt = f"{instructions} \nImages: {image}"
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
             self.evaluation_cost += cost