deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -1,11 +1,16 @@
 from typing import Optional, List, Type, Union
 import asyncio
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+    convert_to_multi_modal_array,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.test_case import (
@@ -57,7 +62,14 @@ class ContextualRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -74,12 +86,16 @@ class ContextualRelevancyMetric(BaseMetric):
                     )
                 )
             else:
+
+                input = test_case.input
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts_list: List[ContextualRelevancyVerdicts] = [
-                    (self._generate_verdicts(test_case.input, context))
-                    for context in test_case.retrieval_context
+                    (self._generate_verdicts(input, context, multimodal))
+                    for context in retrieval_context
                 ]
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -103,7 +119,14 @@ class ContextualRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -112,16 +135,19 @@ class ContextualRelevancyMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts_list: List[ContextualRelevancyVerdicts] = (
                 await asyncio.gather(
                     *[
-                        self._a_generate_verdicts(test_case.input, context)
-                        for context in test_case.retrieval_context
+                        self._a_generate_verdicts(input, context, multimodal)
+                        for context in retrieval_context
                     ]
                 )
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -136,7 +162,7 @@ class ContextualRelevancyMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str):
+    async def _a_generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -154,7 +180,9 @@ class ContextualRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             relevant_statements=relevant_statements,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(
                 prompt, schema=ContextualRelevancyScoreReason
@@ -174,7 +202,7 @@ class ContextualRelevancyMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str):
+    def _generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -192,7 +220,9 @@ class ContextualRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             relevant_statements=relevant_statements,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(
                 prompt, schema=ContextualRelevancyScoreReason
@@ -226,11 +256,12 @@ class ContextualRelevancyMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     async def _a_generate_verdicts(
-        self, input: str, context: List[str]
+        self, input: str, context: List[str], multimodal: bool
     ) -> ContextualRelevancyVerdicts:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input, context=context
+            input=input, context=context, multimodal=multimodal
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(
                 prompt, schema=ContextualRelevancyVerdicts
@@ -249,11 +280,12 @@ class ContextualRelevancyMetric(BaseMetric):
                 return ContextualRelevancyVerdicts(**data)
 
     def _generate_verdicts(
-        self, input: str, context: str
+        self, input: str, context: str, multimodal: bool
     ) -> ContextualRelevancyVerdicts:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input, context=context
+            input=input, context=context, multimodal=multimodal
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(
                 prompt, schema=ContextualRelevancyVerdicts

deepeval/metrics/contextual_relevancy/template.py

@@ -1,4 +1,5 @@
-from typing import List
+from typing import List, Union
+import textwrap
 
 
 class ContextualRelevancyTemplate:
@@ -8,70 +9,98 @@ class ContextualRelevancyTemplate:
         irrelevant_statements: List[str],
         relevant_statements: List[str],
         score: float,
+        multimodal: bool = False,
     ):
-        return f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
-In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
+        # Note: irrelevancies parameter name in multimodal version is kept as irrelevant_statements for consistency
+        return textwrap.dedent(
+            f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
+            In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
 
-** 
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_relevancy_score> because <your_reason>."
-}}
+            ** 
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_relevancy_score> because <your_reason>."
+            }}
 
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-**
+            If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+            **
 
 
-Contextual Relevancy Score:
-{score}
+            Contextual Relevancy Score:
+            {score}
 
-Input:
-{input}
+            Input:
+            {input}
+            
+            Reasons for why the retrieval context is irrelevant to the input:
+            {irrelevant_statements}
 
-Reasons for why the retrieval context is irrelevant to the input:
-{irrelevant_statements}
+            Statement in the retrieval context that is relevant to the input:
+            {relevant_statements}
 
-Statement in the retrieval context that is relevant to the input:
-{relevant_statements}
-
-JSON:
-"""
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(input: str, context: str):
-        return f"""Based on the input and context, please generate a JSON object to indicate whether each statement found in the context is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
-You should first extract statements found in the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
-The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the statement is relevant to the input.
-Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the statement to back up your reason.
-If provided context contains no actual content or statements then: give \"no\" as a \"verdict\",
-put context into \"statement\", and \"No statements found in provided context.\" into \"reason\".
-**
-IMPORTANT: Please make sure to only return in JSON format.
-Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
-Example Input: "What were some of Einstein's achievements?"
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "verdict": "yes",
-            "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
-        }},
-        {{
-            "verdict": "no",
-            "statement": "There was a cat.",
-            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements."
-        }}
-    ]
-}}
-**
-
-Input:
-{input}
-
-Context:
-{context}
-
-JSON:
-"""
+    def generate_verdicts(
+        input: str,
+        context: str,
+        multimodal: bool = False,
+    ):
+        context_type = "context (image or string)" if multimodal else "context"
+        statement_or_image = "statement or image" if multimodal else "statement"
+
+        # Conditional instructions based on mode
+        extraction_instructions = ""
+        if multimodal:
+            extraction_instructions = textwrap.dedent(
+                """
+                If the context is textual, you should first extract the statements found in the context if the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
+                If the context is an image, `statement` should be a description of the image. Do not assume any information not visibly available.
+                """
+            ).strip()
+        else:
+            extraction_instructions = "You should first extract statements found in the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement."
+
+        # Additional instruction for empty context (only in non-multimodal)
+        empty_context_instruction = ""
+        if not multimodal:
+            empty_context_instruction = '\nIf provided context contains no actual content or statements then: give "no" as a "verdict",\nput context into "statement", and "No statements found in provided context." into "reason".'
+
+        return textwrap.dedent(
+            f"""Based on the input and {context_type}, please generate a JSON object to indicate whether {'the context' if multimodal else 'each statement found in the context'} is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
+            {extraction_instructions}
+            The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the {statement_or_image} is relevant to the input.
+            Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the {statement_or_image} to back up your reason.{empty_context_instruction}
+            **
+            IMPORTANT: Please make sure to only return in JSON format.
+            Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
+            Example Input: "What were some of Einstein's achievements?"
+
+            Example:
+            {{
+                "verdicts": [
+                    {{
+                        "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "statement": "There was a cat.",
+                        "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
+                        "verdict": "no"
+                    }}
+                ]
+            }}
+            **
+
+            Input:
+            {input}
+
+            Context:
+            {context}
+
+            JSON:
+            """
+        )

deepeval/metrics/conversation_completeness/template.py

@@ -86,8 +86,8 @@ User wants to tell the assistant something.
 
 Example JSON:
 {{
-    "verdict": "no",
-    "reason": "The user wanted to tell the assistant something but the LLM not only refused to answer but replied 'Oh ok, in that case should you need anything just let me know!', which is completely irrelevant and doesn't satisfy the user at all. "
+    "reason": "The user wanted to tell the assistant something but the LLM not only refused to answer but replied 'Oh ok, in that case should you need anything just let me know!', which is completely irrelevant and doesn't satisfy the user at all.",
+    "verdict": "no"
 }}
 ===== END OF EXAMPLE ======
 

deepeval/metrics/conversational_dag/templates.py

@@ -77,8 +77,8 @@ class ConversationalBinaryJudgementTemplate:
 
                 Example:
                 {{
-                "verdict": true,
-                "reason": "The assistant provided a clear and direct answer in response to every user query."
+                "reason": "The assistant provided a clear and direct answer in response to every user query.",
+                "verdict": true
                 }}
                 **
                 JSON:
@@ -108,8 +108,8 @@ class ConversationalNonBinaryJudgementTemplate:
 
                 Example:
                 {{
-                "verdict": "{options[1]}",
-                "reason": "The assistant partially addressed the user’s issue but missed clarifying their follow-up question."
+                "reason": "The assistant partially addressed the user's issue but missed clarifying their follow-up question.",
+                "verdict": "{options[1]}"
                 }}
                 **
                 JSON:

deepeval/metrics/conversational_g_eval/template.py

@@ -70,7 +70,8 @@ JSON:
     ---
     Example JSON:
     {{
-    "score": 0,
-    "reason": "Your concise and informative reason here."
+        "reason": "Your concise and informative reason here.",
+        "score": 0
     }}
-    """
+
+    JSON:"""

deepeval/metrics/dag/templates.py

@@ -60,11 +60,11 @@ class BinaryJudgementTemplate:
 {text}
 
 **
-IMPORTANT: Please make sure to only return a json with two keys: `verdict` (True or False), and the 'reason' key providing the reason. The verdict must be a boolean only, either True or False.
+IMPORTANT: Please make sure to only return a json with two keys: `verdict` (true or false), and the 'reason' key providing the reason. The verdict must be a boolean only, either true or false.
 Example JSON:
 {{
-    "verdict": True,
-    "reason": "..."
+    "reason": "...",
+    "verdict": true
 }}
 **
 
@@ -85,8 +85,8 @@ class NonBinaryJudgementTemplate:
 IMPORTANT: Please make sure to only return a json with two keys: 'verdict' {options} and 'reason' providing the reason.
 Example JSON:
 {{
-    "verdict": {options},
-    "reason": "..."
+    "reason": "...",
+    "verdict": {options}
 }}
 **
 

deepeval/metrics/faithfulness/faithfulness.py

@@ -1,16 +1,17 @@
 from typing import List, Optional, Union, Type
 import asyncio
 
-from deepeval.test_case import (
-    LLMTestCase,
-    LLMTestCaseParams,
-)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
 from deepeval.metrics import BaseMetric
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
 from deepeval.models import DeepEvalBaseLLM
@@ -67,7 +68,13 @@ class FaithfulnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -84,11 +91,16 @@ class FaithfulnessMetric(BaseMetric):
                     )
                 )
             else:
-                self.truths = self._generate_truths(test_case.retrieval_context)
-                self.claims = self._generate_claims(test_case.actual_output)
-                self.verdicts = self._generate_verdicts()
+                retrieval_context = test_case.retrieval_context
+                actual_output = test_case.actual_output
+
+                self.truths = self._generate_truths(
+                    retrieval_context, multimodal
+                )
+                self.claims = self._generate_claims(actual_output, multimodal)
+                self.verdicts = self._generate_verdicts(multimodal)
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason()
+                self.reason = self._generate_reason(multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -114,7 +126,13 @@ class FaithfulnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -123,13 +141,16 @@ class FaithfulnessMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            retrieval_context = test_case.retrieval_context
+            actual_output = test_case.actual_output
+
             self.truths, self.claims = await asyncio.gather(
-                self._a_generate_truths(test_case.retrieval_context),
-                self._a_generate_claims(test_case.actual_output),
+                self._a_generate_truths(retrieval_context, multimodal),
+                self._a_generate_claims(actual_output, multimodal),
             )
-            self.verdicts = await self._a_generate_verdicts()
+            self.verdicts = await self._a_generate_verdicts(multimodal)
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
+            self.reason = await self._a_generate_reason(multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -146,7 +167,7 @@ class FaithfulnessMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -158,6 +179,7 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -177,7 +199,7 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self) -> str:
+    def _generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -189,6 +211,7 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -208,14 +231,20 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    async def _a_generate_verdicts(self) -> List[FaithfulnessVerdict]:
+    async def _a_generate_verdicts(
+        self, multimodal: bool
+    ) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
         verdicts: List[FaithfulnessVerdict] = []
+
         prompt = self.evaluation_template.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
+            claims=self.claims,
+            retrieval_context="\n\n".join(self.truths),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
@@ -236,14 +265,18 @@ class FaithfulnessMetric(BaseMetric):
                 ]
                 return verdicts
 
-    def _generate_verdicts(self) -> List[FaithfulnessVerdict]:
+    def _generate_verdicts(self, multimodal: bool) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
         verdicts: List[FaithfulnessVerdict] = []
+
         prompt = self.evaluation_template.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
+            claims=self.claims,
+            retrieval_context="\n\n".join(self.truths),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
@@ -262,10 +295,13 @@ class FaithfulnessMetric(BaseMetric):
                 ]
                 return verdicts
 
-    async def _a_generate_truths(self, retrieval_context: str) -> List[str]:
+    async def _a_generate_truths(
+        self, retrieval_context: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_truths(
             retrieval_context="\n\n".join(retrieval_context),
             extraction_limit=self.truths_extraction_limit,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Truths)
@@ -280,10 +316,13 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["truths"]
 
-    def _generate_truths(self, retrieval_context: str) -> List[str]:
+    def _generate_truths(
+        self, retrieval_context: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_truths(
             retrieval_context="\n\n".join(retrieval_context),
             extraction_limit=self.truths_extraction_limit,
+            multimodal=multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Truths)
@@ -298,9 +337,11 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["truths"]
 
-    async def _a_generate_claims(self, actual_output: str) -> List[str]:
+    async def _a_generate_claims(
+        self, actual_output: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_claims(
-            actual_output=actual_output
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Claims)
@@ -315,9 +356,11 @@ class FaithfulnessMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["claims"]
 
-    def _generate_claims(self, actual_output: str) -> List[str]:
+    def _generate_claims(
+        self, actual_output: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_claims(
-            actual_output=actual_output
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Claims)

deepeval/metrics/faithfulness/schema.py

@@ -3,7 +3,7 @@ from pydantic import BaseModel, Field
 
 
 class FaithfulnessVerdict(BaseModel):
-    verdict: Literal["yes", "idk", "no"]
+    verdict: Literal["yes", "no", "idk"]
     reason: Optional[str] = Field(default=None)