deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -1,12 +1,16 @@
 from typing import Optional, List, Type, Union
 import asyncio
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -18,7 +22,10 @@ from deepeval.metrics.contextual_relevancy.template import (
     ContextualRelevancyTemplate,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.contextual_relevancy.schema import *
+from deepeval.metrics.contextual_relevancy.schema import (
+    ContextualRelevancyVerdicts,
+    ContextualRelevancyScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -57,7 +64,17 @@ class ContextualRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -74,12 +91,16 @@ class ContextualRelevancyMetric(BaseMetric):
                     )
                 )
             else:
+
+                input = test_case.input
+                retrieval_context = test_case.retrieval_context
+
                 self.verdicts_list: List[ContextualRelevancyVerdicts] = [
-                    (self._generate_verdicts(test_case.input, context))
-                    for context in test_case.retrieval_context
+                    (self._generate_verdicts(input, context, multimodal))
+                    for context in retrieval_context
                 ]
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -103,7 +124,17 @@ class ContextualRelevancyMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -112,16 +143,19 @@ class ContextualRelevancyMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            retrieval_context = test_case.retrieval_context
+
             self.verdicts_list: List[ContextualRelevancyVerdicts] = (
                 await asyncio.gather(
                     *[
-                        self._a_generate_verdicts(test_case.input, context)
-                        for context in test_case.retrieval_context
+                        self._a_generate_verdicts(input, context, multimodal)
+                        for context in retrieval_context
                     ]
                 )
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -136,7 +170,7 @@ class ContextualRelevancyMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str):
+    async def _a_generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -154,27 +188,18 @@ class ContextualRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             relevant_statements=relevant_statements,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRelevancyScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=ContextualRelevancyScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
-    def _generate_reason(self, input: str):
+    def _generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
             return None
 
@@ -192,23 +217,16 @@ class ContextualRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             relevant_statements=relevant_statements,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
+        )
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRelevancyScoreReason = self.model.generate(
-                    prompt, schema=ContextualRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
     def _calculate_score(self):
         total_verdicts = 0
@@ -226,50 +244,34 @@ class ContextualRelevancyMetric(BaseMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     async def _a_generate_verdicts(
-        self, input: str, context: List[str]
+        self, input: str, context: List[str], multimodal: bool
     ) -> ContextualRelevancyVerdicts:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input, context=context
+            input=input, context=context, multimodal=multimodal
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyVerdicts,
+            extract_schema=lambda r: r,
+            extract_json=lambda data: ContextualRelevancyVerdicts(**data),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRelevancyVerdicts
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = await self.model.a_generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ContextualRelevancyVerdicts(**data)
 
     def _generate_verdicts(
-        self, input: str, context: str
+        self, input: str, context: str, multimodal: bool
     ) -> ContextualRelevancyVerdicts:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input, context=context
+            input=input, context=context, multimodal=multimodal
+        )
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyVerdicts,
+            extract_schema=lambda r: r,
+            extract_json=lambda data: ContextualRelevancyVerdicts(**data),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRelevancyVerdicts
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = self.model.generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ContextualRelevancyVerdicts(**data)
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -277,7 +279,7 @@ class ContextualRelevancyMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/contextual_relevancy/template.py

@@ -1,4 +1,5 @@
-from typing import List
+from typing import List, Union
+import textwrap
 
 
 class ContextualRelevancyTemplate:
@@ -8,70 +9,98 @@ class ContextualRelevancyTemplate:
         irrelevant_statements: List[str],
         relevant_statements: List[str],
         score: float,
+        multimodal: bool = False,
     ):
-        return f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
-In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
+        # Note: irrelevancies parameter name in multimodal version is kept as irrelevant_statements for consistency
+        return textwrap.dedent(
+            f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
+            In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
 
-** 
-IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-Example JSON:
-{{
-    "reason": "The score is <contextual_relevancy_score> because <your_reason>."
-}}
+            ** 
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_relevancy_score> because <your_reason>."
+            }}
 
-If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-**
+            If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
+            **
 
 
-Contextual Relevancy Score:
-{score}
+            Contextual Relevancy Score:
+            {score}
 
-Input:
-{input}
+            Input:
+            {input}
+            
+            Reasons for why the retrieval context is irrelevant to the input:
+            {irrelevant_statements}
 
-Reasons for why the retrieval context is irrelevant to the input:
-{irrelevant_statements}
+            Statement in the retrieval context that is relevant to the input:
+            {relevant_statements}
 
-Statement in the retrieval context that is relevant to the input:
-{relevant_statements}
-
-JSON:
-"""
+            JSON:
+            """
+        )
 
     @staticmethod
-    def generate_verdicts(input: str, context: str):
-        return f"""Based on the input and context, please generate a JSON object to indicate whether each statement found in the context is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
-You should first extract statements found in the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
-The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the statement is relevant to the input.
-Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the statement to back up your reason.
-If provided context contains no actual content or statements then: give \"no\" as a \"verdict\",
-put context into \"statement\", and \"No statements found in provided context.\" into \"reason\".
-**
-IMPORTANT: Please make sure to only return in JSON format.
-Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
-Example Input: "What were some of Einstein's achievements?"
-
-Example:
-{{
-    "verdicts": [
-        {{
-            "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
-            "verdict": "yes"
-        }},
-        {{
-            "statement": "There was a cat.",
-            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
-            "verdict": "no"
-        }}
-    ]
-}}
-**
-
-Input:
-{input}
-
-Context:
-{context}
-
-JSON:
-"""
+    def generate_verdicts(
+        input: str,
+        context: str,
+        multimodal: bool = False,
+    ):
+        context_type = "context (image or string)" if multimodal else "context"
+        statement_or_image = "statement or image" if multimodal else "statement"
+
+        # Conditional instructions based on mode
+        extraction_instructions = ""
+        if multimodal:
+            extraction_instructions = textwrap.dedent(
+                """
+                If the context is textual, you should first extract the statements found in the context if the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
+                If the context is an image, `statement` should be a description of the image. Do not assume any information not visibly available.
+                """
+            ).strip()
+        else:
+            extraction_instructions = "You should first extract statements found in the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement."
+
+        # Additional instruction for empty context (only in non-multimodal)
+        empty_context_instruction = ""
+        if not multimodal:
+            empty_context_instruction = '\nIf provided context contains no actual content or statements then: give "no" as a "verdict",\nput context into "statement", and "No statements found in provided context." into "reason".'
+
+        return textwrap.dedent(
+            f"""Based on the input and {context_type}, please generate a JSON object to indicate whether {'the context' if multimodal else 'each statement found in the context'} is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
+            {extraction_instructions}
+            The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the {statement_or_image} is relevant to the input.
+            Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the {statement_or_image} to back up your reason.{empty_context_instruction}
+            **
+            IMPORTANT: Please make sure to only return in JSON format.
+            Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
+            Example Input: "What were some of Einstein's achievements?"
+
+            Example:
+            {{
+                "verdicts": [
+                    {{
+                        "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
+                        "verdict": "yes"
+                    }},
+                    {{
+                        "statement": "There was a cat.",
+                        "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
+                        "verdict": "no"
+                    }}
+                ]
+            }}
+            **
+
+            Input:
+            {input}
+
+            Context:
+            {context}
+
+            JSON:
+            """
+        )