deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/misuse/misuse.py

@@ -10,12 +10,18 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.misuse.template import MisuseTemplate
-from deepeval.metrics.misuse.schema import *
+from deepeval.metrics.misuse.schema import (
+    Misuses,
+    MisuseVerdict,
+    Verdicts,
+    MisuseScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -57,7 +63,16 @@ class MisuseMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -104,7 +119,16 @@ class MisuseMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -136,7 +160,7 @@ class MisuseMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -150,24 +174,15 @@ class MisuseMetric(BaseMetric):
             score=format(self.score, ".2f"),
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=MisuseScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MisuseScoreReason = await self.model.a_generate(
-                    prompt, schema=MisuseScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=MisuseScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
-    def _generate_reason(self) -> str:
+    def _generate_reason(self) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -181,106 +196,71 @@ class MisuseMetric(BaseMetric):
             score=format(self.score, ".2f"),
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=MisuseScoreReason)
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MisuseScoreReason = self.model.generate(
-                    prompt, schema=MisuseScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=MisuseScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(self) -> List[MisuseVerdict]:
         if len(self.misuses) == 0:
             return []
 
-        verdicts: List[MisuseVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
             misuses=self.misuses, domain=self.domain
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [MisuseVerdict(**item) for item in data["verdicts"]]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                MisuseVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(self) -> List[MisuseVerdict]:
         if len(self.misuses) == 0:
             return []
 
-        verdicts: List[MisuseVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
             misuses=self.misuses, domain=self.domain
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [MisuseVerdict(**item) for item in data["verdicts"]]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                MisuseVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     async def _a_generate_misuses(self, actual_output: str) -> List[str]:
         prompt = self.evaluation_template.generate_misuses(
             actual_output=actual_output, domain=self.domain
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Misuses)
-            self.evaluation_cost += cost
-            return res.misuses
-        else:
-            try:
-                res: Misuses = await self.model.a_generate(
-                    prompt, schema=Misuses
-                )
-                return res.misuses
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["misuses"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Misuses,
+            extract_schema=lambda s: s.misuses,
+            extract_json=lambda data: data["misuses"],
+        )
 
     def _generate_misuses(self, actual_output: str) -> List[str]:
         prompt = self.evaluation_template.generate_misuses(
             actual_output=actual_output, domain=self.domain
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Misuses)
-            self.evaluation_cost += cost
-            return res.misuses
-        else:
-            try:
-                res: Misuses = self.model.generate(prompt, schema=Misuses)
-                return res.misuses
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["misuses"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Misuses,
+            extract_schema=lambda s: s.misuses,
+            extract_json=lambda data: data["misuses"],
+        )
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -301,7 +281,7 @@ class MisuseMetric(BaseMetric):
         else:
             try:
                 self.success = self.score <= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/misuse/template.py

@@ -2,10 +2,20 @@ from typing import List
 
 
 class MisuseTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_reason(misuse_violations: list, score: str):
         return f"""Based on the misuse violations identified: {misuse_violations}, and the misuse score: {score}, please provide a comprehensive reason for the score.
 
+{MisuseTemplate.multimodal_rules}
+
 In your reason, explain why the misuse score is {score} by addressing the specific misuse violations found (if any).
 
 **
@@ -32,6 +42,8 @@ For each misuse statement, return:
 - "verdict": "yes" if it represents inappropriate use of a {domain} chatbot, "no" if it doesn't
 - "reason": Brief explanation of your verdict (only if verdict is "yes")
 
+{MisuseTemplate.multimodal_rules}
+
 **
 IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key. No words or explanation is needed.
 **
@@ -56,6 +68,8 @@ Example JSON:
 
 Text: {actual_output}
 
+{MisuseTemplate.multimodal_rules}
+
 Focus on identifying statements that represent:
 - Queries or requests that fall outside the {domain} domain scope
 - Topics unrelated to {domain} expertise or focus area
@@ -66,6 +80,7 @@ Avoid flagging:
 - Appropriate {domain}-related requests and discussions
 - Queries within the {domain} expertise area
 
+
 Return a list of these misuse statements.
 
 **

deepeval/metrics/multimodal_metrics/__init__.py

@@ -3,4 +3,3 @@ from .image_editing.image_editing import ImageEditingMetric
 from .image_coherence.image_coherence import ImageCoherenceMetric
 from .image_helpfulness.image_helpfulness import ImageHelpfulnessMetric
 from .image_reference.image_reference import ImageReferenceMetric
-from .multimodal_g_eval.multimodal_g_eval import MultimodalGEval

deepeval/metrics/multimodal_metrics/image_coherence/image_coherence.py

@@ -1,16 +1,17 @@
 import asyncio
 from typing import Optional, List, Tuple, Union
 
-from deepeval.metrics import BaseMultimodalMetric
+from deepeval.metrics import BaseMetric
 from deepeval.test_case import LLMTestCaseParams, LLMTestCase, MLLMImage
 from deepeval.metrics.multimodal_metrics.image_coherence.template import (
     ImageCoherenceTemplate,
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
+    check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.multimodal_metrics.image_coherence.schema import (
@@ -23,7 +24,7 @@ from deepeval.utils import (
 )
 
 
-class ImageCoherenceMetric(BaseMultimodalMetric):
+class ImageCoherenceMetric(BaseMetric):
     _required_params: List[LLMTestCaseParams] = [
         LLMTestCaseParams.INPUT,
         LLMTestCaseParams.ACTUAL_OUTPUT,
@@ -53,8 +54,14 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self, self.model
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -155,8 +162,14 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self, self.model
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -261,20 +274,13 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
             context_above, context_below
         )
         prompt = f"{instructions} \nImages: {image}"
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = self.model.generate(
-                    prompt, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     async def a_evaluate_image_coherence(
         self,
@@ -286,20 +292,13 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
             context_above, context_below
         )
         prompt = f"{instructions} \nImages: {image}"
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = await self.model.a_generate(
-                    prompt, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     def get_image_context(
         self, image_index: int, actual_output: List[Union[str, MLLMImage]]
@@ -334,7 +333,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
             if isinstance(element, MLLMImage)
         ]
 
-    def calculate_score(self, scores: List[float]):
+    def calculate_score(self, scores: List[float]) -> float:
         return sum(scores) / len(scores)
 
     def is_successful(self) -> bool:
@@ -343,7 +342,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/multimodal_metrics/image_editing/image_editing.py

@@ -3,7 +3,7 @@ from typing import Optional, List, Tuple, Union
 import math
 import textwrap
 
-from deepeval.metrics import BaseMultimodalMetric
+from deepeval.metrics import BaseMetric
 from deepeval.test_case import LLMTestCaseParams, LLMTestCase, MLLMImage
 from deepeval.metrics.multimodal_metrics.image_editing.template import (
     ImageEditingTemplate,
@@ -14,16 +14,17 @@ from deepeval.utils import (
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
+    check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.multimodal_metrics.image_editing.schema import ReasonScore
 from deepeval.metrics.indicator import metric_progress_indicator
 
 
-class ImageEditingMetric(BaseMultimodalMetric):
+class ImageEditingMetric(BaseMetric):
 
     _required_params: List[LLMTestCaseParams] = [
         LLMTestCaseParams.INPUT,
@@ -52,8 +53,14 @@ class ImageEditingMetric(BaseMultimodalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, 1, 1, self, self.model
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            1,
+            1,
+            self,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -103,7 +110,7 @@ class ImageEditingMetric(BaseMultimodalMetric):
                     steps=[
                         f"Semantic Consistency Scores:\n{self.SC_scores}",
                         f"Semantic Consistency Reasoning:\n{self.SC_reasoning}",
-                        f"Perceptual Quality Scores:\n{self.SC_scores}",
+                        f"Perceptual Quality Scores:\n{self.PQ_scores}",
                         f"Perceptual Quality Reasoning:\n{self.PQ_reasoning}",
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
@@ -117,8 +124,14 @@ class ImageEditingMetric(BaseMultimodalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, 1, 1, self, self.model
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            1,
+            1,
+            self,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -158,7 +171,7 @@ class ImageEditingMetric(BaseMultimodalMetric):
                 steps=[
                     f"Semantic Consistency Scores:\n{self.SC_scores}",
                     f"Semantic Consistency Reasoning:\n{self.SC_reasoning}",
-                    f"Perceptual Quality Scores:\n{self.SC_scores}",
+                    f"Perceptual Quality Scores:\n{self.PQ_scores}",
                     f"Perceptual Quality Reasoning:\n{self.PQ_reasoning}",
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
@@ -190,24 +203,13 @@ class ImageEditingMetric(BaseMultimodalMetric):
                 text_prompt=text_prompt
             )
         ]
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt + images, schema=ReasonScore
-            )
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = await self.model.a_generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = await self.model.a_generate(
-                    prompt + images, input_text=prompt
-                )
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=f"{prompt} {images}",
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     def _evaluate_semantic_consistency(
         self,
@@ -222,20 +224,13 @@ class ImageEditingMetric(BaseMultimodalMetric):
                 text_prompt=text_prompt
             )
         ]
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt + images, schema=ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = self.model.generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = self.model.generate(prompt + images)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=f"{prompt} {images}",
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     async def _a_evaluate_perceptual_quality(
         self, actual_image_output: MLLMImage
@@ -244,22 +239,13 @@ class ImageEditingMetric(BaseMultimodalMetric):
         prompt = [
             ImageEditingTemplate.generate_perceptual_quality_evaluation_results()
         ]
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt + images, schema=ReasonScore
-            )
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = await self.model.a_generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = await self.model.a_generate(prompt + images)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=f"{prompt} {images}",
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     def _evaluate_perceptual_quality(
         self, actual_image_output: MLLMImage
@@ -268,22 +254,15 @@ class ImageEditingMetric(BaseMultimodalMetric):
         prompt = [
             ImageEditingTemplate.generate_perceptual_quality_evaluation_results()
         ]
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt + images, schema=ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = self.model.generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = self.model.generate(prompt + images)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=f"{prompt} {images}",
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
-    def _calculate_score(self) -> List[str]:
+    def _calculate_score(self) -> float:
         min_SC_score = min(self.SC_scores)
         min_PQ_score = min(self.PQ_scores)
         return math.sqrt(min_SC_score * min_PQ_score) / 10
@@ -293,14 +272,14 @@ class ImageEditingMetric(BaseMultimodalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 
     def _generate_reason(
         self,
-    ) -> Tuple[List[float], str]:
+    ) -> str:
         return textwrap.dedent(
             f"""
             The overall score is {self.score:.2f} because the lowest score from semantic consistency was {min(self.SC_scores)}