deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/multimodal_metrics/image_reference/image_reference.py

@@ -1,42 +1,46 @@
 import asyncio
 from typing import Optional, List, Tuple, Union
 
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCaseParams, LLMTestCase, MLLMImage
 from deepeval.metrics.multimodal_metrics.image_reference.template import (
     ImageReferenceTemplate,
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
+    check_llm_test_case_params,
+    initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
-from deepeval.models import DeepEvalBaseMLLM
+from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.multimodal_metrics.image_reference.schema import (
     ReasonScore,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.utils import get_or_create_event_loop
+from deepeval.utils import (
+    get_or_create_event_loop,
+    convert_to_multi_modal_array,
+)
 
 
-class ImageReferenceMetric(BaseMultimodalMetric):
+class ImageReferenceMetric(BaseMetric):
 
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
     ]
 
     def __init__(
         self,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         threshold: float = 0.5,
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
         max_context_size: Optional[int] = None,
     ):
-        self.model, self.using_native_model = initialize_multimodal_model(model)
+        self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.threshold = 1 if strict_mode else threshold
         self.strict_mode = strict_mode
@@ -46,13 +50,19 @@ class ImageReferenceMetric(BaseMultimodalMetric):
 
     def measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -69,7 +79,9 @@ class ImageReferenceMetric(BaseMultimodalMetric):
                     )
                 )
             else:
-                actual_output = test_case.actual_output
+                actual_output = convert_to_multi_modal_array(
+                    test_case.actual_output
+                )
                 self.contexts_above = []
                 self.contexts_below = []
                 self.scores = []
@@ -146,13 +158,19 @@ class ImageReferenceMetric(BaseMultimodalMetric):
 
     async def a_measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -161,7 +179,9 @@ class ImageReferenceMetric(BaseMultimodalMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            actual_output = test_case.actual_output
+            actual_output = convert_to_multi_modal_array(
+                test_case.actual_output
+            )
             self.contexts_above = []
             self.contexts_below = []
             self.scores = []
@@ -254,21 +274,14 @@ class ImageReferenceMetric(BaseMultimodalMetric):
         instructions = ImageReferenceTemplate.evaluate_image_reference(
             context_above, context_below
         )
-        prompt = [instructions] + [image]
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = self.model.generate(
-                    prompt, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        prompt = f"{instructions} \nImages: {image}"
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     async def a_evaluate_image_reference(
         self,
@@ -279,21 +292,14 @@ class ImageReferenceMetric(BaseMultimodalMetric):
         instructions = ImageReferenceTemplate.evaluate_image_reference(
             context_above, context_below
         )
-        prompt = [instructions] + [image]
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = await self.model.a_generate(
-                    prompt, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        prompt = f"{instructions} \nImages: {image}"
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     def get_image_context(
         self, image_index: int, actual_output: List[Union[str, MLLMImage]]
@@ -328,7 +334,7 @@ class ImageReferenceMetric(BaseMultimodalMetric):
             if isinstance(element, MLLMImage)
         ]
 
-    def calculate_score(self, scores: List[float]):
+    def calculate_score(self, scores: List[float]) -> float:
         return sum(scores) / len(scores)
 
     def is_successful(self) -> bool:
@@ -337,7 +343,7 @@ class ImageReferenceMetric(BaseMultimodalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/multimodal_metrics/text_to_image/text_to_image.py

@@ -3,38 +3,42 @@ from typing import Optional, List, Tuple, Union
 import math
 import textwrap
 
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCaseParams, LLMTestCase, MLLMImage
 from deepeval.metrics.multimodal_metrics.text_to_image.template import (
     TextToImageTemplate,
 )
-from deepeval.utils import get_or_create_event_loop
+from deepeval.utils import (
+    get_or_create_event_loop,
+    convert_to_multi_modal_array,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
+    check_llm_test_case_params,
+    initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
-from deepeval.models import DeepEvalBaseMLLM
+from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.multimodal_metrics.text_to_image.schema import ReasonScore
 from deepeval.metrics.indicator import metric_progress_indicator
 
-required_params: List[MLLMTestCaseParams] = [
-    MLLMTestCaseParams.INPUT,
-    MLLMTestCaseParams.ACTUAL_OUTPUT,
+required_params: List[LLMTestCaseParams] = [
+    LLMTestCaseParams.INPUT,
+    LLMTestCaseParams.ACTUAL_OUTPUT,
 ]
 
 
-class TextToImageMetric(BaseMultimodalMetric):
+class TextToImageMetric(BaseMetric):
     def __init__(
         self,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         threshold: float = 0.5,
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
-        self.model, self.using_native_model = initialize_multimodal_model(model)
+        self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.threshold = 1 if strict_mode else threshold
         self.strict_mode = strict_mode
@@ -43,11 +47,19 @@ class TextToImageMetric(BaseMultimodalMetric):
 
     def measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
     ) -> float:
-        check_mllm_test_case_params(test_case, required_params, 0, 1, self)
+        check_llm_test_case_params(
+            test_case,
+            required_params,
+            0,
+            1,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -63,10 +75,12 @@ class TextToImageMetric(BaseMultimodalMetric):
                     )
                 )
             else:
-                input_texts, _ = self.separate_images_from_text(test_case.input)
-                _, output_images = self.separate_images_from_text(
+                input = convert_to_multi_modal_array(test_case.input)
+                actual_output = convert_to_multi_modal_array(
                     test_case.actual_output
                 )
+                input_texts, _ = self.separate_images_from_text(input)
+                _, output_images = self.separate_images_from_text(actual_output)
 
                 self.SC_scores, self.SC_reasoning = (
                     self._evaluate_semantic_consistency(
@@ -90,7 +104,7 @@ class TextToImageMetric(BaseMultimodalMetric):
                     steps=[
                         f"Semantic Consistency Scores:\n{self.SC_scores}",
                         f"Semantic Consistency Reasoning:\n{self.SC_reasoning}",
-                        f"Perceptual Quality Scores:\n{self.SC_scores}",
+                        f"Perceptual Quality Scores:\n{self.PQ_scores}",
                         f"Perceptual Quality Reasoning:\n{self.PQ_reasoning}",
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
@@ -99,11 +113,19 @@ class TextToImageMetric(BaseMultimodalMetric):
 
     async def a_measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
     ) -> float:
-        check_mllm_test_case_params(test_case, required_params, 0, 1, self)
+        check_llm_test_case_params(
+            test_case,
+            required_params,
+            0,
+            1,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -112,10 +134,12 @@ class TextToImageMetric(BaseMultimodalMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            input_texts, _ = self.separate_images_from_text(test_case.input)
-            _, output_images = self.separate_images_from_text(
+            input = convert_to_multi_modal_array(test_case.input)
+            actual_output = convert_to_multi_modal_array(
                 test_case.actual_output
             )
+            input_texts, _ = self.separate_images_from_text(input)
+            _, output_images = self.separate_images_from_text(actual_output)
             (self.SC_scores, self.SC_reasoning), (
                 self.PQ_scores,
                 self.PQ_reasoning,
@@ -139,7 +163,7 @@ class TextToImageMetric(BaseMultimodalMetric):
                 steps=[
                     f"Semantic Consistency Scores:\n{self.SC_scores}",
                     f"Semantic Consistency Reasoning:\n{self.SC_reasoning}",
-                    f"Perceptual Quality Scores:\n{self.SC_scores}",
+                    f"Perceptual Quality Scores:\n{self.PQ_scores}",
                     f"Perceptual Quality Reasoning:\n{self.PQ_reasoning}",
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
@@ -163,106 +187,86 @@ class TextToImageMetric(BaseMultimodalMetric):
         text_prompt: str,
         actual_image_output: MLLMImage,
     ) -> Tuple[List[int], str]:
-        images: List[MLLMImage] = []
-        images.append(actual_image_output)
-        prompt = [
-            TextToImageTemplate.generate_semantic_consistency_evaluation_results(
-                text_prompt=text_prompt
-            )
-        ]
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt + images, ReasonScore
-            )
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = await self.model.a_generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = await self.model.a_generate(
-                    prompt + images, input_text=prompt
+        images: List[MLLMImage] = [actual_image_output]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_semantic_consistency_evaluation_results(
+                    text_prompt=text_prompt
                 )
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+            }
+            Images:
+            {images}
+        """
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     def _evaluate_semantic_consistency(
         self,
         text_prompt: str,
         actual_image_output: MLLMImage,
     ) -> Tuple[List[int], str]:
-        images: List[MLLMImage] = []
-        images.append(actual_image_output)
-        prompt = [
-            TextToImageTemplate.generate_semantic_consistency_evaluation_results(
-                text_prompt=text_prompt
-            )
-        ]
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt + images, ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = self.model.generate(
-                    prompt + images, schema=ReasonScore
+        images: List[MLLMImage] = [actual_image_output]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_semantic_consistency_evaluation_results(
+                    text_prompt=text_prompt
                 )
-                return res.score, res.reasoning
-            except TypeError:
-                res = self.model.generate(prompt + images)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+            }
+            Images:
+            {images}
+        """
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     async def _a_evaluate_perceptual_quality(
         self, actual_image_output: MLLMImage
     ) -> Tuple[List[int], str]:
         images: List[MLLMImage] = [actual_image_output]
-        prompt = [
-            TextToImageTemplate.generate_perceptual_quality_evaluation_results()
-        ]
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt + images, ReasonScore
-            )
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = await self.model.a_generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = await self.model.a_generate(prompt + images)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_perceptual_quality_evaluation_results()
+            }
+            Images:
+            {images}
+        """
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
     def _evaluate_perceptual_quality(
         self, actual_image_output: MLLMImage
     ) -> Tuple[List[int], str]:
         images: List[MLLMImage] = [actual_image_output]
-        prompt = [
-            TextToImageTemplate.generate_perceptual_quality_evaluation_results()
-        ]
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt + images, ReasonScore)
-            self.evaluation_cost += cost
-            return res.score, res.reasoning
-        else:
-            try:
-                res: ReasonScore = self.model.generate(
-                    prompt + images, schema=ReasonScore
-                )
-                return res.score, res.reasoning
-            except TypeError:
-                res = self.model.generate(prompt + images)
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reasoning"]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_perceptual_quality_evaluation_results()
+            }
+            Images:
+            {images}
+        """
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ReasonScore,
+            extract_schema=lambda s: (s.score, s.reasoning),
+            extract_json=lambda data: (data["score"], data["reasoning"]),
+        )
 
-    def _calculate_score(self) -> List[str]:
+    def _calculate_score(self) -> float:
         min_SC_score = min(self.SC_scores)
         min_PQ_score = min(self.PQ_scores)
         return math.sqrt(min_SC_score * min_PQ_score) / 10
@@ -272,14 +276,12 @@ class TextToImageMetric(BaseMultimodalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 
-    def _generate_reason(
-        self,
-    ) -> Tuple[List[float], str]:
+    def _generate_reason(self) -> str:
         return textwrap.dedent(
             f"""
             The overall score is {self.score:.2f} because the lowest score from semantic consistency was {min(self.SC_scores)}