deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_g_eval/template.py

@@ -53,53 +53,45 @@ class MultimodalGEvalTemplate:
             else ""
         )
 
-        return (
-            [
-                textwrap.dedent(
-                    f"""You are an evaluator. Given the following {dependencies}, assess the response below and return a JSON object with two fields:
-
-                - `"score"`: an integer between {score_range[0]} and {score_range[1]}, {score_explanation}.
-                - `"reason"`: a brief explanation for why the score was given. This must mention specific strengths or shortcomings, referencing relevant details from the input. Do **not** quote the score itself in the explanation.
-
-                Your explanation should:
-                - {reasoning_expectation}
-                - Mention key details from the test case parameters.
-                - Be concise, clear, and focused on the evaluation logic.
-
-                Only return valid JSON. Do **not** include any extra commentary or text.
-
-                ---
-
-                Evaluation Steps:
-                {evaluation_steps}
-
-                {rubric_text}
-                Test Case:
-                ************************
-                """
-                )
-            ]
-            + test_case_list
-            + [
-                textwrap.dedent(
-                    f"""
-                ************************
-                \n\n\n
-                Parameters:
-                {parameters}
-                {additional_context}
-
-                ---
-                **Example JSON:**
-                {{
-                    "reason": "your concise and informative reason here",
-                    "score": {score_range[0]}
-                }}
-
-                JSON:
-                """
-                )
-            ]
+        return textwrap.dedent(
+            f"""You are an evaluator. Given the following {dependencies}, assess the response below and return a JSON object with two fields:
+
+            - `"score"`: an integer between {score_range[0]} and {score_range[1]}, {score_explanation}.
+            - `"reason"`: a brief explanation for why the score was given. This must mention specific strengths or shortcomings, referencing relevant details from the input. Do **not** quote the score itself in the explanation.
+
+            Your explanation should:
+            - {reasoning_expectation}
+            - Mention key details from the test case parameters.
+            - Be concise, clear, and focused on the evaluation logic.
+
+            Only return valid JSON. Do **not** include any extra commentary or text.
+
+            ---
+
+            Evaluation Steps:
+            {evaluation_steps}
+
+            {rubric_text}
+            Test Case:
+            ************************
+            
+            {test_case_list}
+
+            ************************
+            \n\n\n
+            Parameters:
+            {parameters}
+            {additional_context}
+
+            ---
+            **Example JSON:**
+            {{
+                "reason": "your concise and informative reason here",
+                "score": {score_range[0]}
+            }}
+
+            JSON:
+            """
         )
 
     @staticmethod
@@ -114,35 +106,28 @@ class MultimodalGEvalTemplate:
             if _additional_context
             else ""
         )
-        return (
-            [
-                textwrap.dedent(
-                    f"""Given the evaluation steps, return a JSON with two keys: 1) a `score` key that is STRICTLY EITHER 1 (follows the criteria 100% outlined in the evaluation steps), OR 0 (does not follow the criteria), and 2) a `reason` key, a reason for the given score, but DO NOT QUOTE THE SCORE in your reason. Please mention specific information from {parameters} in your reason, but be very concise with it!
-
-                Evaluation Steps:
-                {evaluation_steps}
-                ************************
-                """
-                )
-            ]
-            + test_case_list
-            + [
-                textwrap.dedent(
-                    f"""
-                ************************
-                {additional_context}
-                **
-                IMPORTANT: Please make sure to only return in JSON format, with the "score" and "reason" key. No words or explanation is needed.
-
-                Example JSON:
-                {{
-                    "reason": "The text does not follow the evaluation steps provided.",
-                    "score": 0
-                }}
-                **
-
-                JSON:
-                """
-                )
-            ]
+        return textwrap.dedent(
+            f"""Given the evaluation steps, return a JSON with two keys: 1) a `score` key that is STRICTLY EITHER 1 (follows the criteria 100% outlined in the evaluation steps), OR 0 (does not follow the criteria), and 2) a `reason` key, a reason for the given score, but DO NOT QUOTE THE SCORE in your reason. Please mention specific information from {parameters} in your reason, but be very concise with it!
+
+            Evaluation Steps:
+            {evaluation_steps}
+            ************************
+            
+            {test_case_list}
+        
+            ************************
+            {additional_context}
+
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the "score" and "reason" key. No words or explanation is needed.
+
+            Example JSON:
+            {{
+                "reason": "The text does not follow the evaluation steps provided.",
+                "score": 0
+            }}
+            **
+
+            JSON:
+            """
         )

deepeval/metrics/multimodal_metrics/multimodal_g_eval/utils.py

@@ -1,29 +1,26 @@
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, ToolCall
-from deepeval.test_case.mllm_test_case import MLLMImage
-from deepeval.models.mlllms.openai_model import (
+from deepeval.test_case import LLMTestCaseParams, LLMTestCase, ToolCall
+from deepeval.test_case import MLLMImage
+from deepeval.models.llms.openai_model import (
     unsupported_log_probs_multimodal_gpt_models,
 )
-from deepeval.models import (
-    DeepEvalBaseMLLM,
-    MultimodalOpenAIModel,
-)
+from deepeval.models import DeepEvalBaseLLM, GPTModel
 
 from typing import List, Union
 
 
 G_EVAL_PARAMS = {
-    MLLMTestCaseParams.INPUT: "Input",
-    MLLMTestCaseParams.ACTUAL_OUTPUT: "Actual Output",
-    MLLMTestCaseParams.EXPECTED_OUTPUT: "Expected Output",
-    MLLMTestCaseParams.CONTEXT: "Context",
-    MLLMTestCaseParams.RETRIEVAL_CONTEXT: "Retrieval Context",
-    MLLMTestCaseParams.EXPECTED_TOOLS: "Expected Tools",
-    MLLMTestCaseParams.TOOLS_CALLED: "Tools Called",
+    LLMTestCaseParams.INPUT: "Input",
+    LLMTestCaseParams.ACTUAL_OUTPUT: "Actual Output",
+    LLMTestCaseParams.EXPECTED_OUTPUT: "Expected Output",
+    LLMTestCaseParams.CONTEXT: "Context",
+    LLMTestCaseParams.RETRIEVAL_CONTEXT: "Retrieval Context",
+    LLMTestCaseParams.EXPECTED_TOOLS: "Expected Tools",
+    LLMTestCaseParams.TOOLS_CALLED: "Tools Called",
 }
 
 
 def construct_g_eval_params_string(
-    mllm_test_case_params: List[MLLMTestCaseParams],
+    mllm_test_case_params: List[LLMTestCaseParams],
 ):
     g_eval_params = [G_EVAL_PARAMS[param] for param in mllm_test_case_params]
     if len(g_eval_params) == 1:
@@ -39,12 +36,14 @@ def construct_g_eval_params_string(
 
 
 def construct_test_case_list(
-    evaluation_params: List[MLLMTestCaseParams], test_case: MLLMTestCase
+    evaluation_params: List[LLMTestCaseParams], test_case: LLMTestCase
 ) -> List[Union[str, MLLMImage]]:
+    from deepeval.utils import convert_to_multi_modal_array
+
     test_case_list = []
     for param in evaluation_params:
         test_case_param_list = [f"\n\n\n{G_EVAL_PARAMS[param]}:\n"]
-        value = getattr(test_case, param.value)
+        value = convert_to_multi_modal_array(getattr(test_case, param.value))
         for v in value:
             if isinstance(v, ToolCall):
                 test_case_param_list.append(repr(v))
@@ -54,15 +53,16 @@ def construct_test_case_list(
     return test_case_list
 
 
-def no_multimodal_log_prob_support(model: Union[str, DeepEvalBaseMLLM]):
+def no_multimodal_log_prob_support(model: Union[str, DeepEvalBaseLLM]):
     if (
         isinstance(model, str)
         and model in unsupported_log_probs_multimodal_gpt_models
     ):
         return True
     elif (
-        isinstance(model, MultimodalOpenAIModel)
-        and model.model_name in unsupported_log_probs_multimodal_gpt_models
+        isinstance(model, GPTModel)
+        and model.get_model_name()
+        in unsupported_log_probs_multimodal_gpt_models
     ):
         return True
     return False

deepeval/metrics/multimodal_metrics/text_to_image/text_to_image.py

@@ -4,37 +4,40 @@ import math
 import textwrap
 
 from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
+from deepeval.test_case import LLMTestCaseParams, LLMTestCase, MLLMImage
 from deepeval.metrics.multimodal_metrics.text_to_image.template import (
     TextToImageTemplate,
 )
-from deepeval.utils import get_or_create_event_loop
+from deepeval.utils import (
+    get_or_create_event_loop,
+    convert_to_multi_modal_array,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_mllm_test_case_params,
-    initialize_multimodal_model,
+    initialize_model,
 )
-from deepeval.models import DeepEvalBaseMLLM
+from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.multimodal_metrics.text_to_image.schema import ReasonScore
 from deepeval.metrics.indicator import metric_progress_indicator
 
-required_params: List[MLLMTestCaseParams] = [
-    MLLMTestCaseParams.INPUT,
-    MLLMTestCaseParams.ACTUAL_OUTPUT,
+required_params: List[LLMTestCaseParams] = [
+    LLMTestCaseParams.INPUT,
+    LLMTestCaseParams.ACTUAL_OUTPUT,
 ]
 
 
 class TextToImageMetric(BaseMultimodalMetric):
     def __init__(
         self,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         threshold: float = 0.5,
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
-        self.model, self.using_native_model = initialize_multimodal_model(model)
+        self.model, self.using_native_model = initialize_model(model)
         self.evaluation_model = self.model.get_model_name()
         self.threshold = 1 if strict_mode else threshold
         self.strict_mode = strict_mode
@@ -43,11 +46,13 @@ class TextToImageMetric(BaseMultimodalMetric):
 
     def measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
     ) -> float:
-        check_mllm_test_case_params(test_case, required_params, 0, 1, self)
+        check_mllm_test_case_params(
+            test_case, required_params, 0, 1, self, self.model
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -63,10 +68,12 @@ class TextToImageMetric(BaseMultimodalMetric):
                     )
                 )
             else:
-                input_texts, _ = self.separate_images_from_text(test_case.input)
-                _, output_images = self.separate_images_from_text(
+                input = convert_to_multi_modal_array(test_case.input)
+                actual_output = convert_to_multi_modal_array(
                     test_case.actual_output
                 )
+                input_texts, _ = self.separate_images_from_text(input)
+                _, output_images = self.separate_images_from_text(actual_output)
 
                 self.SC_scores, self.SC_reasoning = (
                     self._evaluate_semantic_consistency(
@@ -99,11 +106,13 @@ class TextToImageMetric(BaseMultimodalMetric):
 
     async def a_measure(
         self,
-        test_case: MLLMTestCase,
+        test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
     ) -> float:
-        check_mllm_test_case_params(test_case, required_params, 0, 1, self)
+        check_mllm_test_case_params(
+            test_case, required_params, 0, 1, self, self.model
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -112,10 +121,12 @@ class TextToImageMetric(BaseMultimodalMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            input_texts, _ = self.separate_images_from_text(test_case.input)
-            _, output_images = self.separate_images_from_text(
+            input = convert_to_multi_modal_array(test_case.input)
+            actual_output = convert_to_multi_modal_array(
                 test_case.actual_output
             )
+            input_texts, _ = self.separate_images_from_text(input)
+            _, output_images = self.separate_images_from_text(actual_output)
             (self.SC_scores, self.SC_reasoning), (
                 self.PQ_scores,
                 self.PQ_reasoning,
@@ -165,27 +176,27 @@ class TextToImageMetric(BaseMultimodalMetric):
     ) -> Tuple[List[int], str]:
         images: List[MLLMImage] = []
         images.append(actual_image_output)
-        prompt = [
-            TextToImageTemplate.generate_semantic_consistency_evaluation_results(
-                text_prompt=text_prompt
-            )
-        ]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_semantic_consistency_evaluation_results(
+                    text_prompt=text_prompt
+                )
+            }
+            Images:
+            {images}
+        """
         if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt + images, ReasonScore
-            )
+            res, cost = await self.model.a_generate(prompt, ReasonScore)
             self.evaluation_cost += cost
             return res.score, res.reasoning
         else:
             try:
                 res: ReasonScore = await self.model.a_generate(
-                    prompt + images, schema=ReasonScore
+                    prompt, schema=ReasonScore
                 )
                 return res.score, res.reasoning
             except TypeError:
-                res = await self.model.a_generate(
-                    prompt + images, input_text=prompt
-                )
+                res = await self.model.a_generate(prompt, input_text=prompt)
                 data = trimAndLoadJson(res, self)
                 return data["score"], data["reasoning"]
 
@@ -196,23 +207,27 @@ class TextToImageMetric(BaseMultimodalMetric):
     ) -> Tuple[List[int], str]:
         images: List[MLLMImage] = []
         images.append(actual_image_output)
-        prompt = [
-            TextToImageTemplate.generate_semantic_consistency_evaluation_results(
-                text_prompt=text_prompt
-            )
-        ]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_semantic_consistency_evaluation_results(
+                    text_prompt=text_prompt
+                )
+            }
+            Images:
+            {images}
+        """
         if self.using_native_model:
-            res, cost = self.model.generate(prompt + images, ReasonScore)
+            res, cost = self.model.generate(prompt, ReasonScore)
             self.evaluation_cost += cost
             return res.score, res.reasoning
         else:
             try:
                 res: ReasonScore = self.model.generate(
-                    prompt + images, schema=ReasonScore
+                    prompt, schema=ReasonScore
                 )
                 return res.score, res.reasoning
             except TypeError:
-                res = self.model.generate(prompt + images)
+                res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return data["score"], data["reasoning"]
 
@@ -220,23 +235,25 @@ class TextToImageMetric(BaseMultimodalMetric):
         self, actual_image_output: MLLMImage
     ) -> Tuple[List[int], str]:
         images: List[MLLMImage] = [actual_image_output]
-        prompt = [
-            TextToImageTemplate.generate_perceptual_quality_evaluation_results()
-        ]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_perceptual_quality_evaluation_results()
+            }
+            Images:
+            {images}
+        """
         if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt + images, ReasonScore
-            )
+            res, cost = await self.model.a_generate(prompt, ReasonScore)
             self.evaluation_cost += cost
             return res.score, res.reasoning
         else:
             try:
                 res: ReasonScore = await self.model.a_generate(
-                    prompt + images, schema=ReasonScore
+                    prompt, schema=ReasonScore
                 )
                 return res.score, res.reasoning
             except TypeError:
-                res = await self.model.a_generate(prompt + images)
+                res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return data["score"], data["reasoning"]
 
@@ -244,9 +261,13 @@ class TextToImageMetric(BaseMultimodalMetric):
         self, actual_image_output: MLLMImage
     ) -> Tuple[List[int], str]:
         images: List[MLLMImage] = [actual_image_output]
-        prompt = [
-            TextToImageTemplate.generate_perceptual_quality_evaluation_results()
-        ]
+        prompt = f"""
+            {
+                TextToImageTemplate.generate_perceptual_quality_evaluation_results()
+            }
+            Images:
+            {images}
+        """
         if self.using_native_model:
             res, cost = self.model.generate(prompt + images, ReasonScore)
             self.evaluation_cost += cost
@@ -254,11 +275,11 @@ class TextToImageMetric(BaseMultimodalMetric):
         else:
             try:
                 res: ReasonScore = self.model.generate(
-                    prompt + images, schema=ReasonScore
+                    prompt, schema=ReasonScore
                 )
                 return res.score, res.reasoning
             except TypeError:
-                res = self.model.generate(prompt + images)
+                res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return data["score"], data["reasoning"]
 

deepeval/metrics/ragas.py

@@ -10,7 +10,7 @@ from deepeval.telemetry import capture_metric_type
 
 # check langchain availability
 try:
-    import langchain_core
+    import langchain_core  # noqa: F401
     from langchain_core.language_models import BaseChatModel
     from langchain_core.embeddings import Embeddings
 
@@ -501,7 +501,7 @@ class RagasMetric(BaseMetric):
     def measure(self, test_case: LLMTestCase):
         # sends to server
         try:
-            from ragas import evaluate
+            from ragas import evaluate  # noqa: F401
         except ModuleNotFoundError:
             raise ModuleNotFoundError(
                 "Please install ragas to use this metric. `pip install ragas`."
@@ -509,7 +509,7 @@ class RagasMetric(BaseMetric):
 
         try:
             # How do i make sure this isn't just huggingface dataset
-            from datasets import Dataset
+            from datasets import Dataset  # noqa: F401
         except ModuleNotFoundError:
             raise ModuleNotFoundError("Please install dataset")
 

deepeval/metrics/tool_correctness/tool_correctness.py

@@ -83,7 +83,7 @@ class ToolCorrectnessMetric(BaseMetric):
                 self.tools_called: List[ToolCall] = test_case.tools_called
                 self.expected_tools: List[ToolCall] = test_case.expected_tools
                 tool_calling_score = self._calculate_score()
-                if self.available_tools:
+                if self.available_tools and not test_case.multimodal:
                     tool_selection_score = self._get_tool_selection_score(
                         test_case.input,
                         test_case.tools_called,
@@ -177,7 +177,7 @@ class ToolCorrectnessMetric(BaseMetric):
             self.tools_called: List[ToolCall] = test_case.tools_called
             self.expected_tools: List[ToolCall] = test_case.expected_tools
             tool_calling_score = self._calculate_score()
-            if self.available_tools:
+            if self.available_tools and not test_case.multimodal:
                 tool_selection_score = await self._a_get_tool_selection_score(
                     test_case.input,
                     test_case.tools_called,

deepeval/metrics/turn_contextual_precision/schema.py

@@ -0,0 +1,21 @@
+from typing import List
+from pydantic import BaseModel
+
+
+class ContextualPrecisionVerdict(BaseModel):
+    verdict: str
+    reason: str
+
+
+class Verdicts(BaseModel):
+    verdicts: List[ContextualPrecisionVerdict]
+
+
+class ContextualPrecisionScoreReason(BaseModel):
+    reason: str
+
+
+class InteractionContextualPrecisionScore(BaseModel):
+    score: float
+    reason: str
+    verdicts: List[ContextualPrecisionVerdict]