deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/g_eval/template.py

@@ -3,11 +3,23 @@ import textwrap
 
 
 class GEvalTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def generate_evaluation_steps(parameters: str, criteria: str):
+    def generate_evaluation_steps(
+        parameters: str, criteria: str, multimodal: bool = False
+    ):
         return textwrap.dedent(
             f"""Given an evaluation criteria which outlines how you should judge the {parameters}, generate 3-4 concise evaluation steps based on the criteria below. You MUST make it clear how to evaluate {parameters} in relation to one another.
 
+            {GEvalTemplate.multimodal_rules if multimodal else ""}
+
             Evaluation Criteria:
             {criteria}
 
@@ -31,6 +43,7 @@ class GEvalTemplate:
         rubric: Optional[str] = None,
         score_range: Tuple[int, int] = (0, 10),
         _additional_context: Optional[str] = None,
+        multimodal: bool = False,
     ):
         rubric_text = f"Rubric:\n{rubric}\n" if rubric else ""
         dependencies = (
@@ -62,6 +75,7 @@ class GEvalTemplate:
             - {reasoning_expectation}
             - Mention key details from the test case parameters.
             - Be concise, clear, and focused on the evaluation logic.
+            {GEvalTemplate.multimodal_rules if multimodal else ""}
 
             Only return valid JSON. Do **not** include any extra commentary or text.
 
@@ -95,6 +109,7 @@ class GEvalTemplate:
         test_case_content: str,
         parameters: str,
         _additional_context: Optional[str] = None,
+        multimodal: bool = False,
     ):
         additional_context = (
             f"\n\nAdditional Context:\n{_additional_context}\n"
@@ -104,6 +119,8 @@ class GEvalTemplate:
         return textwrap.dedent(
             f"""Given the evaluation steps, return a JSON with two keys: 1) a `score` key that is STRICTLY EITHER 1 (follows the criteria 100% outlined in the evaluation steps), OR 0 (does not follow the criteria), and 2) a `reason` key, a reason for the given score, but DO NOT QUOTE THE SCORE in your reason. Please mention specific information from {parameters} in your reason, but be very concise with it!
 
+            {GEvalTemplate.multimodal_rules if multimodal else ""}
+
             Evaluation Steps:
             {evaluation_steps}
 

deepeval/metrics/g_eval/utils.py

@@ -9,8 +9,8 @@ from deepeval.test_case import (
     LLMTestCase,
     ToolCall,
 )
-from deepeval.models.llms.openai_model import unsupported_log_probs_gpt_models
 from pydantic import BaseModel, field_validator
+from deepeval.models.llms.constants import OPENAI_MODELS_DATA
 
 from deepeval.test_case.conversational_test_case import ConversationalTestCase
 
@@ -114,16 +114,17 @@ def format_rubrics(rubrics: Optional[List[Rubric]]) -> Optional[str]:
 
 def no_log_prob_support(model: Union[str, DeepEvalBaseLLM]):
 
-    if isinstance(model, str) and model in unsupported_log_probs_gpt_models:
-        return True
+    if isinstance(model, str):
+        model_data = OPENAI_MODELS_DATA.get(model)
+        if not model_data.supports_log_probs:
+            return True
     elif (
-        isinstance(model, GPTModel)
-        and model.model_name in unsupported_log_probs_gpt_models
+        isinstance(model, GPTModel) and not model.model_data.supports_log_probs
     ):
         return True
     elif (
         isinstance(model, AzureOpenAIModel)
-        and model.model_name in unsupported_log_probs_gpt_models
+        and not model.model_data.supports_log_probs
     ):
         return True
 

deepeval/metrics/goal_accuracy/goal_accuracy.py

@@ -3,11 +3,12 @@ import asyncio
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     get_unit_interactions,
     print_tools_called,
     check_conversational_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
 from deepeval.metrics import BaseConversationalMetric
@@ -55,8 +56,14 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            None,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -80,17 +87,21 @@ class GoalAccuracyMetric(BaseConversationalMetric):
                 )
                 goal_scores = [
                     self._get_goal_accuracy_score(
-                        task.user_goal, task.steps_taken
+                        task.user_goal, task.steps_taken, multimodal
                     )
                     for task in goal_and_steps_taken
                 ]
                 plan_scores = [
-                    self._get_plan_scores(task.user_goal, task.steps_taken)
+                    self._get_plan_scores(
+                        task.user_goal, task.steps_taken, multimodal
+                    )
                     for task in goal_and_steps_taken
                 ]
                 self.score = self._calculate_score(goal_scores, plan_scores)
                 self.success = self.score >= self.threshold
-                self.reason = self._generate_reason(goal_scores, plan_scores)
+                self.reason = self._generate_reason(
+                    goal_scores, plan_scores, multimodal
+                )
 
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -117,8 +128,14 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            None,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -134,21 +151,23 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             goal_scores = await asyncio.gather(
                 *[
                     self._a_get_goal_accuracy_score(
-                        task.user_goal, task.steps_taken
+                        task.user_goal, task.steps_taken, multimodal
                     )
                     for task in goal_and_steps_taken
                 ]
             )
             plan_scores = await asyncio.gather(
                 *[
-                    self._a_get_plan_scores(task.user_goal, task.steps_taken)
+                    self._a_get_plan_scores(
+                        task.user_goal, task.steps_taken, multimodal
+                    )
                     for task in goal_and_steps_taken
                 ]
             )
             self.score = self._calculate_score(goal_scores, plan_scores)
             self.success = self.score >= self.threshold
             self.reason = await self._a_generate_reason(
-                goal_scores, plan_scores
+                goal_scores, plan_scores, multimodal
             )
 
             self.verbose_logs = construct_verbose_logs(
@@ -191,41 +210,31 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             goal_and_steps_taken.append(new_goal_steps)
         return goal_and_steps_taken
 
-    def _get_plan_scores(self, user_goal, steps_taken):
+    def _get_plan_scores(self, user_goal, steps_taken, multimodal: bool):
         prompt = GoalAccuracyTemplate.get_plan_evaluation_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanScore(**data),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=PlanScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: PlanScore = self.model.generate(prompt, schema=PlanScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanScore(**data)
 
-    async def _a_get_plan_scores(self, user_goal, steps_taken):
+    async def _a_get_plan_scores(
+        self, user_goal, steps_taken, multimodal: bool
+    ):
         prompt = GoalAccuracyTemplate.get_plan_evaluation_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanScore(**data),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=PlanScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: PlanScore = await self.model.a_generate(
-                    prompt, schema=PlanScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanScore(**data)
 
     def _calculate_score(
         self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
@@ -240,7 +249,10 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     def _generate_reason(
-        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+        self,
+        goal_scores: List[GoalScore],
+        plan_scores: List[PlanScore],
+        multimodal: bool,
     ):
         goal_evaluations = ""
         for goal_score in goal_scores:
@@ -254,18 +266,25 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             )
 
         prompt = GoalAccuracyTemplate.get_final_reason(
-            self.score, self.threshold, goal_evaluations, plan_evalautions
+            self.score,
+            self.threshold,
+            goal_evaluations,
+            plan_evalautions,
+            multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
+            self._accrue_cost(cost)
             return res
         else:
             res = self.model.generate(prompt)
             return res
 
     async def _a_generate_reason(
-        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+        self,
+        goal_scores: List[GoalScore],
+        plan_scores: List[PlanScore],
+        multimodal: bool,
     ):
         goal_evaluations = ""
         for goal_score in goal_scores:
@@ -279,51 +298,47 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             )
 
         prompt = GoalAccuracyTemplate.get_final_reason(
-            self.score, self.threshold, goal_evaluations, plan_evalautions
+            self.score,
+            self.threshold,
+            goal_evaluations,
+            plan_evalautions,
+            multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
+            self._accrue_cost(cost)
             return res
         else:
             res = await self.model.a_generate(prompt)
             return res
 
-    def _get_goal_accuracy_score(self, user_goal, steps_taken):
+    def _get_goal_accuracy_score(
+        self, user_goal, steps_taken, multimodal: bool
+    ):
         prompt = GoalAccuracyTemplate.get_accuracy_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=GoalScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: GoalScore(**data),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=GoalScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: GoalScore = self.model.generate(prompt, schema=GoalScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return GoalScore(**data)
 
-    async def _a_get_goal_accuracy_score(self, user_goal, steps_taken):
+    async def _a_get_goal_accuracy_score(
+        self, user_goal, steps_taken, multimodal: bool
+    ):
         prompt = GoalAccuracyTemplate.get_accuracy_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=GoalScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: GoalScore(**data),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=GoalScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: GoalScore = await self.model.a_generate(
-                    prompt, schema=GoalScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return GoalScore(**data)
 
     def print_goals_and_steps_taken(self, goals_and_steps):
         final_goals_and_steps = ""
@@ -340,7 +355,7 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/goal_accuracy/template.py

@@ -3,8 +3,16 @@ import textwrap
 
 
 class GoalAccuracyTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def get_accuracy_score(task, steps_taken):
+    def get_accuracy_score(task, steps_taken, multimodal: bool = False):
         return textwrap.dedent(
             f"""You are an expert evaluator assessing the **goal accuracy** of an AI assistant's single interaction.
 
@@ -36,6 +44,8 @@ class GoalAccuracyTemplate:
                 - When uncertain, assume the goal was **not achieved**.
                 - The metric is designed to fail unless the assistant's output is precise, complete, and user-visible.
 
+                {GoalAccuracyTemplate.multimodal_rules if multimodal else ""}
+
                 SCORING GUIDE:
 
                 - **1.0** → Goal completely and correctly achieved; all required outputs visible to the user.  
@@ -102,7 +112,7 @@ class GoalAccuracyTemplate:
         )
 
     @staticmethod
-    def get_plan_evaluation_score(task, steps_taken):
+    def get_plan_evaluation_score(task, steps_taken, multimodal: bool = False):
         return textwrap.dedent(
             f"""You are an expert evaluator assessing the **planning quality** and **plan adherence** of an AI agent tasked with fulfilling a user's request.
 
@@ -132,6 +142,8 @@ class GoalAccuracyTemplate:
                 - Tool use should be coherent within the plan, not ad hoc or speculative.  
                 - This evaluation excludes correctness or efficiency — focus solely on plan and adherence.
 
+                {GoalAccuracyTemplate.multimodal_rules if multimodal else ""}
+
                 SCORING GUIDE:
 
                 - **1.0** → Complete, clear, and logical plan **fully followed** with all steps aligned to the user's goal.  
@@ -188,7 +200,11 @@ class GoalAccuracyTemplate:
 
     @staticmethod
     def get_final_reason(
-        final_score, threshold, goal_evaluations, plan_evalautions
+        final_score,
+        threshold,
+        goal_evaluations,
+        plan_evalautions,
+        multimodal: bool = False,
     ):
         return textwrap.dedent(
             f"""You are an expert evaluator providing a **final justification** for whether an AI agent has passed or failed an evaluation metric.
@@ -213,6 +229,8 @@ class GoalAccuracyTemplate:
                 - If the agent **failed**, explain which aspects (task or plan or both) led to the failure.
                 - Avoid vague praise or criticism — ground the reason in the actual scores and justifications.
 
+                {GoalAccuracyTemplate.multimodal_rules if multimodal else ""}
+
                 ---
 
                 FORMAT:

deepeval/metrics/hallucination/hallucination.py

@@ -8,14 +8,19 @@ from deepeval.metrics import BaseMetric
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.hallucination.template import HallucinationTemplate
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.hallucination.schema import *
+from deepeval.metrics.hallucination.schema import (
+    HallucinationVerdict,
+    Verdicts,
+    HallucinationScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -55,7 +60,16 @@ class HallucinationMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -102,7 +116,16 @@ class HallucinationMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -150,22 +173,13 @@ class HallucinationMetric(BaseMetric):
             score=format(self.score, ".2f"),
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=HallucinationScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: HallucinationScoreReason = await self.model.a_generate(
-                    prompt, schema=HallucinationScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=HallucinationScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self):
         if self.include_reason is False:
@@ -185,74 +199,45 @@ class HallucinationMetric(BaseMetric):
             score=format(self.score, ".2f"),
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=HallucinationScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: HallucinationScoreReason = self.model.generate(
-                    prompt, schema=HallucinationScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=HallucinationScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, actual_output: str, contexts: List[str]
     ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
             actual_output=actual_output, contexts=contexts
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                HallucinationVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(
         self, actual_output: str, contexts: List[str]
     ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
             actual_output=actual_output, contexts=contexts
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                HallucinationVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -273,7 +258,7 @@ class HallucinationMetric(BaseMetric):
         else:
             try:
                 self.success = self.score <= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/hallucination/template.py

@@ -2,9 +2,20 @@ from typing import List
 
 
 class HallucinationTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_verdicts(actual_output: str, contexts: List[str]):
         return f"""For each context in contexts, which is a list of strings, please generate a list of JSON objects to indicate whether the given 'actual output' agrees with EACH context. The JSON will have 2 fields: 'verdict' and 'reason'.
+
+{HallucinationTemplate.multimodal_rules}
+
 The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the given text agrees with the context. 
 The 'reason' is the reason for the verdict. When the answer is 'no', try to provide a correction in the reason. 
 
@@ -46,6 +57,8 @@ JSON:
     ):
         return f"""Given a list of factual alignments and contradictions, which highlights alignment/contradictions between the `actual output` and `contexts, use it to provide a reason for the hallucination score in a CONCISELY. Note that The hallucination score ranges from 0 - 1, and the lower the better.
 
+{HallucinationTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON: