deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/g_eval/g_eval.py

@@ -15,6 +15,8 @@ from deepeval.metrics.utils import (
     trimAndLoadJson,
     initialize_model,
     check_llm_test_case_params,
+    generate_with_schema_and_extract,
+    a_generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
@@ -82,7 +84,19 @@ class GEval(BaseMetric):
         _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
-        check_llm_test_case_params(test_case, self.evaluation_params, self)
+
+        multimodal = test_case.multimodal
+
+        check_llm_test_case_params(
+            test_case,
+            self.evaluation_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
+
         self.evaluation_cost = 0 if self.using_native_model else None
 
         with metric_progress_indicator(
@@ -96,18 +110,25 @@ class GEval(BaseMetric):
                     _in_component=_in_component,
                     _additional_context=_additional_context,
                 )
+                settings = get_settings()
                 loop.run_until_complete(
                     asyncio.wait_for(
                         coro,
-                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                        timeout=(
+                            None
+                            if settings.DEEPEVAL_DISABLE_TIMEOUTS
+                            else settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+                        ),
                     )
                 )
             else:
                 self.evaluation_steps: List[str] = (
-                    self._generate_evaluation_steps()
+                    self._generate_evaluation_steps(multimodal)
                 )
                 g_score, reason = self._evaluate(
-                    test_case, _additional_context=_additional_context
+                    test_case,
+                    _additional_context=_additional_context,
+                    multimodal=multimodal,
                 )
                 self.score = (
                     (float(g_score) - self.score_range[0])
@@ -143,7 +164,18 @@ class GEval(BaseMetric):
         _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
-        check_llm_test_case_params(test_case, self.evaluation_params, self)
+
+        multimodal = test_case.multimodal
+
+        check_llm_test_case_params(
+            test_case,
+            self.evaluation_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -153,10 +185,12 @@ class GEval(BaseMetric):
             _in_component=_in_component,
         ):
             self.evaluation_steps: List[str] = (
-                await self._a_generate_evaluation_steps()
+                await self._a_generate_evaluation_steps(multimodal)
             )
             g_score, reason = await self._a_evaluate(
-                test_case, _additional_context=_additional_context
+                test_case,
+                _additional_context=_additional_context,
+                multimodal=multimodal,
             )
             self.score = (
                 (float(g_score) - self.score_range[0]) / self.score_range_span
@@ -182,7 +216,7 @@ class GEval(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_evaluation_steps(self) -> List[str]:
+    async def _a_generate_evaluation_steps(self, multimodal: bool) -> List[str]:
         if self.evaluation_steps:
             return self.evaluation_steps
 
@@ -190,25 +224,19 @@ class GEval(BaseMetric):
             self.evaluation_params
         )
         prompt = self.evaluation_template.generate_evaluation_steps(
-            criteria=self.criteria, parameters=g_eval_params_str
+            criteria=self.criteria,
+            parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=gschema.Steps,
+            extract_schema=lambda s: s.steps,
+            extract_json=lambda d: d["steps"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["steps"]
-        else:
-            try:
-                res: gschema.Steps = await self.model.a_generate(
-                    prompt, schema=gschema.Steps
-                )
-                return res.steps
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["steps"]
 
-    def _generate_evaluation_steps(self) -> List[str]:
+    def _generate_evaluation_steps(self, multimodal: bool) -> List[str]:
         if self.evaluation_steps:
             return self.evaluation_steps
 
@@ -216,26 +244,23 @@ class GEval(BaseMetric):
             self.evaluation_params
         )
         prompt = self.evaluation_template.generate_evaluation_steps(
-            criteria=self.criteria, parameters=g_eval_params_str
+            criteria=self.criteria,
+            parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=gschema.Steps,
+            extract_schema=lambda s: s.steps,
+            extract_json=lambda d: d["steps"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["steps"]
-        else:
-            try:
-                res: gschema.Steps = self.model.generate(
-                    prompt, schema=gschema.Steps
-                )
-                return res.steps
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["steps"]
 
     async def _a_evaluate(
-        self, test_case: LLMTestCase, _additional_context: Optional[str] = None
+        self,
+        test_case: LLMTestCase,
+        multimodal: bool,
+        _additional_context: Optional[str] = None,
     ) -> Tuple[Union[int, float], str]:
         test_case_content = construct_test_case_string(
             self.evaluation_params, test_case
@@ -252,6 +277,7 @@ class GEval(BaseMetric):
                 rubric=rubric_str,
                 score_range=self.score_range,
                 _additional_context=_additional_context,
+                multimodal=multimodal,
             )
         else:
             prompt = (
@@ -262,6 +288,7 @@ class GEval(BaseMetric):
                     test_case_content=test_case_content,
                     parameters=g_eval_params_str,
                     _additional_context=_additional_context,
+                    multimodal=multimodal,
                 )
             )
         try:
@@ -275,8 +302,7 @@ class GEval(BaseMetric):
                 prompt, top_logprobs=self.top_logprobs
             )
 
-            if self.evaluation_cost is not None:
-                self.evaluation_cost += cost
+            self._accrue_cost(cost)
 
             data = trimAndLoadJson(res.choices[0].message.content, self)
 
@@ -292,27 +318,21 @@ class GEval(BaseMetric):
                 return weighted_summed_score, reason
             except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
-        except (
-            AttributeError
-        ):  # This catches the case where a_generate_raw_response doesn't exist.
-            if self.using_native_model:
-                res, cost = await self.model.a_generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reason"]
-            else:
-                try:
-                    res: gschema.ReasonScore = await self.model.a_generate(
-                        prompt, schema=gschema.ReasonScore
-                    )
-                    return res.score, res.reason
-                except TypeError:
-                    res = await self.model.a_generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    return data["score"], data["reason"]
+        except AttributeError:
+            # This catches the case where a_generate_raw_response doesn't exist.
+            return await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=gschema.ReasonScore,
+                extract_schema=lambda s: (s.score, s.reason),
+                extract_json=lambda d: (d["score"], d["reason"]),
+            )
 
     def _evaluate(
-        self, test_case: LLMTestCase, _additional_context: Optional[str] = None
+        self,
+        test_case: LLMTestCase,
+        multimodal: bool,
+        _additional_context: Optional[str] = None,
     ) -> Tuple[Union[int, float], str]:
         test_case_content = construct_test_case_string(
             self.evaluation_params, test_case
@@ -330,6 +350,7 @@ class GEval(BaseMetric):
                 rubric=rubric_str,
                 score_range=self.score_range,
                 _additional_context=_additional_context,
+                multimodal=multimodal,
             )
         else:
             prompt = (
@@ -340,6 +361,7 @@ class GEval(BaseMetric):
                     test_case_content=test_case_content,
                     parameters=g_eval_params_str,
                     _additional_context=_additional_context,
+                    multimodal=multimodal,
                 )
             )
 
@@ -351,7 +373,7 @@ class GEval(BaseMetric):
             res, cost = self.model.generate_raw_response(
                 prompt, top_logprobs=self.top_logprobs
             )
-            self.evaluation_cost += cost
+            self._accrue_cost(cost)
             data = trimAndLoadJson(res.choices[0].message.content, self)
 
             reason = data["reason"]
@@ -368,21 +390,13 @@ class GEval(BaseMetric):
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
-            if self.using_native_model:
-                res, cost = self.model.generate(prompt)
-                self.evaluation_cost += cost
-                data = trimAndLoadJson(res, self)
-                return data["score"], data["reason"]
-            else:
-                try:
-                    res: gschema.ReasonScore = self.model.generate(
-                        prompt, schema=gschema.ReasonScore
-                    )
-                    return res.score, res.reason
-                except TypeError:
-                    res = self.model.generate(prompt)
-                    data = trimAndLoadJson(res, self)
-                    return data["score"], data["reason"]
+            return generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=gschema.ReasonScore,
+                extract_schema=lambda s: (s.score, s.reason),
+                extract_json=lambda d: (d["score"], d["reason"]),
+            )
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/g_eval/template.py

@@ -3,11 +3,23 @@ import textwrap
 
 
 class GEvalTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def generate_evaluation_steps(parameters: str, criteria: str):
+    def generate_evaluation_steps(
+        parameters: str, criteria: str, multimodal: bool = False
+    ):
         return textwrap.dedent(
             f"""Given an evaluation criteria which outlines how you should judge the {parameters}, generate 3-4 concise evaluation steps based on the criteria below. You MUST make it clear how to evaluate {parameters} in relation to one another.
 
+            {GEvalTemplate.multimodal_rules if multimodal else ""}
+
             Evaluation Criteria:
             {criteria}
 
@@ -31,6 +43,7 @@ class GEvalTemplate:
         rubric: Optional[str] = None,
         score_range: Tuple[int, int] = (0, 10),
         _additional_context: Optional[str] = None,
+        multimodal: bool = False,
     ):
         rubric_text = f"Rubric:\n{rubric}\n" if rubric else ""
         dependencies = (
@@ -62,6 +75,7 @@ class GEvalTemplate:
             - {reasoning_expectation}
             - Mention key details from the test case parameters.
             - Be concise, clear, and focused on the evaluation logic.
+            {GEvalTemplate.multimodal_rules if multimodal else ""}
 
             Only return valid JSON. Do **not** include any extra commentary or text.
 
@@ -95,6 +109,7 @@ class GEvalTemplate:
         test_case_content: str,
         parameters: str,
         _additional_context: Optional[str] = None,
+        multimodal: bool = False,
     ):
         additional_context = (
             f"\n\nAdditional Context:\n{_additional_context}\n"
@@ -104,6 +119,8 @@ class GEvalTemplate:
         return textwrap.dedent(
             f"""Given the evaluation steps, return a JSON with two keys: 1) a `score` key that is STRICTLY EITHER 1 (follows the criteria 100% outlined in the evaluation steps), OR 0 (does not follow the criteria), and 2) a `reason` key, a reason for the given score, but DO NOT QUOTE THE SCORE in your reason. Please mention specific information from {parameters} in your reason, but be very concise with it!
 
+            {GEvalTemplate.multimodal_rules if multimodal else ""}
+
             Evaluation Steps:
             {evaluation_steps}
 

deepeval/metrics/g_eval/utils.py

@@ -9,8 +9,8 @@ from deepeval.test_case import (
     LLMTestCase,
     ToolCall,
 )
-from deepeval.models.llms.openai_model import unsupported_log_probs_gpt_models
 from pydantic import BaseModel, field_validator
+from deepeval.models.llms.constants import OPENAI_MODELS_DATA
 
 from deepeval.test_case.conversational_test_case import ConversationalTestCase
 
@@ -114,16 +114,17 @@ def format_rubrics(rubrics: Optional[List[Rubric]]) -> Optional[str]:
 
 def no_log_prob_support(model: Union[str, DeepEvalBaseLLM]):
 
-    if isinstance(model, str) and model in unsupported_log_probs_gpt_models:
-        return True
+    if isinstance(model, str):
+        model_data = OPENAI_MODELS_DATA.get(model)
+        if not model_data.supports_log_probs:
+            return True
     elif (
-        isinstance(model, GPTModel)
-        and model.get_model_name() in unsupported_log_probs_gpt_models
+        isinstance(model, GPTModel) and not model.model_data.supports_log_probs
     ):
         return True
     elif (
         isinstance(model, AzureOpenAIModel)
-        and model.get_model_name() in unsupported_log_probs_gpt_models
+        and not model.model_data.supports_log_probs
     ):
         return True
 

deepeval/metrics/goal_accuracy/goal_accuracy.py

@@ -3,11 +3,12 @@ import asyncio
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     get_unit_interactions,
     print_tools_called,
     check_conversational_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
 from deepeval.metrics import BaseConversationalMetric
@@ -55,8 +56,14 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            None,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -80,17 +87,21 @@ class GoalAccuracyMetric(BaseConversationalMetric):
                 )
                 goal_scores = [
                     self._get_goal_accuracy_score(
-                        task.user_goal, task.steps_taken
+                        task.user_goal, task.steps_taken, multimodal
                     )
                     for task in goal_and_steps_taken
                 ]
                 plan_scores = [
-                    self._get_plan_scores(task.user_goal, task.steps_taken)
+                    self._get_plan_scores(
+                        task.user_goal, task.steps_taken, multimodal
+                    )
                     for task in goal_and_steps_taken
                 ]
                 self.score = self._calculate_score(goal_scores, plan_scores)
                 self.success = self.score >= self.threshold
-                self.reason = self._generate_reason(goal_scores, plan_scores)
+                self.reason = self._generate_reason(
+                    goal_scores, plan_scores, multimodal
+                )
 
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -117,8 +128,14 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            None,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -134,21 +151,23 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             goal_scores = await asyncio.gather(
                 *[
                     self._a_get_goal_accuracy_score(
-                        task.user_goal, task.steps_taken
+                        task.user_goal, task.steps_taken, multimodal
                     )
                     for task in goal_and_steps_taken
                 ]
             )
             plan_scores = await asyncio.gather(
                 *[
-                    self._a_get_plan_scores(task.user_goal, task.steps_taken)
+                    self._a_get_plan_scores(
+                        task.user_goal, task.steps_taken, multimodal
+                    )
                     for task in goal_and_steps_taken
                 ]
             )
             self.score = self._calculate_score(goal_scores, plan_scores)
             self.success = self.score >= self.threshold
             self.reason = await self._a_generate_reason(
-                goal_scores, plan_scores
+                goal_scores, plan_scores, multimodal
             )
 
             self.verbose_logs = construct_verbose_logs(
@@ -191,41 +210,31 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             goal_and_steps_taken.append(new_goal_steps)
         return goal_and_steps_taken
 
-    def _get_plan_scores(self, user_goal, steps_taken):
+    def _get_plan_scores(self, user_goal, steps_taken, multimodal: bool):
         prompt = GoalAccuracyTemplate.get_plan_evaluation_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanScore(**data),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=PlanScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: PlanScore = self.model.generate(prompt, schema=PlanScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanScore(**data)
 
-    async def _a_get_plan_scores(self, user_goal, steps_taken):
+    async def _a_get_plan_scores(
+        self, user_goal, steps_taken, multimodal: bool
+    ):
         prompt = GoalAccuracyTemplate.get_plan_evaluation_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanScore(**data),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=PlanScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: PlanScore = await self.model.a_generate(
-                    prompt, schema=PlanScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanScore(**data)
 
     def _calculate_score(
         self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
@@ -240,7 +249,10 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         return 0 if self.strict_mode and score < self.threshold else score
 
     def _generate_reason(
-        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+        self,
+        goal_scores: List[GoalScore],
+        plan_scores: List[PlanScore],
+        multimodal: bool,
     ):
         goal_evaluations = ""
         for goal_score in goal_scores:
@@ -254,18 +266,25 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             )
 
         prompt = GoalAccuracyTemplate.get_final_reason(
-            self.score, self.threshold, goal_evaluations, plan_evalautions
+            self.score,
+            self.threshold,
+            goal_evaluations,
+            plan_evalautions,
+            multimodal,
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
+            self._accrue_cost(cost)
             return res
         else:
             res = self.model.generate(prompt)
             return res
 
     async def _a_generate_reason(
-        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+        self,
+        goal_scores: List[GoalScore],
+        plan_scores: List[PlanScore],
+        multimodal: bool,
     ):
         goal_evaluations = ""
         for goal_score in goal_scores:
@@ -279,51 +298,47 @@ class GoalAccuracyMetric(BaseConversationalMetric):
             )
 
         prompt = GoalAccuracyTemplate.get_final_reason(
-            self.score, self.threshold, goal_evaluations, plan_evalautions
+            self.score,
+            self.threshold,
+            goal_evaluations,
+            plan_evalautions,
+            multimodal,
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
+            self._accrue_cost(cost)
             return res
         else:
             res = await self.model.a_generate(prompt)
             return res
 
-    def _get_goal_accuracy_score(self, user_goal, steps_taken):
+    def _get_goal_accuracy_score(
+        self, user_goal, steps_taken, multimodal: bool
+    ):
         prompt = GoalAccuracyTemplate.get_accuracy_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=GoalScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: GoalScore(**data),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=GoalScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: GoalScore = self.model.generate(prompt, schema=GoalScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return GoalScore(**data)
 
-    async def _a_get_goal_accuracy_score(self, user_goal, steps_taken):
+    async def _a_get_goal_accuracy_score(
+        self, user_goal, steps_taken, multimodal: bool
+    ):
         prompt = GoalAccuracyTemplate.get_accuracy_score(
-            user_goal, "\n".join(steps_taken)
+            user_goal, "\n".join(steps_taken), multimodal
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=GoalScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: GoalScore(**data),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=GoalScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: GoalScore = await self.model.a_generate(
-                    prompt, schema=GoalScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return GoalScore(**data)
 
     def print_goals_and_steps_taken(self, goals_and_steps):
         final_goals_and_steps = ""
@@ -340,7 +355,7 @@ class GoalAccuracyMetric(BaseConversationalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success