deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/goal_accuracy/template.py

@@ -3,8 +3,16 @@ import textwrap
 
 
 class GoalAccuracyTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def get_accuracy_score(task, steps_taken):
+    def get_accuracy_score(task, steps_taken, multimodal: bool = False):
         return textwrap.dedent(
             f"""You are an expert evaluator assessing the **goal accuracy** of an AI assistant's single interaction.
 
@@ -36,6 +44,8 @@ class GoalAccuracyTemplate:
                 - When uncertain, assume the goal was **not achieved**.
                 - The metric is designed to fail unless the assistant's output is precise, complete, and user-visible.
 
+                {GoalAccuracyTemplate.multimodal_rules if multimodal else ""}
+
                 SCORING GUIDE:
 
                 - **1.0** → Goal completely and correctly achieved; all required outputs visible to the user.  
@@ -102,7 +112,7 @@ class GoalAccuracyTemplate:
         )
 
     @staticmethod
-    def get_plan_evaluation_score(task, steps_taken):
+    def get_plan_evaluation_score(task, steps_taken, multimodal: bool = False):
         return textwrap.dedent(
             f"""You are an expert evaluator assessing the **planning quality** and **plan adherence** of an AI agent tasked with fulfilling a user's request.
 
@@ -132,6 +142,8 @@ class GoalAccuracyTemplate:
                 - Tool use should be coherent within the plan, not ad hoc or speculative.  
                 - This evaluation excludes correctness or efficiency — focus solely on plan and adherence.
 
+                {GoalAccuracyTemplate.multimodal_rules if multimodal else ""}
+
                 SCORING GUIDE:
 
                 - **1.0** → Complete, clear, and logical plan **fully followed** with all steps aligned to the user's goal.  
@@ -188,7 +200,11 @@ class GoalAccuracyTemplate:
 
     @staticmethod
     def get_final_reason(
-        final_score, threshold, goal_evaluations, plan_evalautions
+        final_score,
+        threshold,
+        goal_evaluations,
+        plan_evalautions,
+        multimodal: bool = False,
     ):
         return textwrap.dedent(
             f"""You are an expert evaluator providing a **final justification** for whether an AI agent has passed or failed an evaluation metric.
@@ -213,6 +229,8 @@ class GoalAccuracyTemplate:
                 - If the agent **failed**, explain which aspects (task or plan or both) led to the failure.
                 - Avoid vague praise or criticism — ground the reason in the actual scores and justifications.
 
+                {GoalAccuracyTemplate.multimodal_rules if multimodal else ""}
+
                 ---
 
                 FORMAT:

deepeval/metrics/hallucination/hallucination.py

@@ -8,14 +8,19 @@ from deepeval.metrics import BaseMetric
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.hallucination.template import HallucinationTemplate
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.hallucination.schema import *
+from deepeval.metrics.hallucination.schema import (
+    HallucinationVerdict,
+    Verdicts,
+    HallucinationScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -55,7 +60,16 @@ class HallucinationMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -102,7 +116,16 @@ class HallucinationMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -150,22 +173,13 @@ class HallucinationMetric(BaseMetric):
             score=format(self.score, ".2f"),
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=HallucinationScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: HallucinationScoreReason = await self.model.a_generate(
-                    prompt, schema=HallucinationScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=HallucinationScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self):
         if self.include_reason is False:
@@ -185,74 +199,45 @@ class HallucinationMetric(BaseMetric):
             score=format(self.score, ".2f"),
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=HallucinationScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: HallucinationScoreReason = self.model.generate(
-                    prompt, schema=HallucinationScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=HallucinationScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, actual_output: str, contexts: List[str]
     ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
             actual_output=actual_output, contexts=contexts
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                HallucinationVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(
         self, actual_output: str, contexts: List[str]
     ) -> List[HallucinationVerdict]:
-        verdicts: List[HallucinationVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
             actual_output=actual_output, contexts=contexts
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    HallucinationVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                HallucinationVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -273,7 +258,7 @@ class HallucinationMetric(BaseMetric):
         else:
             try:
                 self.success = self.score <= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/hallucination/template.py

@@ -2,9 +2,20 @@ from typing import List
 
 
 class HallucinationTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_verdicts(actual_output: str, contexts: List[str]):
         return f"""For each context in contexts, which is a list of strings, please generate a list of JSON objects to indicate whether the given 'actual output' agrees with EACH context. The JSON will have 2 fields: 'verdict' and 'reason'.
+
+{HallucinationTemplate.multimodal_rules}
+
 The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the given text agrees with the context. 
 The 'reason' is the reason for the verdict. When the answer is 'no', try to provide a correction in the reason. 
 
@@ -46,6 +57,8 @@ JSON:
     ):
         return f"""Given a list of factual alignments and contradictions, which highlights alignment/contradictions between the `actual output` and `contexts, use it to provide a reason for the hallucination score in a CONCISELY. Note that The hallucination score ranges from 0 - 1, and the lower the better.
 
+{HallucinationTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON:

deepeval/metrics/indicator.py

@@ -1,24 +1,24 @@
+import asyncio
+import logging
+import sys
+import time
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 from contextlib import contextmanager
-import sys
 from typing import List, Optional, Union
-import time
-import asyncio
 
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.metrics import (
     BaseMetric,
     BaseConversationalMetric,
-    BaseMultimodalMetric,
     BaseArenaMetric,
 )
 from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.test_run.cache import CachedTestCase, Cache
 from deepeval.telemetry import capture_metric_type
 from deepeval.utils import update_pbar
+from deepeval.config.settings import get_settings
 
-import logging
 
 logger = logging.getLogger(__name__)
 
@@ -74,7 +74,7 @@ def metric_progress_indicator(
 async def measure_metric_task(
     task_id,
     progress,
-    metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
+    metric: Union[BaseMetric, BaseConversationalMetric],
     test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
@@ -156,9 +156,7 @@ async def measure_metric_task(
 
 
 async def measure_metrics_with_indicator(
-    metrics: List[
-        Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric]
-    ],
+    metrics: List[Union[BaseMetric, BaseConversationalMetric]],
     test_case: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     cached_test_case: Union[CachedTestCase, None],
     ignore_errors: bool,
@@ -238,7 +236,7 @@ async def measure_metrics_with_indicator(
 
 
 async def safe_a_measure(
-    metric: Union[BaseMetric, BaseMultimodalMetric, BaseConversationalMetric],
+    metric: Union[BaseMetric, BaseConversationalMetric],
     tc: Union[LLMTestCase, LLMTestCase, ConversationalTestCase],
     ignore_errors: bool,
     skip_on_missing_params: bool,
@@ -263,6 +261,9 @@ async def safe_a_measure(
             "Timed out/cancelled while evaluating metric. "
             "Increase DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE or set "
             "DEEPEVAL_LOG_STACK_TRACES=1 for full traceback."
+            if not get_settings().DEEPEVAL_DISABLE_TIMEOUTS
+            else "Cancelled while evaluating metric (DeepEval timeouts are disabled; this likely came from upstream orchestration or the provider/network layer). "
+            "Set DEEPEVAL_LOG_STACK_TRACES=1 for full traceback."
         )
         metric.success = False
 

deepeval/metrics/json_correctness/json_correctness.py

@@ -11,7 +11,8 @@ from deepeval.metrics.utils import (
     construct_verbose_logs,
     check_llm_test_case_params,
     initialize_model,
-    trimAndLoadJson,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
@@ -46,6 +47,7 @@ class JsonCorrectnessMetric(BaseMetric):
         self.async_mode = async_mode
         self.verbose_mode = verbose_mode
         self.expected_schema = expected_schema
+        self.evaluation_model = self.model.get_model_name()
 
     def measure(
         self,
@@ -55,7 +57,16 @@ class JsonCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -77,7 +88,7 @@ class JsonCorrectnessMetric(BaseMetric):
                     self.expected_schema.model_validate_json(
                         test_case.actual_output
                     )
-                except ValidationError as e:
+                except ValidationError:
                     valid_json = False
 
                 self.score = 1 if valid_json else 0
@@ -106,7 +117,16 @@ class JsonCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -120,7 +140,7 @@ class JsonCorrectnessMetric(BaseMetric):
                 self.expected_schema.model_validate_json(
                     test_case.actual_output
                 )
-            except ValidationError as e:
+            except ValidationError:
                 valid_json = False
 
             self.score = 1 if valid_json else 0
@@ -156,22 +176,13 @@ class JsonCorrectnessMetric(BaseMetric):
             is_valid_json=is_valid_json,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=JsonCorrectnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: JsonCorrectnessScoreReason = await self.model.a_generate(
-                    prompt, schema=JsonCorrectnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=JsonCorrectnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def generate_reason(self, actual_output: str) -> str:
         if self.include_reason is False:
@@ -189,22 +200,13 @@ class JsonCorrectnessMetric(BaseMetric):
             is_valid_json=is_valid_json,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=JsonCorrectnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: JsonCorrectnessScoreReason = self.model.generate(
-                    prompt, schema=JsonCorrectnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=JsonCorrectnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -212,7 +214,7 @@ class JsonCorrectnessMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/json_correctness/template.py

@@ -2,12 +2,22 @@ from typing import Optional
 
 
 class JsonCorrectnessTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_reason(
         actual_output: str, expected_schema: str, is_valid_json: bool
     ):
         return f"""Based on the given generated json, generated by an LLM, and a boolean stating whether it is a valid JSON based on the expected json schema, give a reason why it is OR is not a valid Json.
 
+{JsonCorrectnessTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON: