deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/dag/nodes.py

@@ -18,7 +18,11 @@ from deepeval.metrics.dag.templates import (
 from deepeval.metrics.base_metric import BaseMetric
 from deepeval.metrics.g_eval.g_eval import GEval
 from deepeval.metrics.g_eval.utils import G_EVAL_PARAMS
-from deepeval.metrics.utils import copy_metrics, trimAndLoadJson
+from deepeval.metrics.utils import (
+    copy_metrics,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
+)
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams, ToolCall
 from deepeval.utils import prettify_list
 
@@ -222,20 +226,13 @@ class VerdictNode(BaseNode):
             score=metric.score,
             name=metric.__name__,
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(prompt, schema=MetricScoreReason)
-            metric.evaluation_cost += cost
-        else:
-            try:
-                res: MetricScoreReason = metric.model.generate(
-                    prompt, schema=MetricScoreReason
-                )
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                res = MetricScoreReason(**data)
-
-        return res.reason
+        return generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=MetricScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(self, metric: BaseMetric):
         prompt = VerdictNodeTemplate.generate_reason(
@@ -243,22 +240,13 @@ class VerdictNode(BaseNode):
             score=metric.score,
             name=metric.__name__,
         )
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=MetricScoreReason
-            )
-            metric.evaluation_cost += cost
-        else:
-            try:
-                res: MetricScoreReason = await metric.model.a_generate(
-                    prompt, schema=MetricScoreReason
-                )
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                res = MetricScoreReason(**data)
-
-        return res.reason
+        return await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=MetricScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
 
 @dataclass
@@ -317,20 +305,13 @@ class TaskNode(BaseNode):
             instructions=self.instructions,
             text=text,
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(prompt, schema=TaskNodeOutput)
-            metric.evaluation_cost += cost
-            self._output = res.output
-        else:
-            try:
-                res: TaskNodeOutput = metric.model.generate(
-                    prompt, schema=TaskNodeOutput
-                )
-                self._output = res.output
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._output = TaskNodeOutput(**data).output
+        self._output = generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=TaskNodeOutput,
+            extract_schema=lambda s: s.output,
+            extract_json=lambda data: data["output"],
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -371,22 +352,13 @@ class TaskNode(BaseNode):
             text=text,
         )
 
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=TaskNodeOutput
-            )
-            metric.evaluation_cost += cost
-            self._output = res.output
-        else:
-            try:
-                res: TaskNodeOutput = await metric.model.a_generate(
-                    prompt, schema=TaskNodeOutput
-                )
-                self._output = res.output
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._output = TaskNodeOutput(**data).output
+        self._output = await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=TaskNodeOutput,
+            extract_schema=lambda s: s.output,
+            extract_json=lambda data: data["output"],
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -470,23 +442,13 @@ class BinaryJudgementNode(BaseNode):
             criteria=self.criteria,
             text=text,
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(
-                prompt, schema=BinaryJudgementVerdict
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: BinaryJudgementVerdict = metric.model.generate(
-                    prompt, schema=BinaryJudgementVerdict
-                )
-                self._verdict = res
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = BinaryJudgementVerdict(**data)
-
+        self._verdict = generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=BinaryJudgementVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: BinaryJudgementVerdict(**data),
+        )
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
         )
@@ -520,22 +482,13 @@ class BinaryJudgementNode(BaseNode):
             criteria=self.criteria,
             text=text,
         )
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=BinaryJudgementVerdict
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: BinaryJudgementVerdict = await metric.model.a_generate(
-                    prompt, schema=BinaryJudgementVerdict
-                )
-                self._verdict = res
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = BinaryJudgementVerdict(**data)
+        self._verdict = await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=BinaryJudgementVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: BinaryJudgementVerdict(**data),
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -629,22 +582,14 @@ class NonBinaryJudgementNode(BaseNode):
         prompt = NonBinaryJudgementTemplate.generate_non_binary_verdict(
             criteria=self.criteria, text=text, options=self._verdict_options
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(
-                prompt, schema=self._verdict_schema
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: self._verdict_schema = metric.model.generate(
-                    prompt, schema=self._verdict_schema
-                )
-                self._verdict = res
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = self._verdict_schema(**data)
+
+        self._verdict = generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=self._verdict_schema,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: self._verdict_schema(**data),
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -678,22 +623,14 @@ class NonBinaryJudgementNode(BaseNode):
         prompt = NonBinaryJudgementTemplate.generate_non_binary_verdict(
             criteria=self.criteria, text=text, options=self._verdict_options
         )
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=self._verdict_schema
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: self._verdict_schema = await metric.model.a_generate(
-                    prompt, schema=self._verdict_schema
-                )
-                self._verdict = res
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = self._verdict_schema(**data)
+
+        self._verdict = await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=self._verdict_schema,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: self._verdict_schema(**data),
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)

deepeval/metrics/dag/templates.py

@@ -1,5 +1,13 @@
 from typing import List
 
+multimodal_rules = """
+    --- MULTIMODAL INPUT RULES ---
+    - Treat image content as factual evidence.
+    - Only reference visual details that are explicitly and clearly visible.
+    - Do not infer or guess objects, text, or details not visibly present.
+    - If an image is unclear or ambiguous, mark uncertainty explicitly.
+"""
+
 
 class VerdictNodeTemplate:
     @staticmethod
@@ -34,6 +42,8 @@ class TaskNodeTemplate:
     def generate_task_output(instructions: str, text: str):
         return f"""Given the following instructions, generate an output.
 
+{multimodal_rules}
+
 {instructions}
 
 {text}
@@ -57,14 +67,16 @@ class BinaryJudgementTemplate:
     def generate_binary_verdict(criteria: str, text: str):
         return f"""{criteria}
 
+{multimodal_rules}
+
 {text}
 
 **
-IMPORTANT: Please make sure to only return a json with two keys: `verdict` (True or False), and the 'reason' key providing the reason. The verdict must be a boolean only, either True or False.
+IMPORTANT: Please make sure to only return a json with two keys: `verdict` (true or false), and the 'reason' key providing the reason. The verdict must be a boolean only, either true or false.
 Example JSON:
 {{
     "reason": "...",
-    "verdict": True
+    "verdict": true
 }}
 **
 
@@ -79,6 +91,8 @@ class NonBinaryJudgementTemplate:
     ):
         return f"""{criteria}
 
+{multimodal_rules}
+
 {text}
 
 **

deepeval/metrics/exact_match/exact_match.py

@@ -32,7 +32,15 @@ class ExactMatchMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            None,
+            test_case.multimodal,
+        )
 
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component