deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/dag/nodes.py

@@ -18,7 +18,11 @@ from deepeval.metrics.dag.templates import (
 from deepeval.metrics.base_metric import BaseMetric
 from deepeval.metrics.g_eval.g_eval import GEval
 from deepeval.metrics.g_eval.utils import G_EVAL_PARAMS
-from deepeval.metrics.utils import copy_metrics, trimAndLoadJson
+from deepeval.metrics.utils import (
+    copy_metrics,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
+)
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams, ToolCall
 from deepeval.utils import prettify_list
 
@@ -222,20 +226,13 @@ class VerdictNode(BaseNode):
             score=metric.score,
             name=metric.__name__,
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(prompt, schema=MetricScoreReason)
-            metric.evaluation_cost += cost
-        else:
-            try:
-                res: MetricScoreReason = metric.model.generate(
-                    prompt, schema=MetricScoreReason
-                )
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                res = MetricScoreReason(**data)
-
-        return res.reason
+        return generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=MetricScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(self, metric: BaseMetric):
         prompt = VerdictNodeTemplate.generate_reason(
@@ -243,22 +240,13 @@ class VerdictNode(BaseNode):
             score=metric.score,
             name=metric.__name__,
         )
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=MetricScoreReason
-            )
-            metric.evaluation_cost += cost
-        else:
-            try:
-                res: MetricScoreReason = await metric.model.a_generate(
-                    prompt, schema=MetricScoreReason
-                )
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                res = MetricScoreReason(**data)
-
-        return res.reason
+        return await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=MetricScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
 
 @dataclass
@@ -317,20 +305,13 @@ class TaskNode(BaseNode):
             instructions=self.instructions,
             text=text,
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(prompt, schema=TaskNodeOutput)
-            metric.evaluation_cost += cost
-            self._output = res.output
-        else:
-            try:
-                res: TaskNodeOutput = metric.model.generate(
-                    prompt, schema=TaskNodeOutput
-                )
-                self._output = res.output
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._output = TaskNodeOutput(**data).output
+        self._output = generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=TaskNodeOutput,
+            extract_schema=lambda s: s.output,
+            extract_json=lambda data: data["output"],
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -371,22 +352,13 @@ class TaskNode(BaseNode):
             text=text,
         )
 
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=TaskNodeOutput
-            )
-            metric.evaluation_cost += cost
-            self._output = res.output
-        else:
-            try:
-                res: TaskNodeOutput = await metric.model.a_generate(
-                    prompt, schema=TaskNodeOutput
-                )
-                self._output = res.output
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._output = TaskNodeOutput(**data).output
+        self._output = await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=TaskNodeOutput,
+            extract_schema=lambda s: s.output,
+            extract_json=lambda data: data["output"],
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -470,23 +442,13 @@ class BinaryJudgementNode(BaseNode):
             criteria=self.criteria,
             text=text,
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(
-                prompt, schema=BinaryJudgementVerdict
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: BinaryJudgementVerdict = metric.model.generate(
-                    prompt, schema=BinaryJudgementVerdict
-                )
-                self._verdict = res
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = BinaryJudgementVerdict(**data)
-
+        self._verdict = generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=BinaryJudgementVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: BinaryJudgementVerdict(**data),
+        )
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
         )
@@ -520,22 +482,13 @@ class BinaryJudgementNode(BaseNode):
             criteria=self.criteria,
             text=text,
         )
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=BinaryJudgementVerdict
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: BinaryJudgementVerdict = await metric.model.a_generate(
-                    prompt, schema=BinaryJudgementVerdict
-                )
-                self._verdict = res
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = BinaryJudgementVerdict(**data)
+        self._verdict = await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=BinaryJudgementVerdict,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: BinaryJudgementVerdict(**data),
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -629,22 +582,14 @@ class NonBinaryJudgementNode(BaseNode):
         prompt = NonBinaryJudgementTemplate.generate_non_binary_verdict(
             criteria=self.criteria, text=text, options=self._verdict_options
         )
-        if metric.using_native_model:
-            res, cost = metric.model.generate(
-                prompt, schema=self._verdict_schema
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: self._verdict_schema = metric.model.generate(
-                    prompt, schema=self._verdict_schema
-                )
-                self._verdict = res
-            except TypeError:
-                res = metric.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = self._verdict_schema(**data)
+
+        self._verdict = generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=self._verdict_schema,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: self._verdict_schema(**data),
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)
@@ -678,22 +623,14 @@ class NonBinaryJudgementNode(BaseNode):
         prompt = NonBinaryJudgementTemplate.generate_non_binary_verdict(
             criteria=self.criteria, text=text, options=self._verdict_options
         )
-        if metric.using_native_model:
-            res, cost = await metric.model.a_generate(
-                prompt, schema=self._verdict_schema
-            )
-            metric.evaluation_cost += cost
-            self._verdict = res
-        else:
-            try:
-                res: self._verdict_schema = await metric.model.a_generate(
-                    prompt, schema=self._verdict_schema
-                )
-                self._verdict = res
-            except TypeError:
-                res = await metric.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                self._verdict = self._verdict_schema(**data)
+
+        self._verdict = await a_generate_with_schema_and_extract(
+            metric=metric,
+            prompt=prompt,
+            schema_cls=self._verdict_schema,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: self._verdict_schema(**data),
+        )
 
         metric._verbose_steps.append(
             construct_node_verbose_log(self, self._depth)

deepeval/metrics/dag/templates.py

@@ -1,5 +1,13 @@
 from typing import List
 
+multimodal_rules = """
+    --- MULTIMODAL INPUT RULES ---
+    - Treat image content as factual evidence.
+    - Only reference visual details that are explicitly and clearly visible.
+    - Do not infer or guess objects, text, or details not visibly present.
+    - If an image is unclear or ambiguous, mark uncertainty explicitly.
+"""
+
 
 class VerdictNodeTemplate:
     @staticmethod
@@ -34,6 +42,8 @@ class TaskNodeTemplate:
     def generate_task_output(instructions: str, text: str):
         return f"""Given the following instructions, generate an output.
 
+{multimodal_rules}
+
 {instructions}
 
 {text}
@@ -57,6 +67,8 @@ class BinaryJudgementTemplate:
     def generate_binary_verdict(criteria: str, text: str):
         return f"""{criteria}
 
+{multimodal_rules}
+
 {text}
 
 **
@@ -79,6 +91,8 @@ class NonBinaryJudgementTemplate:
     ):
         return f"""{criteria}
 
+{multimodal_rules}
+
 {text}
 
 **

deepeval/metrics/exact_match/exact_match.py

@@ -32,7 +32,15 @@ class ExactMatchMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            None,
+            test_case.multimodal,
+        )
 
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component

deepeval/metrics/faithfulness/faithfulness.py

@@ -1,7 +1,7 @@
 from typing import List, Optional, Union, Type
 import asyncio
 
-from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
 from deepeval.metrics import BaseMetric
 from deepeval.utils import (
     get_or_create_event_loop,
@@ -9,10 +9,10 @@ from deepeval.utils import (
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
-    check_mllm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.faithfulness.template import FaithfulnessTemplate
@@ -69,12 +69,15 @@ class FaithfulnessMetric(BaseMetric):
     ) -> float:
 
         multimodal = test_case.multimodal
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -127,12 +130,15 @@ class FaithfulnessMetric(BaseMetric):
     ) -> float:
 
         multimodal = test_case.multimodal
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -182,22 +188,13 @@ class FaithfulnessMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=FaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: FaithfulnessScoreReason = await self.model.a_generate(
-                    prompt, schema=FaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
@@ -214,22 +211,13 @@ class FaithfulnessMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=FaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: FaithfulnessScoreReason = self.model.generate(
-                    prompt, schema=FaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, multimodal: bool
@@ -237,63 +225,41 @@ class FaithfulnessMetric(BaseMetric):
         if len(self.claims) == 0:
             return []
 
-        verdicts: List[FaithfulnessVerdict] = []
-
         prompt = self.evaluation_template.generate_verdicts(
             claims=self.claims,
             retrieval_context="\n\n".join(self.truths),
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                FaithfulnessVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(self, multimodal: bool) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
-        verdicts: List[FaithfulnessVerdict] = []
-
         prompt = self.evaluation_template.generate_verdicts(
             claims=self.claims,
             retrieval_context="\n\n".join(self.truths),
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                FaithfulnessVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     async def _a_generate_truths(
         self, retrieval_context: str, multimodal: bool
@@ -303,18 +269,13 @@ class FaithfulnessMetric(BaseMetric):
             extraction_limit=self.truths_extraction_limit,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = await self.model.a_generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
+        )
 
     def _generate_truths(
         self, retrieval_context: str, multimodal: bool
@@ -324,18 +285,13 @@ class FaithfulnessMetric(BaseMetric):
             extraction_limit=self.truths_extraction_limit,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = self.model.generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
+        )
 
     async def _a_generate_claims(
         self, actual_output: str, multimodal: bool
@@ -343,18 +299,13 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_claims(
             actual_output=actual_output, multimodal=multimodal
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = await self.model.a_generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
+        )
 
     def _generate_claims(
         self, actual_output: str, multimodal: bool
@@ -362,18 +313,13 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_claims(
             actual_output=actual_output, multimodal=multimodal
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = self.model.generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
+        )
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -400,7 +346,7 @@ class FaithfulnessMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success