deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/faithfulness/faithfulness.py

@@ -1,17 +1,18 @@
 from typing import List, Optional, Union, Type
 import asyncio
 
-from deepeval.test_case import (
-    LLMTestCase,
-    LLMTestCaseParams,
-)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
 from deepeval.metrics import BaseMetric
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.faithfulness.template import FaithfulnessTemplate
@@ -67,7 +68,16 @@ class FaithfulnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -84,11 +94,16 @@ class FaithfulnessMetric(BaseMetric):
                     )
                 )
             else:
-                self.truths = self._generate_truths(test_case.retrieval_context)
-                self.claims = self._generate_claims(test_case.actual_output)
-                self.verdicts = self._generate_verdicts()
+                retrieval_context = test_case.retrieval_context
+                actual_output = test_case.actual_output
+
+                self.truths = self._generate_truths(
+                    retrieval_context, multimodal
+                )
+                self.claims = self._generate_claims(actual_output, multimodal)
+                self.verdicts = self._generate_verdicts(multimodal)
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason()
+                self.reason = self._generate_reason(multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -114,7 +129,16 @@ class FaithfulnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -123,13 +147,16 @@ class FaithfulnessMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            retrieval_context = test_case.retrieval_context
+            actual_output = test_case.actual_output
+
             self.truths, self.claims = await asyncio.gather(
-                self._a_generate_truths(test_case.retrieval_context),
-                self._a_generate_claims(test_case.actual_output),
+                self._a_generate_truths(retrieval_context, multimodal),
+                self._a_generate_claims(actual_output, multimodal),
             )
-            self.verdicts = await self._a_generate_verdicts()
+            self.verdicts = await self._a_generate_verdicts(multimodal)
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
+            self.reason = await self._a_generate_reason(multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -146,7 +173,7 @@ class FaithfulnessMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -158,26 +185,18 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=FaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: FaithfulnessScoreReason = await self.model.a_generate(
-                    prompt, schema=FaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
-    def _generate_reason(self) -> str:
+    def _generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -189,148 +208,118 @@ class FaithfulnessMetric(BaseMetric):
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=FaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: FaithfulnessScoreReason = self.model.generate(
-                    prompt, schema=FaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=FaithfulnessScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
-    async def _a_generate_verdicts(self) -> List[FaithfulnessVerdict]:
+    async def _a_generate_verdicts(
+        self, multimodal: bool
+    ) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
-        verdicts: List[FaithfulnessVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
+            claims=self.claims,
+            retrieval_context="\n\n".join(self.truths),
+            multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(self) -> List[FaithfulnessVerdict]:
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                FaithfulnessVerdict(**item) for item in data["verdicts"]
+            ],
+        )
+
+    def _generate_verdicts(self, multimodal: bool) -> List[FaithfulnessVerdict]:
         if len(self.claims) == 0:
             return []
 
-        verdicts: List[FaithfulnessVerdict] = []
         prompt = self.evaluation_template.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
+            claims=self.claims,
+            retrieval_context="\n\n".join(self.truths),
+            multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    async def _a_generate_truths(self, retrieval_context: str) -> List[str]:
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                FaithfulnessVerdict(**item) for item in data["verdicts"]
+            ],
+        )
+
+    async def _a_generate_truths(
+        self, retrieval_context: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_truths(
             retrieval_context="\n\n".join(retrieval_context),
             extraction_limit=self.truths_extraction_limit,
+            multimodal=multimodal,
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = await self.model.a_generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
 
-    def _generate_truths(self, retrieval_context: str) -> List[str]:
+    def _generate_truths(
+        self, retrieval_context: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_truths(
             retrieval_context="\n\n".join(retrieval_context),
             extraction_limit=self.truths_extraction_limit,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Truths,
+            extract_schema=lambda s: s.truths,
+            extract_json=lambda data: data["truths"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = self.model.generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
 
-    async def _a_generate_claims(self, actual_output: str) -> List[str]:
+    async def _a_generate_claims(
+        self, actual_output: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_claims(
-            actual_output=actual_output
+            actual_output=actual_output, multimodal=multimodal
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = await self.model.a_generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
 
-    def _generate_claims(self, actual_output: str) -> List[str]:
+    def _generate_claims(
+        self, actual_output: str, multimodal: bool
+    ) -> List[str]:
         prompt = self.evaluation_template.generate_claims(
-            actual_output=actual_output
+            actual_output=actual_output, multimodal=multimodal
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Claims,
+            extract_schema=lambda s: s.claims,
+            extract_json=lambda data: data["claims"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = self.model.generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -357,7 +346,7 @@ class FaithfulnessMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/faithfulness/schema.py

@@ -3,7 +3,7 @@ from pydantic import BaseModel, Field
 
 
 class FaithfulnessVerdict(BaseModel):
-    verdict: Literal["yes", "idk", "no"]
+    verdict: Literal["yes", "no", "idk"]
     reason: Optional[str] = Field(default=None)