deepeval 3.7.5__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -3,14 +3,13 @@ from typing import Optional, List, Type, Union
 from deepeval.utils import (
     get_or_create_event_loop,
     prettify_list,
-    convert_to_multi_modal_array,
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
-    check_mllm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -20,7 +19,11 @@ from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.contextual_recall.template import ContextualRecallTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.contextual_recall.schema import *
+from deepeval.metrics.contextual_recall.schema import (
+    ContextualRecallVerdict,
+    Verdicts,
+    ContextualRecallScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -62,12 +65,15 @@ class ContextualRecallMetric(BaseMetric):
     ) -> float:
         multimodal = test_case.multimodal
 
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -118,12 +124,15 @@ class ContextualRecallMetric(BaseMetric):
 
         multimodal = test_case.multimodal
 
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -178,22 +187,13 @@ class ContextualRecallMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRecallScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRecallScoreReason = await self.model.a_generate(
-                    prompt, schema=ContextualRecallScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRecallScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self, expected_output: str, multimodal: bool):
         if self.include_reason is False:
@@ -215,22 +215,13 @@ class ContextualRecallMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRecallScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRecallScoreReason = self.model.generate(
-                    prompt, schema=ContextualRecallScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRecallScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(self):
         number_of_verdicts = len(self.verdicts)
@@ -256,25 +247,15 @@ class ContextualRecallMetric(BaseMetric):
             retrieval_context=retrieval_context,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                ContextualRecallVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(
         self,
@@ -287,23 +268,15 @@ class ContextualRecallMetric(BaseMetric):
             retrieval_context=retrieval_context,
             multimodal=multimodal,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                ContextualRecallVerdict(**item) for item in data["verdicts"]
+            ],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -311,7 +284,7 @@ class ContextualRecallMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -4,14 +4,13 @@ import asyncio
 from deepeval.utils import (
     get_or_create_event_loop,
     prettify_list,
-    convert_to_multi_modal_array,
 )
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
-    check_mllm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -23,7 +22,10 @@ from deepeval.metrics.contextual_relevancy.template import (
     ContextualRelevancyTemplate,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.contextual_relevancy.schema import *
+from deepeval.metrics.contextual_relevancy.schema import (
+    ContextualRelevancyVerdicts,
+    ContextualRelevancyScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -64,12 +66,15 @@ class ContextualRelevancyMetric(BaseMetric):
 
         multimodal = test_case.multimodal
 
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -121,12 +126,15 @@ class ContextualRelevancyMetric(BaseMetric):
 
         multimodal = test_case.multimodal
 
-        if multimodal:
-            check_mllm_test_case_params(
-                test_case, self._required_params, None, None, self, self.model
-            )
-        else:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -183,24 +191,13 @@ class ContextualRelevancyMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRelevancyScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=ContextualRelevancyScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _generate_reason(self, input: str, multimodal: bool):
         if self.include_reason is False:
@@ -223,22 +220,13 @@ class ContextualRelevancyMetric(BaseMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualRelevancyScoreReason = self.model.generate(
-                    prompt, schema=ContextualRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(self):
         total_verdicts = 0
@@ -262,22 +250,13 @@ class ContextualRelevancyMetric(BaseMetric):
             input=input, context=context, multimodal=multimodal
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRelevancyVerdicts
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = await self.model.a_generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ContextualRelevancyVerdicts(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyVerdicts,
+            extract_schema=lambda r: r,
+            extract_json=lambda data: ContextualRelevancyVerdicts(**data),
+        )
 
     def _generate_verdicts(
         self, input: str, context: str, multimodal: bool
@@ -286,22 +265,13 @@ class ContextualRelevancyMetric(BaseMetric):
             input=input, context=context, multimodal=multimodal
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRelevancyVerdicts
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = self.model.generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ContextualRelevancyVerdicts(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualRelevancyVerdicts,
+            extract_schema=lambda r: r,
+            extract_json=lambda data: ContextualRelevancyVerdicts(**data),
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -309,7 +279,7 @@ class ContextualRelevancyMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success