deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_contextual_precision/multimodal_contextual_precision.py

@@ -1,7 +1,7 @@
 from typing import Optional, List, Union
 
 from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
+from deepeval.test_case import MLLMTestCase
 from deepeval.metrics.multimodal_metrics.multimodal_contextual_precision.template import (
     MultiModalContextualPrecisionTemplate,
 )
@@ -14,7 +14,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.test_case import LLMTestCaseParams
 from deepeval.models import DeepEvalBaseMLLM
-from deepeval.metrics.multimodal_metrics.multimodal_contextual_precision.schema import *
+import deepeval.metrics.multimodal_metrics.multimodal_contextual_precision.schema as mcpschema
 from deepeval.metrics.indicator import metric_progress_indicator
 
 
@@ -49,6 +49,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -56,7 +57,9 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
+            self,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
@@ -65,10 +68,11 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
-                self.verdicts: List[ContextualPrecisionVerdict] = (
+                self.verdicts: List[mcpschema.ContextualPrecisionVerdict] = (
                     self._generate_verdicts(
                         test_case.input,
                         test_case.expected_output,
@@ -93,6 +97,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -105,7 +110,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: List[ContextualPrecisionVerdict] = (
+            self.verdicts: List[mcpschema.ContextualPrecisionVerdict] = (
                 await self._a_generate_verdicts(
                     test_case.input,
                     test_case.expected_output,
@@ -125,12 +130,12 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
 
             return self.score
 
-    async def _a_generate_reason(self, input: str):
+    async def _a_generate_reason(self, input: str) -> Optional[str]:
         if self.include_reason is False:
             return None
 
         retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            {"verdict": verdict.verdict, "reason": verdict.reason}
             for verdict in self.verdicts
         ]
         prompt = MultiModalContextualPrecisionTemplate.generate_reason(
@@ -141,15 +146,17 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
 
         if self.using_native_model:
             res, cost = await self.model.a_generate(
-                prompt, schema=MultimodelContextualPrecisionScoreReason
+                prompt,
+                schema=mcpschema.MultimodelContextualPrecisionScoreReason,
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: MultimodelContextualPrecisionScoreReason = (
+                res: mcpschema.MultimodelContextualPrecisionScoreReason = (
                     await self.model.a_generate(
-                        prompt, schema=MultimodelContextualPrecisionScoreReason
+                        prompt,
+                        schema=mcpschema.MultimodelContextualPrecisionScoreReason,
                     )
                 )
                 return res.reason
@@ -158,12 +165,12 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str):
+    def _generate_reason(self, input: str) -> Optional[str]:
         if self.include_reason is False:
             return None
 
         retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            {"verdict": verdict.verdict, "reason": verdict.reason}
             for verdict in self.verdicts
         ]
         prompt = MultiModalContextualPrecisionTemplate.generate_reason(
@@ -174,15 +181,17 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
 
         if self.using_native_model:
             res, cost = self.model.generate(
-                prompt, schema=MultimodelContextualPrecisionScoreReason
+                prompt,
+                schema=mcpschema.MultimodelContextualPrecisionScoreReason,
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: MultimodelContextualPrecisionScoreReason = (
+                res: mcpschema.MultimodelContextualPrecisionScoreReason = (
                     self.model.generate(
-                        prompt, schema=MultimodelContextualPrecisionScoreReason
+                        prompt,
+                        schema=mcpschema.MultimodelContextualPrecisionScoreReason,
                     )
                 )
                 return res.reason
@@ -193,21 +202,23 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
 
     async def _a_generate_verdicts(
         self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
+    ) -> List[mcpschema.ContextualPrecisionVerdict]:
         prompt = MultiModalContextualPrecisionTemplate.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            res, cost = await self.model.a_generate(
+                prompt, schema=mcpschema.Verdicts
+            )
             self.evaluation_cost += cost
             verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
+                res: mcpschema.Verdicts = await self.model.a_generate(
+                    prompt, schema=mcpschema.Verdicts
                 )
                 verdicts = [item for item in res.verdicts]
                 return verdicts
@@ -215,34 +226,36 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 verdicts = [
-                    ContextualPrecisionVerdict(**item)
+                    mcpschema.ContextualPrecisionVerdict(**item)
                     for item in data["verdicts"]
                 ]
                 return verdicts
 
     def _generate_verdicts(
         self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
+    ) -> List[mcpschema.ContextualPrecisionVerdict]:
         prompt = MultiModalContextualPrecisionTemplate.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
+            res, cost = self.model.generate(prompt, schema=mcpschema.Verdicts)
             self.evaluation_cost += cost
             verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                res: mcpschema.Verdicts = self.model.generate(
+                    prompt, schema=mcpschema.Verdicts
+                )
                 verdicts = [item for item in res.verdicts]
                 return verdicts
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 verdicts = [
-                    ContextualPrecisionVerdict(**item)
+                    mcpschema.ContextualPrecisionVerdict(**item)
                     for item in data["verdicts"]
                 ]
                 return verdicts
@@ -279,7 +292,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/multimodal_metrics/multimodal_contextual_recall/multimodal_contextual_recall.py

@@ -48,6 +48,7 @@ class MultimodalContextualRecallMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -64,6 +65,7 @@ class MultimodalContextualRecallMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -90,6 +92,7 @@ class MultimodalContextualRecallMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_contextual_relevancy/multimodal_contextual_relevancy.py

@@ -49,6 +49,7 @@ class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -65,6 +66,7 @@ class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -90,6 +92,7 @@ class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_faithfulness/multimodal_faithfulness.py

@@ -53,6 +53,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -71,6 +72,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -97,6 +99,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py

@@ -78,6 +78,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
 
@@ -96,6 +97,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                         _additional_context=_additional_context,
                     )
                 )
@@ -132,6 +134,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         _show_indicator: bool = True,
         _in_component: bool = False,
         _additional_context: Optional[str] = None,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_mllm_test_case_params(

deepeval/metrics/multimodal_metrics/multimodal_tool_correctness/multimodal_tool_correctness.py

@@ -3,7 +3,7 @@ from typing import List, Dict
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    check_llm_test_case_params,
+    check_mllm_test_case_params,
 )
 from deepeval.test_case import (
     MLLMTestCase,
@@ -11,10 +11,10 @@ from deepeval.test_case import (
     ToolCallParams,
     ToolCall,
 )
-from deepeval.metrics import BaseMetric
+from deepeval.metrics import BaseMultimodalMetric
 
 
-class MultimodalToolCorrectnessMetric(BaseMetric):
+class MultimodalToolCorrectnessMetric(BaseMultimodalMetric):
 
     _required_params: List[MLLMTestCaseParams] = [
         MLLMTestCaseParams.INPUT,
@@ -46,8 +46,11 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_mllm_test_case_params(
+            test_case, self._required_params, None, None, self
+        )
         self.test_case = test_case
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component
@@ -90,11 +93,13 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         return self.measure(
             test_case,
             _show_indicator=_show_indicator,
             _in_component=_in_component,
+            _log_metric_to_confident=_log_metric_to_confident,
         )
 
     ##################################################
@@ -278,7 +283,7 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
 
     @property
     def __name__(self):
-        return "Tool Correctness"
+        return "Multi Modal Tool Correctness"
 
     def indent_multiline_string(self, s, indent_level=4):
         indent = " " * indent_level

deepeval/metrics/non_advice/non_advice.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.non_advice.template import NonAdviceTemplate
 from deepeval.metrics.non_advice.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class NonAdviceMetric(BaseMetric):
@@ -58,6 +59,7 @@ class NonAdviceMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -73,6 +75,7 @@ class NonAdviceMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -93,6 +96,10 @@ class NonAdviceMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -101,6 +108,7 @@ class NonAdviceMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -129,6 +137,10 @@ class NonAdviceMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.pii_leakage.template import PIILeakageTemplate
 from deepeval.metrics.pii_leakage.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class PIILeakageMetric(BaseMetric):
@@ -49,6 +50,7 @@ class PIILeakageMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -64,6 +66,7 @@ class PIILeakageMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class PIILeakageMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -92,6 +99,7 @@ class PIILeakageMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -120,7 +128,10 @@ class PIILeakageMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/plan_adherence/__init__.py

@@ -0,0 +1 @@
+from .plan_adherence import PlanAdherenceMetric