deepeval 3.6.5__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/multimodal_answer_relevancy.py

@@ -46,13 +46,16 @@ class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
+            self,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
@@ -61,6 +64,7 @@ class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +93,7 @@ class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_contextual_precision/multimodal_contextual_precision.py

@@ -49,6 +49,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -56,7 +57,9 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
+            self,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
@@ -65,6 +68,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -93,6 +97,7 @@ class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_contextual_recall/multimodal_contextual_recall.py

@@ -48,6 +48,7 @@ class MultimodalContextualRecallMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -64,6 +65,7 @@ class MultimodalContextualRecallMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -90,6 +92,7 @@ class MultimodalContextualRecallMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_contextual_relevancy/multimodal_contextual_relevancy.py

@@ -49,6 +49,7 @@ class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -65,6 +66,7 @@ class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -90,6 +92,7 @@ class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_faithfulness/multimodal_faithfulness.py

@@ -53,6 +53,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -71,6 +72,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -97,6 +99,7 @@ class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py

@@ -78,6 +78,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
 
@@ -96,6 +97,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                         _additional_context=_additional_context,
                     )
                 )
@@ -132,6 +134,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         _show_indicator: bool = True,
         _in_component: bool = False,
         _additional_context: Optional[str] = None,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_mllm_test_case_params(

deepeval/metrics/multimodal_metrics/multimodal_tool_correctness/multimodal_tool_correctness.py

@@ -3,7 +3,7 @@ from typing import List, Dict
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    check_llm_test_case_params,
+    check_mllm_test_case_params,
 )
 from deepeval.test_case import (
     MLLMTestCase,
@@ -11,10 +11,10 @@ from deepeval.test_case import (
     ToolCallParams,
     ToolCall,
 )
-from deepeval.metrics import BaseMetric
+from deepeval.metrics import BaseMultimodalMetric
 
 
-class MultimodalToolCorrectnessMetric(BaseMetric):
+class MultimodalToolCorrectnessMetric(BaseMultimodalMetric):
 
     _required_params: List[MLLMTestCaseParams] = [
         MLLMTestCaseParams.INPUT,
@@ -46,8 +46,11 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_mllm_test_case_params(
+            test_case, self._required_params, None, None, self
+        )
         self.test_case = test_case
         with metric_progress_indicator(
             self, _show_indicator=_show_indicator, _in_component=_in_component
@@ -90,11 +93,13 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         return self.measure(
             test_case,
             _show_indicator=_show_indicator,
             _in_component=_in_component,
+            _log_metric_to_confident=_log_metric_to_confident,
         )
 
     ##################################################
@@ -278,7 +283,7 @@ class MultimodalToolCorrectnessMetric(BaseMetric):
 
     @property
     def __name__(self):
-        return "Tool Correctness"
+        return "Multi Modal Tool Correctness"
 
     def indent_multiline_string(self, s, indent_level=4):
         indent = " " * indent_level

deepeval/metrics/non_advice/non_advice.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.non_advice.template import NonAdviceTemplate
 from deepeval.metrics.non_advice.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class NonAdviceMetric(BaseMetric):
@@ -58,6 +59,7 @@ class NonAdviceMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -73,6 +75,7 @@ class NonAdviceMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -93,6 +96,10 @@ class NonAdviceMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -101,6 +108,7 @@ class NonAdviceMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -129,6 +137,10 @@ class NonAdviceMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.pii_leakage.template import PIILeakageTemplate
 from deepeval.metrics.pii_leakage.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class PIILeakageMetric(BaseMetric):
@@ -49,6 +50,7 @@ class PIILeakageMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -64,6 +66,7 @@ class PIILeakageMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class PIILeakageMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -92,6 +99,7 @@ class PIILeakageMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -120,7 +128,10 @@ class PIILeakageMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/prompt_alignment/prompt_alignment.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from typing import Optional, List, Union
 
 from deepeval.utils import get_or_create_event_loop, prettify_list
@@ -15,7 +17,10 @@ from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.prompt_alignment.template import PromptAlignmentTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.prompt_alignment.schema import *
+from deepeval.metrics.prompt_alignment import schema as paschema
+from deepeval.config.settings import get_settings
+
+from deepeval.metrics.api import metric_data_manager
 
 
 class PromptAlignmentMetric(BaseMetric):
@@ -52,6 +57,7 @@ class PromptAlignmentMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -62,15 +68,19 @@ class PromptAlignmentMetric(BaseMetric):
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
+                coro = self.a_measure(
+                    test_case,
+                    _show_indicator=False,
+                    _in_component=_in_component,
+                )
                 loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
+                    asyncio.wait_for(
+                        coro,
+                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
                     )
                 )
             else:
-                self.verdicts: Verdicts = self._generate_verdicts(
+                self.verdicts: paschema.Verdicts = self._generate_verdicts(
                     test_case.input, test_case.actual_output
                 )
                 self.score = self._calculate_score()
@@ -86,6 +96,10 @@ class PromptAlignmentMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -94,6 +108,7 @@ class PromptAlignmentMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -105,7 +120,7 @@ class PromptAlignmentMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: Verdicts = await self._a_generate_verdicts(
+            self.verdicts: paschema.Verdicts = await self._a_generate_verdicts(
                 test_case.input, test_case.actual_output
             )
             self.score = self._calculate_score()
@@ -121,7 +136,10 @@ class PromptAlignmentMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str, actual_output: str) -> str:
@@ -141,14 +159,17 @@ class PromptAlignmentMetric(BaseMetric):
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(
-                prompt, schema=PromptAlignmentScoreReason
+                prompt, schema=paschema.PromptAlignmentScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: PromptAlignmentScoreReason = await self.model.a_generate(
-                    prompt=prompt, schema=PromptAlignmentScoreReason
+                res: paschema.PromptAlignmentScoreReason = (
+                    await self.model.a_generate(
+                        prompt=prompt,
+                        schema=paschema.PromptAlignmentScoreReason,
+                    )
                 )
                 return res.reason
             except TypeError:
@@ -173,14 +194,14 @@ class PromptAlignmentMetric(BaseMetric):
         )
         if self.using_native_model:
             res, cost = self.model.generate(
-                prompt, schema=PromptAlignmentScoreReason
+                prompt, schema=paschema.PromptAlignmentScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: PromptAlignmentScoreReason = self.model.generate(
-                    prompt=prompt, schema=PromptAlignmentScoreReason
+                res: paschema.PromptAlignmentScoreReason = self.model.generate(
+                    prompt=prompt, schema=paschema.PromptAlignmentScoreReason
                 )
                 return res.reason
             except TypeError:
@@ -190,48 +211,56 @@ class PromptAlignmentMetric(BaseMetric):
 
     async def _a_generate_verdicts(
         self, input: str, actual_output: str
-    ) -> Verdicts:
+    ) -> paschema.Verdicts:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            res, cost = await self.model.a_generate(
+                prompt, schema=paschema.Verdicts
+            )
             self.evaluation_cost += cost
             return [item for item in res.verdicts]
         else:
             try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
+                res: paschema.Verdicts = await self.model.a_generate(
+                    prompt, schema=paschema.Verdicts
                 )
                 return [item for item in res.verdicts]
             except TypeError:
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return [
-                    PromptAlignmentVerdict(**item) for item in data["verdicts"]
+                    paschema.PromptAlignmentVerdict(**item)
+                    for item in data["verdicts"]
                 ]
 
-    def _generate_verdicts(self, input: str, actual_output: str) -> Verdicts:
+    def _generate_verdicts(
+        self, input: str, actual_output: str
+    ) -> paschema.Verdicts:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
+            res, cost = self.model.generate(prompt, schema=paschema.Verdicts)
             self.evaluation_cost += cost
             return [item for item in res.verdicts]
         else:
             try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                res: paschema.Verdicts = self.model.generate(
+                    prompt, schema=paschema.Verdicts
+                )
                 return [item for item in res.verdicts]
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return [
-                    PromptAlignmentVerdict(**item) for item in data["verdicts"]
+                    paschema.PromptAlignmentVerdict(**item)
+                    for item in data["verdicts"]
                 ]
 
     def _calculate_score(self):
@@ -253,7 +282,7 @@ class PromptAlignmentMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/role_adherence/role_adherence.py

@@ -1,6 +1,7 @@
 from typing import Optional, Union, List
 
 from deepeval.metrics import BaseConversationalMetric
+from deepeval.metrics.api import metric_data_manager
 from deepeval.metrics.role_adherence.schema import (
     OutOfCharacterResponseVerdicts,
 )
@@ -44,6 +45,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case,
@@ -63,6 +65,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -82,6 +85,10 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -89,6 +96,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case,
@@ -124,6 +132,10 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, role: str) -> str:

deepeval/metrics/role_violation/role_violation.py

@@ -17,6 +17,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.role_violation.template import RoleViolationTemplate
 from deepeval.metrics.role_violation.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class RoleViolationMetric(BaseMetric):
@@ -58,6 +59,7 @@ class RoleViolationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -73,6 +75,7 @@ class RoleViolationMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -94,6 +97,10 @@ class RoleViolationMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -102,6 +109,7 @@ class RoleViolationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -131,6 +139,10 @@ class RoleViolationMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score