deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/indicator.py

@@ -100,6 +100,7 @@ async def measure_metric_task(
                     test_case,
                     _show_indicator=False,
                     _in_component=_in_component,
+                    _log_metric_to_confident=False,
                 )
                 finish_text = "Done"
             except MissingTestCaseParamsError as e:
@@ -116,7 +117,9 @@ async def measure_metric_task(
             except TypeError:
                 try:
                     await metric.a_measure(
-                        test_case, _in_component=_in_component
+                        test_case,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=False,
                     )
                     finish_text = "Done"
                 except MissingTestCaseParamsError as e:
@@ -241,7 +244,10 @@ async def safe_a_measure(
 ):
     try:
         await metric.a_measure(
-            tc, _show_indicator=False, _in_component=_in_component
+            tc,
+            _show_indicator=False,
+            _in_component=_in_component,
+            _log_metric_to_confident=False,
         )
         update_pbar(progress, pbar_eval_id)
     except MissingTestCaseParamsError as e:

deepeval/metrics/json_correctness/json_correctness.py

@@ -18,6 +18,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.json_correctness.template import JsonCorrectnessTemplate
 from deepeval.metrics.json_correctness.schema import JsonCorrectnessScoreReason
 from deepeval.utils import get_or_create_event_loop
+from deepeval.metrics.api import metric_data_manager
 
 DEFAULT_CORRECT_REASON = "The generated Json matches and is syntactically correct to the expected schema."
 
@@ -51,6 +52,7 @@ class JsonCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -66,6 +68,7 @@ class JsonCorrectnessMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -88,6 +91,10 @@ class JsonCorrectnessMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -96,6 +103,7 @@ class JsonCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -126,7 +134,10 @@ class JsonCorrectnessMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def a_generate_reason(self, actual_output: str) -> str:

deepeval/metrics/knowledge_retention/knowledge_retention.py

@@ -20,6 +20,7 @@ from deepeval.metrics.knowledge_retention.schema import (
     KnowledgeRetentionScoreReason,
 )
 from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.metrics.api import metric_data_manager
 
 
 class KnowledgeRetentionMetric(BaseConversationalMetric):
@@ -47,6 +48,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -63,6 +65,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -91,6 +98,7 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -120,6 +128,10 @@ class KnowledgeRetentionMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/mcp/mcp_task_completion.py

@@ -16,6 +16,7 @@ from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.mcp.schema import Task, TaskScore
 from deepeval.metrics.mcp.template import MCPTaskCompletionTemplate
 from deepeval.errors import MissingTestCaseParamsError
+from deepeval.metrics.api import metric_data_manager
 
 
 class MCPTaskCompletionMetric(BaseConversationalMetric):
@@ -46,6 +47,7 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -62,6 +64,7 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -90,6 +93,10 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
                         f"Score: {self.score}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -97,6 +104,7 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -104,7 +112,10 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, async_mode=True, _show_indicator=_show_indicator
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             if not test_case.mcp_servers:
                 error_str = "'mcp_servers' in a conversational test case cannot be empty for the 'MCPTaskCompletionMetric' metric."
@@ -131,6 +142,11 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
                     f"Score: {self.score}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
         return self.score
 
     def _generate_reason(self, task_scores: List[TaskScore]) -> str:
@@ -228,8 +244,10 @@ class MCPTaskCompletionMetric(BaseConversationalMetric):
         return tasks
 
     def _calculate_score(self, scores: List[TaskScore]) -> float:
+        score_divsor = len(scores) if len(scores) > 0 else 1
         total_score = sum(score.score for score in scores)
-        return total_score / len(scores)
+        score = total_score / score_divsor
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/mcp/multi_turn_mcp_use_metric.py

@@ -16,6 +16,7 @@ from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.mcp.schema import Task, ArgsScore, ToolScore
 from deepeval.metrics.mcp.template import MCPTaskCompletionTemplate
 from deepeval.errors import MissingTestCaseParamsError
+from deepeval.metrics.api import metric_data_manager
 
 
 class MultiTurnMCPUseMetric(BaseConversationalMetric):
@@ -46,6 +47,7 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -62,6 +64,7 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -102,6 +105,11 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
                         f"Score: {self.score}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
             return self.score
 
     async def a_measure(
@@ -109,6 +117,7 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -116,7 +125,10 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, async_mode=True, _show_indicator=_show_indicator
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             if not test_case.mcp_servers:
                 error_str = "'mcp_servers' in a conversational test case cannot be empty for the 'MultiTurnMCPUseMetric' metric."
@@ -161,6 +173,10 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
                     f"Score: {self.score}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
         return self.score
 
     def _get_tool_accuracy_score(
@@ -299,13 +315,20 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         tool_accuracy_score: List[ToolScore],
         args_accuracy_score: List[ArgsScore],
     ) -> float:
-        tool_score = sum(score.score for score in tool_accuracy_score) / len(
-            tool_accuracy_score
+        tool_divisor = (
+            len(tool_accuracy_score) if len(tool_accuracy_score) > 0 else 1
+        )
+        args_divisor = (
+            len(args_accuracy_score) if len(args_accuracy_score) > 0 else 1
+        )
+        tool_score = (
+            sum(score.score for score in tool_accuracy_score) / tool_divisor
         )
-        args_score = sum(score.score for score in args_accuracy_score) / len(
-            args_accuracy_score
+        args_score = (
+            sum(score.score for score in args_accuracy_score) / args_divisor
         )
-        return min(tool_score, args_score)
+        score = min(tool_score, args_score)
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def _generate_reason(
         self,

deepeval/metrics/mcp_use_metric/mcp_use_metric.py

@@ -20,6 +20,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 from .template import MCPUseMetricTemplate
 from .schema import MCPPrimitivesScore, MCPArgsScore
+from deepeval.metrics.api import metric_data_manager
 
 
 class MCPUseMetric(BaseMetric):
@@ -51,6 +52,7 @@ class MCPUseMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_llm_test_case_params(test_case, self._required_params, self)
 
@@ -65,6 +67,7 @@ class MCPUseMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -104,6 +107,10 @@ class MCPUseMetric(BaseMetric):
                     self,
                     steps=steps,
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
                 return self.score
 
@@ -112,6 +119,7 @@ class MCPUseMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_llm_test_case_params(test_case, self._required_params, self)
 
@@ -154,7 +162,10 @@ class MCPUseMetric(BaseMetric):
                 self,
                 steps=steps,
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     def _get_primitives_used_score(
@@ -260,9 +271,10 @@ class MCPUseMetric(BaseMetric):
         primitives_used_score: MCPPrimitivesScore,
         argument_correctness_score: MCPArgsScore,
     ) -> float:
-        return min(
+        score = min(
             primitives_used_score.score, argument_correctness_score.score
         )
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def _get_reason(
         self,

deepeval/metrics/misuse/misuse.py

@@ -16,6 +16,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.metrics.misuse.template import MisuseTemplate
 from deepeval.metrics.misuse.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class MisuseMetric(BaseMetric):
@@ -53,6 +54,7 @@ class MisuseMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -68,6 +70,7 @@ class MisuseMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -86,6 +89,10 @@ class MisuseMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -94,6 +101,7 @@ class MisuseMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -122,7 +130,10 @@ class MisuseMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/multimodal_metrics/image_coherence/image_coherence.py

@@ -48,6 +48,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -63,6 +64,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -146,6 +148,7 @@ class ImageCoherenceMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/image_editing/image_editing.py

@@ -47,6 +47,7 @@ class ImageEditingMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, 1, 1, self
@@ -63,6 +64,7 @@ class ImageEditingMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -108,6 +110,7 @@ class ImageEditingMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, 1, 1, self

deepeval/metrics/multimodal_metrics/image_helpfulness/image_helpfulness.py

@@ -49,6 +49,7 @@ class ImageHelpfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -64,6 +65,7 @@ class ImageHelpfulnessMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -147,6 +149,7 @@ class ImageHelpfulnessMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/image_reference/image_reference.py

@@ -49,6 +49,7 @@ class ImageReferenceMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
@@ -64,6 +65,7 @@ class ImageReferenceMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -147,6 +149,7 @@ class ImageReferenceMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self

deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/multimodal_answer_relevancy.py

@@ -46,13 +46,16 @@ class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self
         )
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
+            self,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
@@ -61,6 +64,7 @@ class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +93,7 @@ class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
         test_case: MLLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_mllm_test_case_params(
             test_case, self._required_params, None, None, self