deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/bias/bias.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Type, Union
 
 from deepeval.metrics import BaseMetric
+from deepeval.metrics.api import metric_data_manager
 from deepeval.test_case import (
     LLMTestCase,
     LLMTestCaseParams,
@@ -48,8 +49,8 @@ class BiasMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -63,6 +64,7 @@ class BiasMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -81,7 +83,10 @@ class BiasMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -89,8 +94,8 @@ class BiasMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -116,6 +121,10 @@ class BiasMetric(BaseMetric):
                 ],
             )
 
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -17,7 +17,8 @@ from deepeval.metrics.contextual_precision.template import (
     ContextualPrecisionTemplate,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.contextual_precision.schema import *
+import deepeval.metrics.contextual_precision.schema as cpschema
+from deepeval.metrics.api import metric_data_manager
 
 
 class ContextualPrecisionMetric(BaseMetric):
@@ -53,8 +54,8 @@ class ContextualPrecisionMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -68,10 +69,11 @@ class ContextualPrecisionMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
-                self.verdicts: List[ContextualPrecisionVerdict] = (
+                self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                     self._generate_verdicts(
                         test_case.input,
                         test_case.expected_output,
@@ -88,7 +90,10 @@ class ContextualPrecisionMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +101,7 @@ class ContextualPrecisionMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -107,7 +113,7 @@ class ContextualPrecisionMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: List[ContextualPrecisionVerdict] = (
+            self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                 await self._a_generate_verdicts(
                     test_case.input,
                     test_case.expected_output,
@@ -124,7 +130,10 @@ class ContextualPrecisionMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str):
@@ -132,7 +141,7 @@ class ContextualPrecisionMetric(BaseMetric):
             return None
 
         retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            {"verdict": verdict.verdict, "reason": verdict.reason}
             for verdict in self.verdicts
         ]
         prompt = self.evaluation_template.generate_reason(
@@ -143,15 +152,15 @@ class ContextualPrecisionMetric(BaseMetric):
 
         if self.using_native_model:
             res, cost = await self.model.a_generate(
-                prompt, schema=ContextualPrecisionScoreReason
+                prompt, schema=cpschema.ContextualPrecisionScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: ContextualPrecisionScoreReason = (
+                res: cpschema.ContextualPrecisionScoreReason = (
                     await self.model.a_generate(
-                        prompt, schema=ContextualPrecisionScoreReason
+                        prompt, schema=cpschema.ContextualPrecisionScoreReason
                     )
                 )
                 return res.reason
@@ -165,7 +174,7 @@ class ContextualPrecisionMetric(BaseMetric):
             return None
 
         retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            {"verdict": verdict.verdict, "reason": verdict.reason}
             for verdict in self.verdicts
         ]
         prompt = self.evaluation_template.generate_reason(
@@ -176,14 +185,16 @@ class ContextualPrecisionMetric(BaseMetric):
 
         if self.using_native_model:
             res, cost = self.model.generate(
-                prompt, schema=ContextualPrecisionScoreReason
+                prompt, schema=cpschema.ContextualPrecisionScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: ContextualPrecisionScoreReason = self.model.generate(
-                    prompt, schema=ContextualPrecisionScoreReason
+                res: cpschema.ContextualPrecisionScoreReason = (
+                    self.model.generate(
+                        prompt, schema=cpschema.ContextualPrecisionScoreReason
+                    )
                 )
                 return res.reason
             except TypeError:
@@ -193,21 +204,23 @@ class ContextualPrecisionMetric(BaseMetric):
 
     async def _a_generate_verdicts(
         self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
+    ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            res, cost = await self.model.a_generate(
+                prompt, schema=cpschema.Verdicts
+            )
             self.evaluation_cost += cost
             verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
+                res: cpschema.Verdicts = await self.model.a_generate(
+                    prompt, schema=cpschema.Verdicts
                 )
                 verdicts = [item for item in res.verdicts]
                 return verdicts
@@ -215,34 +228,36 @@ class ContextualPrecisionMetric(BaseMetric):
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 verdicts = [
-                    ContextualPrecisionVerdict(**item)
+                    cpschema.ContextualPrecisionVerdict(**item)
                     for item in data["verdicts"]
                 ]
                 return verdicts
 
     def _generate_verdicts(
         self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
+    ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
+            res, cost = self.model.generate(prompt, schema=cpschema.Verdicts)
             self.evaluation_cost += cost
             verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                res: cpschema.Verdicts = self.model.generate(
+                    prompt, schema=cpschema.Verdicts
+                )
                 verdicts = [item for item in res.verdicts]
                 return verdicts
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 verdicts = [
-                    ContextualPrecisionVerdict(**item)
+                    cpschema.ContextualPrecisionVerdict(**item)
                     for item in data["verdicts"]
                 ]
                 return verdicts
@@ -279,7 +294,7 @@ class ContextualPrecisionMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -16,6 +16,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.contextual_recall.template import ContextualRecallTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.contextual_recall.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ContextualRecallMetric(BaseMetric):
@@ -52,8 +53,8 @@ class ContextualRecallMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -67,6 +68,7 @@ class ContextualRecallMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -85,7 +87,10 @@ class ContextualRecallMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -93,6 +98,7 @@ class ContextualRecallMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -121,7 +127,10 @@ class ContextualRecallMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, expected_output: str):

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -19,6 +19,7 @@ from deepeval.metrics.contextual_relevancy.template import (
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.contextual_relevancy.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ContextualRelevancyMetric(BaseMetric):
@@ -53,6 +54,7 @@ class ContextualRelevancyMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -68,6 +70,7 @@ class ContextualRelevancyMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -85,6 +88,10 @@ class ContextualRelevancyMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -93,6 +100,7 @@ class ContextualRelevancyMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -122,7 +130,10 @@ class ContextualRelevancyMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str):

deepeval/metrics/conversation_completeness/conversation_completeness.py

@@ -19,6 +19,7 @@ from deepeval.test_case import TurnParams
 from deepeval.test_case.conversational_test_case import Turn
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.conversation_completeness.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ConversationCompletenessMetric(BaseConversationalMetric):
@@ -48,6 +49,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -64,6 +66,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +92,10 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +103,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -129,6 +137,10 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/conversational_dag/conversational_dag.py

@@ -18,6 +18,7 @@ from deepeval.metrics.dag.utils import (
     extract_required_params,
     copy_graph,
 )
+from deepeval.metrics.api import metric_data_manager
 
 
 class ConversationalDAGMetric(BaseConversationalMetric):
@@ -59,6 +60,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case,
@@ -77,6 +79,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +92,10 @@ class ConversationalDAGMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +103,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case,
@@ -119,6 +127,10 @@ class ConversationalDAGMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     def is_successful(self) -> bool:

deepeval/metrics/conversational_dag/nodes.py

@@ -141,7 +141,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_convo_g_eval = ConversationalGEval(**convo_g_eval_args)
 
                 copied_convo_g_eval.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_convo_g_eval)
@@ -157,7 +159,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_metric.verbose_mode = False
 
                 copied_metric.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)
@@ -213,7 +217,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_convo_g_eval = ConversationalGEval(**convo_g_eval_args)
 
                 await copied_convo_g_eval.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_convo_g_eval)
@@ -229,7 +235,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_metric.verbose_mode = False
 
                 await copied_metric.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)

deepeval/metrics/conversational_g_eval/__init__.py

@@ -0,0 +1,3 @@
+from .template import ConversationalGEvalTemplate
+
+__all__ = ["ConversationalGEvalTemplate"]