deepeval 3.6.6__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -16,6 +16,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.contextual_recall.template import ContextualRecallTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.contextual_recall.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ContextualRecallMetric(BaseMetric):
@@ -52,8 +53,8 @@ class ContextualRecallMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -67,6 +68,7 @@ class ContextualRecallMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -85,7 +87,10 @@ class ContextualRecallMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -93,6 +98,7 @@ class ContextualRecallMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -121,7 +127,10 @@ class ContextualRecallMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, expected_output: str):

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -19,6 +19,7 @@ from deepeval.metrics.contextual_relevancy.template import (
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.contextual_relevancy.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ContextualRelevancyMetric(BaseMetric):
@@ -53,6 +54,7 @@ class ContextualRelevancyMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -68,6 +70,7 @@ class ContextualRelevancyMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -85,6 +88,10 @@ class ContextualRelevancyMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -93,6 +100,7 @@ class ContextualRelevancyMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -122,7 +130,10 @@ class ContextualRelevancyMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str):

deepeval/metrics/conversation_completeness/conversation_completeness.py

@@ -19,6 +19,7 @@ from deepeval.test_case import TurnParams
 from deepeval.test_case.conversational_test_case import Turn
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.conversation_completeness.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ConversationCompletenessMetric(BaseConversationalMetric):
@@ -48,6 +49,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -64,6 +66,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +92,10 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +103,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self._required_test_case_params, self
@@ -129,6 +137,10 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/conversational_dag/conversational_dag.py

@@ -18,6 +18,7 @@ from deepeval.metrics.dag.utils import (
     extract_required_params,
     copy_graph,
 )
+from deepeval.metrics.api import metric_data_manager
 
 
 class ConversationalDAGMetric(BaseConversationalMetric):
@@ -59,6 +60,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case,
@@ -77,6 +79,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +92,10 @@ class ConversationalDAGMetric(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +103,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case,
@@ -119,6 +127,10 @@ class ConversationalDAGMetric(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     def is_successful(self) -> bool:

deepeval/metrics/conversational_dag/nodes.py

@@ -141,7 +141,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_convo_g_eval = ConversationalGEval(**convo_g_eval_args)
 
                 copied_convo_g_eval.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_convo_g_eval)
@@ -157,7 +159,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_metric.verbose_mode = False
 
                 copied_metric.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)
@@ -213,7 +217,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_convo_g_eval = ConversationalGEval(**convo_g_eval_args)
 
                 await copied_convo_g_eval.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_convo_g_eval)
@@ -229,7 +235,9 @@ class ConversationalVerdictNode(ConversationalBaseNode):
                 copied_metric.verbose_mode = False
 
                 await copied_metric.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)

deepeval/metrics/conversational_g_eval/conversational_g_eval.py

@@ -11,7 +11,6 @@ from deepeval.metrics.g_eval.utils import (
     format_rubrics,
 )
 from deepeval.test_case import (
-    Turn,
     TurnParams,
     ConversationalTestCase,
 )
@@ -28,7 +27,8 @@ from deepeval.metrics.utils import (
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.conversational_g_eval.schema import *
+import deepeval.metrics.conversational_g_eval.schema as cgschema
+from deepeval.metrics.api import metric_data_manager
 
 
 class ConversationalGEval(BaseConversationalMetric):
@@ -92,6 +92,7 @@ class ConversationalGEval(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self.evaluation_params, self
@@ -108,6 +109,7 @@ class ConversationalGEval(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -132,6 +134,10 @@ class ConversationalGEval(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -140,6 +146,7 @@ class ConversationalGEval(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self.evaluation_params, self
@@ -173,6 +180,10 @@ class ConversationalGEval(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 
@@ -187,12 +198,16 @@ class ConversationalGEval(BaseConversationalMetric):
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Steps)
+            res, cost = await self.model.a_generate(
+                prompt, schema=cgschema.Steps
+            )
             self.evaluation_cost += cost
             return res.steps
         else:
             try:
-                res: Steps = await self.model.a_generate(prompt, schema=Steps)
+                res: cgschema.Steps = await self.model.a_generate(
+                    prompt, schema=cgschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = await self.model.a_generate(prompt)
@@ -210,12 +225,14 @@ class ConversationalGEval(BaseConversationalMetric):
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Steps)
+            res, cost = self.model.generate(prompt, schema=cgschema.Steps)
             self.evaluation_cost += cost
             return res.steps
         else:
             try:
-                res: Steps = self.model.generate(prompt, schema=Steps)
+                res: cgschema.Steps = self.model.generate(
+                    prompt, schema=cgschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = self.model.generate(prompt)
@@ -270,21 +287,21 @@ class ConversationalGEval(BaseConversationalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except (
             AttributeError
         ):  # This catches the case where a_generate_raw_response doesn't exist.
             if self.using_native_model:
                 res, cost = await self.model.a_generate(
-                    prompt, schema=ReasonScore
+                    prompt, schema=cgschema.ReasonScore
                 )
                 self.evaluation_cost += cost
                 return res.score, res.reason
             else:
                 try:
-                    res: ReasonScore = await self.model.a_generate(
-                        prompt, schema=ReasonScore
+                    res: cgschema.ReasonScore = await self.model.a_generate(
+                        prompt, schema=cgschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -340,18 +357,20 @@ class ConversationalGEval(BaseConversationalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
             if self.using_native_model:
-                res, cost = self.model.generate(prompt, schema=ReasonScore)
+                res, cost = self.model.generate(
+                    prompt, schema=cgschema.ReasonScore
+                )
                 self.evaluation_cost += cost
                 return res.score, res.reason
             else:
                 try:
-                    res: ReasonScore = self.model.generate(
-                        prompt, schema=ReasonScore
+                    res: cgschema.ReasonScore = self.model.generate(
+                        prompt, schema=cgschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -362,49 +381,44 @@ class ConversationalGEval(BaseConversationalMetric):
     def generate_weighted_summed_score(
         self, raw_score: int, raw_response: ChatCompletion
     ) -> Union[int, float]:
-        try:
-            generated_logprobs = raw_response.choices[0].logprobs.content
-            # First, locate the token that we care for logprobs, i.e., the token matching the score
-            score_logprobs = None
-            for token_logprobs in generated_logprobs:
-                if token_logprobs.token == str(raw_score):
-                    score_logprobs = token_logprobs
-                    break
-            # Then, calculate the score based on the logprobs
-            token_linear_probability: Dict[int, float] = {}
-            sum_linear_probability = 0
-            # Filter out tokens with <1% linear probability, i.e., logprobs < math.log(0.01)
-            min_logprob = math.log(0.01)
-            for token_logprob in score_logprobs.top_logprobs:
-                logprob = token_logprob.logprob
-
-                # Filter out low probability tokens
-                if logprob < min_logprob:
-                    continue
-                # Filter out non-decimal token to prevent errors in later int(token) conversion
-                if not token_logprob.token.isdecimal():
-                    continue
-
-                # Calculate the linear probability
-                linear_prob = math.exp(logprob)
-                token_score = int(token_logprob.token)
-                if token_linear_probability.get(token_score):
-                    token_linear_probability[token_score] += linear_prob
-                else:
-                    token_linear_probability[token_score] = linear_prob
-                sum_linear_probability += linear_prob
-
-            sum_of_weighted_scores = 0.0
-            for score, prob in token_linear_probability.items():
-                sum_of_weighted_scores += score * prob
-
-            # Scale the sum of linear probability to 1
-            weighted_summed_score = (
-                sum_of_weighted_scores / sum_linear_probability
-            )
-            return weighted_summed_score
-        except:
-            raise
+        generated_logprobs = raw_response.choices[0].logprobs.content
+        # First, locate the token that we care for logprobs, i.e., the token matching the score
+        score_logprobs = None
+        for token_logprobs in generated_logprobs:
+            if token_logprobs.token == str(raw_score):
+                score_logprobs = token_logprobs
+                break
+        # Then, calculate the score based on the logprobs
+        token_linear_probability: Dict[int, float] = {}
+        sum_linear_probability = 0
+        # Filter out tokens with <1% linear probability, i.e., logprobs < math.log(0.01)
+        min_logprob = math.log(0.01)
+        for token_logprob in score_logprobs.top_logprobs:
+            logprob = token_logprob.logprob
+
+            # Filter out low probability tokens
+            if logprob < min_logprob:
+                continue
+            # Filter out non-decimal token to prevent errors in later int(token) conversion
+            if not token_logprob.token.isdecimal():
+                continue
+
+            # Calculate the linear probability
+            linear_prob = math.exp(logprob)
+            token_score = int(token_logprob.token)
+            if token_linear_probability.get(token_score):
+                token_linear_probability[token_score] += linear_prob
+            else:
+                token_linear_probability[token_score] = linear_prob
+            sum_linear_probability += linear_prob
+
+        sum_of_weighted_scores = 0.0
+        for score, prob in token_linear_probability.items():
+            sum_of_weighted_scores += score * prob
+
+        # Scale the sum of linear probability to 1
+        weighted_summed_score = sum_of_weighted_scores / sum_linear_probability
+        return weighted_summed_score
 
     def number_evaluation_steps(self):
         evaluation_steps = """"""
@@ -417,8 +431,8 @@ class ConversationalGEval(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/dag/dag.py

@@ -18,6 +18,7 @@ from deepeval.metrics.dag.utils import (
     is_valid_dag_from_roots,
     extract_required_params,
 )
+from deepeval.metrics.api import metric_data_manager
 
 
 class DAGMetric(BaseMetric):
@@ -59,6 +60,7 @@ class DAGMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_llm_test_case_params(
             test_case,
@@ -77,6 +79,7 @@ class DAGMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +92,10 @@ class DAGMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +103,7 @@ class DAGMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_llm_test_case_params(
             test_case,
@@ -119,6 +127,10 @@ class DAGMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     def is_successful(self) -> bool:

deepeval/metrics/dag/nodes.py

@@ -111,7 +111,9 @@ class VerdictNode(BaseNode):
                 copied_g_eval = GEval(**g_eval_args)
 
                 copied_g_eval.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_g_eval)
@@ -124,7 +126,9 @@ class VerdictNode(BaseNode):
                 copied_metric.verbose_mode = False
 
                 copied_metric.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)
@@ -174,7 +178,9 @@ class VerdictNode(BaseNode):
                 copied_g_eval = GEval(**g_eval_args)
 
                 await copied_g_eval.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_g_eval)
@@ -188,7 +194,9 @@ class VerdictNode(BaseNode):
                 copied_metric.verbose_mode = False
 
                 await copied_metric.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)

deepeval/metrics/faithfulness/faithfulness.py

@@ -23,6 +23,7 @@ from deepeval.metrics.faithfulness.schema import (
     Truths,
     Claims,
 )
+from deepeval.metrics.api import metric_data_manager
 
 
 class FaithfulnessMetric(BaseMetric):
@@ -63,6 +64,7 @@ class FaithfulnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -78,6 +80,7 @@ class FaithfulnessMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -96,6 +99,10 @@ class FaithfulnessMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -104,6 +111,7 @@ class FaithfulnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -132,7 +140,10 @@ class FaithfulnessMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str: