deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/conversational_g_eval/conversational_g_eval.py

@@ -1,7 +1,7 @@
 """A slightly modified tailored version of the LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
 from openai.types.chat.chat_completion import ChatCompletion
-from typing import Optional, List, Tuple, Union, Dict
+from typing import Optional, List, Tuple, Union, Dict, Type
 import math
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.metrics.g_eval.utils import (
@@ -11,7 +11,6 @@ from deepeval.metrics.g_eval.utils import (
     format_rubrics,
 )
 from deepeval.test_case import (
-    Turn,
     TurnParams,
     ConversationalTestCase,
 )
@@ -28,7 +27,8 @@ from deepeval.metrics.utils import (
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.conversational_g_eval.schema import *
+import deepeval.metrics.conversational_g_eval.schema as cgschema
+from deepeval.metrics.api import metric_data_manager
 
 
 class ConversationalGEval(BaseConversationalMetric):
@@ -44,6 +44,9 @@ class ConversationalGEval(BaseConversationalMetric):
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
+        evaluation_template: Type[
+            ConversationalGEvalTemplate
+        ] = ConversationalGEvalTemplate,
         _include_g_eval_suffix: bool = True,
     ):
         if evaluation_params is not None and len(evaluation_params) == 0:
@@ -85,6 +88,7 @@ class ConversationalGEval(BaseConversationalMetric):
         self.strict_mode = strict_mode
         self.async_mode = async_mode
         self.verbose_mode = verbose_mode
+        self.evaluation_template = evaluation_template
         self._include_g_eval_suffix = _include_g_eval_suffix
 
     def measure(
@@ -92,6 +96,7 @@ class ConversationalGEval(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self.evaluation_params, self
@@ -108,6 +113,7 @@ class ConversationalGEval(BaseConversationalMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -132,6 +138,10 @@ class ConversationalGEval(BaseConversationalMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -140,6 +150,7 @@ class ConversationalGEval(BaseConversationalMetric):
         test_case: ConversationalTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_conversational_test_case_params(
             test_case, self.evaluation_params, self
@@ -173,6 +184,10 @@ class ConversationalGEval(BaseConversationalMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
 
             return self.score
 
@@ -183,16 +198,20 @@ class ConversationalGEval(BaseConversationalMetric):
         g_eval_params_str = construct_conversational_g_eval_turn_params_string(
             self.evaluation_params
         )
-        prompt = ConversationalGEvalTemplate.generate_evaluation_steps(
+        prompt = self.evaluation_template.generate_evaluation_steps(
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Steps)
+            res, cost = await self.model.a_generate(
+                prompt, schema=cgschema.Steps
+            )
             self.evaluation_cost += cost
             return res.steps
         else:
             try:
-                res: Steps = await self.model.a_generate(prompt, schema=Steps)
+                res: cgschema.Steps = await self.model.a_generate(
+                    prompt, schema=cgschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = await self.model.a_generate(prompt)
@@ -206,16 +225,18 @@ class ConversationalGEval(BaseConversationalMetric):
         g_eval_params_str = construct_conversational_g_eval_turn_params_string(
             self.evaluation_params
         )
-        prompt = ConversationalGEvalTemplate.generate_evaluation_steps(
+        prompt = self.evaluation_template.generate_evaluation_steps(
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Steps)
+            res, cost = self.model.generate(prompt, schema=cgschema.Steps)
             self.evaluation_cost += cost
             return res.steps
         else:
             try:
-                res: Steps = self.model.generate(prompt, schema=Steps)
+                res: cgschema.Steps = self.model.generate(
+                    prompt, schema=cgschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = self.model.generate(prompt)
@@ -233,7 +254,7 @@ class ConversationalGEval(BaseConversationalMetric):
         )
         if not self.strict_mode:
             rubric_str = format_rubrics(self.rubric) if self.rubric else None
-            prompt = ConversationalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=self.number_evaluation_steps(),
                 test_case_content=test_case_content,
                 turns=[
@@ -244,7 +265,7 @@ class ConversationalGEval(BaseConversationalMetric):
                 rubric=rubric_str,
             )
         else:
-            prompt = ConversationalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=self.number_evaluation_steps(),
                 test_case_content=test_case_content,
                 turns=[
@@ -270,21 +291,21 @@ class ConversationalGEval(BaseConversationalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except (
             AttributeError
         ):  # This catches the case where a_generate_raw_response doesn't exist.
             if self.using_native_model:
                 res, cost = await self.model.a_generate(
-                    prompt, schema=ReasonScore
+                    prompt, schema=cgschema.ReasonScore
                 )
                 self.evaluation_cost += cost
                 return res.score, res.reason
             else:
                 try:
-                    res: ReasonScore = await self.model.a_generate(
-                        prompt, schema=ReasonScore
+                    res: cgschema.ReasonScore = await self.model.a_generate(
+                        prompt, schema=cgschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -303,7 +324,7 @@ class ConversationalGEval(BaseConversationalMetric):
         )
         if not self.strict_mode:
             rubric_str = format_rubrics(self.rubric) if self.rubric else None
-            prompt = ConversationalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=self.number_evaluation_steps(),
                 test_case_content=test_case_content,
                 turns=[
@@ -314,7 +335,7 @@ class ConversationalGEval(BaseConversationalMetric):
                 rubric=rubric_str,
             )
         else:
-            prompt = ConversationalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=self.number_evaluation_steps(),
                 test_case_content=test_case_content,
                 turns=[
@@ -340,18 +361,20 @@ class ConversationalGEval(BaseConversationalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
             if self.using_native_model:
-                res, cost = self.model.generate(prompt, schema=ReasonScore)
+                res, cost = self.model.generate(
+                    prompt, schema=cgschema.ReasonScore
+                )
                 self.evaluation_cost += cost
                 return res.score, res.reason
             else:
                 try:
-                    res: ReasonScore = self.model.generate(
-                        prompt, schema=ReasonScore
+                    res: cgschema.ReasonScore = self.model.generate(
+                        prompt, schema=cgschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -362,49 +385,44 @@ class ConversationalGEval(BaseConversationalMetric):
     def generate_weighted_summed_score(
         self, raw_score: int, raw_response: ChatCompletion
     ) -> Union[int, float]:
-        try:
-            generated_logprobs = raw_response.choices[0].logprobs.content
-            # First, locate the token that we care for logprobs, i.e., the token matching the score
-            score_logprobs = None
-            for token_logprobs in generated_logprobs:
-                if token_logprobs.token == str(raw_score):
-                    score_logprobs = token_logprobs
-                    break
-            # Then, calculate the score based on the logprobs
-            token_linear_probability: Dict[int, float] = {}
-            sum_linear_probability = 0
-            # Filter out tokens with <1% linear probability, i.e., logprobs < math.log(0.01)
-            min_logprob = math.log(0.01)
-            for token_logprob in score_logprobs.top_logprobs:
-                logprob = token_logprob.logprob
-
-                # Filter out low probability tokens
-                if logprob < min_logprob:
-                    continue
-                # Filter out non-decimal token to prevent errors in later int(token) conversion
-                if not token_logprob.token.isdecimal():
-                    continue
-
-                # Calculate the linear probability
-                linear_prob = math.exp(logprob)
-                token_score = int(token_logprob.token)
-                if token_linear_probability.get(token_score):
-                    token_linear_probability[token_score] += linear_prob
-                else:
-                    token_linear_probability[token_score] = linear_prob
-                sum_linear_probability += linear_prob
-
-            sum_of_weighted_scores = 0.0
-            for score, prob in token_linear_probability.items():
-                sum_of_weighted_scores += score * prob
-
-            # Scale the sum of linear probability to 1
-            weighted_summed_score = (
-                sum_of_weighted_scores / sum_linear_probability
-            )
-            return weighted_summed_score
-        except:
-            raise
+        generated_logprobs = raw_response.choices[0].logprobs.content
+        # First, locate the token that we care for logprobs, i.e., the token matching the score
+        score_logprobs = None
+        for token_logprobs in generated_logprobs:
+            if token_logprobs.token == str(raw_score):
+                score_logprobs = token_logprobs
+                break
+        # Then, calculate the score based on the logprobs
+        token_linear_probability: Dict[int, float] = {}
+        sum_linear_probability = 0
+        # Filter out tokens with <1% linear probability, i.e., logprobs < math.log(0.01)
+        min_logprob = math.log(0.01)
+        for token_logprob in score_logprobs.top_logprobs:
+            logprob = token_logprob.logprob
+
+            # Filter out low probability tokens
+            if logprob < min_logprob:
+                continue
+            # Filter out non-decimal token to prevent errors in later int(token) conversion
+            if not token_logprob.token.isdecimal():
+                continue
+
+            # Calculate the linear probability
+            linear_prob = math.exp(logprob)
+            token_score = int(token_logprob.token)
+            if token_linear_probability.get(token_score):
+                token_linear_probability[token_score] += linear_prob
+            else:
+                token_linear_probability[token_score] = linear_prob
+            sum_linear_probability += linear_prob
+
+        sum_of_weighted_scores = 0.0
+        for score, prob in token_linear_probability.items():
+            sum_of_weighted_scores += score * prob
+
+        # Scale the sum of linear probability to 1
+        weighted_summed_score = sum_of_weighted_scores / sum_linear_probability
+        return weighted_summed_score
 
     def number_evaluation_steps(self):
         evaluation_steps = """"""
@@ -417,8 +435,8 @@ class ConversationalGEval(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/dag/dag.py

@@ -18,6 +18,7 @@ from deepeval.metrics.dag.utils import (
     is_valid_dag_from_roots,
     extract_required_params,
 )
+from deepeval.metrics.api import metric_data_manager
 
 
 class DAGMetric(BaseMetric):
@@ -59,6 +60,7 @@ class DAGMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_llm_test_case_params(
             test_case,
@@ -77,6 +79,7 @@ class DAGMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -89,6 +92,10 @@ class DAGMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +103,7 @@ class DAGMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
         check_llm_test_case_params(
             test_case,
@@ -119,6 +127,10 @@ class DAGMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     def is_successful(self) -> bool:

deepeval/metrics/dag/nodes.py

@@ -111,7 +111,9 @@ class VerdictNode(BaseNode):
                 copied_g_eval = GEval(**g_eval_args)
 
                 copied_g_eval.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_g_eval)
@@ -124,7 +126,9 @@ class VerdictNode(BaseNode):
                 copied_metric.verbose_mode = False
 
                 copied_metric.measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)
@@ -174,7 +178,9 @@ class VerdictNode(BaseNode):
                 copied_g_eval = GEval(**g_eval_args)
 
                 await copied_g_eval.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_g_eval)
@@ -188,7 +194,9 @@ class VerdictNode(BaseNode):
                 copied_metric.verbose_mode = False
 
                 await copied_metric.a_measure(
-                    test_case=test_case, _show_indicator=False
+                    test_case=test_case,
+                    _show_indicator=False,
+                    _log_metric_to_confident=False,
                 )
                 metric._verbose_steps.append(
                     construct_node_verbose_log(self, depth, copied_metric)

deepeval/metrics/dag/schema.py

@@ -11,7 +11,7 @@ class TaskNodeOutput(BaseModel):
 
 
 class BinaryJudgementVerdict(BaseModel):
-    verdict: Literal[True, False]
+    verdict: bool
     reason: str
 
 

deepeval/metrics/dag/templates.py

@@ -60,10 +60,10 @@ class BinaryJudgementTemplate:
 {text}
 
 **
-IMPORTANT: Please make sure to only return a json with two keys: `verdict` (true or false), and the 'reason' key providing the reason.
+IMPORTANT: Please make sure to only return a json with two keys: `verdict` (True or False), and the 'reason' key providing the reason. The verdict must be a boolean only, either True or False.
 Example JSON:
 {{
-    "verdict": true,
+    "verdict": True,
     "reason": "..."
 }}
 **

deepeval/metrics/faithfulness/faithfulness.py

@@ -23,6 +23,7 @@ from deepeval.metrics.faithfulness.schema import (
     Truths,
     Claims,
 )
+from deepeval.metrics.api import metric_data_manager
 
 
 class FaithfulnessMetric(BaseMetric):
@@ -63,6 +64,7 @@ class FaithfulnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -78,6 +80,7 @@ class FaithfulnessMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -96,6 +99,10 @@ class FaithfulnessMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -104,6 +111,7 @@ class FaithfulnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -132,7 +140,10 @@ class FaithfulnessMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/g_eval/g_eval.py

@@ -31,6 +31,7 @@ from deepeval.metrics.g_eval.utils import (
     number_evaluation_steps,
     get_score_range,
 )
+from deepeval.metrics.api import metric_data_manager
 from deepeval.config.settings import get_settings
 
 
@@ -74,6 +75,7 @@ class GEval(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
         check_llm_test_case_params(test_case, self.evaluation_params, self)
@@ -122,6 +124,10 @@ class GEval(BaseMetric):
                         f"Reason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -130,6 +136,7 @@ class GEval(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
         _additional_context: Optional[str] = None,
     ) -> float:
         check_llm_test_case_params(test_case, self.evaluation_params, self)
@@ -165,6 +172,10 @@ class GEval(BaseMetric):
                     f"Reason: {self.reason}",
                 ],
             )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_evaluation_steps(self) -> List[str]:

deepeval/metrics/goal_accuracy/__init__.py

@@ -0,0 +1 @@
+from .goal_accuracy import GoalAccuracyMetric