deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/goal_accuracy/goal_accuracy.py

@@ -0,0 +1,349 @@
+from typing import Optional, List, Union
+import asyncio
+from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    get_unit_interactions,
+    print_tools_called,
+    check_conversational_test_case_params,
+    initialize_model,
+)
+from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
+from deepeval.metrics import BaseConversationalMetric
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.goal_accuracy.template import (
+    GoalAccuracyTemplate,
+)
+from deepeval.metrics.goal_accuracy.schema import (
+    GoalSteps,
+    GoalScore,
+    PlanScore,
+)
+from deepeval.metrics.api import metric_data_manager
+
+
+class GoalAccuracyMetric(BaseConversationalMetric):
+
+    _required_test_case_params = [
+        TurnParams.ROLE,
+        TurnParams.CONTENT,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+
+    def measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case, self._required_test_case_params, self
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                unit_interactions = get_unit_interactions(test_case.turns)
+                goal_and_steps_taken = self._goal_and_steps_taken(
+                    unit_interactions
+                )
+                goal_scores = [
+                    self._get_goal_accuracy_score(
+                        task.user_goal, task.steps_taken
+                    )
+                    for task in goal_and_steps_taken
+                ]
+                plan_scores = [
+                    self._get_plan_scores(task.user_goal, task.steps_taken)
+                    for task in goal_and_steps_taken
+                ]
+                self.score = self._calculate_score(goal_scores, plan_scores)
+                self.success = self.score >= self.threshold
+                self.reason = self._generate_reason(goal_scores, plan_scores)
+
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Goals and steps taken: \n{self.print_goals_and_steps_taken(goal_and_steps_taken)} \n",
+                        f"Goal evaluations: {prettify_list(goal_scores)} \n\n"
+                        f"Plan evaluations: {prettify_list(plan_scores)} \n\n"
+                        f"Final Score: {self.score}",
+                        f"Final Reason: {self.reason}",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case, self._required_test_case_params, self
+        )
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            unit_interactions = get_unit_interactions(test_case.turns)
+            goal_and_steps_taken = self._goal_and_steps_taken(unit_interactions)
+            goal_scores = await asyncio.gather(
+                *[
+                    self._a_get_goal_accuracy_score(
+                        task.user_goal, task.steps_taken
+                    )
+                    for task in goal_and_steps_taken
+                ]
+            )
+            plan_scores = await asyncio.gather(
+                *[
+                    self._a_get_plan_scores(task.user_goal, task.steps_taken)
+                    for task in goal_and_steps_taken
+                ]
+            )
+            self.score = self._calculate_score(goal_scores, plan_scores)
+            self.success = self.score >= self.threshold
+            self.reason = await self._a_generate_reason(
+                goal_scores, plan_scores
+            )
+
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"Goals and steps taken: \n{self.print_goals_and_steps_taken(goal_and_steps_taken)} \n",
+                    f"Goal evaluations: {prettify_list(goal_scores)} \n\n"
+                    f"Plan evaluations: {prettify_list(plan_scores)} \n\n"
+                    f"Final Score: {self.score}",
+                    f"Final Reason: {self.reason}",
+                ],
+            )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    def _goal_and_steps_taken(
+        self, unit_interactions: List[List[Turn]]
+    ) -> List[GoalSteps]:
+        goal_and_steps_taken = []
+        for unit_interaction in unit_interactions:
+            user_messages = "User messages: \n"
+            for turn in unit_interaction:
+                if turn.role == "user":
+                    user_messages += turn.content + "\n"
+                else:
+                    break
+            new_goal_steps = GoalSteps(user_goal=user_messages, steps_taken=[])
+            assistant_messages = "Assistant messages: \n"
+            for turn in unit_interaction[1:]:
+                if turn.role == "assistant":
+                    assistant_messages += f"{turn.content} \n"
+                    if turn.tools_called:
+                        assistant_messages += f"Tools called: \n{print_tools_called(turn.tools_called)} \n"
+                    new_goal_steps.steps_taken.append(assistant_messages)
+            goal_and_steps_taken.append(new_goal_steps)
+        return goal_and_steps_taken
+
+    def _get_plan_scores(self, user_goal, steps_taken):
+        prompt = GoalAccuracyTemplate.get_plan_evaluation_score(
+            user_goal, "\n".join(steps_taken)
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=PlanScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: PlanScore = self.model.generate(prompt, schema=PlanScore)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return PlanScore(**data)
+
+    async def _a_get_plan_scores(self, user_goal, steps_taken):
+        prompt = GoalAccuracyTemplate.get_plan_evaluation_score(
+            user_goal, "\n".join(steps_taken)
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=PlanScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: PlanScore = await self.model.a_generate(
+                    prompt, schema=PlanScore
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return PlanScore(**data)
+
+    def _calculate_score(
+        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+    ):
+        goal_scores = [goal_score.score for goal_score in goal_scores]
+        plan_scores = [plan_score.score for plan_score in plan_scores]
+        goal_score_divisor = len(goal_scores) if len(goal_scores) > 0 else 1
+        plan_score_divisor = len(plan_scores) if len(plan_scores) > 0 else 1
+        goal_avg = sum(goal_scores) / goal_score_divisor
+        plan_avg = sum(plan_scores) / plan_score_divisor
+        score = (goal_avg + plan_avg) / 2
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    def _generate_reason(
+        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+    ):
+        goal_evaluations = ""
+        for goal_score in goal_scores:
+            goal_evaluations += (
+                f"Score: {goal_score.score}, Reason: {goal_score.reason}"
+            )
+        plan_evalautions = ""
+        for plan_score in plan_scores:
+            plan_evalautions += (
+                f"Score: {plan_score.score}, Reason: {plan_score.reason} \n"
+            )
+
+        prompt = GoalAccuracyTemplate.get_final_reason(
+            self.score, self.threshold, goal_evaluations, plan_evalautions
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = self.model.generate(prompt)
+            return res
+
+    async def _a_generate_reason(
+        self, goal_scores: List[GoalScore], plan_scores: List[PlanScore]
+    ):
+        goal_evaluations = ""
+        for goal_score in goal_scores:
+            goal_evaluations += (
+                f"Score: {goal_score.score}, Reason: {goal_score.reason}"
+            )
+        plan_evalautions = ""
+        for plan_score in plan_scores:
+            plan_evalautions += (
+                f"Score: {plan_score.score}, Reason: {plan_score.reason} \n"
+            )
+
+        prompt = GoalAccuracyTemplate.get_final_reason(
+            self.score, self.threshold, goal_evaluations, plan_evalautions
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = await self.model.a_generate(prompt)
+            return res
+
+    def _get_goal_accuracy_score(self, user_goal, steps_taken):
+        prompt = GoalAccuracyTemplate.get_accuracy_score(
+            user_goal, "\n".join(steps_taken)
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=GoalScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: GoalScore = self.model.generate(prompt, schema=GoalScore)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return GoalScore(**data)
+
+    async def _a_get_goal_accuracy_score(self, user_goal, steps_taken):
+        prompt = GoalAccuracyTemplate.get_accuracy_score(
+            user_goal, "\n".join(steps_taken)
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=GoalScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: GoalScore = await self.model.a_generate(
+                    prompt, schema=GoalScore
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return GoalScore(**data)
+
+    def print_goals_and_steps_taken(self, goals_and_steps):
+        final_goals_and_steps = ""
+        for goal_step in goals_and_steps:
+            final_goals_and_steps += f"{goal_step.user_goal} \n"
+            final_goals_and_steps += (
+                f"c{prettify_list(goal_step.steps_taken)} \n\n"
+            )
+        return final_goals_and_steps
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Goal Accuracy"

deepeval/metrics/goal_accuracy/schema.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class GoalSteps(BaseModel):
+    user_goal: str
+    steps_taken: List[str]
+
+
+class GoalScore(BaseModel):
+    score: float
+    reason: str
+
+
+class PlanScore(BaseModel):
+    score: float
+    reason: str

deepeval/metrics/goal_accuracy/template.py

@@ -0,0 +1,235 @@
+from typing import List
+import textwrap
+
+
+class GoalAccuracyTemplate:
+    @staticmethod
+    def get_accuracy_score(task, steps_taken):
+        return textwrap.dedent(
+            f"""You are an expert evaluator assessing the **goal accuracy** of an AI assistant's single interaction.
+
+                PURPOSE:
+
+                Evaluate whether the assistant's **visible output** (what the user actually saw) **fully and correctly achieved the user's stated goal.  
+                Ignore internal reasoning, hidden tool calls, or retriever outputs unless their results were explicitly surfaced to the user.
+
+                The evaluation must be **strict and adversarial** — if the goal is not *clearly, fully, and correctly achieved*, assign a low score.
+
+                EVALUATION RULES
+
+                1. **User-visible fulfillment only**
+                - Base your judgment solely on what the user would see in the assistant's message.
+                - Ignore hidden or internal steps unless their results were explicitly communicated.
+
+                2. **Goal completion**
+                - The assistant must explicitly provide everything the user asked for.
+                - If even one subpart of the task is missing, incomplete, or vague, the score must be **≤ 0.5**.
+
+                3. **Correctness and relevance**
+                - The information provided must be factually correct and directly relevant to the task.
+                - Hallucinated or unrelated content automatically lowers the score.
+
+                4. **Self-sufficiency**
+                - The visible response must stand on its own; the user should not need prior context or follow-up clarification.
+
+                5. **Strict bias toward failure**
+                - When uncertain, assume the goal was **not achieved**.
+                - The metric is designed to fail unless the assistant's output is precise, complete, and user-visible.
+
+                SCORING GUIDE:
+
+                - **1.0** → Goal completely and correctly achieved; all required outputs visible to the user.  
+                - **0.75** → Mostly achieved; minor omissions or trivial inaccuracies.  
+                - **0.5** → Partially achieved; core goal addressed, but key parts missing or incorrect.  
+                - **0.25** → Weak attempt; loosely related but fails to satisfy the user’s request.  
+                - **0.0** → Goal not achieved at all; irrelevant, wrong, or missing answer.
+
+                *When in doubt, choose the lower score.*
+
+                OUTPUT FORMAT:
+
+                Return only a valid JSON object with this structure:
+
+                {{
+                    "score": 0.0,
+                    "reason": "1-3 factual sentences explaining what parts of the user's goal were or were not achieved."
+                }}
+
+                The reason must:
+                - Be objective and concise.
+                - Refer to **specific missing or incorrect elements**.
+                - Avoid vague language (“somewhat correct”, “pretty accurate”).
+
+                EXAMPLES:
+
+                **Example 1**
+                Task: "Translate 'good night' into French."  
+                Assistant Reply: "Bonne nuit."  
+                →  
+                {{
+                    "score": 1.0,
+                    "reason": "The assistant provided the exact, correct translation requested by the user."
+                }}
+
+                **Example 2**
+                Task: "List three renewable energy sources."  
+                Assistant Reply: "Solar and wind energy."  
+                →  
+                {{
+                    "score": 0.5,
+                    "reason": "The assistant only listed two sources instead of three, so the goal was partially achieved."
+                }}
+
+                **Example 3**
+                Task: "Summarize this paragraph."  
+                Assistant Reply: "It talks about technology."  
+                →  
+                {{
+                    "score": 0.25,
+                    "reason": "The summary is too vague and fails to convey key information from the text."
+                }}
+
+                *** END OF EXAMPLES ***
+
+                USER TASK:
+                {task}
+
+                AGENT STEPS:
+                {steps_taken}
+
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def get_plan_evaluation_score(task, steps_taken):
+        return textwrap.dedent(
+            f"""You are an expert evaluator assessing the **planning quality** and **plan adherence** of an AI agent tasked with fulfilling a user's request.
+
+                OBJECTIVE:
+
+                Evaluate:
+
+                1. **Plan Quality** — Was the agent's plan clear, complete, and logically structured to fully address the user's task?  
+                2. **Plan Adherence** — Did the agent consistently follow that plan without unjustified deviations, omissions, or extraneous steps?
+
+                Your judgment must be strict: a plan must be well-formed and execution must align with it for a high score.
+
+                EVALUATION CRITERIA
+
+                - Plan Quality:  
+                - The plan should explicitly or implicitly outline all necessary steps to fulfill the user's task.  
+                - It must be logically ordered, neither vague nor overly generic.  
+                - Missing critical components or unclear structuring lowers the score drastically.
+
+                - Plan Adherence:  
+                - Execution must closely match the planned steps.  
+                - Any skipped, added, or rearranged steps without clear justification count as plan deviations.  
+                - Minor, justified variations are acceptable but reduce the score slightly.
+
+                - General Rules:  
+                - If no discernible plan exists, score ≤ 0.5 regardless of task completion.  
+                - Tool use should be coherent within the plan, not ad hoc or speculative.  
+                - This evaluation excludes correctness or efficiency — focus solely on plan and adherence.
+
+                SCORING GUIDE:
+
+                - **1.0** → Complete, clear, and logical plan **fully followed** with all steps aligned to the user's goal.  
+                - **0.75** → Mostly clear plan with minor omissions or small execution deviations that do not impact the overall strategy.  
+                - **0.5** → Partial plan exists but is incomplete, vague, or only partially followed; notable deviations present.  
+                - **0.25** → Weak or fragmented plan; execution frequently diverges or lacks coherence with any strategy.  
+                - **0.0** → No evidence of a plan; execution appears random or unrelated to the user's task.
+
+                INSTRUCTIONS:
+
+                1. Identify the agent's plan from the steps taken (explicit plans stated or implicit structure).  
+                2. Assess plan completeness and logical order relative to the user's task.  
+                3. Compare execution steps against the plan to check for adherence, noting any unjustified deviations.  
+                4. Deduct points for vagueness, missing critical steps, or inconsistent execution.
+
+                OUTPUT FORMAT:
+
+                Return only a valid JSON object with exactly two fields:
+
+                {{
+                    "score": 0.0,
+                    "reason": "1-3 concise sentences explaining the quality of the plan and how well execution matched it. Specify missing or extra steps, plan clarity, and adherence issues."
+                }}
+
+                EXAMPLE:
+
+                User Task: "Plan a business trip including booking a flight, hotel, and preparing an agenda."
+
+                Agent Steps include:
+                - Outlined flight, hotel, and agenda steps explicitly.
+                - Executed flight and hotel booking steps.
+                - Skipped agenda preparation despite mentioning it in the plan.
+
+                Example JSON:
+
+                {{
+                    "score": 0.75,
+                    "reason": "The agent formed a clear plan covering flights, hotel, and agenda, but failed to execute the agenda preparation step, reducing adherence."
+                }}
+
+                **** END OF EXAMPLE ****
+
+                INPUTS:
+                
+                USER TASK:
+                {task}
+
+                AGENT STEPS:
+                {steps_taken}
+
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def get_final_reason(
+        final_score, threshold, goal_evaluations, plan_evalautions
+    ):
+        return textwrap.dedent(
+            f"""You are an expert evaluator providing a **final justification** for whether an AI agent has passed or failed an evaluation metric.
+
+                You are given:
+                - An agent's goal execution scores and reasons.
+                - The agent's plan evaluation scores and reasons.
+                - The **final combined score**.
+                - The **threshold** required to pass.
+                - Whether the result is a **pass** or **fail**.
+
+                Your job is to write a short, precise explanation of **why** the agent passed or failed — taking into account the quality of execution and planning, and the threshold.
+
+                ---
+
+                INSTRUCTIONS:
+
+                - Write 2-4 clear, objective sentences explaining the overall result.
+                - Explicitly reference both the task and plan performance — **both must be addressed**.
+                - Mention how the final score compares to the threshold.
+                - If the agent **passed**, highlight how both task execution and planning were sufficient to meet the goal.
+                - If the agent **failed**, explain which aspects (task or plan or both) led to the failure.
+                - Avoid vague praise or criticism — ground the reason in the actual scores and justifications.
+
+                ---
+
+                FORMAT:
+                Return only a single string. Do **not** include JSON or any extra formatting.
+
+                ---
+
+                Goal evaluations:
+                {goal_evaluations}
+
+                Plan evaluations:
+                {plan_evalautions}
+
+                Final Score: {final_score}
+                Threshold: {threshold}
+                Result: {"PASS" if final_score >= threshold else "FAIL"}
+
+                Final Reason:
+            """
+        )

deepeval/metrics/hallucination/hallucination.py

@@ -16,15 +16,16 @@ from deepeval.metrics.hallucination.template import HallucinationTemplate
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.hallucination.schema import *
-
-required_params: List[LLMTestCaseParams] = [
-    LLMTestCaseParams.INPUT,
-    LLMTestCaseParams.ACTUAL_OUTPUT,
-    LLMTestCaseParams.CONTEXT,
-]
+from deepeval.metrics.api import metric_data_manager
 
 
 class HallucinationMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.CONTEXT,
+    ]
+
     def __init__(
         self,
         threshold: float = 0.5,
@@ -51,9 +52,10 @@ class HallucinationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, required_params, self)
+        check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -66,6 +68,7 @@ class HallucinationMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -84,6 +87,10 @@ class HallucinationMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -92,9 +99,10 @@ class HallucinationMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, required_params, self)
+        check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -118,7 +126,10 @@ class HallucinationMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self):