deepeval 3.6.7__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/metrics/plan_adherence/plan_adherence.py

@@ -0,0 +1,292 @@
+from typing import Optional, List, Union, Dict
+
+from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    check_llm_test_case_params,
+    initialize_model,
+)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+from deepeval.metrics import BaseMetric
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.step_efficiency.template import (
+    StepEfficiencyTemplate,
+)
+from deepeval.metrics.step_efficiency.schema import Task
+from deepeval.metrics.plan_adherence.schema import (
+    AgentPlan,
+    PlanAdherenceScore,
+)
+from deepeval.metrics.plan_adherence.template import (
+    PlanAdherenceTemplate,
+)
+from deepeval.metrics.api import metric_data_manager
+
+
+class PlanAdherenceMetric(BaseMetric):
+
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.TOOLS_CALLED,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+        self.requires_trace = True
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        has_trace: bool = isinstance(test_case._trace_dict, Dict)
+        if not has_trace:
+            check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                task = self._extract_task_from_trace(test_case)
+                agent_plan = self._extract_plan_from_trace(test_case)
+                if len(agent_plan.plan) == 0:
+                    self.score = 1
+                    self.reason = "There were no plans to evaluate within the trace of your agent's execution. Please check if the agent's planning or reasoning or thinking is stored in any one of the trace attributes."
+                else:
+                    plan_adherence_score = self._get_plan_adherence_score(
+                        task, agent_plan.plan, test_case
+                    )
+                    self.score = (
+                        0
+                        if self.strict_mode
+                        and plan_adherence_score.score < self.threshold
+                        else plan_adherence_score.score
+                    )
+                    self.reason = plan_adherence_score.reason
+                self.success = self.score >= self.threshold
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Task: {task} \n",
+                        f"Agent Plan: \n{prettify_list(agent_plan.plan)} \n",
+                        f"Final Score: {self.score} \n",
+                        f"Final Reason: {self.reason} \n",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        has_trace: bool = isinstance(test_case._trace_dict, Dict)
+        if not has_trace:
+            check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            task = await self._a_extract_task_from_trace(test_case)
+            agent_plan = await self._a_extract_plan_from_trace(test_case)
+            if len(agent_plan.plan) == 0:
+                self.score = 1
+                self.reason = "There were no plans to evaluate within the trace of your agent's execution. Please check if the agent's planning or reasoning or thinking is stored in any one of the trace attributes."
+            else:
+                plan_adherence_score = await self._a_get_plan_adherence_score(
+                    task, agent_plan.plan, test_case
+                )
+                self.score = (
+                    0
+                    if self.strict_mode
+                    and plan_adherence_score.score < self.threshold
+                    else plan_adherence_score.score
+                )
+                self.reason = plan_adherence_score.reason
+            self.success = self.score >= self.threshold
+
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"Task: {task} \n",
+                    f"Agent Plan: \n{prettify_list(agent_plan.plan)} \n",
+                    f"Final Score: {self.score} \n",
+                    f"Final Reason: {self.reason} \n",
+                ],
+            )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    def _get_plan_adherence_score(self, task, plan, test_case):
+        prompt = PlanAdherenceTemplate.evaluate_adherence(
+            task, "\n".join(plan), test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=PlanAdherenceScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = self.model.generate(
+                    prompt, schema=PlanAdherenceScore
+                )
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return PlanAdherenceScore(**data)
+
+    async def _a_get_plan_adherence_score(self, task, plan, test_case):
+        prompt = PlanAdherenceTemplate.evaluate_adherence(
+            task, "\n".join(plan), test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=PlanAdherenceScore
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = await self.model.a_generate(
+                    prompt, schema=PlanAdherenceScore
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return PlanAdherenceScore(**data)
+
+    def _extract_plan_from_trace(self, test_case: LLMTestCase) -> AgentPlan:
+        prompt = PlanAdherenceTemplate.extract_plan_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=AgentPlan)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = self.model.generate(prompt, schema=AgentPlan)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return AgentPlan(**data)
+
+    async def _a_extract_plan_from_trace(
+        self, test_case: LLMTestCase
+    ) -> AgentPlan:
+        prompt = PlanAdherenceTemplate.extract_plan_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=AgentPlan)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = await self.model.a_generate(
+                    prompt, schema=AgentPlan
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return AgentPlan(**data)
+
+    def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
+        prompt = StepEfficiencyTemplate.extract_task_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=Task)
+            self.evaluation_cost += cost
+            return res.task
+        else:
+            try:
+                res: Task = self.model.generate(prompt, schema=Task)
+                return res.task
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["task"]
+
+    async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> str:
+        prompt = StepEfficiencyTemplate.extract_task_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=Task)
+            self.evaluation_cost += cost
+            return res.task
+        else:
+            try:
+                res: Task = await self.model.a_generate(prompt, schema=Task)
+                return res.task
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["task"]
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Plan Adherence"

deepeval/metrics/plan_adherence/schema.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import List, Dict, Literal
+
+
+class AgentPlan(BaseModel):
+    plan: List[str]
+
+
+class PlanAdherenceScore(BaseModel):
+    score: float
+    reason: str

deepeval/metrics/plan_adherence/template.py

@@ -0,0 +1,170 @@
+import textwrap
+import json
+from deepeval.tracing.utils import make_json_serializable
+
+
+class PlanAdherenceTemplate:
+
+    @staticmethod
+    def extract_plan_from_trace(trace: dict) -> str:
+        return textwrap.dedent(
+            f"""You are a **systems analyst** evaluating an AI agent's execution trace.
+
+                Your sole task is to extract the **explicit or clearly implied plan** the agent followed or intended to follow — *only if that plan is directly evidenced in the trace*.
+
+                STRICT RULES TO FOLLOW:
+
+                1. Source Evidence Requirement
+                - Every plan step you include **must** be directly supported by explicit text in the trace.  
+                - Acceptable evidence sources:
+                    - `"reasoning"` or `"thought"` fields inside tool calls or function invocations.
+                    - Explicit plan-like statements or lists written by the agent (e.g., “My plan is to…”).
+                - If no evidence exists for a step, DO NOT infer or invent it.
+
+                2. No Hallucination Policy 
+                - You must *not* create or rephrase steps that aren't explicitly or strongly implied by the trace.  
+                - If there is no coherent plan present, output an empty list.
+
+                3. Focus on Intent, Not Outcomes
+                - If the agent's plan is stated but execution differs, still extract the intended steps — but only if those intended steps are traceable.
+
+                4. Granularity
+                - Each step should represent a single distinct action or intention.
+                - Avoid merging multiple intentions into one step or splitting one intention into multiple steps.
+
+                5. Neutral Language
+                - Reproduce the plan steps in **neutral, minimal paraphrasing**.  
+                - Do not interpret motivation, quality, or success of actions.
+
+                OUTPUT FORMAT:
+
+                Return a JSON object with exactly this structure:
+                {{
+                    "plan": [
+                        "step 1",
+                        "step 2",
+                        ...
+                    ]
+                }}
+
+                If no plan is evidenced in the trace, return:
+                {{
+                    "plan": []
+                }}
+
+                Do not include commentary, confidence scores, or explanations.
+
+                TRACE:
+
+                {json.dumps(trace, indent=2, default=str)}
+
+                JSON:
+            """
+        )
+
+    @staticmethod
+    def evaluate_adherence(
+        user_task: str, agent_plan: str, execution_trace: dict
+    ) -> str:
+        return textwrap.dedent(
+            f"""You are an **adversarial plan adherence evaluator**. Your goal is to assign the **lowest justifiable score** based on how strictly the agent's actions in the execution trace align with its declared plan.
+
+                INPUTS:
+
+                - **User Task:** The original request or objective.
+                - **Agent Plan:** The explicit step-by-step plan the agent was supposed to follow.
+                - **Execution Trace:** A detailed record of all agent actions, reasoning, tool calls, and outputs.
+
+                EVALUATION OBJECTIVE:
+
+                Determine whether the agent **exactly and exclusively** followed its plan.  
+                You are not evaluating success, correctness, or usefulness — **only plan obedience**.
+
+                Assume **non-adherence by default** unless clear, direct evidence in the trace proves that 
+                each planned step was executed *as written* and *no additional actions occurred*.
+
+                ### STRICT ADHERENCE RULES
+
+                1. Step Verification
+                - Every step in the plan must correspond to a **verifiable, explicit** action or reasoning entry in the trace.
+                - Each step must appear in the same logical order as the plan.
+                - If a step is missing, only implied, or ambiguous, treat as **not followed**.
+
+                2. No Extraneous Actions
+                - If the trace includes **any** major action, tool call, or reasoning segment not clearly present in the plan, immediately lower the score to as low as possible.
+                - Extra or unnecessary steps are considered serious violations.
+
+                3. Order Consistency
+                - If the agent performed steps in a different order than the plan specifies, the score must be close to 0, regardless of other alignment.
+
+                4. Completeness
+                - If even one planned step is missing, skipped, or only partially reflected in the trace, the score must be lowest possible.
+
+                5. Ambiguity Handling
+                - If it is unclear whether a trace action corresponds to a plan step, treat that step as **not executed**.
+                - When uncertain, assign the **lower score**.
+
+                6. Focus Exclusively on Plan Compliance
+                - Ignore task success, reasoning quality, or correctness of outcomes. 
+                - Evaluate *only* whether the trace reflects the exact plan execution.
+
+
+                SCORING SCALE
+
+                - **1.0 — Perfect adherence**
+                - Every planned step is explicitly and verifiably present in the trace, in correct order.
+                - No skipped or added steps.
+                - No ambiguity in matching.
+
+                - **0.75 — Strong adherence**
+                - All or nearly all steps are executed in order.
+                - At most one minor deviation (e.g., a trivial reordering or minor redundant step) that does not change the plan’s structure.
+
+                - **0.5 — Partial adherence**
+                - Some steps clearly match, but others are missing, out of order, or replaced.
+                - At least one extra or ambiguous action appears.
+                - *This should be the highest score possible when there are any deviations.*
+
+                - **0.25 — Weak adherence**
+                - Only a few steps from the plan appear in the trace, or multiple extraneous actions occur.
+                - The structure or sequence of the plan is mostly lost.
+
+                - **0.0 — No adherence**
+                - The trace shows little or no resemblance to the plan.
+                - Steps are ignored, replaced, or executed in an entirely different order.
+
+                Always err toward the **lower score** when evidence is partial, ambiguous, or contradictory.
+
+                OUTPUT FORMAT:
+
+                Return a JSON object with exactly this structure:
+
+                {{
+                    "score": 0.0,
+                    "reason": "1-3 concise, factual sentences citing specific matched, missing, or extra steps."
+                }}
+
+                Requirements for `"reason"`:
+                - Reference specific plan step numbers or phrases.
+                - Mention concrete trace evidence of mismatches or additions.
+                - Avoid subjective adjectives (e.g., “mostly”, “close”, “reasonable”).
+                - Be precise and neutral.
+
+                ---
+
+                INPUTS:
+
+                User Task:
+                {user_task}
+
+                Agent Plan:
+                {agent_plan}
+
+                Execution Trace:
+                {json.dumps(execution_trace, indent=2, default=make_json_serializable)}
+
+                ---
+
+                JSON:
+            """
+        )

deepeval/metrics/plan_quality/__init__.py

@@ -0,0 +1 @@
+from .plan_quality import PlanQualityMetric