deepeval 3.6.7__py3-none-any.whl → 3.6.9__py3-none-any.whl

deepeval/metrics/plan_quality/plan_quality.py

@@ -0,0 +1,292 @@
+from typing import Optional, List, Union, Dict
+
+from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    check_llm_test_case_params,
+    initialize_model,
+)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+from deepeval.metrics import BaseMetric
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.step_efficiency.template import (
+    StepEfficiencyTemplate,
+)
+from deepeval.metrics.step_efficiency.schema import Task
+from deepeval.metrics.plan_quality.schema import (
+    AgentPlan,
+    PlanQualityScore,
+)
+from deepeval.metrics.plan_quality.template import (
+    PlanQualityTemplate,
+)
+from deepeval.metrics.plan_adherence.template import (
+    PlanAdherenceTemplate,
+)
+from deepeval.metrics.api import metric_data_manager
+
+
+class PlanQualityMetric(BaseMetric):
+
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.TOOLS_CALLED,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+        self.requires_trace = True
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        has_trace: bool = isinstance(test_case._trace_dict, Dict)
+        if not has_trace:
+            check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                task = self._extract_task_from_trace(test_case)
+                agent_plan = self._extract_plan_from_trace(test_case)
+                if len(agent_plan.plan) == 0:
+                    self.score = 1
+                    self.reason = "There were no plans to evaluate within the trace of your agent's execution. Please check if the agent's planning or reasoning or thinking is stored in any one of the trace attributes."
+                else:
+                    plan_quality_score = self._get_plan_quality_score(
+                        task, agent_plan.plan
+                    )
+                    self.score = (
+                        0
+                        if self.strict_mode
+                        and plan_quality_score.score < self.threshold
+                        else plan_quality_score.score
+                    )
+                    self.reason = plan_quality_score.reason
+                self.success = self.score >= self.threshold
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Task: {task} \n",
+                        f"Agent Plan: \n{prettify_list(agent_plan.plan)} \n",
+                        f"Final Score Score: {self.score} \n",
+                        f"Final Score Reason: {self.reason} \n",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        has_trace: bool = isinstance(test_case._trace_dict, Dict)
+        if not has_trace:
+            check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            task = await self._a_extract_task_from_trace(test_case)
+            agent_plan = await self._a_extract_plan_from_trace(test_case)
+            if len(agent_plan.plan) == 0:
+                self.score = 1
+                self.reason = "There are no plans to evaluate within the trace of your agent's execution. Please check if the agent's planning or reasoning or thinking is stored in the trace attributes."
+            else:
+                plan_quality_score = await self._a_get_plan_quality_score(
+                    task, agent_plan.plan
+                )
+                self.score = (
+                    0
+                    if self.strict_mode
+                    and plan_quality_score.score < self.threshold
+                    else plan_quality_score.score
+                )
+                self.reason = plan_quality_score.reason
+            self.success = self.score >= self.threshold
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"Task: {task} \n",
+                    f"Agent Plan: \n{prettify_list(agent_plan.plan)} \n",
+                    f"Final Score: {self.score} \n",
+                    f"Final Reason: {self.reason} \n",
+                ],
+            )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    def _get_plan_quality_score(self, task, plan):
+        prompt = PlanQualityTemplate.evaluate_plan_quality(
+            task, "\n".join(plan)
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=PlanQualityScore)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = self.model.generate(prompt, schema=PlanQualityScore)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return PlanQualityScore(**data)
+
+    async def _a_get_plan_quality_score(self, task, plan):
+        prompt = PlanQualityTemplate.evaluate_plan_quality(
+            task, "\n".join(plan)
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=PlanQualityScore
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = await self.model.a_generate(
+                    prompt, schema=PlanQualityScore
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return PlanQualityScore(**data)
+
+    def _extract_plan_from_trace(self, test_case: LLMTestCase) -> AgentPlan:
+        prompt = PlanAdherenceTemplate.extract_plan_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=AgentPlan)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = self.model.generate(prompt, schema=AgentPlan)
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return AgentPlan(**data)
+
+    async def _a_extract_plan_from_trace(
+        self, test_case: LLMTestCase
+    ) -> AgentPlan:
+        prompt = PlanAdherenceTemplate.extract_plan_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=AgentPlan)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = await self.model.a_generate(
+                    prompt, schema=AgentPlan
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return AgentPlan(**data)
+
+    def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
+        prompt = StepEfficiencyTemplate.extract_task_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=Task)
+            self.evaluation_cost += cost
+            return res.task
+        else:
+            try:
+                res: Task = self.model.generate(prompt, schema=Task)
+                return res.task
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["task"]
+
+    async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> str:
+        prompt = StepEfficiencyTemplate.extract_task_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=Task)
+            self.evaluation_cost += cost
+            return res.task
+        else:
+            try:
+                res: Task = await self.model.a_generate(prompt, schema=Task)
+                return res.task
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["task"]
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Plan Quality"

deepeval/metrics/plan_quality/schema.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import List, Dict, Literal
+
+
+class AgentPlan(BaseModel):
+    plan: List[str]
+
+
+class PlanQualityScore(BaseModel):
+    score: float
+    reason: str

deepeval/metrics/plan_quality/template.py

@@ -0,0 +1,101 @@
+import textwrap
+import json
+from deepeval.tracing.utils import make_json_serializable
+
+
+class PlanQualityTemplate:
+
+    @staticmethod
+    def evaluate_plan_quality(user_task: str, agent_plan: list) -> str:
+        return textwrap.dedent(
+            f"""You are a **plan quality evaluator**. Your task is to critically assess the **quality, completeness, and optimality** of an AI agent's plan to accomplish the given user task.
+
+                INPUTS:
+
+                - **User Task:** The user's explicit goal or instruction.
+                - **Agent Plan:** The ordered list of steps the agent intends to follow to achieve that goal.
+
+                EVALUATION OBJECTIVE:
+
+                Judge the **intrinsic quality** of the plan — whether the plan itself is strong enough to fully and efficiently achieve the user's task.
+
+                The evaluation must be **strict**.  If the plan is incomplete, inefficient, redundant, or missing critical details, assign a very low score.
+
+                STRICT EVALUATION CRITERIA:
+
+                1. Completeness (Most Important)
+                - The plan must fully address all major requirements of the user task.  
+                - Missing even one critical subtask or dependency should reduce the score sharply.
+                - The plan must include all prerequisite actions necessary for the final outcome.
+
+                2. Logical Coherence
+                - Steps must follow a clear, rational sequence that leads directly to completing the task.  
+                - Disordered, redundant, or circular reasoning should be penalized heavily.
+                - Every step must have a clear purpose; no filler or irrelevant actions.
+
+                3. Optimality and Efficiency
+                - The plan must be **minimal but sufficient** — no unnecessary or repetitive steps.
+                - If a more direct, simpler, or logically superior plan could achieve the same outcome, the current plan should receive a lower score.
+
+                4. Level of Detail
+                - Each step should be specific enough for an agent to execute it reliably without ambiguity.  
+                - Vague steps (e.g., “Do research”, “Handle results”) that lack operational clarity 
+                    lower the score.
+
+                5. Alignment with Task
+                - The plan must explicitly and directly target the user's stated goal.  
+                - If any step diverges from the main objective, the score should drop significantly.
+
+                ---
+
+                SCORING SCALE (STRICT)
+
+                - **1.0 — Excellent plan**
+                - Fully complete, logically ordered, and optimally efficient.  
+                - No missing, redundant, or ambiguous steps.  
+                - Directly fulfills every aspect of the user task.
+
+                - **0.75 — Good plan**
+                - Covers nearly all aspects of the task with clear logic.  
+                - Minor gaps or small inefficiencies that do not block task completion.
+
+                - **0.5 — Adequate but flawed plan**
+                - Partially complete; key details missing or step order inefficient.  
+                - Some ambiguity or redundancy that would likely affect execution success.
+
+                - **0.25 — Weak plan**
+                - Major missing steps or unclear logic.  
+                - The plan would likely fail to complete the task as written.
+
+                - **0.0 — Inadequate plan**
+                - Irrelevant, incoherent, or severely incomplete plan.  
+                - Does not align with the user’s task or cannot plausibly achieve it.
+
+                *When in doubt, assign the lower score.*
+
+                OUTPUT FORMAT:
+
+                Return a JSON object with this exact structure:
+
+                {{
+                    "score": 0.0,
+                    "reason": "1-3 short, precise sentences explaining what the plan lacks or how it could fail."
+                }}
+
+                The `"reason"` must:
+                - Reference specific missing, unclear, or inefficient steps.
+                - Avoid vague language (“seems fine”, “mostly works”).
+                - Use objective terms describing gaps or weaknesses.
+
+                PROVIDED DATA
+
+                User Task:
+                {user_task}
+
+                Agent Plan:
+                {agent_plan}
+
+
+                JSON:
+            """
+        )

deepeval/metrics/step_efficiency/__init__.py

@@ -0,0 +1 @@
+from .step_efficiency import StepEfficiencyMetric

deepeval/metrics/step_efficiency/schema.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel
+from typing import List, Dict, Literal
+
+
+class Task(BaseModel):
+    task: str
+
+
+class EfficiencyVerdict(BaseModel):
+    score: float
+    reason: str

deepeval/metrics/step_efficiency/step_efficiency.py

@@ -0,0 +1,234 @@
+from typing import Optional, List, Union, Dict
+
+from deepeval.utils import get_or_create_event_loop
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    check_llm_test_case_params,
+    initialize_model,
+)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+from deepeval.metrics import BaseMetric
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.step_efficiency.template import (
+    StepEfficiencyTemplate,
+)
+from deepeval.metrics.step_efficiency.schema import Task, EfficiencyVerdict
+from deepeval.metrics.api import metric_data_manager
+
+
+class StepEfficiencyMetric(BaseMetric):
+
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+        LLMTestCaseParams.TOOLS_CALLED,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+        self.requires_trace = True
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        has_trace: bool = isinstance(test_case._trace_dict, Dict)
+        if not has_trace:
+            check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                task = self._extract_task_from_trace(test_case)
+                efficiency_verdict = self._get_score(task, test_case)
+                self.score = (
+                    0
+                    if self.strict_mode
+                    and efficiency_verdict.score < self.threshold
+                    else efficiency_verdict.score
+                )
+                self.reason = efficiency_verdict.reason
+                self.success = self.score >= self.threshold
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Task: {task} \n",
+                        f"Efficiency Score: {self.score}",
+                        f"Efficiency Reason: {self.reason}",
+                    ],
+                )
+
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+                return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        has_trace: bool = isinstance(test_case._trace_dict, Dict)
+        if not has_trace:
+            check_llm_test_case_params(test_case, self._required_params, self)
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            task = await self._a_extract_task_from_trace(test_case)
+            efficiency_verdict = await self._a_get_score(task, test_case)
+            self.score = (
+                0
+                if self.strict_mode
+                and efficiency_verdict.score < self.threshold
+                else efficiency_verdict.score
+            )
+            self.reason = efficiency_verdict.reason
+            self.success = self.score >= self.threshold
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    f"Task: {task} \n",
+                    f"Efficiency Score: {self.score}",
+                    f"Efficiency Reason: {self.reason}",
+                ],
+            )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    def _get_score(self, task: str, test_case: LLMTestCase):
+        if test_case._trace_dict is not None:
+            prompt = StepEfficiencyTemplate.get_execution_efficiency(
+                task, test_case._trace_dict
+            )
+
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=EfficiencyVerdict)
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = self.model.generate(
+                    prompt, schema=EfficiencyVerdict
+                )
+                return res
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return EfficiencyVerdict(**data)
+
+    async def _a_get_score(self, task: str, test_case: LLMTestCase):
+        if test_case._trace_dict is not None:
+            prompt = StepEfficiencyTemplate.get_execution_efficiency(
+                task, test_case._trace_dict
+            )
+
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=EfficiencyVerdict
+            )
+            self.evaluation_cost += cost
+            return res
+        else:
+            try:
+                res: Task = await self.model.a_generate(
+                    prompt, schema=EfficiencyVerdict
+                )
+                return res
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return EfficiencyVerdict(**data)
+
+    def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
+        prompt = StepEfficiencyTemplate.extract_task_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=Task)
+            self.evaluation_cost += cost
+            return res.task
+        else:
+            try:
+                res: Task = self.model.generate(prompt, schema=Task)
+                return res.task
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["task"]
+
+    async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> str:
+        prompt = StepEfficiencyTemplate.extract_task_from_trace(
+            test_case._trace_dict
+        )
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=Task)
+            self.evaluation_cost += cost
+            return res.task
+        else:
+            try:
+                res: Task = await self.model.a_generate(prompt, schema=Task)
+                return res.task
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["task"]
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Execution Efficiency"