deepeval 3.7.5__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/plan_adherence/plan_adherence.py

@@ -3,9 +3,10 @@ from typing import Optional, List, Union, Dict
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams
 from deepeval.metrics import BaseMetric
@@ -30,7 +31,6 @@ class PlanAdherenceMetric(BaseMetric):
     _required_params: List[LLMTestCaseParams] = [
         LLMTestCaseParams.INPUT,
         LLMTestCaseParams.ACTUAL_OUTPUT,
-        LLMTestCaseParams.TOOLS_CALLED,
     ]
 
     def __init__(
@@ -58,9 +58,15 @@ class PlanAdherenceMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -118,9 +124,15 @@ class PlanAdherenceMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
 
@@ -169,58 +181,37 @@ class PlanAdherenceMetric(BaseMetric):
         prompt = PlanAdherenceTemplate.evaluate_adherence(
             task, "\n".join(plan), test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=PlanAdherenceScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = self.model.generate(
-                    prompt, schema=PlanAdherenceScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanAdherenceScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanAdherenceScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanAdherenceScore(**data),
+        )
 
     async def _a_get_plan_adherence_score(self, task, plan, test_case):
         prompt = PlanAdherenceTemplate.evaluate_adherence(
             task, "\n".join(plan), test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=PlanAdherenceScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = await self.model.a_generate(
-                    prompt, schema=PlanAdherenceScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanAdherenceScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanAdherenceScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanAdherenceScore(**data),
+        )
 
     def _extract_plan_from_trace(self, test_case: LLMTestCase) -> AgentPlan:
         prompt = PlanAdherenceTemplate.extract_plan_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=AgentPlan)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = self.model.generate(prompt, schema=AgentPlan)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return AgentPlan(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AgentPlan,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: AgentPlan(**data),
+        )
 
     async def _a_extract_plan_from_trace(
         self, test_case: LLMTestCase
@@ -228,54 +219,37 @@ class PlanAdherenceMetric(BaseMetric):
         prompt = PlanAdherenceTemplate.extract_plan_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=AgentPlan)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = await self.model.a_generate(
-                    prompt, schema=AgentPlan
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return AgentPlan(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AgentPlan,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: AgentPlan(**data),
+        )
 
     def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
         prompt = StepEfficiencyTemplate.extract_task_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Task)
-            self.evaluation_cost += cost
-            return res.task
-        else:
-            try:
-                res: Task = self.model.generate(prompt, schema=Task)
-                return res.task
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Task,
+            extract_schema=lambda s: s.task,
+            extract_json=lambda data: data["task"],
+        )
 
     async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> str:
         prompt = StepEfficiencyTemplate.extract_task_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Task)
-            self.evaluation_cost += cost
-            return res.task
-        else:
-            try:
-                res: Task = await self.model.a_generate(prompt, schema=Task)
-                return res.task
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Task,
+            extract_schema=lambda s: s.task,
+            extract_json=lambda data: data["task"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -283,7 +257,7 @@ class PlanAdherenceMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/plan_adherence/template.py

@@ -4,6 +4,13 @@ from deepeval.tracing.utils import make_json_serializable
 
 
 class PlanAdherenceTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
 
     @staticmethod
     def extract_plan_from_trace(trace: dict) -> str:
@@ -36,6 +43,8 @@ class PlanAdherenceTemplate:
                 - Reproduce the plan steps in **neutral, minimal paraphrasing**.  
                 - Do not interpret motivation, quality, or success of actions.
 
+                {PlanAdherenceTemplate.multimodal_rules}
+
                 OUTPUT FORMAT:
 
                 Return a JSON object with exactly this structure:
@@ -108,6 +117,8 @@ class PlanAdherenceTemplate:
                 - Ignore task success, reasoning quality, or correctness of outcomes. 
                 - Evaluate *only* whether the trace reflects the exact plan execution.
 
+                {PlanAdherenceTemplate.multimodal_rules}
+
 
                 SCORING SCALE
 

deepeval/metrics/plan_quality/plan_quality.py

@@ -3,9 +3,10 @@ from typing import Optional, List, Union, Dict
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams
 from deepeval.metrics import BaseMetric
@@ -33,7 +34,6 @@ class PlanQualityMetric(BaseMetric):
     _required_params: List[LLMTestCaseParams] = [
         LLMTestCaseParams.INPUT,
         LLMTestCaseParams.ACTUAL_OUTPUT,
-        LLMTestCaseParams.TOOLS_CALLED,
     ]
 
     def __init__(
@@ -61,9 +61,15 @@ class PlanQualityMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -121,9 +127,15 @@ class PlanQualityMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
-        has_trace: bool = isinstance(test_case._trace_dict, Dict)
-        if not has_trace:
-            check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
 
@@ -171,56 +183,37 @@ class PlanQualityMetric(BaseMetric):
         prompt = PlanQualityTemplate.evaluate_plan_quality(
             task, "\n".join(plan)
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=PlanQualityScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = self.model.generate(prompt, schema=PlanQualityScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanQualityScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanQualityScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanQualityScore(**data),
+        )
 
     async def _a_get_plan_quality_score(self, task, plan):
         prompt = PlanQualityTemplate.evaluate_plan_quality(
             task, "\n".join(plan)
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=PlanQualityScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = await self.model.a_generate(
-                    prompt, schema=PlanQualityScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return PlanQualityScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=PlanQualityScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: PlanQualityScore(**data),
+        )
 
     def _extract_plan_from_trace(self, test_case: LLMTestCase) -> AgentPlan:
         prompt = PlanAdherenceTemplate.extract_plan_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=AgentPlan)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = self.model.generate(prompt, schema=AgentPlan)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return AgentPlan(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AgentPlan,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: AgentPlan(**data),
+        )
 
     async def _a_extract_plan_from_trace(
         self, test_case: LLMTestCase
@@ -228,54 +221,37 @@ class PlanQualityMetric(BaseMetric):
         prompt = PlanAdherenceTemplate.extract_plan_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=AgentPlan)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: Task = await self.model.a_generate(
-                    prompt, schema=AgentPlan
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return AgentPlan(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=AgentPlan,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: AgentPlan(**data),
+        )
 
     def _extract_task_from_trace(self, test_case: LLMTestCase) -> str:
         prompt = StepEfficiencyTemplate.extract_task_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Task)
-            self.evaluation_cost += cost
-            return res.task
-        else:
-            try:
-                res: Task = self.model.generate(prompt, schema=Task)
-                return res.task
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Task,
+            extract_schema=lambda s: s.task,
+            extract_json=lambda data: data["task"],
+        )
 
     async def _a_extract_task_from_trace(self, test_case: LLMTestCase) -> str:
         prompt = StepEfficiencyTemplate.extract_task_from_trace(
             test_case._trace_dict
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Task)
-            self.evaluation_cost += cost
-            return res.task
-        else:
-            try:
-                res: Task = await self.model.a_generate(prompt, schema=Task)
-                return res.task
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["task"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Task,
+            extract_schema=lambda s: s.task,
+            extract_json=lambda data: data["task"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:
@@ -283,7 +259,7 @@ class PlanQualityMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/plan_quality/template.py

@@ -4,6 +4,13 @@ from deepeval.tracing.utils import make_json_serializable
 
 
 class PlanQualityTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
 
     @staticmethod
     def evaluate_plan_quality(user_task: str, agent_plan: list) -> str:
@@ -46,6 +53,8 @@ class PlanQualityTemplate:
                 - The plan must explicitly and directly target the user's stated goal.  
                 - If any step diverges from the main objective, the score should drop significantly.
 
+                {PlanQualityTemplate.multimodal_rules}
+
                 ---
 
                 SCORING SCALE (STRICT)