deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/mcp/multi_turn_mcp_use_metric.py

@@ -7,8 +7,9 @@ from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
     get_unit_interactions,
-    trimAndLoadJson,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import ConversationalTestCase, TurnParams
@@ -50,7 +51,12 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -120,7 +126,12 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         _log_metric_to_confident: bool = True,
     ):
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -185,18 +196,13 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         prompt = MCPTaskCompletionTemplate.get_tool_correctness_score(
             task, test_case.mcp_servers
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ToolScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ToolScore = self.model.generate(prompt, schema=ToolScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolScore(**data),
+        )
 
     async def _a_get_tool_accuracy_score(
         self, task: Task, test_case: ConversationalTestCase
@@ -204,20 +210,13 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         prompt = MCPTaskCompletionTemplate.get_tool_correctness_score(
             task, test_case.mcp_servers
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=ToolScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ToolScore = await self.model.a_generate(
-                    prompt, schema=ToolScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ToolScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ToolScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ToolScore(**data),
+        )
 
     def _get_args_score(
         self, task: Task, test_case: ConversationalTestCase
@@ -225,18 +224,13 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         prompt = MCPTaskCompletionTemplate.get_args_correctness_score(
             task, test_case.mcp_servers
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=ArgsScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ArgsScore = self.model.generate(prompt, schema=ArgsScore)
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ArgsScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgsScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ArgsScore(**data),
+        )
 
     async def _a_get_args_score(
         self, task: Task, test_case: ConversationalTestCase
@@ -244,20 +238,13 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         prompt = MCPTaskCompletionTemplate.get_args_correctness_score(
             task, test_case.mcp_servers
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=ArgsScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ArgsScore = await self.model.a_generate(
-                    prompt, schema=ArgsScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ArgsScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgsScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: ArgsScore(**data),
+        )
 
     def _get_tasks(self, unit_interactions: List) -> List[Task]:
         tasks = []
@@ -334,32 +321,63 @@ class MultiTurnMCPUseMetric(BaseConversationalMetric):
         self,
         tool_accuracy_score: List[ToolScore],
         args_accuracy_score: List[ArgsScore],
-    ) -> str:
-        reason = "["
+    ) -> Optional[str]:
+        if not self.include_reason:
+            return None
+
+        reasons = []
         for task_score in tool_accuracy_score:
-            if task_score.score < self.threshold:
-                reason += "\nPrimitives Used\n"
-                reason += (
-                    f"Score: {task_score.score}\n"
-                    f"Reason: {task_score.reason}\n"
-                )
-        for task_score in args_accuracy_score:
-            if task_score.score < self.threshold:
-                reason += "\nArguments Generated\n"
-                reason += (
-                    f"Score: {task_score.score}\n"
-                    f"Reason: {task_score.reason}\n"
-                )
-        reason += "]"
-        return reason
+            reasons.append(task_score.reason)
+
+        for arg_score in args_accuracy_score:
+            reasons.append(arg_score.reason)
+
+        prompt = MCPTaskCompletionTemplate.generate_final_reason(
+            self.score, self.success, reasons
+        )
+
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = self.model.generate(prompt)
+            return res
+
+    async def _a_generate_reason(
+        self,
+        tool_accuracy_score: List[ToolScore],
+        args_accuracy_score: List[ArgsScore],
+    ) -> Optional[str]:
+        if not self.include_reason:
+            return None
+
+        reasons = []
+        for task_score in tool_accuracy_score:
+            reasons.append(task_score.reason)
+
+        for arg_score in args_accuracy_score:
+            reasons.append(arg_score.reason)
+
+        prompt = MCPTaskCompletionTemplate.generate_final_reason(
+            self.score, self.success, reasons
+        )
+
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = await self.model.a_generate(prompt)
+            return res
 
     def is_successful(self) -> bool:
         if self.error is not None:
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/mcp/template.py

@@ -4,6 +4,14 @@ from deepeval.test_case import MCPServer
 
 
 class MCPTaskCompletionTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def get_args_correctness_score(task: Task, mcp_servers: List[MCPServer]):
         available_tools = [data.available_tools for data in mcp_servers]
@@ -12,6 +20,8 @@ class MCPTaskCompletionTemplate:
         steps_taken = "\n".join(task.steps_taken)
         return f"""Evaluate whether the arguments (inputs) provided by the agent to the tools, resources, and prompts were correct and aligned with their respective input schemas. Your job is to determine if the agent supplied appropriate, complete, and well-formatted arguments for each invocation.
 
+{MCPTaskCompletionTemplate.multimodal_rules}
+
 Output a JSON object with exactly two fields: 'score' and 'reason'.
 
 Scoring:
@@ -55,6 +65,8 @@ JSON:
         steps_taken = "\n".join(task.steps_taken)
         return f"""Evaluate whether the tools, resources, and prompts used by the agent were appropriate and optimal, based strictly on the list of available tools and resources provided. Your job is to determine whether the agent selected the most suitable tools and prompts for the task at hand. Output a JSON object with exactly two fields: 'score' and 'reason'.
 
+{MCPTaskCompletionTemplate.multimodal_rules}
+
 Scoring:
 - 'score' is a float between 0 and 1 inclusive.
 - Use intermediate values (e.g., 0.25, 0.5, 0.75) to reflect partially appropriate tool use, suboptimal decisions, or missed better alternatives.
@@ -92,6 +104,9 @@ JSON:
     def get_task_completion_score(task: Task):
         steps_taken = "\n".join(task.steps_taken)
         return f"""Evaluate whether the user's task has been successfully completed by the agent, based strictly on what the user can see in the agent's responses. You must return a JSON object with exactly two fields: 'score' and 'reason'.
+
+{MCPTaskCompletionTemplate.multimodal_rules}
+
 Scoring:
 - 'score' is a float between 0 and 1 inclusive.
 - Use intermediate values (e.g., 0.25, 0.5, 0.75) to reflect partial task success or missing/inaccurate information.
@@ -123,3 +138,40 @@ Example Output:
 
 JSON:
 """
+
+    @staticmethod
+    def generate_final_reason(
+        final_score: float, success: bool, reasons: List[str]
+    ):
+        return f"""You are an AI evaluator producing a single final explanation for the an MCP application's evaluation results using the provided reasons.
+
+        Context:
+        The reasons are from metrics that were used to evaluate an MCP application by determining whether the model accurately completed a task or called toos and resources with the right arguments.
+
+        Inputs:
+        - final_score: the averaged score across all interactions.
+        - success: whether the metric passed or failed
+        - reasons: a list of textual reasons generated from individual interactions.
+
+        Instructions:
+        1. Read all reasons and synthesize them into one unified explanation.
+        2. Do not repeat every reason; merge them into a concise, coherent narrative.
+        4. If the metric failed, state the dominant failure reasons. If it passed, state why the application has passed.
+        5. Output a single paragraph with no lists, no bullets, no markup.
+
+        Output:
+        A single paragraph explaining the final outcome.
+
+        Here's the inputs:
+
+        Final Score: {final_score}
+        
+        Reasons: 
+        {reasons}
+
+        Success: {success}
+
+        Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
+
+        The final reason:
+        """

deepeval/metrics/mcp_use_metric/mcp_use_metric.py

@@ -3,9 +3,10 @@ from typing import Optional, List, Union
 from deepeval.utils import get_or_create_event_loop
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -54,7 +55,16 @@ class MCPUseMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -121,11 +131,23 @@ class MCPUseMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+        multimodal = test_case.multimodal
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             available_primitives, primitives_used = (
                 self._get_mcp_interaction_text(
@@ -177,20 +199,13 @@ class MCPUseMetric(BaseMetric):
         prompt = MCPUseMetricTemplate.get_primitive_correctness_prompt(
             test_case, available_primitives, primitives_used
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=MCPPrimitivesScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: MCPPrimitivesScore = self.model.generate(
-                    prompt, schema=MCPPrimitivesScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return MCPPrimitivesScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=MCPPrimitivesScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: MCPPrimitivesScore(**data),
+        )
 
     async def _a_get_primitives_used_score(
         self,
@@ -201,22 +216,13 @@ class MCPUseMetric(BaseMetric):
         prompt = MCPUseMetricTemplate.get_primitive_correctness_prompt(
             test_case, available_primitives, primitives_used
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=MCPPrimitivesScore
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: MCPPrimitivesScore = await self.model.a_generate(
-                    prompt, schema=MCPPrimitivesScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return MCPPrimitivesScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=MCPPrimitivesScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: MCPPrimitivesScore(**data),
+        )
 
     def _get_argument_correctness_score(
         self,
@@ -227,20 +233,13 @@ class MCPUseMetric(BaseMetric):
         prompt = MCPUseMetricTemplate.get_mcp_argument_correctness_prompt(
             test_case, available_primitives, primitives_used
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=MCPArgsScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: MCPArgsScore = self.model.generate(
-                    prompt, schema=MCPArgsScore
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return MCPArgsScore(**data)
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=MCPArgsScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: MCPArgsScore(**data),
+        )
 
     async def _a_get_argument_correctness_score(
         self,
@@ -251,20 +250,13 @@ class MCPUseMetric(BaseMetric):
         prompt = MCPUseMetricTemplate.get_mcp_argument_correctness_prompt(
             test_case, available_primitives, primitives_used
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=MCPArgsScore)
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: MCPArgsScore = await self.model.a_generate(
-                    prompt, schema=MCPArgsScore
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return MCPArgsScore(**data)
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=MCPArgsScore,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: MCPArgsScore(**data),
+        )
 
     def _calculate_score(
         self,
@@ -280,7 +272,9 @@ class MCPUseMetric(BaseMetric):
         self,
         primitives_used_score: MCPPrimitivesScore,
         argument_correctness_score: MCPArgsScore,
-    ) -> str:
+    ) -> Optional[str]:
+        if not self.include_reason:
+            return None
         return (
             f"[\n"
             f"\t{primitives_used_score.reason}\n"
@@ -390,7 +384,7 @@ class MCPUseMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/mcp_use_metric/template.py

@@ -3,6 +3,14 @@ import textwrap
 
 
 class MCPUseMetricTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def get_mcp_argument_correctness_prompt(
         test_case: LLMTestCase,
@@ -12,6 +20,8 @@ class MCPUseMetricTemplate:
         return textwrap.dedent(
             f"""Evaluate whether the arguments passed to each tool (primitive) used by the agent were appropriate and correct for the intended purpose. Focus on whether the input types, formats, and contents match the expectations of the tools and are suitable given the user's request.
 
+            {MCPUseMetricTemplate.multimodal_rules}
+
             You must return a JSON object with exactly two fields: 'score' and 'reason'.
 
             Scoring:
@@ -68,6 +78,8 @@ class MCPUseMetricTemplate:
         return textwrap.dedent(
             f"""Evaluate whether the tools (primitives) selected and used by the agent were appropriate and correct for fulfilling the user’s request. Base your judgment on the user input, the agent’s visible output, and the tools that were available to the agent. You must return a JSON object with exactly two fields: 'score' and 'reason'.
 
+            {MCPUseMetricTemplate.multimodal_rules}
+
             Scoring:
             - 'score' is a float between 0 and 1 inclusive.
             - Use intermediate values (e.g., 0.25, 0.5, 0.75) to reflect cases where the tools used were partially correct, suboptimal, or only somewhat relevant.