deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/arena_g_eval/arena_g_eval.py

@@ -14,12 +14,17 @@ from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     check_arena_test_case_params,
     construct_verbose_logs,
-    trimAndLoadJson,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.arena_g_eval.schema import *
+from deepeval.metrics.arena_g_eval.schema import (
+    RewrittenReason,
+    Winner,
+    Steps,
+)
 from deepeval.metrics.g_eval.utils import (
     construct_g_eval_params_string,
     validate_criteria_and_evaluation_steps,
@@ -62,7 +67,13 @@ class ArenaGEval(BaseArenaMetric):
         _progress: Optional[Progress] = None,
         _pbar_id: Optional[int] = None,
     ) -> str:
-        check_arena_test_case_params(test_case, self.evaluation_params, self)
+        check_arena_test_case_params(
+            test_case,
+            self.evaluation_params,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
         self.evaluation_cost = 0 if self.using_native_model else None
 
         with metric_progress_indicator(self, _show_indicator=_show_indicator):
@@ -76,12 +87,12 @@ class ArenaGEval(BaseArenaMetric):
                 )
             else:
                 self.evaluation_steps: List[str] = (
-                    self._generate_evaluation_steps()
+                    self._generate_evaluation_steps(test_case.multimodal)
                 )
                 if _progress:
                     update_pbar(_progress, _pbar_id)
                 masked_winner, masked_reason, dummy_to_real_names = (
-                    self._compare(test_case)
+                    self._compare(test_case, test_case.multimodal)
                 )
                 if _progress:
                     update_pbar(_progress, _pbar_id)
@@ -111,7 +122,13 @@ class ArenaGEval(BaseArenaMetric):
         _progress: Optional[Progress] = None,
         _pbar_id: Optional[int] = None,
     ) -> str:
-        check_arena_test_case_params(test_case, self.evaluation_params, self)
+        check_arena_test_case_params(
+            test_case,
+            self.evaluation_params,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
         self.evaluation_cost = 0 if self.using_native_model else None
 
         with metric_progress_indicator(
@@ -120,12 +137,12 @@ class ArenaGEval(BaseArenaMetric):
             _show_indicator=_show_indicator,
         ):
             self.evaluation_steps: List[str] = (
-                await self._a_generate_evaluation_steps()
+                await self._a_generate_evaluation_steps(test_case.multimodal)
             )
             if _progress:
                 update_pbar(_progress, _pbar_id)
             masked_winner, masked_reason, dummy_to_real_names = (
-                await self._a_compare(test_case)
+                await self._a_compare(test_case, test_case.multimodal)
             )
             if _progress:
                 update_pbar(_progress, _pbar_id)
@@ -147,7 +164,7 @@ class ArenaGEval(BaseArenaMetric):
             )
             return self.winner
 
-    async def _a_generate_evaluation_steps(self) -> List[str]:
+    async def _a_generate_evaluation_steps(self, multimodal: bool) -> List[str]:
         if self.evaluation_steps:
             return self.evaluation_steps
 
@@ -155,23 +172,20 @@ class ArenaGEval(BaseArenaMetric):
             self.evaluation_params
         )
         prompt = ArenaGEvalTemplate.generate_evaluation_steps(
-            criteria=self.criteria, parameters=g_eval_params_str
+            criteria=self.criteria,
+            parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            self,
+            prompt,
+            Steps,
+            extract_schema=lambda s: s.steps,
+            extract_json=lambda data: data["steps"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["steps"]
-        else:
-            try:
-                res: Steps = await self.model.a_generate(prompt, schema=Steps)
-                return res.steps
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["steps"]
 
-    def _generate_evaluation_steps(self) -> List[str]:
+    def _generate_evaluation_steps(self, multimodal: bool) -> List[str]:
         if self.evaluation_steps:
             return self.evaluation_steps
 
@@ -179,25 +193,20 @@ class ArenaGEval(BaseArenaMetric):
             self.evaluation_params
         )
         prompt = ArenaGEvalTemplate.generate_evaluation_steps(
-            criteria=self.criteria, parameters=g_eval_params_str
+            criteria=self.criteria,
+            parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            self,
+            prompt,
+            Steps,
+            extract_schema=lambda s: s.steps,
+            extract_json=lambda data: data["steps"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["steps"]
-        else:
-            try:
-                res: Steps = self.model.generate(prompt, schema=Steps)
-                return res.steps
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["steps"]
 
     async def _a_compare(
-        self,
-        test_case: ArenaTestCase,
+        self, test_case: ArenaTestCase, multimodal: bool
     ) -> Tuple[str, str, Dict[str, str]]:
         formatted_test_case, dummy_to_real_names = format_arena_test_case(
             self.evaluation_params, test_case
@@ -209,23 +218,27 @@ class ArenaGEval(BaseArenaMetric):
             evaluation_steps=number_evaluation_steps(self.evaluation_steps),
             test_case_contents=formatted_test_case,
             parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            self,
+            prompt,
+            Winner,
+            extract_schema=lambda s: (
+                s.winner,
+                s.reason,
+                dummy_to_real_names,
+            ),
+            extract_json=lambda data: (
+                data["winner"],
+                data["reason"],
+                dummy_to_real_names,
+            ),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Winner)
-            self.evaluation_cost += cost
-            return res.winner, res.reason, dummy_to_real_names
-        else:
-            try:
-                res: Winner = await self.model.a_generate(prompt, schema=Winner)
-                return res.winner, res.reason, dummy_to_real_names
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["winner"], data["reason"], dummy_to_real_names
 
     def _compare(
-        self,
-        test_case: ArenaTestCase,
+        self, test_case: ArenaTestCase, multimodal: bool
     ) -> Tuple[str, str, Dict[str, str]]:
         formatted_test_case, dummy_to_real_names = format_arena_test_case(
             self.evaluation_params, test_case
@@ -237,19 +250,23 @@ class ArenaGEval(BaseArenaMetric):
             evaluation_steps=number_evaluation_steps(self.evaluation_steps),
             test_case_contents=formatted_test_case,
             parameters=g_eval_params_str,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            self,
+            prompt,
+            Winner,
+            extract_schema=lambda s: (
+                s.winner,
+                s.reason,
+                dummy_to_real_names,
+            ),
+            extract_json=lambda data: (
+                data["winner"],
+                data["reason"],
+                dummy_to_real_names,
+            ),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Winner)
-            self.evaluation_cost += cost
-            return res.winner, res.reason, dummy_to_real_names
-        else:
-            try:
-                res: Winner = self.model.generate(prompt, schema=Winner)
-                return res.winner, res.reason, dummy_to_real_names
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["winner"], data["reason"], dummy_to_real_names
 
     async def _a_generate_rewritten_reason(
         self,
@@ -260,22 +277,14 @@ class ArenaGEval(BaseArenaMetric):
             reason=reason,
             dummy_to_real_names=dummy_to_real_names,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=RewrittenReason
-            )
-            self.evaluation_cost += cost
-            return res.rewritten_reason
-        else:
-            try:
-                res: RewrittenReason = await self.model.a_generate(
-                    prompt, schema=RewrittenReason
-                )
-                return res.rewritten_reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["rewritten_reason"]
+
+        return await a_generate_with_schema_and_extract(
+            self,
+            prompt,
+            RewrittenReason,
+            extract_schema=lambda s: s.rewritten_reason,
+            extract_json=lambda data: data["rewritten_reason"],
+        )
 
     def _generate_rewritten_reason(
         self,
@@ -286,20 +295,13 @@ class ArenaGEval(BaseArenaMetric):
             reason=reason,
             dummy_to_real_names=dummy_to_real_names,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=RewrittenReason)
-            self.evaluation_cost += cost
-            return res.rewritten_reason
-        else:
-            try:
-                res: RewrittenReason = self.model.generate(
-                    prompt, schema=RewrittenReason
-                )
-                return res.rewritten_reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["rewritten_reason"]
+        return generate_with_schema_and_extract(
+            self,
+            prompt,
+            RewrittenReason,
+            extract_schema=lambda s: s.rewritten_reason,
+            extract_json=lambda data: data["rewritten_reason"],
+        )
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/arena_g_eval/template.py

@@ -3,11 +3,23 @@ import textwrap
 
 
 class ArenaGEvalTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def generate_evaluation_steps(parameters: str, criteria: str):
+    def generate_evaluation_steps(
+        parameters: str, criteria: str, multimodal: Optional[bool]
+    ):
         return textwrap.dedent(
             f"""Given an evaluation criteria which outlines how you should choose the winner out of all contestants based on the {parameters}, generate 3-4 concise evaluation steps based on the criteria below. You MUST make it clear how to evaluate {parameters} in relation to one another.
 
+            {ArenaGEvalTemplate.multimodal_rules if multimodal else ""}
+
             Evaluation Criteria:
             {criteria}
 
@@ -28,6 +40,7 @@ class ArenaGEvalTemplate:
         evaluation_steps: str,
         test_case_contents: List[str],
         parameters: str,
+        multimodal: Optional[bool],
     ):
         reasoning_expectation = (
             "Be specific and grounded in the evaluation steps."
@@ -36,6 +49,9 @@ class ArenaGEvalTemplate:
         return textwrap.dedent(
             f"""
             You are a judge. Given the following evaluation steps, select the single contestant that best aligns with the evaluation steps.
+
+            {ArenaGEvalTemplate.multimodal_rules if multimodal else ""}
+
             Return a JSON object with three fields:
 
             - `"winner"`: the contestant that is best aligned with the evaluation steps.

deepeval/metrics/argument_correctness/argument_correctness.py

@@ -3,9 +3,10 @@ from typing import Optional, List, Type, Union
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -18,7 +19,11 @@ from deepeval.metrics.argument_correctness.template import (
     ArgumentCorrectnessTemplate,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.argument_correctness.schema import *
+from deepeval.metrics.argument_correctness.schema import (
+    ArgumentCorrectnessVerdict,
+    Verdicts,
+    ArgumentCorrectnessScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -57,7 +62,15 @@ class ArgumentCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -81,11 +94,15 @@ class ArgumentCorrectnessMetric(BaseMetric):
                 else:
                     self.verdicts: List[ArgumentCorrectnessVerdict] = (
                         self._generate_verdicts(
-                            test_case.input, test_case.tools_called
+                            test_case.input,
+                            test_case.tools_called,
+                            test_case.multimodal,
                         )
                     )
                     self.score = self._calculate_score()
-                    self.reason = self._generate_reason(test_case.input)
+                    self.reason = self._generate_reason(
+                        test_case.input, test_case.multimodal
+                    )
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -108,7 +125,15 @@ class ArgumentCorrectnessMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -124,11 +149,15 @@ class ArgumentCorrectnessMetric(BaseMetric):
             else:
                 self.verdicts: List[ArgumentCorrectnessVerdict] = (
                     await self._a_generate_verdicts(
-                        test_case.input, test_case.tools_called
+                        test_case.input,
+                        test_case.tools_called,
+                        test_case.multimodal,
                     )
                 )
                 self.score = self._calculate_score()
-                self.reason = await self._a_generate_reason(test_case.input)
+                self.reason = await self._a_generate_reason(
+                    test_case.input, test_case.multimodal
+                )
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -143,7 +172,7 @@ class ArgumentCorrectnessMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str) -> str:
+    async def _a_generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -156,27 +185,18 @@ class ArgumentCorrectnessMetric(BaseMetric):
             incorrect_tool_calls_reasons=incorrect_tool_calls_reasons,
             input=input,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgumentCorrectnessScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ArgumentCorrectnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ArgumentCorrectnessScoreReason = (
-                    await self.model.a_generate(
-                        prompt=prompt, schema=ArgumentCorrectnessScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
-    def _generate_reason(self, input: str) -> str:
+    def _generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -189,76 +209,50 @@ class ArgumentCorrectnessMetric(BaseMetric):
             incorrect_tool_calls_reasons=incorrect_tool_calls_reasons,
             input=input,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ArgumentCorrectnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ArgumentCorrectnessScoreReason = self.model.generate(
-                    prompt=prompt, schema=ArgumentCorrectnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ArgumentCorrectnessScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
-        self,
-        input: str,
-        tools_called: List[ToolCall],
+        self, input: str, tools_called: List[ToolCall], multimodal: bool
     ) -> List[ArgumentCorrectnessVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input,
-            tools_called=tools_called,
+            input=input, tools_called=tools_called, multimodal=multimodal
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                ArgumentCorrectnessVerdict(**item) for item in data["verdicts"]
+            ],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    ArgumentCorrectnessVerdict(**item)
-                    for item in data["verdicts"]
-                ]
 
     def _generate_verdicts(
-        self,
-        input: str,
-        tools_called: List[ToolCall],
+        self, input: str, tools_called: List[ToolCall], multimodal: bool
     ) -> List[ArgumentCorrectnessVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
-            input=input,
-            tools_called=tools_called,
+            input=input, tools_called=tools_called, multimodal=multimodal
+        )
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda r: list(r.verdicts),
+            extract_json=lambda data: [
+                ArgumentCorrectnessVerdict(**item) for item in data["verdicts"]
+            ],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    ArgumentCorrectnessVerdict(**item)
-                    for item in data["verdicts"]
-                ]
 
     def _calculate_score(self):
         number_of_verdicts = len(self.verdicts)
@@ -279,7 +273,7 @@ class ArgumentCorrectnessMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/argument_correctness/template.py

@@ -4,8 +4,18 @@ import textwrap
 
 
 class ArgumentCorrectnessTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def generate_verdicts(input: str, tools_called: List[ToolCall]):
+    def generate_verdicts(
+        input: str, tools_called: List[ToolCall], multimodal: bool = False
+    ):
 
         stringified_tools_called = repr(tools_called)
 
@@ -19,6 +29,8 @@ class ArgumentCorrectnessTemplate:
             Provide a 'reason' ONLY if the answer is 'no'. 
             If there is no input parameter, answer 'no' for the verdict and provide the reason as "No input parameter provided".
 
+            {ArgumentCorrectnessTemplate.multimodal_rules if multimodal else ""}
+
             **
             IMPORTANT: Please make sure to only return in valid and parseable JSON format, with the 'verdicts' key mapping to a list of JSON objects. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.
             Example input: 
@@ -92,12 +104,17 @@ class ArgumentCorrectnessTemplate:
 
     @staticmethod
     def generate_reason(
-        incorrect_tool_calls_reasons: List[str], input: str, score: float
+        incorrect_tool_calls_reasons: List[str],
+        input: str,
+        score: float,
+        multimodal: bool = False,
     ):
         return textwrap.dedent(
             f"""Given the argument correctness score, the list of reasons of incorrect tool calls, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score. You can mention tool calls or input, but do not mention an output or a response.
             If there is nothing incorrect, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
 
+            {ArgumentCorrectnessTemplate.multimodal_rules if multimodal else ""}
+
             **
             IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason. Ensure all strings are closed appropriately. Repair any invalid JSON before you output it.