deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/prompt_alignment/prompt_alignment.py

@@ -2,12 +2,17 @@ import asyncio
 
 from typing import Optional, List, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+    get_per_task_timeout,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
-    trimAndLoadJson,
     check_llm_test_case_params,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.test_case import (
     LLMTestCase,
@@ -18,7 +23,6 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.prompt_alignment.template import PromptAlignmentTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.prompt_alignment import schema as paschema
-from deepeval.config.settings import get_settings
 
 from deepeval.metrics.api import metric_data_manager
 
@@ -60,7 +64,15 @@ class PromptAlignmentMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -72,16 +84,19 @@ class PromptAlignmentMetric(BaseMetric):
                     test_case,
                     _show_indicator=False,
                     _in_component=_in_component,
+                    _log_metric_to_confident=_log_metric_to_confident,
                 )
                 loop.run_until_complete(
                     asyncio.wait_for(
                         coro,
-                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                        timeout=get_per_task_timeout(),
                     )
                 )
             else:
-                self.verdicts: paschema.Verdicts = self._generate_verdicts(
-                    test_case.input, test_case.actual_output
+                self.verdicts: List[paschema.PromptAlignmentVerdict] = (
+                    self._generate_verdicts(
+                        test_case.input, test_case.actual_output
+                    )
                 )
                 self.score = self._calculate_score()
                 self.reason = self._generate_reason(
@@ -111,7 +126,15 @@ class PromptAlignmentMetric(BaseMetric):
         _log_metric_to_confident: bool = True,
     ) -> float:
 
-        check_llm_test_case_params(test_case, self._required_params, self)
+        check_llm_test_case_params(
+            test_case,
+            self._required_params,
+            None,
+            None,
+            self,
+            self.model,
+            test_case.multimodal,
+        )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -120,8 +143,10 @@ class PromptAlignmentMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: paschema.Verdicts = await self._a_generate_verdicts(
-                test_case.input, test_case.actual_output
+            self.verdicts: List[paschema.PromptAlignmentVerdict] = (
+                await self._a_generate_verdicts(
+                    test_case.input, test_case.actual_output
+                )
             )
             self.score = self._calculate_score()
             self.reason = await self._a_generate_reason(
@@ -142,7 +167,9 @@ class PromptAlignmentMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str, actual_output: str) -> str:
+    async def _a_generate_reason(
+        self, input: str, actual_output: str
+    ) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -157,27 +184,16 @@ class PromptAlignmentMetric(BaseMetric):
             actual_output=actual_output,
             score=format(self.score, ".2f"),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=paschema.PromptAlignmentScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: paschema.PromptAlignmentScoreReason = (
-                    await self.model.a_generate(
-                        prompt=prompt,
-                        schema=paschema.PromptAlignmentScoreReason,
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
-    def _generate_reason(self, input: str, actual_output: str) -> str:
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=paschema.PromptAlignmentScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
+
+    def _generate_reason(self, input: str, actual_output: str) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -192,78 +208,54 @@ class PromptAlignmentMetric(BaseMetric):
             actual_output=actual_output,
             score=format(self.score, ".2f"),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=paschema.PromptAlignmentScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: paschema.PromptAlignmentScoreReason = self.model.generate(
-                    prompt=prompt, schema=paschema.PromptAlignmentScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=paschema.PromptAlignmentScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_verdicts(
         self, input: str, actual_output: str
-    ) -> paschema.Verdicts:
+    ) -> List[paschema.PromptAlignmentVerdict]:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=paschema.Verdicts
-            )
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: paschema.Verdicts = await self.model.a_generate(
-                    prompt, schema=paschema.Verdicts
-                )
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    paschema.PromptAlignmentVerdict(**item)
-                    for item in data["verdicts"]
-                ]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=paschema.Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                paschema.PromptAlignmentVerdict(**item)
+                for item in data["verdicts"]
+            ],
+        )
 
     def _generate_verdicts(
         self, input: str, actual_output: str
-    ) -> paschema.Verdicts:
+    ) -> List[paschema.PromptAlignmentVerdict]:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=paschema.Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: paschema.Verdicts = self.model.generate(
-                    prompt, schema=paschema.Verdicts
-                )
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    paschema.PromptAlignmentVerdict(**item)
-                    for item in data["verdicts"]
-                ]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=paschema.Verdicts,
+            extract_schema=lambda s: list(s.verdicts),
+            extract_json=lambda data: [
+                paschema.PromptAlignmentVerdict(**item)
+                for item in data["verdicts"]
+            ],
+        )
 
-    def _calculate_score(self):
+    def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
         if number_of_verdicts == 0:
             return 1

deepeval/metrics/prompt_alignment/template.py

@@ -2,6 +2,14 @@ from typing import List
 
 
 class PromptAlignmentTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_verdicts(
         prompt_instructions: List[str], input: str, actual_output: str
@@ -14,6 +22,8 @@ The 'reason' is the reason for the verdict.
 Provide a 'reason' ONLY if the answer is 'no'. 
 The provided prompt instructions are the instructions to be followed in the prompt, which you have no access to.
 
+{PromptAlignmentTemplate.multimodal_rules}
+
 **
 IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key mapping to a list of JSON objects.
 Example input: What number is the stars of the sky?
@@ -63,6 +73,8 @@ The unalignments represent prompt instructions that are not followed by the LLM
 If there no unaligments, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
 Don't have to talk about whether the actual output is a good fit for the input, access ENTIRELY based on the unalignment reasons.
 
+{PromptAlignmentTemplate.multimodal_rules}
+
 **
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON:

deepeval/metrics/role_adherence/role_adherence.py

@@ -4,20 +4,21 @@ from deepeval.metrics import BaseConversationalMetric
 from deepeval.metrics.api import metric_data_manager
 from deepeval.metrics.role_adherence.schema import (
     OutOfCharacterResponseVerdicts,
+    RoleAdherenceScoreReason,
 )
 from deepeval.metrics.role_adherence.template import RoleAdherenceTemplate
 from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
     convert_turn_to_dict,
-    trimAndLoadJson,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.test_case import Turn, ConversationalTestCase, TurnParams
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.role_adherence.schema import *
 
 
 class RoleAdherenceMetric(BaseConversationalMetric):
@@ -51,7 +52,9 @@ class RoleAdherenceMetric(BaseConversationalMetric):
             test_case,
             self._required_test_case_params,
             self,
-            require_chatbot_role=True,
+            True,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -102,7 +105,9 @@ class RoleAdherenceMetric(BaseConversationalMetric):
             test_case,
             self._required_test_case_params,
             self,
-            require_chatbot_role=True,
+            True,
+            self.model,
+            test_case.multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -138,7 +143,7 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, role: str) -> str:
+    async def _a_generate_reason(self, role: str) -> Optional[str]:
         if self.include_reason is False:
             return None
 
@@ -150,24 +155,17 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                 for verdict in self.out_of_character_verdicts.verdicts
             ],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=RoleAdherenceScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: RoleAdherenceScoreReason = await self.model.a_generate(
-                    prompt, schema=RoleAdherenceScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=RoleAdherenceScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
-    def _generate_reason(self, role: str) -> str:
+    def _generate_reason(self, role: str) -> Optional[str]:
+        if self.include_reason is False:
+            return None
         prompt = RoleAdherenceTemplate.generate_reason(
             score=self.score,
             role=role,
@@ -176,22 +174,13 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                 for verdict in self.out_of_character_verdicts.verdicts
             ],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=RoleAdherenceScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: RoleAdherenceScoreReason = self.model.generate(
-                    prompt, schema=RoleAdherenceScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=RoleAdherenceScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_extract_out_of_character_verdicts(
         self, turns: List[Turn], role: str
@@ -202,28 +191,23 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                 role=role,
             )
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=OutOfCharacterResponseVerdicts
+        res: OutOfCharacterResponseVerdicts = (
+            await a_generate_with_schema_and_extract(
+                metric=self,
+                prompt=prompt,
+                schema_cls=OutOfCharacterResponseVerdicts,
+                extract_schema=lambda s: s,
+                extract_json=lambda data: OutOfCharacterResponseVerdicts(
+                    **data
+                ),
             )
-            self.evaluation_cost += cost
-        else:
-            try:
-                res: OutOfCharacterResponseVerdicts = (
-                    await self.model.a_generate(
-                        prompt, schema=OutOfCharacterResponseVerdicts
-                    )
-                )
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                res = OutOfCharacterResponseVerdicts(**data)
+        )
 
         for verdict in res.verdicts:
             try:
                 index = verdict.index
                 verdict.ai_message = f"{turns[index].content} (turn #{index+1})"
-            except:
+            except Exception:
                 pass
         return res
 
@@ -236,26 +220,19 @@ class RoleAdherenceMetric(BaseConversationalMetric):
                 role=role,
             )
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=OutOfCharacterResponseVerdicts
-            )
-            self.evaluation_cost += cost
-        else:
-            try:
-                res: OutOfCharacterResponseVerdicts = self.model.generate(
-                    prompt, schema=OutOfCharacterResponseVerdicts
-                )
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                res = OutOfCharacterResponseVerdicts(**data)
+        res: OutOfCharacterResponseVerdicts = generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=OutOfCharacterResponseVerdicts,
+            extract_schema=lambda s: s,
+            extract_json=lambda data: OutOfCharacterResponseVerdicts(**data),
+        )
 
         for verdict in res.verdicts:
             try:
                 index = verdict.index
                 verdict.ai_message = f"{turns[index].content} (turn #{index+1})"
-            except:
+            except Exception:
                 pass
         return res
 
@@ -278,8 +255,8 @@ class RoleAdherenceMetric(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/role_adherence/template.py

@@ -2,11 +2,22 @@ from typing import List, Dict
 
 
 class RoleAdherenceTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def extract_out_of_character_response_verdicts(
         turns: List[Dict], role: str
     ):
         return f"""Based on the given list of message exchanges between a user and an LLM chatbot, generate a JSON object to specify which `ai_message` did not adhere to the specified chatbot role. 
+
+{RoleAdherenceTemplate.multimodal_rules}
+
 The JSON will have 1 field: "verdicts", which is a list of verdicts specifying the indices and reasons of the LLM ai_message/responses that did NOT adhere to the chatbot role.
 You MUST USE look at all messages provided in the list of messages to make an informed judgement on role adherence.
 
@@ -72,6 +83,9 @@ JSON:
         return f"""Below is a list of LLM chatbot responses (ai_message) that is out of character with respect to the specified chatbot role. It is drawn from a list of messages in a conversation, which you have minimal knowledge of.
 Given the role adherence score, which is a 0-1 score indicating how well the chatbot responses has adhered to the given role through a conversation, with 1 being the best and 0 being worst, provide a reason by quoting the out of character responses to justify the score. 
 
+
+{RoleAdherenceTemplate.multimodal_rules}
+
 ** 
 IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
 Example JSON: