deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/conversation_completeness/conversation_completeness.py

@@ -8,9 +8,10 @@ from deepeval.metrics.conversation_completeness.template import (
 from deepeval.metrics.utils import (
     check_conversational_test_case_params,
     construct_verbose_logs,
-    trimAndLoadJson,
     initialize_model,
     convert_turn_to_dict,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
@@ -18,7 +19,11 @@ from deepeval.test_case import ConversationalTestCase
 from deepeval.test_case import TurnParams
 from deepeval.test_case.conversational_test_case import Turn
 from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.conversation_completeness.schema import *
+from deepeval.metrics.conversation_completeness.schema import (
+    UserIntentions,
+    ConversationCompletenessVerdict,
+    ConversationCompletenessScoreReason,
+)
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -51,8 +56,15 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ):
+
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -71,17 +83,19 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                 )
             else:
                 self.user_intentions = self._extract_user_intentions(
-                    test_case.turns
+                    test_case.turns, multimodal=multimodal
                 )
                 self.verdicts = [
                     self._generate_verdict(
-                        turns=test_case.turns, intention=user_intention
+                        turns=test_case.turns,
+                        intention=user_intention,
+                        multimodal=multimodal,
                     )
                     for user_intention in self.user_intentions
                 ]
 
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason()
+                self.reason = self._generate_reason(multimodal=multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -105,28 +119,40 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
+
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
-            test_case, self._required_test_case_params, self
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
-            self, async_mode=True, _show_indicator=_show_indicator
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
         ):
             self.user_intentions = await self._a_extract_user_intentions(
-                test_case.turns
+                test_case.turns, multimodal=multimodal
             )
             self.verdicts = await asyncio.gather(
                 *[
                     self._a_generate_verdict(
-                        turns=test_case.turns, intention=user_intention
+                        turns=test_case.turns,
+                        intention=user_intention,
+                        multimodal=multimodal,
                     )
                     for user_intention in self.user_intentions
                 ]
             )
 
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
+            self.reason = await self._a_generate_reason(multimodal=multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -143,7 +169,7 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self) -> str:
+    async def _a_generate_reason(self, multimodal: bool) -> str:
         incompletenesses: List[str] = []
         for verdict in self.verdicts:
             if verdict.verdict.strip().lower() == "no":
@@ -153,27 +179,17 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
             score=self.score,
             incompletenesses=incompletenesses,
             intentions=self.user_intentions,
+            multimodal=multimodal,
+        )
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ConversationCompletenessScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ConversationCompletenessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ConversationCompletenessScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=ConversationCompletenessScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
-    def _generate_reason(self) -> str:
+    def _generate_reason(self, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -186,113 +202,79 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
             score=self.score,
             incompletenesses=incompletenesses,
             intentions=self.user_intentions,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ConversationCompletenessScoreReason,
+            extract_schema=lambda score_reason: score_reason.reason,
+            extract_json=lambda data: data["reason"],
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ConversationCompletenessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ConversationCompletenessScoreReason = self.model.generate(
-                    prompt, schema=ConversationCompletenessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
 
     async def _a_generate_verdict(
-        self, turns: List[Turn], intention: str
+        self, turns: List[Turn], intention: str, multimodal: bool
     ) -> ConversationCompletenessVerdict:
         prompt = ConversationCompletenessTemplate.generate_verdicts(
             turns=[convert_turn_to_dict(turn) for turn in turns],
             intention=intention,
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ConversationCompletenessVerdict,
+            extract_schema=lambda r: r,
+            extract_json=lambda data: ConversationCompletenessVerdict(**data),
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ConversationCompletenessVerdict
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ConversationCompletenessVerdict = (
-                    await self.model.a_generate(
-                        prompt, schema=ConversationCompletenessVerdict
-                    )
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ConversationCompletenessVerdict(**data)
 
     def _generate_verdict(
-        self, turns: List[Turn], intention: str
+        self, turns: List[Turn], intention: str, multimodal: bool
     ) -> ConversationCompletenessVerdict:
         prompt = ConversationCompletenessTemplate.generate_verdicts(
             turns=[convert_turn_to_dict(turn) for turn in turns],
             intention=intention,
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ConversationCompletenessVerdict,
+            extract_schema=lambda r: r,
+            extract_json=lambda data: ConversationCompletenessVerdict(**data),
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ConversationCompletenessVerdict
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res: ConversationCompletenessVerdict = self.model.generate(
-                    prompt, schema=ConversationCompletenessVerdict
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ConversationCompletenessVerdict(**data)
 
-    async def _a_extract_user_intentions(self, turns: List[Turn]) -> List[str]:
+    async def _a_extract_user_intentions(
+        self, turns: List[Turn], multimodal: bool
+    ) -> List[str]:
         prompt = ConversationCompletenessTemplate.extract_user_intentions(
-            turns=[convert_turn_to_dict(turn) for turn in turns]
+            turns=[convert_turn_to_dict(turn) for turn in turns],
+            multimodal=multimodal,
+        )
+
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=UserIntentions,
+            extract_schema=lambda r: r.intentions,
+            extract_json=lambda data: UserIntentions(**data).intentions,
         )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=UserIntentions
-            )
-            self.evaluation_cost += cost
-            return res.intentions
-        else:
-            try:
-                res: UserIntentions = await self.model.a_generate(
-                    prompt, schema=UserIntentions
-                )
-                return res.intentions
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return UserIntentions(**data).intentions
 
-    def _extract_user_intentions(self, turns: List[Turn]) -> List[str]:
+    def _extract_user_intentions(
+        self, turns: List[Turn], multimodal: bool
+    ) -> List[str]:
         prompt = ConversationCompletenessTemplate.extract_user_intentions(
-            turns=[convert_turn_to_dict(turn) for turn in turns]
+            turns=[convert_turn_to_dict(turn) for turn in turns],
+            multimodal=multimodal,
+        )
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=UserIntentions,
+            extract_schema=lambda r: r.intentions,
+            extract_json=lambda data: UserIntentions(**data).intentions,
         )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=UserIntentions)
-            self.evaluation_cost += cost
-            return res.intentions
-        else:
-            try:
-                res: UserIntentions = self.model.generate(
-                    prompt, schema=UserIntentions
-                )
-                return res.intentions
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return UserIntentions(**data).intentions
 
     def _calculate_score(self) -> float:
         number_of_verdicts = len(self.verdicts)
@@ -312,8 +294,8 @@ class ConversationCompletenessMetric(BaseConversationalMetric):
             self.success = False
         else:
             try:
-                self.score >= self.threshold
-            except:
+                self.success = self.score >= self.threshold
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/conversation_completeness/template.py

@@ -2,11 +2,21 @@ from typing import List, Dict
 
 
 class ConversationCompletenessTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
-    def extract_user_intentions(turns: List[Dict]):
+    def extract_user_intentions(turns: List[Dict], multimodal: bool = False):
         return f"""Based on the given list of message exchanges between a user and an LLM, generate a JSON object to extract all user intentions in the conversation. The JSON will have 1 field: 'intentions'.
 You should ONLY consider the overall intention, and not dwell too much on the specifics, as we are more concerned about the overall objective of the conversation.
 
+{ConversationCompletenessTemplate.multimodal_rules if multimodal else ""}
+
 **
 IMPORTANT: Please make sure to only return in JSON format.
 Example Turns:
@@ -49,8 +59,13 @@ JSON:
 """
 
     @staticmethod
-    def generate_verdicts(turns: List[Dict], intention: str):
+    def generate_verdicts(
+        turns: List[Dict], intention: str, multimodal: bool = False
+    ):
         return f"""Based on the given list of message exchanges between a user and an LLM, generate a JSON object to indicate whether given user intention was satisfied from the conversation messages. The JSON will have 2 fields: 'verdict' and 'reason'.
+
+{ConversationCompletenessTemplate.multimodal_rules if multimodal else ""}
+
 The 'verdict' key should STRICTLY be either 'yes' or 'no', which states whether the user intention was satisfied or not.
 Provide a 'reason' ONLY if the answer is 'no'.
 You MUST USE look at all messages provided in the list of messages to make an informed judgement on satisfaction.
@@ -106,8 +121,13 @@ JSON:
 """
 
     @staticmethod
-    def generate_reason(score, incompletenesses, intentions):
+    def generate_reason(
+        score, incompletenesses, intentions, multimodal: bool = False
+    ):
         return f"""Below is a list of incompletenesses drawn from some messages in a conversation, which you have minimal knowledge of. It is a list of strings explaining why an LLM 'actual_output' is incomplete to satisfy the user `input` for a particular message.
+
+{ConversationCompletenessTemplate.multimodal_rules if multimodal else ""}
+
 Given the completeness score, which is a 0-1 score indicating how incomplete the OVERALL `actual_output`s are to the user intentions found in the `input`s of a conversation (higher the better), CONCISELY summarize the incompletenesses to justify the score. 
 
 ** 

deepeval/metrics/conversational_dag/conversational_dag.py

@@ -1,4 +1,4 @@
-from typing import Optional, Union
+from typing import List, Optional, Union
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.test_case import (
     ConversationalTestCase,
@@ -11,7 +11,6 @@ from deepeval.metrics.utils import (
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.g_eval.schema import *
 from deepeval.metrics import DeepAcyclicGraph
 from deepeval.metrics.dag.utils import (
     is_valid_dag_from_roots,
@@ -35,11 +34,8 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         verbose_mode: bool = False,
         _include_dag_suffix: bool = True,
     ):
-        if (
-            is_valid_dag_from_roots(
-                root_nodes=dag.root_nodes, multiturn=dag.multiturn
-            )
-            == False
+        if not is_valid_dag_from_roots(
+            root_nodes=dag.root_nodes, multiturn=dag.multiturn
         ):
             raise ValueError("Cycle detected in DAG graph.")
 
@@ -62,10 +58,14 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
             test_case,
             extract_required_params(self.dag.root_nodes, multiturn=True),
             self,
+            False,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -105,10 +105,14 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
+        multimodal = test_case.multimodal
         check_conversational_test_case_params(
             test_case,
             extract_required_params(self.dag.root_nodes, multiturn=True),
             self,
+            False,
+            self.model,
+            multimodal,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -139,7 +143,7 @@ class ConversationalDAGMetric(BaseConversationalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success