deepeval 3.7.5__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/turn_contextual_relevancy/turn_contextual_relevancy.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Union, Type, Tuple
 import asyncio
-
+import itertools
 from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.utils import (
@@ -12,6 +12,7 @@ from deepeval.metrics.utils import (
     trimAndLoadJson,
     check_conversational_test_case_params,
     get_unit_interactions,
+    get_turns_in_sliding_window,
     initialize_model,
 )
 from deepeval.models import DeepEvalBaseLLM
@@ -30,6 +31,7 @@ from deepeval.metrics.api import metric_data_manager
 
 class TurnContextualRelevancyMetric(BaseConversationalMetric):
     _required_test_case_params: List[TurnParams] = [
+        TurnParams.ROLE,
         TurnParams.CONTENT,
         TurnParams.RETRIEVAL_CONTEXT,
     ]
@@ -42,6 +44,7 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
+        window_size: int = 10,
         evaluation_template: Type[
             TurnContextualRelevancyTemplate
         ] = TurnContextualRelevancyTemplate,
@@ -53,6 +56,7 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
         self.async_mode = async_mode
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
+        self.window_size = window_size
         self.evaluation_template = evaluation_template
 
     def measure(
@@ -89,9 +93,19 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
                 )
             else:
                 unit_interactions = get_unit_interactions(test_case.turns)
-                scores = self._get_contextual_relevancy_scores(
-                    unit_interactions, multimodal
-                )
+                turns_windows: List[List[Turn]] = [
+                    list(itertools.chain(*window))
+                    for window in get_turns_in_sliding_window(
+                        unit_interactions, self.window_size
+                    )
+                ]
+                scores = []
+                for window in turns_windows:
+                    scores.extend(
+                        self._get_contextual_relevancy_scores(
+                            window, multimodal
+                        )
+                    )
                 self.score = self._calculate_score(scores)
                 self.success = self.score >= self.threshold
                 self.reason = self._generate_reason(scores)
@@ -137,9 +151,25 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             _in_component=_in_component,
         ):
             unit_interactions = get_unit_interactions(test_case.turns)
-            scores = await self._a_get_contextual_relevancy_scores(
-                unit_interactions, multimodal
-            )
+            turns_windows: List[List[Turn]] = [
+                list(itertools.chain(*window))
+                for window in get_turns_in_sliding_window(
+                    unit_interactions, self.window_size
+                )
+            ]
+            scores = []
+            tasks = []
+
+            async def get_individual_scores(window):
+                scores.extend(
+                    await self._a_get_contextual_relevancy_scores(
+                        window, multimodal
+                    )
+                )
+
+            for window in turns_windows:
+                tasks.append(get_individual_scores(window))
+            await asyncio.gather(*tasks)
             self.score = self._calculate_score(scores)
             self.success = self.score >= self.threshold
             self.reason = await self._a_generate_reason(scores)
@@ -160,69 +190,63 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             return self.score
 
     async def _a_get_contextual_relevancy_scores(
-        self, unit_interactions: List[List[Turn]], multimodal: bool
+        self, turns_window: List[Turn], multimodal: bool
     ):
-        async def get_interaction_score(unit_interaction: List[Turn]):
-            user_content = "User Message: "
-            retrieval_context = []
-            for turn in unit_interaction:
-                if turn.role == "user":
-                    user_content += f"\n{turn.content} "
-                else:
-                    retrieval_context.extend(turn.retrieval_context)
+        windows_scores = []
 
-            # Generate verdicts for each retrieval context
-            verdicts = await self._a_generate_verdicts(
-                user_content, retrieval_context, multimodal
-            )
-            score, reason = await self._a_get_interaction_score_and_reason(
-                user_content, verdicts, multimodal
-            )
-            interaction_score = InteractionContextualRelevancyScore(
-                score=score,
-                reason=reason,
-                verdicts=verdicts,
-            )
-            return interaction_score
+        user_content = ""
+        retrieval_context = []
+        for turn in turns_window:
+            if turn.role == "user":
+                user_content += f"\n{turn.content} "
+            else:
+                if turn.retrieval_context is not None:
+                    retrieval_context.extend(turn.retrieval_context)
 
-        final_scores = await asyncio.gather(
-            *[
-                get_interaction_score(unit_interaction)
-                for unit_interaction in unit_interactions
-            ]
+        verdicts = await self._a_generate_verdicts(
+            user_content, retrieval_context, multimodal
+        )
+        score, reason = await self._a_get_interaction_score_and_reason(
+            user_content, verdicts, multimodal
+        )
+        interaction_score = InteractionContextualRelevancyScore(
+            score=score,
+            reason=reason,
+            verdicts=verdicts,
         )
 
-        return final_scores
+        windows_scores.append(interaction_score)
+
+        return windows_scores
 
     def _get_contextual_relevancy_scores(
-        self, unit_interactions: List[List[Turn]], multimodal: bool
+        self, turns_window: List[Turn], multimodal: bool
     ):
-        interaction_scores = []
-
-        for unit_interaction in unit_interactions:
-            user_content = "User Message: "
-            retrieval_context = []
-            for turn in unit_interaction:
-                if turn.role == "user":
-                    user_content += f"\n{turn.content} "
-                else:
+        windows_scores = []
+
+        user_content = ""
+        retrieval_context = []
+        for turn in turns_window:
+            if turn.role == "user":
+                user_content += f"\n{turn.content} "
+            else:
+                if turn.retrieval_context is not None:
                     retrieval_context.extend(turn.retrieval_context)
 
-            # Generate verdicts for each retrieval context
-            verdicts = self._generate_verdicts(
-                user_content, retrieval_context, multimodal
-            )
-            score, reason = self._get_interaction_score_and_reason(
-                user_content, verdicts, multimodal
-            )
-            interaction_score = InteractionContextualRelevancyScore(
-                score=score,
-                reason=reason,
-                verdicts=verdicts,
-            )
-            interaction_scores.append(interaction_score)
+        verdicts = self._generate_verdicts(
+            user_content, retrieval_context, multimodal
+        )
+        score, reason = self._get_interaction_score_and_reason(
+            user_content, verdicts, multimodal
+        )
+        interaction_score = InteractionContextualRelevancyScore(
+            score=score,
+            reason=reason,
+            verdicts=verdicts,
+        )
+        windows_scores.append(interaction_score)
 
-        return interaction_scores
+        return windows_scores
 
     async def _a_generate_verdicts(
         self, input: str, retrieval_context: List[str], multimodal: bool
@@ -313,7 +337,10 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
         multimodal: bool,
     ) -> Tuple[float, str]:
         if len(verdicts) == 0:
-            return 1, None
+            return (
+                1,
+                "There were no retrieval contexts in the given turns to evaluate the contextual relevancy.",
+            )
 
         score = self._calculate_interaction_score(verdicts)
         reason = await self._a_get_interaction_reason(
@@ -332,7 +359,10 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
         multimodal: bool,
     ) -> Tuple[float, str]:
         if len(verdicts) == 0:
-            return 1, None
+            return (
+                1,
+                "There were no retrieval contexts in the given turns to evaluate the contextual relevancy.",
+            )
 
         score = self._calculate_interaction_score(verdicts)
         reason = self._get_interaction_reason(
@@ -377,7 +407,6 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
             if verdict.verdict.strip().lower() == "yes":
                 relevant_statements.append(verdict.statement)
             else:
-                # Include the reason for irrelevance
                 irrelevant_statements.append(
                     f"{verdict.statement}: {verdict.reason}"
                 )
@@ -458,12 +487,12 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
                 return data["reason"]
 
     def _get_verbose_steps(
-        self, interaction_scores: List[InteractionContextualRelevancyScore]
+        self, windows_scores: List[InteractionContextualRelevancyScore]
     ):
         steps = []
-        for index, interaction_score in enumerate(interaction_scores):
+        for index, interaction_score in enumerate(windows_scores):
             interaction_steps = [
-                f"Interaction {index + 1} \n",
+                f"Window {index + 1} \n",
                 f"Verdicts: {prettify_list(interaction_score.verdicts)} \n",
                 f"Score: {interaction_score.score} \n",
                 f"Reason: {interaction_score.reason} \n",
@@ -474,6 +503,12 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
     def _generate_reason(
         self, scores: List[InteractionContextualRelevancyScore]
     ) -> str:
+        if self.include_reason is False:
+            return None
+
+        if len(scores) == 0:
+            return "There were no retrieval contexts in your turns to evaluate, hence the score is 1"
+
         reasons = []
         for score in scores:
             reasons.append(score.reason)
@@ -493,6 +528,12 @@ class TurnContextualRelevancyMetric(BaseConversationalMetric):
     async def _a_generate_reason(
         self, scores: List[InteractionContextualRelevancyScore]
     ) -> str:
+        if self.include_reason is False:
+            return None
+
+        if len(scores) == 0:
+            return "There were no retrieval contexts in your turns to evaluate, hence the score is 1"
+
         reasons = []
         for score in scores:
             reasons.append(score.reason)

deepeval/metrics/turn_faithfulness/schema.py

@@ -25,7 +25,7 @@ class FaithfulnessScoreReason(BaseModel):
 
 class InteractionFaithfulnessScore(BaseModel):
     score: float
-    reason: str
+    reason: Optional[str]
     claims: List[str]
     truths: List[str]
     verdicts: List[FaithfulnessVerdict]

deepeval/metrics/turn_faithfulness/turn_faithfulness.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Union, Type, Tuple
 import asyncio
-
+import itertools
 from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.utils import (
@@ -12,6 +12,7 @@ from deepeval.metrics.utils import (
     trimAndLoadJson,
     check_conversational_test_case_params,
     get_unit_interactions,
+    get_turns_in_sliding_window,
     initialize_model,
 )
 from deepeval.models import DeepEvalBaseLLM
@@ -32,6 +33,7 @@ from deepeval.metrics.api import metric_data_manager
 
 class TurnFaithfulnessMetric(BaseConversationalMetric):
     _required_test_case_params: List[TurnParams] = [
+        TurnParams.ROLE,
         TurnParams.CONTENT,
         TurnParams.RETRIEVAL_CONTEXT,
     ]
@@ -46,6 +48,7 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
         verbose_mode: bool = False,
         truths_extraction_limit: Optional[int] = None,
         penalize_ambiguous_claims: bool = False,
+        window_size: int = 10,
         evaluation_template: Type[
             TurnFaithfulnessTemplate
         ] = TurnFaithfulnessTemplate,
@@ -59,6 +62,7 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
         self.verbose_mode = verbose_mode
         self.evaluation_template = evaluation_template
         self.penalize_ambiguous_claims = penalize_ambiguous_claims
+        self.window_size = window_size
 
         self.truths_extraction_limit = truths_extraction_limit
         if self.truths_extraction_limit is not None:
@@ -98,9 +102,17 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
                 )
             else:
                 unit_interactions = get_unit_interactions(test_case.turns)
-                scores = self._get_faithfulness_scores(
-                    unit_interactions, multimodal
-                )
+                turns_windows: List[List[Turn]] = [
+                    list(itertools.chain(*window))
+                    for window in get_turns_in_sliding_window(
+                        unit_interactions, self.window_size
+                    )
+                ]
+                scores = []
+                for window in turns_windows:
+                    scores.extend(
+                        self._get_faithfulness_scores(window, multimodal)
+                    )
                 self.score = self._calculate_score(scores)
                 self.success = self.score >= self.threshold
                 self.reason = self._generate_reason(scores)
@@ -146,9 +158,23 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             _in_component=_in_component,
         ):
             unit_interactions = get_unit_interactions(test_case.turns)
-            scores = await self._a_get_faithfulness_scores(
-                unit_interactions, multimodal
-            )
+            turns_windows: List[List[Turn]] = [
+                list(itertools.chain(*window))
+                for window in get_turns_in_sliding_window(
+                    unit_interactions, self.window_size
+                )
+            ]
+            scores = []
+            tasks = []
+
+            async def get_individual_scores(window):
+                scores.extend(
+                    await self._a_get_faithfulness_scores(window, multimodal)
+                )
+
+            for window in turns_windows:
+                tasks.append(get_individual_scores(window))
+            await asyncio.gather(*tasks)
             self.score = self._calculate_score(scores)
             self.success = self.score >= self.threshold
             self.reason = await self._a_generate_reason(scores)
@@ -169,82 +195,75 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
             return self.score
 
     async def _a_get_faithfulness_scores(
-        self, unit_interactions: List[List[Turn]], multimodal: bool
+        self, turns_window: List[Turn], multimodal: bool
     ):
 
-        async def get_interaction_score(unit_interaction: List[Turn]):
-            user_content = "User Message: "
-            retrieval_context = []
-            assistant_content = "Assistant Message: "
-            for turn in unit_interaction:
-                if turn.role == "user":
-                    user_content += f"\n{turn.content} "
-                else:
-                    assistant_content += f"\n{turn.content} "
+        windows_scores = []
+
+        user_content = ""
+        assistant_content = ""
+        retrieval_context = []
+        for turn in turns_window:
+            if turn.role == "user":
+                user_content += f"\n{turn.content} "
+            else:
+                assistant_content += f"\n{turn.content}"
+                if turn.retrieval_context is not None:
                     retrieval_context.extend(turn.retrieval_context)
-            truths = await self._a_generate_truths(
-                retrieval_context, multimodal
-            )
-            claims = await self._a_generate_claims(
-                user_content, assistant_content, multimodal
-            )
-            verdicts = await self._a_generate_verdicts(
-                claims, truths, multimodal
-            )
-            score, reason = self._get_interaction_score_and_reason(
-                verdicts, multimodal
-            )
-            interaction_score = InteractionFaithfulnessScore(
-                score=score,
-                reason=reason,
-                claims=claims,
-                truths=truths,
-                verdicts=verdicts,
-            )
-            return interaction_score
 
-        final_scores = await asyncio.gather(
-            *[
-                get_interaction_score(unit_interaction)
-                for unit_interaction in unit_interactions
-            ]
+        truths = await self._a_generate_truths(retrieval_context, multimodal)
+        claims = await self._a_generate_claims(
+            user_content, assistant_content, multimodal
+        )
+        verdicts = await self._a_generate_verdicts(claims, truths, multimodal)
+        score, reason = self._get_interaction_score_and_reason(
+            verdicts, multimodal
         )
+        interaction_score = InteractionFaithfulnessScore(
+            score=score,
+            reason=reason,
+            claims=claims,
+            truths=truths,
+            verdicts=verdicts,
+        )
+        windows_scores.append(interaction_score)
 
-        return final_scores
+        return windows_scores
 
     def _get_faithfulness_scores(
-        self, unit_interactions: List[List[Turn]], multimodal: bool
+        self, turns_window: List[Turn], multimodal: bool
     ):
-        interaction_scores = []
-
-        for unit_interaction in unit_interactions:
-            user_content = "User Message: "
-            retrieval_context = []
-            assistant_content = "Assistant Message: "
-            for turn in unit_interaction:
-                if turn.role == "user":
-                    user_content += f"\n{turn.content} "
-                else:
-                    assistant_content += f"\n{turn.content} "
+        windows_scores = []
+
+        user_content = ""
+        assistant_content = ""
+        retrieval_context = []
+        for turn in turns_window:
+            if turn.role == "user":
+                user_content += f"\n{turn.content} "
+            else:
+                assistant_content += f"\n{turn.content}"
+                if turn.retrieval_context is not None:
                     retrieval_context.extend(turn.retrieval_context)
-            truths = self._generate_truths(retrieval_context, multimodal)
-            claims = self._generate_claims(
-                user_content, assistant_content, multimodal
-            )
-            verdicts = self._generate_verdicts(claims, truths, multimodal)
-            score, reason = self._get_interaction_score_and_reason(
-                verdicts, multimodal
-            )
-            interaction_score = InteractionFaithfulnessScore(
-                score=score,
-                reason=reason,
-                claims=claims,
-                truths=truths,
-                verdicts=verdicts,
-            )
-            interaction_scores.append(interaction_score)
 
-        return interaction_scores
+        truths = self._generate_truths(retrieval_context, multimodal)
+        claims = self._generate_claims(
+            user_content, assistant_content, multimodal
+        )
+        verdicts = self._generate_verdicts(claims, truths, multimodal)
+        score, reason = self._get_interaction_score_and_reason(
+            verdicts, multimodal
+        )
+        interaction_score = InteractionFaithfulnessScore(
+            score=score,
+            reason=reason,
+            claims=claims,
+            truths=truths,
+            verdicts=verdicts,
+        )
+        windows_scores.append(interaction_score)
+
+        return windows_scores
 
     async def _a_generate_truths(
         self, retrieval_context: str, multimodal: bool
@@ -522,7 +541,7 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
         steps = []
         for index, interaction_score in enumerate(interaction_scores):
             interaction_steps = [
-                f"Interaction {index + 1} \n",
+                f"Window {index + 1} \n",
                 f"Truths: {prettify_list(interaction_score.truths)} \n",
                 f"Claims: {prettify_list(interaction_score.claims)} \n",
                 f"Verdicts: {prettify_list(interaction_score.verdicts)} \n",
@@ -535,6 +554,12 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
     def _generate_reason(
         self, scores: List[InteractionFaithfulnessScore]
     ) -> str:
+        if self.include_reason is False:
+            return None
+
+        if len(scores) == 0:
+            return "There were no retrieval contexts in your turns to evaluate, hence the score is 1"
+
         reasons = []
         for score in scores:
             reasons.append(score.reason)
@@ -554,6 +579,12 @@ class TurnFaithfulnessMetric(BaseConversationalMetric):
     async def _a_generate_reason(
         self, scores: List[InteractionFaithfulnessScore]
     ) -> str:
+        if self.include_reason is False:
+            return None
+
+        if len(scores) == 0:
+            return "There were no retrieval contexts in your turns to evaluate, hence the score is 1"
+
         reasons = []
         for score in scores:
             reasons.append(score.reason)

deepeval/metrics/turn_relevancy/template.py

@@ -2,9 +2,20 @@ from typing import List, Dict
 
 
 class TurnRelevancyTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
+
     @staticmethod
     def generate_verdicts(sliding_window: List[Dict]):
         return f"""Based on the given list of message exchanges between a user and an LLM, generate a JSON object to indicate whether the LAST `assistant` message is relevant to context in messages. The JSON will have 2 fields: 'verdict' and 'reason'.
+
+{TurnRelevancyTemplate.multimodal_rules}
+
 The 'verdict' key should STRICTLY be either 'yes' or 'no', which states whether the last `assistant` message is relevant according to the context in messages 
 Provide a 'reason' ONLY if the answer is 'no'. 
 You MUST USE the previous messages (if any) provided in the list of messages to make an informed judgement on relevancy.
@@ -52,6 +63,9 @@ JSON:
     @staticmethod
     def generate_reason(score, irrelevancies):
         return f"""Below is a list of irrelevancies drawn from some messages in a conversation, which you have minimal knowledge of. It is a list of strings explaining why the 'assistant' messages are irrelevant to the 'user' messages.
+
+{TurnRelevancyTemplate.multimodal_rules}
+
 Given the relevancy score, which is a 0-1 score indicating how irrelevant the OVERALL AI messages are in a conversation (higher the better), CONCISELY summarize the irrelevancies to justify the score. 
 
 **