deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/metrics/turn_contextual_precision/turn_contextual_precision.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Union, Type, Tuple
 import asyncio
-
+import itertools
 from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.utils import (
@@ -12,7 +12,10 @@ from deepeval.metrics.utils import (
     trimAndLoadJson,
     check_conversational_test_case_params,
     get_unit_interactions,
+    get_turns_in_sliding_window,
     initialize_model,
+    a_generate_with_schema_and_extract,
+    generate_with_schema_and_extract,
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.turn_contextual_precision.template import (
@@ -30,6 +33,7 @@ from deepeval.metrics.api import metric_data_manager
 
 class TurnContextualPrecisionMetric(BaseConversationalMetric):
     _required_test_case_params: List[TurnParams] = [
+        TurnParams.ROLE,
         TurnParams.CONTENT,
         TurnParams.RETRIEVAL_CONTEXT,
         TurnParams.EXPECTED_OUTCOME,
@@ -43,6 +47,7 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
+        window_size: int = 10,
         evaluation_template: Type[
             TurnContextualPrecisionTemplate
         ] = TurnContextualPrecisionTemplate,
@@ -54,6 +59,7 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
         self.async_mode = async_mode
         self.strict_mode = strict_mode
         self.verbose_mode = verbose_mode
+        self.window_size = window_size
         self.evaluation_template = evaluation_template
 
     def measure(
@@ -90,9 +96,19 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
                 )
             else:
                 unit_interactions = get_unit_interactions(test_case.turns)
-                scores = self._get_contextual_precision_scores(
-                    unit_interactions, test_case.expected_outcome, multimodal
-                )
+                turns_windows: List[List[Turn]] = [
+                    list(itertools.chain(*window))
+                    for window in get_turns_in_sliding_window(
+                        unit_interactions, self.window_size
+                    )
+                ]
+                scores = []
+                for window in turns_windows:
+                    scores.extend(
+                        self._get_contextual_precision_scores(
+                            window, test_case.expected_outcome, multimodal
+                        )
+                    )
                 self.score = self._calculate_score(scores)
                 self.success = self.score >= self.threshold
                 self.reason = self._generate_reason(scores)
@@ -138,9 +154,25 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             _in_component=_in_component,
         ):
             unit_interactions = get_unit_interactions(test_case.turns)
-            scores = await self._a_get_contextual_precision_scores(
-                unit_interactions, test_case.expected_outcome, multimodal
-            )
+            turns_windows: List[List[Turn]] = [
+                list(itertools.chain(*window))
+                for window in get_turns_in_sliding_window(
+                    unit_interactions, self.window_size
+                )
+            ]
+            scores = []
+            tasks = []
+
+            async def get_individual_scores(window):
+                scores.extend(
+                    await self._a_get_contextual_precision_scores(
+                        window, test_case.expected_outcome, multimodal
+                    )
+                )
+
+            for window in turns_windows:
+                tasks.append(get_individual_scores(window))
+            await asyncio.gather(*tasks)
             self.score = self._calculate_score(scores)
             self.success = self.score >= self.threshold
             self.reason = await self._a_generate_reason(scores)
@@ -162,78 +194,73 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
 
     async def _a_get_contextual_precision_scores(
         self,
-        unit_interactions: List[List[Turn]],
-        _expected_outcome: str,
+        turns_window: List[Turn],
+        expected_outcome: str,
         multimodal: bool,
     ):
-        async def get_interaction_score(unit_interaction: List[Turn]):
-            user_content = "User Message: "
-            retrieval_context = []
-            expected_outcome = (
-                f"Expected Assistant Message: \n{_expected_outcome}"
-            )
-            for turn in unit_interaction:
-                if turn.role == "user":
-                    user_content += f"\n{turn.content} "
-                else:
-                    retrieval_context.extend(turn.retrieval_context)
+        windows_scores = []
 
-            verdicts = await self._a_generate_verdicts(
-                user_content, expected_outcome, retrieval_context, multimodal
-            )
-            score, reason = await self._a_get_interaction_score_and_reason(
-                user_content, verdicts, multimodal
-            )
-            interaction_score = InteractionContextualPrecisionScore(
-                score=score,
-                reason=reason,
-                verdicts=verdicts,
-            )
-            return interaction_score
+        user_content = ""
+        retrieval_context = []
+        for turn in turns_window:
+            if turn.role == "user":
+                user_content += f"\n{turn.content} "
+            else:
+                if turn.retrieval_context is not None:
+                    retrieval_context.extend(turn.retrieval_context)
 
-        final_scores = await asyncio.gather(
-            *[
-                get_interaction_score(unit_interaction)
-                for unit_interaction in unit_interactions
-            ]
+        verdicts = await self._a_generate_verdicts(
+            user_content,
+            expected_outcome,
+            retrieval_context,
+            multimodal,
         )
+        score, reason = await self._a_get_interaction_score_and_reason(
+            user_content, verdicts, multimodal
+        )
+        interaction_score = InteractionContextualPrecisionScore(
+            score=score,
+            reason=reason,
+            verdicts=verdicts,
+        )
+        windows_scores.append(interaction_score)
 
-        return final_scores
+        return windows_scores
 
     def _get_contextual_precision_scores(
         self,
-        unit_interactions: List[List[Turn]],
-        _expected_outcome: str,
+        turns_window: List[Turn],
+        expected_outcome: str,
         multimodal: bool,
     ):
-        interaction_scores = []
+        windows_scores = []
 
-        for unit_interaction in unit_interactions:
-            user_content = "User Message: "
-            retrieval_context = []
-            expected_outcome = (
-                f"Expected Assistant Message: \n{_expected_outcome}"
-            )
-            for turn in unit_interaction:
-                if turn.role == "user":
-                    user_content += f"\n{turn.content} "
-                else:
+        user_content = ""
+        retrieval_context = []
+        for turn in turns_window:
+            if turn.role == "user":
+                user_content += f"\n{turn.content} "
+            else:
+                if turn.retrieval_context is not None:
                     retrieval_context.extend(turn.retrieval_context)
 
-            verdicts = self._generate_verdicts(
-                user_content, expected_outcome, retrieval_context, multimodal
-            )
-            score, reason = self._get_interaction_score_and_reason(
-                user_content, verdicts, multimodal
-            )
-            interaction_score = InteractionContextualPrecisionScore(
-                score=score,
-                reason=reason,
-                verdicts=verdicts,
-            )
-            interaction_scores.append(interaction_score)
+        verdicts = self._generate_verdicts(
+            user_content,
+            expected_outcome,
+            retrieval_context,
+            multimodal,
+        )
+        score, reason = self._get_interaction_score_and_reason(
+            user_content, verdicts, multimodal
+        )
+        interaction_score = InteractionContextualPrecisionScore(
+            score=score,
+            reason=reason,
+            verdicts=verdicts,
+        )
+        windows_scores.append(interaction_score)
 
-        return interaction_scores
+        return windows_scores
 
     async def _a_generate_verdicts(
         self,
@@ -254,26 +281,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     def _generate_verdicts(
         self,
@@ -294,24 +308,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=Verdicts,
+            extract_schema=lambda s: s.verdicts,
+            extract_json=lambda data: data["verdicts"],
+        )
 
     async def _a_get_interaction_score_and_reason(
         self,
@@ -320,7 +323,10 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
         multimodal: bool,
     ) -> Tuple[float, str]:
         if len(verdicts) == 0:
-            return 1, None
+            return (
+                1,
+                "There were no retrieval contexts in the given turns to evaluate the contextual precision.",
+            )
 
         score = self._calculate_interaction_score(verdicts)
         reason = await self._a_get_interaction_reason(
@@ -339,7 +345,10 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
         multimodal: bool,
     ) -> Tuple[float, str]:
         if len(verdicts) == 0:
-            return 1, None
+            return (
+                1,
+                "There were no retrieval contexts in the given turns to evaluate the contextual precision.",
+            )
 
         score = self._calculate_interaction_score(verdicts)
         reason = self._get_interaction_reason(
@@ -376,7 +385,6 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
         if relevant_nodes_count == 0:
             return 0
 
-        # Calculate Average Precision
         score = sum_weighted_precision_at_k / relevant_nodes_count
         return 0 if self.strict_mode and score < self.threshold else score
 
@@ -408,24 +416,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualPrecisionScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualPrecisionScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=ContextualPrecisionScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_interaction_reason(
         self,
@@ -455,22 +452,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             multimodal=multimodal,
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualPrecisionScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: ContextualPrecisionScoreReason = self.model.generate(
-                    prompt, schema=ContextualPrecisionScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _get_verbose_steps(
         self, interaction_scores: List[InteractionContextualPrecisionScore]
@@ -478,7 +466,7 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
         steps = []
         for index, interaction_score in enumerate(interaction_scores):
             interaction_steps = [
-                f"Interaction {index + 1} \n",
+                f"Window {index + 1} \n",
                 f"Verdicts: {prettify_list(interaction_score.verdicts)} \n",
                 f"Score: {interaction_score.score} \n",
                 f"Reason: {interaction_score.reason} \n",
@@ -489,6 +477,12 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
     def _generate_reason(
         self, scores: List[InteractionContextualPrecisionScore]
     ) -> str:
+        if self.include_reason is False:
+            return None
+
+        if len(scores) == 0:
+            return "There were no retrieval contexts in your turns to evaluate, hence the score is 1"
+
         reasons = []
         for score in scores:
             reasons.append(score.reason)
@@ -497,17 +491,23 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = self.model.generate(prompt)
-            return res
+        return generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     async def _a_generate_reason(
         self, scores: List[InteractionContextualPrecisionScore]
     ) -> str:
+        if self.include_reason is False:
+            return None
+
+        if len(scores) == 0:
+            return "There were no retrieval contexts in your turns to evaluate, hence the score is 1"
+
         reasons = []
         for score in scores:
             reasons.append(score.reason)
@@ -516,13 +516,13 @@ class TurnContextualPrecisionMetric(BaseConversationalMetric):
             self.score, self.success, reasons
         )
 
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
-            self.evaluation_cost += cost
-            return res
-        else:
-            res = await self.model.a_generate(prompt)
-            return res
+        return await a_generate_with_schema_and_extract(
+            metric=self,
+            prompt=prompt,
+            schema_cls=ContextualPrecisionScoreReason,
+            extract_schema=lambda s: s.reason,
+            extract_json=lambda data: data["reason"],
+        )
 
     def _calculate_score(
         self, scores: List[InteractionContextualPrecisionScore]

deepeval/metrics/turn_contextual_recall/schema.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 from pydantic import BaseModel
 
 
@@ -17,5 +17,5 @@ class ContextualRecallScoreReason(BaseModel):
 
 class InteractionContextualRecallScore(BaseModel):
     score: float
-    reason: str
-    verdicts: List[ContextualRecallVerdict]
+    reason: Optional[str]
+    verdicts: Optional[List[ContextualRecallVerdict]]

deepeval/metrics/turn_contextual_recall/template.py

@@ -125,6 +125,13 @@ class TurnContextualRecallTemplate:
             Context:
             This metric evaluates conversational contextual recall by determining whether sentences in the assistant output can be attributed to the retrieval context for each interaction. Each interaction yields a reason indicating which sentences were supported or unsupported. You are given all those reasons.
 
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_recall_score> because <your_reason>."
+            }}
+
             Inputs:
             - final_score: the averaged score across all interactions.
             - success: whether the metric passed or failed
@@ -151,7 +158,7 @@ class TurnContextualRecallTemplate:
 
             Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
 
-            The final reason:
+            JSON:
             """
         )