deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/turn_contextual_recall/template.py

@@ -0,0 +1,178 @@
+from typing import List, Union
+import textwrap
+from deepeval.test_case import MLLMImage
+
+
+class TurnContextualRecallTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+        - When evaluating claims, compare them to BOTH textual and visual evidence.
+        - If the claim references something not clearly visible, respond with 'idk'.
+    """
+
+    @staticmethod
+    def generate_reason(
+        expected_outcome: str,
+        supportive_reasons: str,
+        unsupportive_reasons: str,
+        score: float,
+        multimodal: bool = False,
+    ):
+        content_type = "sentence or image" if multimodal else "sentence"
+
+        return textwrap.dedent(
+            f"""Given the original assistant output, a list of supportive reasons, and a list of unsupportive reasons ({'which is' if multimodal else 'which are'} deduced directly from the {'"assistant output"' if multimodal else 'original assistant output'}), and a contextual recall score (closer to 1 the better), summarize a CONCISE reason for the score.
+            A supportive reason is the reason why a certain {content_type} in the original assistant output can be attributed to the node in the retrieval context.
+            An unsupportive reason is the reason why a certain {content_type} in the original assistant output cannot be attributed to anything in the retrieval context.
+            In your reason, you should {'related' if multimodal else 'relate'} supportive/unsupportive reasons to the {content_type} number in assistant output, and {'info' if multimodal else 'include info'} regarding the node number in retrieval context to support your final reason. The first mention of "node(s)" should specify "node(s) in retrieval context{')' if multimodal else ''}.
+
+            {TurnContextualRecallTemplate.multimodal_rules if multimodal else ""}
+            
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
+            Example JSON:
+            {{
+                "reason": "The score is <contextual_recall_score> because <your_reason>."
+            }}
+
+            DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
+            If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it{',' if multimodal else ''} otherwise it gets annoying).
+            **
+
+            Contextual Recall Score:
+            {score}
+
+            Assistant Output:
+            {expected_outcome}
+
+            Supportive Reasons:
+            {supportive_reasons}
+
+            Unsupportive Reasons:
+            {unsupportive_reasons}
+
+            JSON:
+            """
+        )
+
+    @staticmethod
+    def generate_verdicts(
+        expected_outcome: str,
+        retrieval_context: List[Union[str, MLLMImage]],
+        multimodal: bool = False,
+    ):
+        content_type = "sentence and image" if multimodal else "sentence"
+        content_type_plural = (
+            "sentences and images" if multimodal else "sentences"
+        )
+        content_or = "sentence or image" if multimodal else "sentence"
+
+        # For multimodal, we need to annotate the retrieval context with node IDs
+        context_to_display = (
+            TurnContextualRecallTemplate.id_retrieval_context(retrieval_context)
+            if multimodal
+            else retrieval_context
+        )
+
+        node_instruction = ""
+        if multimodal:
+            node_instruction = " A node is either a string or image, but not both (so do not group images and texts in the same nodes)."
+
+        return textwrap.dedent(
+            f"""For EACH {content_type} in the given assistant output below, determine whether the {content_or} can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: `verdict` and `reason`.
+            The `verdict` key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the {content_or} can be attributed to any parts of the retrieval context, else answer 'no'.
+            The `reason` key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said {content_or}.{node_instruction} You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
+
+            {TurnContextualRecallTemplate.multimodal_rules if multimodal else ""}
+            
+            **
+            IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: `verdict` and `reason`.
+
+            {{
+                "verdicts": [
+                    {{
+                        "reason": "...",
+                        "verdict": "yes"
+                    }},
+                    ...
+                ]  
+            }}
+
+            Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of {content_type_plural} in {'the' if multimodal else '`assistant output`'}{' `assistant output`' if multimodal else ''}.
+            **
+
+            Assistant Output:
+            {expected_outcome}
+
+            Retrieval Context:
+            {context_to_display}
+
+            JSON:
+            """
+        )
+
+    @staticmethod
+    def generate_final_reason(
+        final_score: float, success: bool, reasons: List[str]
+    ):
+        return textwrap.dedent(
+            f"""You are an AI evaluator producing a single final explanation for the TurnContextualRecallMetric result.
+
+            Context:
+            This metric evaluates conversational contextual recall by determining whether sentences in the assistant output can be attributed to the retrieval context for each interaction. Each interaction yields a reason indicating which sentences were supported or unsupported. You are given all those reasons.
+
+            Inputs:
+            - final_score: the averaged score across all interactions.
+            - success: whether the metric passed or failed
+            - reasons: a list of textual reasons generated from individual interactions.
+
+            Instructions:
+            1. Read all reasons and synthesize them into one unified explanation.
+            2. Describe patterns of unsupported sentences, missing context coverage, or well-attributed outputs if present.
+            3. Do not repeat every reason; merge them into a concise, coherent narrative.
+            4. If the metric failed, state the dominant failure modes. If it passed, state why the assistant output was well-supported by retrieval context.
+            5. Output a single paragraph with no lists, no bullets, no markup.
+
+            Output:
+            A single paragraph explaining the final outcome.
+
+            Here's the inputs:
+
+            Final Score: {final_score}
+            
+            Reasons: 
+            {reasons}
+
+            Success: {success}
+
+            Now give me a final reason that explains why the metric passed or failed. Output ONLY the reason and nothing else.
+
+            The final reason:
+            """
+        )
+
+    @staticmethod
+    def id_retrieval_context(
+        retrieval_context: List[Union[str, MLLMImage]],
+    ) -> List[Union[str, MLLMImage]]:
+        """
+        Annotates retrieval context with node IDs for multimodal processing.
+
+        Args:
+            retrieval_context: List of contexts (can be strings or MLLMImages)
+
+        Returns:
+            Annotated list with "Node X:" prefixes
+        """
+        annotated_retrieval_context = []
+        for i, context in enumerate(retrieval_context):
+            if isinstance(context, str):
+                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
+            elif isinstance(context, MLLMImage):
+                annotated_retrieval_context.append(f"Node {i + 1}:")
+                annotated_retrieval_context.append(context)
+        return annotated_retrieval_context

deepeval/metrics/turn_contextual_recall/turn_contextual_recall.py

@@ -0,0 +1,520 @@
+from typing import List, Optional, Union, Type, Tuple
+import asyncio
+
+from deepeval.test_case import ConversationalTestCase, TurnParams, Turn
+from deepeval.metrics import BaseConversationalMetric
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
+from deepeval.metrics.utils import (
+    construct_verbose_logs,
+    trimAndLoadJson,
+    check_conversational_test_case_params,
+    get_unit_interactions,
+    initialize_model,
+)
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.turn_contextual_recall.template import (
+    TurnContextualRecallTemplate,
+)
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.turn_contextual_recall.schema import (
+    ContextualRecallVerdict,
+    Verdicts,
+    ContextualRecallScoreReason,
+    InteractionContextualRecallScore,
+)
+from deepeval.metrics.api import metric_data_manager
+
+
+class TurnContextualRecallMetric(BaseConversationalMetric):
+    _required_test_case_params: List[TurnParams] = [
+        TurnParams.CONTENT,
+        TurnParams.RETRIEVAL_CONTEXT,
+        TurnParams.EXPECTED_OUTCOME,
+    ]
+
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+        include_reason: bool = True,
+        async_mode: bool = True,
+        strict_mode: bool = False,
+        verbose_mode: bool = False,
+        evaluation_template: Type[
+            TurnContextualRecallTemplate
+        ] = TurnContextualRecallTemplate,
+    ):
+        self.threshold = 1 if strict_mode else threshold
+        self.model, self.using_native_model = initialize_model(model)
+        self.evaluation_model = self.model.get_model_name()
+        self.include_reason = include_reason
+        self.async_mode = async_mode
+        self.strict_mode = strict_mode
+        self.verbose_mode = verbose_mode
+        self.evaluation_template = evaluation_template
+
+    def measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ):
+        check_conversational_test_case_params(
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
+        )
+
+        multimodal = test_case.multimodal
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            if self.async_mode:
+                loop = get_or_create_event_loop()
+                loop.run_until_complete(
+                    self.a_measure(
+                        test_case,
+                        _show_indicator=False,
+                        _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
+                    )
+                )
+            else:
+                unit_interactions = get_unit_interactions(test_case.turns)
+                scores = self._get_contextual_recall_scores(
+                    unit_interactions, test_case.expected_outcome, multimodal
+                )
+                self.score = self._calculate_score(scores)
+                self.success = self.score >= self.threshold
+                self.reason = self._generate_reason(scores)
+                verbose_steps = self._get_verbose_steps(scores)
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        *verbose_steps,
+                        f"Final Score: {self.score}\n",
+                        f"Final Reason: {self.reason}\n",
+                    ],
+                )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: ConversationalTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_conversational_test_case_params(
+            test_case,
+            self._required_test_case_params,
+            self,
+            False,
+            self.model,
+            test_case.multimodal,
+        )
+
+        multimodal = test_case.multimodal
+
+        self.evaluation_cost = 0 if self.using_native_model else None
+        with metric_progress_indicator(
+            self,
+            async_mode=True,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        ):
+            unit_interactions = get_unit_interactions(test_case.turns)
+            scores = await self._a_get_contextual_recall_scores(
+                unit_interactions, test_case.expected_outcome, multimodal
+            )
+            self.score = self._calculate_score(scores)
+            self.success = self.score >= self.threshold
+            self.reason = await self._a_generate_reason(scores)
+            verbose_steps = self._get_verbose_steps(scores)
+            self.verbose_logs = construct_verbose_logs(
+                self,
+                steps=[
+                    *verbose_steps,
+                    f"Final Score: {self.score}\n",
+                    f"Final Reason: {self.reason}\n",
+                ],
+            )
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def _a_get_contextual_recall_scores(
+        self,
+        unit_interactions: List[List[Turn]],
+        _expected_outcome: str,
+        multimodal: bool,
+    ):
+        async def get_interaction_score(unit_interaction: List[Turn]):
+            retrieval_context = []
+            expected_outcome = (
+                f"Expected Assistant Message: \n{_expected_outcome}"
+            )
+            for turn in unit_interaction:
+                if turn.role == "assistant":
+                    retrieval_context.extend(turn.retrieval_context)
+
+            verdicts = await self._a_generate_verdicts(
+                expected_outcome, retrieval_context, multimodal
+            )
+            score, reason = await self._a_get_interaction_score_and_reason(
+                expected_outcome, verdicts, multimodal
+            )
+            interaction_score = InteractionContextualRecallScore(
+                score=score,
+                reason=reason,
+                verdicts=verdicts,
+            )
+            return interaction_score
+
+        final_scores = await asyncio.gather(
+            *[
+                get_interaction_score(unit_interaction)
+                for unit_interaction in unit_interactions
+            ]
+        )
+
+        return final_scores
+
+    def _get_contextual_recall_scores(
+        self,
+        unit_interactions: List[List[Turn]],
+        _expected_outcome: str,
+        multimodal: bool,
+    ):
+        interaction_scores = []
+
+        for unit_interaction in unit_interactions:
+            retrieval_context = []
+            expected_outcome = (
+                f"Expected Assistant Message: \n{_expected_outcome}"
+            )
+            for turn in unit_interaction:
+                if turn.role == "assistant":
+                    retrieval_context.extend(turn.retrieval_context)
+
+            verdicts = self._generate_verdicts(
+                expected_outcome, retrieval_context, multimodal
+            )
+            score, reason = self._get_interaction_score_and_reason(
+                expected_outcome, verdicts, multimodal
+            )
+            interaction_score = InteractionContextualRecallScore(
+                score=score,
+                reason=reason,
+                verdicts=verdicts,
+            )
+            interaction_scores.append(interaction_score)
+
+        return interaction_scores
+
+    async def _a_generate_verdicts(
+        self,
+        expected_outcome: str,
+        retrieval_context: List[str],
+        multimodal: bool,
+    ) -> List[ContextualRecallVerdict]:
+        if len(retrieval_context) == 0:
+            return []
+
+        verdicts: List[ContextualRecallVerdict] = []
+
+        prompt = self.evaluation_template.generate_verdicts(
+            expected_outcome=expected_outcome,
+            retrieval_context=retrieval_context,
+            multimodal=multimodal,
+        )
+
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            self.evaluation_cost += cost
+            verdicts = [item for item in res.verdicts]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = await self.model.a_generate(
+                    prompt, schema=Verdicts
+                )
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                verdicts = [
+                    ContextualRecallVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    def _generate_verdicts(
+        self,
+        expected_outcome: str,
+        retrieval_context: List[str],
+        multimodal: bool,
+    ) -> List[ContextualRecallVerdict]:
+        if len(retrieval_context) == 0:
+            return []
+
+        verdicts: List[ContextualRecallVerdict] = []
+
+        prompt = self.evaluation_template.generate_verdicts(
+            expected_outcome=expected_outcome,
+            retrieval_context=retrieval_context,
+            multimodal=multimodal,
+        )
+
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt, schema=Verdicts)
+            self.evaluation_cost += cost
+            verdicts = [item for item in res.verdicts]
+            return verdicts
+        else:
+            try:
+                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                verdicts = [item for item in res.verdicts]
+                return verdicts
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                verdicts = [
+                    ContextualRecallVerdict(**item) for item in data["verdicts"]
+                ]
+                return verdicts
+
+    async def _a_get_interaction_score_and_reason(
+        self,
+        expected_outcome: str,
+        verdicts: List[ContextualRecallVerdict],
+        multimodal: bool,
+    ) -> Tuple[float, str]:
+        if len(verdicts) == 0:
+            return 1, None
+
+        score = self._calculate_interaction_score(verdicts)
+        reason = await self._a_get_interaction_reason(
+            expected_outcome, score, verdicts, multimodal
+        )
+        return (
+            (0, reason)
+            if self.strict_mode and score < self.threshold
+            else (score, reason)
+        )
+
+    def _get_interaction_score_and_reason(
+        self,
+        expected_outcome: str,
+        verdicts: List[ContextualRecallVerdict],
+        multimodal: bool,
+    ) -> Tuple[float, str]:
+        if len(verdicts) == 0:
+            return 1, None
+
+        score = self._calculate_interaction_score(verdicts)
+        reason = self._get_interaction_reason(
+            expected_outcome, score, verdicts, multimodal
+        )
+        return (
+            (0, reason)
+            if self.strict_mode and score < self.threshold
+            else (score, reason)
+        )
+
+    def _calculate_interaction_score(
+        self, verdicts: List[ContextualRecallVerdict]
+    ) -> float:
+        number_of_verdicts = len(verdicts)
+        if number_of_verdicts == 0:
+            return 1
+
+        attributable_count = 0
+        for verdict in verdicts:
+            if verdict.verdict.strip().lower() == "yes":
+                attributable_count += 1
+
+        score = attributable_count / number_of_verdicts
+        return 0 if self.strict_mode and score < self.threshold else score
+
+    async def _a_get_interaction_reason(
+        self,
+        expected_outcome: str,
+        score: float,
+        verdicts: List[ContextualRecallVerdict],
+        multimodal: bool,
+    ) -> str:
+        if self.include_reason is False:
+            return None
+
+        # Prepare verdicts with node information for reasoning
+        supportive_reasons = []
+        unsupportive_reasons = []
+        for verdict in verdicts:
+            if verdict.verdict.lower() == "yes":
+                supportive_reasons.append(verdict.reason)
+            else:
+                unsupportive_reasons.append(verdict.reason)
+
+        prompt = self.evaluation_template.generate_reason(
+            expected_outcome=expected_outcome,
+            supportive_reasons=supportive_reasons,
+            unsupportive_reasons=unsupportive_reasons,
+            score=format(score, ".2f"),
+            multimodal=multimodal,
+        )
+
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(
+                prompt, schema=ContextualRecallScoreReason
+            )
+            self.evaluation_cost += cost
+            return res.reason
+        else:
+            try:
+                res: ContextualRecallScoreReason = await self.model.a_generate(
+                    prompt, schema=ContextualRecallScoreReason
+                )
+                return res.reason
+            except TypeError:
+                res = await self.model.a_generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["reason"]
+
+    def _get_interaction_reason(
+        self,
+        expected_outcome: str,
+        score: float,
+        verdicts: List[ContextualRecallVerdict],
+        multimodal: bool,
+    ) -> str:
+        if self.include_reason is False:
+            return None
+
+        # Prepare verdicts with node information for reasoning
+        supportive_reasons = []
+        unsupportive_reasons = []
+        for verdict in verdicts:
+            if verdict.verdict.lower() == "yes":
+                supportive_reasons.append(verdict.reason)
+            else:
+                unsupportive_reasons.append(verdict.reason)
+
+        prompt = self.evaluation_template.generate_reason(
+            expected_outcome=expected_outcome,
+            supportive_reasons=supportive_reasons,
+            unsupportive_reasons=unsupportive_reasons,
+            score=format(score, ".2f"),
+            multimodal=multimodal,
+        )
+
+        if self.using_native_model:
+            res, cost = self.model.generate(
+                prompt, schema=ContextualRecallScoreReason
+            )
+            self.evaluation_cost += cost
+            return res.reason
+        else:
+            try:
+                res: ContextualRecallScoreReason = self.model.generate(
+                    prompt, schema=ContextualRecallScoreReason
+                )
+                return res.reason
+            except TypeError:
+                res = self.model.generate(prompt)
+                data = trimAndLoadJson(res, self)
+                return data["reason"]
+
+    def _get_verbose_steps(
+        self, interaction_scores: List[InteractionContextualRecallScore]
+    ):
+        steps = []
+        for index, interaction_score in enumerate(interaction_scores):
+            interaction_steps = [
+                f"Interaction {index + 1} \n",
+                f"Verdicts: {prettify_list(interaction_score.verdicts)} \n",
+                f"Score: {interaction_score.score} \n",
+                f"Reason: {interaction_score.reason} \n",
+            ]
+            steps.extend(interaction_steps)
+        return steps
+
+    def _generate_reason(
+        self, scores: List[InteractionContextualRecallScore]
+    ) -> str:
+        reasons = []
+        for score in scores:
+            reasons.append(score.reason)
+
+        prompt = self.evaluation_template.generate_final_reason(
+            self.score, self.success, reasons
+        )
+
+        if self.using_native_model:
+            res, cost = self.model.generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = self.model.generate(prompt)
+            return res
+
+    async def _a_generate_reason(
+        self, scores: List[InteractionContextualRecallScore]
+    ) -> str:
+        reasons = []
+        for score in scores:
+            reasons.append(score.reason)
+
+        prompt = self.evaluation_template.generate_final_reason(
+            self.score, self.success, reasons
+        )
+
+        if self.using_native_model:
+            res, cost = await self.model.a_generate(prompt)
+            self.evaluation_cost += cost
+            return res
+        else:
+            res = await self.model.a_generate(prompt)
+            return res
+
+    def _calculate_score(
+        self, scores: List[InteractionContextualRecallScore]
+    ) -> float:
+        number_of_scores = len(scores)
+        if number_of_scores == 0:
+            return 1
+        total_score = 0
+        for score in scores:
+            total_score += score.score
+        return total_score / number_of_scores
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Turn Contextual Recall"