deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_contextual_recall/multimodal_contextual_recall.py

@@ -1,285 +0,0 @@
-from typing import Optional, List, Union
-
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
-from deepeval.metrics.multimodal_metrics.multimodal_contextual_recall.template import (
-    MultimodalContextualRecallTemplate,
-)
-from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.utils import (
-    construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
-)
-from deepeval.models import DeepEvalBaseMLLM
-from deepeval.metrics.multimodal_metrics.multimodal_contextual_recall.schema import *
-from deepeval.metrics.indicator import metric_progress_indicator
-
-
-class MultimodalContextualRecallMetric(BaseMultimodalMetric):
-
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
-        MLLMTestCaseParams.RETRIEVAL_CONTEXT,
-        MLLMTestCaseParams.EXPECTED_OUTPUT,
-    ]
-
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        self.threshold = 1 if strict_mode else threshold
-        self.model, self.using_native_model = initialize_multimodal_model(model)
-        self.evaluation_model = self.model.get_model_name()
-        self.include_reason = include_reason
-        self.async_mode = async_mode
-        self.strict_mode = strict_mode
-        self.verbose_mode = verbose_mode
-
-    def measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _log_metric_to_confident=_log_metric_to_confident,
-                    )
-                )
-            else:
-                self.verdicts: List[ContextualRecallVerdict] = (
-                    self._generate_verdicts(
-                        test_case.expected_output, test_case.retrieval_context
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.expected_output)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = construct_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{prettify_list(self.verdicts)}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            async_mode=True,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            self.verdicts: List[ContextualRecallVerdict] = (
-                await self._a_generate_verdicts(
-                    test_case.expected_output, test_case.retrieval_context
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(
-                test_case.expected_output
-            )
-            self.success = self.score >= self.threshold
-            self.verbose_logs = construct_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{prettify_list(self.verdicts)}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(
-        self, expected_output: List[Union[str, MLLMImage]]
-    ):
-        if self.include_reason is False:
-            return None
-
-        supportive_reasons = []
-        unsupportive_reasons = []
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                supportive_reasons.append(verdict.reason)
-            else:
-                unsupportive_reasons.append(verdict.reason)
-
-        prompt = MultimodalContextualRecallTemplate.generate_reason(
-            expected_output=expected_output,
-            supportive_reasons=supportive_reasons,
-            unsupportive_reasons=unsupportive_reasons,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=MultimodalContextualRecallScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodalContextualRecallScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=MultimodalContextualRecallScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, expected_output: List[Union[str, MLLMImage]]):
-        if self.include_reason is False:
-            return None
-
-        supportive_reasons = []
-        unsupportive_reasons = []
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                supportive_reasons.append(verdict.reason)
-            else:
-                unsupportive_reasons.append(verdict.reason)
-
-        prompt = MultimodalContextualRecallTemplate.generate_reason(
-            expected_output=expected_output,
-            supportive_reasons=supportive_reasons,
-            unsupportive_reasons=unsupportive_reasons,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=MultimodalContextualRecallScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodalContextualRecallScoreReason = (
-                    self.model.generate(
-                        prompt, schema=MultimodalContextualRecallScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _calculate_score(self):
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        justified_sentences = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.lower() == "yes":
-                justified_sentences += 1
-
-        score = justified_sentences / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    async def _a_generate_verdicts(
-        self,
-        expected_output: List[Union[str, MLLMImage]],
-        retrieval_context: List[Union[str, MLLMImage]],
-    ) -> List[ContextualRecallVerdict]:
-        prompt = MultimodalContextualRecallTemplate.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts: Verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self,
-        expected_output: List[Union[str, MLLMImage]],
-        retrieval_context: List[Union[str, MLLMImage]],
-    ) -> List[ContextualRecallVerdict]:
-        prompt = MultimodalContextualRecallTemplate.generate_verdicts(
-            expected_output=expected_output, retrieval_context=retrieval_context
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts: Verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts: Verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    ContextualRecallVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def is_successful(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Multimodal Contextual Recall"

deepeval/metrics/multimodal_metrics/multimodal_contextual_recall/schema.py

@@ -1,15 +0,0 @@
-from typing import List, Optional
-from pydantic import BaseModel, Field
-
-
-class ContextualRecallVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[ContextualRecallVerdict]
-
-
-class MultimodalContextualRecallScoreReason(BaseModel):
-    reason: str

deepeval/metrics/multimodal_metrics/multimodal_contextual_recall/template.py

@@ -1,112 +0,0 @@
-from typing import Union, List
-import textwrap
-
-from deepeval.test_case import MLLMImage
-
-
-class MultimodalContextualRecallTemplate:
-    @staticmethod
-    def generate_reason(
-        expected_output, supportive_reasons, unsupportive_reasons, score
-    ) -> List[Union[str, MLLMImage]]:
-        return (
-            [
-                textwrap.dedent(
-                    f"""Given the original expected output, a list of supportive reasons, and a list of unsupportive reasons (which is deduced directly from the 'expected output'), and a contextual recall score (closer to 1 the better), summarize a CONCISE reason for the score.
-                    A supportive reason is the reason why a certain sentence or image in the original expected output can be attributed to the node in the retrieval context.
-                    An unsupportive reason is the reason why a certain sentence or image in the original expected output cannot be attributed to anything in the retrieval context.
-                    In your reason, you should related supportive/unsupportive reasons to the sentence or image number in expected output, and info regarding the node number in retrieval context to support your final reason. The first mention of "node(s)" should specify "node(s) in retrieval context)".
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-                    Example JSON:
-                    {{
-                        "reason": "The score is <contextual_recall_score> because <your_reason>."
-                    }}
-
-                    DO NOT mention 'supportive reasons' and 'unsupportive reasons' in your reason, these terms are just here for you to understand the broader scope of things.
-                    If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-                    **
-
-                    Contextual Recall Score:
-                    {score}
-
-                    Expected Output:
-                    """
-                )
-            ]
-            + expected_output
-            + [
-                textwrap.dedent(
-                    f"""Supportive Reasons:
-                    {supportive_reasons}
-
-                    Unsupportive Reasons:
-                    {unsupportive_reasons}
-
-                    JSON:"""
-                )
-            ]
-        )
-
-    @staticmethod
-    def generate_verdicts(
-        expected_output, retrieval_context
-    ) -> List[Union[str, MLLMImage]]:
-        return (
-            [
-                textwrap.dedent(
-                    f"""For EACH sentence and image in the given expected output below, determine whether the sentence or image can be attributed to the nodes of retrieval contexts. Please generate a list of JSON with two keys: `verdict` and `reason`.
-                    The `verdict` key should STRICTLY be either a 'yes' or 'no'. Answer 'yes' if the sentence or image can be attributed to any parts of the retrieval context, else answer 'no'.
-                    The `reason` key should provide a reason why to the verdict. In the reason, you should aim to include the node(s) count in the retrieval context (eg., 1st node, and 2nd node in the retrieval context) that is attributed to said sentence or image. A node is either a string or image, but not both (so do not group images and texts in the same nodes). You should also aim to quote the specific part of the retrieval context to justify your verdict, but keep it extremely concise and cut short the quote with an ellipsis if possible. 
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects, each with two keys: `verdict` and `reason`.
-
-                    {{
-                        "verdicts": [
-                            {{
-                                "reason": "...",
-                                "verdict": "yes"
-                            }},
-                            ...
-                        ]  
-                    }}
-
-                    Since you are going to generate a verdict for each sentence, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of sentences and images in the `expected output`.
-                    **
-
-                    Expected Output:
-                    """
-                )
-            ]
-            + expected_output
-            + [
-                textwrap.dedent(
-                    """
-                    Retrieval Context:
-                    """
-                )
-            ]
-            + MultimodalContextualRecallTemplate.id_retrieval_context(
-                retrieval_context
-            )
-            + [
-                textwrap.dedent(
-                    """
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def id_retrieval_context(retrieval_context) -> List[Union[str, MLLMImage]]:
-        annotated_retrieval_context = []
-        for i, context in enumerate(retrieval_context):
-            if isinstance(context, str):
-                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
-            elif isinstance(context, MLLMImage):
-                annotated_retrieval_context.append(f"Node {i + 1}:")
-                annotated_retrieval_context.append(context)
-        return annotated_retrieval_context

deepeval/metrics/multimodal_metrics/multimodal_contextual_relevancy/multimodal_contextual_relevancy.py

@@ -1,282 +0,0 @@
-from typing import Optional, List, Union
-import asyncio
-
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
-from deepeval.metrics.multimodal_metrics.multimodal_contextual_relevancy.template import (
-    MultimodalContextualRelevancyTemplate,
-)
-from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.utils import (
-    construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
-)
-
-from deepeval.models import DeepEvalBaseMLLM
-from deepeval.metrics.multimodal_metrics.multimodal_contextual_relevancy.schema import *
-from deepeval.metrics.indicator import metric_progress_indicator
-
-
-class MultimodalContextualRelevancyMetric(BaseMultimodalMetric):
-
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
-        MLLMTestCaseParams.RETRIEVAL_CONTEXT,
-    ]
-
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        self.threshold = 1 if strict_mode else threshold
-        self.model, self.using_native_model = initialize_multimodal_model(model)
-        self.evaluation_model = self.model.get_model_name()
-        self.include_reason = include_reason
-        self.async_mode = async_mode
-        self.strict_mode = strict_mode
-        self.verbose_mode = verbose_mode
-
-    def measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _log_metric_to_confident=_log_metric_to_confident,
-                    )
-                )
-            else:
-                self.verdicts_list: List[ContextualRelevancyVerdicts] = [
-                    (self._generate_verdicts(test_case.input, context))
-                    for context in test_case.retrieval_context
-                ]
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = construct_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{prettify_list(self.verdicts_list)}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            async_mode=True,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            self.verdicts_list: List[ContextualRelevancyVerdicts] = (
-                await asyncio.gather(
-                    *[
-                        self._a_generate_verdicts(test_case.input, context)
-                        for context in test_case.retrieval_context
-                    ]
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
-            self.success = self.score >= self.threshold
-            self.verbose_logs = construct_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{prettify_list(self.verdicts_list)}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(self, input: List[Union[str, MLLMImage]]):
-        if self.include_reason is False:
-            return None
-
-        irrelevancies = []
-        relevant_statements = []
-        for verdicts in self.verdicts_list:
-            for verdict in verdicts.verdicts:
-                if verdict.verdict.lower() == "no":
-                    irrelevancies.append(verdict.reason)
-                else:
-                    relevant_statements.append(verdict.statement)
-
-        prompt: dict = MultimodalContextualRelevancyTemplate.generate_reason(
-            input=input,
-            irrelevancies=irrelevancies,
-            relevant_statements=relevant_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=MultimodelContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodelContextualRelevancyScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=MultimodelContextualRelevancyScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, input: List[Union[str, MLLMImage]]):
-        if self.include_reason is False:
-            return None
-
-        irrelevancies = []
-        relevant_statements = []
-        for verdicts in self.verdicts_list:
-            for verdict in verdicts.verdicts:
-                if verdict.verdict.lower() == "no":
-                    irrelevancies.append(verdict.reason)
-                else:
-                    relevant_statements.append(verdict.statement)
-
-        prompt: dict = MultimodalContextualRelevancyTemplate.generate_reason(
-            input=input,
-            irrelevancies=irrelevancies,
-            relevant_statements=relevant_statements,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=MultimodelContextualRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodelContextualRelevancyScoreReason = (
-                    self.model.generate(
-                        prompt, schema=MultimodelContextualRelevancyScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _calculate_score(self):
-        total_verdicts = 0
-        relevant_statements = 0
-        for verdicts in self.verdicts_list:
-            for verdict in verdicts.verdicts:
-                total_verdicts += 1
-                if verdict.verdict.lower() == "yes":
-                    relevant_statements += 1
-
-        if total_verdicts == 0:
-            return 0
-
-        score = relevant_statements / total_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    async def _a_generate_verdicts(
-        self,
-        input: List[Union[str, MLLMImage]],
-        context: List[Union[str, MLLMImage]],
-    ) -> ContextualRelevancyVerdicts:
-        prompt = MultimodalContextualRelevancyTemplate.generate_verdicts(
-            input=input, context=context
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=ContextualRelevancyVerdicts
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = await self.model.a_generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ContextualRelevancyVerdicts(**data)
-
-    def _generate_verdicts(
-        self,
-        input: List[Union[str, MLLMImage]],
-        context: List[Union[str, MLLMImage]],
-    ) -> ContextualRelevancyVerdicts:
-        prompt = MultimodalContextualRelevancyTemplate.generate_verdicts(
-            input=input, context=context
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=ContextualRelevancyVerdicts
-            )
-            self.evaluation_cost += cost
-            return res
-        else:
-            try:
-                res = self.model.generate(
-                    prompt, schema=ContextualRelevancyVerdicts
-                )
-                return res
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return ContextualRelevancyVerdicts(**data)
-
-    def is_successful(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Multimodal Contextual Relevancy"