deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/multimodal_answer_relevancy.py

@@ -1,343 +0,0 @@
-from typing import Optional, List, Union
-
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
-from deepeval.metrics.multimodal_metrics.multimodal_answer_relevancy.template import (
-    MultimodalAnswerRelevancyTemplate,
-)
-from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.utils import (
-    construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
-)
-from deepeval.models import DeepEvalBaseMLLM
-from deepeval.metrics.multimodal_metrics.multimodal_answer_relevancy.schema import *
-from deepeval.metrics.indicator import metric_progress_indicator
-
-
-class MultimodalAnswerRelevancyMetric(BaseMultimodalMetric):
-
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
-    ]
-
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        self.threshold = 1 if strict_mode else threshold
-        self.model, self.using_native_model = initialize_multimodal_model(model)
-        self.evaluation_model = self.model.get_model_name()
-        self.include_reason = include_reason
-        self.async_mode = async_mode
-        self.strict_mode = strict_mode
-        self.verbose_mode = verbose_mode
-
-    def measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _log_metric_to_confident=_log_metric_to_confident,
-                    )
-                )
-            else:
-                self.statements: List[str] = self._generate_statements(
-                    test_case.actual_output
-                )
-                self.verdicts: List[AnswerRelevancyVerdict] = (
-                    self._generate_verdicts(test_case.input)
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = construct_verbose_logs(
-                    self,
-                    steps=[
-                        f"Statements:\n{prettify_list(self.statements)}",
-                        f"Verdicts:\n{prettify_list(self.verdicts)}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            async_mode=True,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            self.statements: List[str] = await self._a_generate_statements(
-                test_case.actual_output
-            )
-            self.verdicts: List[AnswerRelevancyVerdict] = (
-                await self._a_generate_verdicts(test_case.input)
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
-            self.success = self.score >= self.threshold
-            self.verbose_logs = construct_verbose_logs(
-                self,
-                steps=[
-                    f"Statements:\n{prettify_list(self.statements)}",
-                    f"Verdicts:\n{prettify_list(self.verdicts)}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(
-        self,
-        input: List[Union[str, MLLMImage]],
-    ) -> str:
-        if self.include_reason is False:
-            return None
-
-        irrelevant_statements = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                irrelevant_statements.append(verdict.reason)
-
-        prompt = MultimodalAnswerRelevancyTemplate.generate_reason(
-            irrelevant_statements=irrelevant_statements,
-            input=input,
-            score=format(self.score, ".2f"),
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=MultimodelAnswerRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodelAnswerRelevancyScoreReason = (
-                    await self.model.a_generate(
-                        prompt=prompt,
-                        schema=MultimodelAnswerRelevancyScoreReason,
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _generate_reason(
-        self,
-        input: List[Union[str, MLLMImage]],
-    ) -> str:
-        if self.include_reason is False:
-            return None
-
-        irrelevant_statements = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                irrelevant_statements.append(verdict.reason)
-
-        prompt = MultimodalAnswerRelevancyTemplate.generate_reason(
-            irrelevant_statements=irrelevant_statements,
-            input=input,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=MultimodelAnswerRelevancyScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodelAnswerRelevancyScoreReason = self.model.generate(
-                    prompt=prompt, schema=MultimodelAnswerRelevancyScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(
-        self,
-        input: List[Union[str, MLLMImage]],
-    ) -> List[AnswerRelevancyVerdict]:
-        if len(self.statements) == 0:
-            return []
-
-        prompt = MultimodalAnswerRelevancyTemplate.generate_verdicts(
-            input=input,
-            actual_output=self.statements,
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    AnswerRelevancyVerdict(**item) for item in data["verdicts"]
-                ]
-
-    def _generate_verdicts(
-        self, input: List[Union[str, MLLMImage]]
-    ) -> List[AnswerRelevancyVerdict]:
-        if len(self.statements) == 0:
-            return []
-
-        prompt = MultimodalAnswerRelevancyTemplate.generate_verdicts(
-            input=input,
-            actual_output=self.statements,
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            return [item for item in res.verdicts]
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                return [item for item in res.verdicts]
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return [
-                    AnswerRelevancyVerdict(**item) for item in data["verdicts"]
-                ]
-
-    async def _a_generate_statements(
-        self,
-        actual_output: List[Union[str, MLLMImage]],
-    ) -> List[str]:
-        prompt = MultimodalAnswerRelevancyTemplate.generate_statements(
-            actual_output=[
-                ele for ele in actual_output if isinstance(ele, str)
-            ],
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Statements)
-            self.evaluation_cost += cost
-            statements: List[str] = res.statements + [
-                ele for ele in actual_output if isinstance(ele, MLLMImage)
-            ]
-            return statements
-        else:
-            try:
-                res: Statements = await self.model.a_generate(
-                    prompt, schema=Statements
-                )
-                statements: List[str] = res.statements + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                statements = data["statements"] + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-
-    def _generate_statements(
-        self,
-        actual_output: List[Union[str, MLLMImage]],
-    ) -> List[str]:
-        prompt = MultimodalAnswerRelevancyTemplate.generate_statements(
-            actual_output=[
-                ele for ele in actual_output if isinstance(ele, str)
-            ],
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Statements)
-            self.evaluation_cost += cost
-            statements = res.statements + [
-                ele for ele in actual_output if isinstance(ele, MLLMImage)
-            ]
-            return statements
-        else:
-            try:
-                res: Statements = self.model.generate(prompt, schema=Statements)
-                statements = res.statements + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                statements = data["statements"] + [
-                    ele for ele in actual_output if isinstance(ele, MLLMImage)
-                ]
-                return statements
-
-    def _calculate_score(self):
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 1
-
-        relevant_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() != "no":
-                relevant_count += 1
-
-        score = relevant_count / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def is_successful(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Multimodal Answer Relevancy"

deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/schema.py

@@ -1,19 +0,0 @@
-from typing import List, Optional
-from pydantic import BaseModel, Field
-
-
-class Statements(BaseModel):
-    statements: List[str]
-
-
-class AnswerRelevancyVerdict(BaseModel):
-    verdict: str
-    reason: Optional[str] = Field(default=None)
-
-
-class Verdicts(BaseModel):
-    verdicts: List[AnswerRelevancyVerdict]
-
-
-class MultimodelAnswerRelevancyScoreReason(BaseModel):
-    reason: str

deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/template.py

@@ -1,122 +0,0 @@
-from typing import Union, List
-import textwrap
-
-from deepeval.test_case import MLLMImage
-
-
-class MultimodalAnswerRelevancyTemplate:
-    @staticmethod
-    def generate_statements(actual_output: List[str]):
-
-        return textwrap.dedent(
-            f"""Given the text, breakdown and generate a list of statements presented. Ambiguous statements and single words can also be considered as statements.
-
-            Example:
-            Example text: Shoes. The shoes can be refunded at no extra cost. Thanks for asking the question!
-
-            {{
-                "statements": ["Shoes.", "Shoes can be refunded at no extra cost", "Thanks for asking the question!"]
-            }}
-            ===== END OF EXAMPLE ======
-                    
-            **
-            IMPORTANT: Please make sure to only return in JSON format, with the "statements" key mapping to a list of strings. No words or explanation is needed.
-            **
-
-            Text:
-            {actual_output}
-
-            JSON:
-            """
-        )
-
-    @staticmethod
-    def generate_verdicts(input, actual_output):
-        return (
-            [
-                textwrap.dedent(
-                    f"""For the provided list of statements (which can contain images), determine whether each statement or image is relevant to address the input.
-                    Please generate a list of JSON with two keys: `verdict` and `reason`.
-                    The 'verdict' key should STRICTLY be either a 'yes', 'idk' or 'no'. Answer 'yes' if the statement or image is relevant to addressing the original input, 'no' if the statement or image is irrelevant, and 'idk' if it is ambiguous (eg., not directly relevant but could be used as a supporting point to address the input).
-                    The 'reason' is the reason for the verdict.
-                    Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
-                    The provided statements are statements and images generated in the actual output.
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key mapping to a list of JSON objects.
-                    Example input: What should I do if there is an earthquake?
-                    Example statements: ["Shoes.", "Thanks for asking the question!", "Is there anything else I can help you with?", "Duck and hide"]
-                    Example JSON:
-                    {{
-                        "verdicts": [
-                            {{
-                                "reason": "The 'Shoes.' statement made in the actual output is completely irrelevant to the input, which asks about what to do in the event of an earthquake.",
-                                "verdict": "no"
-                            }},
-                            {{
-                                "reason": "The statement thanking the user for asking the question is not directly relevant to the input, but is not entirely irrelevant.",
-                                "verdict": "idk"
-                            }},
-                            {{
-                                "reason": "The question about whether there is anything else the user can help with is not directly relevant to the input, but is not entirely irrelevant.",
-                                "verdict": "idk"
-                            }},
-                            {{
-                                "verdict": "yes"
-                            }}
-                        ]  
-                    }}
-
-                    Since you are going to generate a verdict for each statement and image, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to the number of `statements`.
-                    **          
-
-                    Input:
-                    """
-                )
-            ]
-            + input
-            + [
-                textwrap.dedent(
-                    """
-                    Statements:
-                    """
-                )
-            ]
-            + actual_output
-            + [
-                textwrap.dedent(
-                    """
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def generate_reason(irrelevant_statements, input, score):
-        return textwrap.dedent(
-            f"""Given the answer relevancy score, the list of reasons of irrelevant statements made in the actual output, and the input, provide a CONCISE reason for the score. Explain why it is not higher, but also why it is at its current score.
-            The irrelevant statements represent things in the actual output that is irrelevant to addressing whatever is asked/talked about in the input.
-            If there is nothing irrelevant, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-
-
-            **
-            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-            Example JSON:
-            {{
-                "reason": "The score is <answer_relevancy_score> because <your_reason>."
-            }}
-            **
-
-            Answer Relevancy Score:
-            {score}
-
-            Reasons why the score can't be higher based on irrelevant statements in the actual output:
-            {irrelevant_statements}
-
-            Input:
-            {input}
-
-            JSON:
-            """
-        )