deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_contextual_relevancy/template.py

@@ -1,102 +0,0 @@
-from typing import Union, List
-import textwrap
-
-from deepeval.test_case import MLLMImage
-
-
-class MultimodalContextualRelevancyTemplate:
-    @staticmethod
-    def generate_reason(
-        input: List[Union[str, MLLMImage]],
-        irrelevancies: List[str],
-        relevant_statements: List[str],
-        score: float,
-    ):
-        return (
-            [
-                textwrap.dedent(
-                    f"""Based on the given input, reasons for why the retrieval context is irrelevant to the input, the statements in the retrieval context that is actually relevant to the retrieval context, and the contextual relevancy score (the closer to 1 the better), please generate a CONCISE reason for the score.
-                    In your reason, you should quote data provided in the reasons for irrelevancy and relevant statements to support your point.
-
-                    ** 
-                    IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-                    Example JSON:
-                    {{
-                        "reason": "The score is <contextual_relevancy_score> because <your_reason>."
-                    }}
-
-                    If the score is 1, keep it short and say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-                    **
-
-
-                    Contextual Relevancy Score:
-                    {score}
-
-                    Input:
-                    """
-                )
-            ]
-            + input
-            + [
-                textwrap.dedent(
-                    f"""
-                    Reasons for why the retrieval context is irrelevant to the input:
-                    {irrelevancies}
-
-                    Statement in the retrieval context that is relevant to the input:
-                    {relevant_statements}
-
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def generate_verdicts(
-        input: List[Union[str, MLLMImage]], context: List[Union[str, MLLMImage]]
-    ) -> List[Union[str, MLLMImage]]:
-        return (
-            [
-                textwrap.dedent(
-                    f"""Based on the input and context (image or string), please generate a JSON object to indicate whether the context is relevant to the provided input. The JSON will be a list of 'verdicts', with 2 mandatory fields: 'verdict' and 'statement', and 1 optional field: 'reason'.
-                If the context is textual, you should first extract the statements found in the context if the context, which are high level information found in the context, before deciding on a verdict and optionally a reason for each statement.
-                If the context is an image, `statement` should be a description of the image. Do not assume any information not visibly available.
-                The 'verdict' key should STRICTLY be either 'yes' or 'no', and states whether the statement or image is relevant to the input.
-                Provide a 'reason' ONLY IF verdict is no. You MUST quote the irrelevant parts of the statement or image to back up your reason.
-
-                **
-                IMPORTANT: Please make sure to only return in JSON format.
-                Example Context: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. He won the Nobel Prize in 1968. There was a cat."
-                Example Input: "What were some of Einstein's achievements?"
-
-                Example:
-                {{
-                    "verdicts": [
-                        {{
-                            "statement": "Einstein won the Nobel Prize for his discovery of the photoelectric effect in 1968",
-                            "verdict": "yes"
-                        }},
-                        {{
-                            "statement": "There was a cat.",
-                            "reason": "The retrieval context contained the information 'There was a cat' when it has nothing to do with Einstein's achievements.",
-                            "verdict": "no"
-                        }}
-                    ]
-                }}
-                **
-
-                Input:
-                """
-                )
-            ]
-            + input
-            + [
-                textwrap.dedent(
-                    """
-                Context:
-                """
-                )
-            ]
-            + [context]
-        )

deepeval/metrics/multimodal_metrics/multimodal_faithfulness/multimodal_faithfulness.py

@@ -1,356 +0,0 @@
-from typing import List, Optional, Union
-import asyncio
-
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCaseParams, MLLMTestCase, MLLMImage
-from deepeval.metrics.multimodal_metrics.multimodal_faithfulness.template import (
-    MultimodalFaithfulnessTemplate,
-)
-from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.utils import (
-    construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
-)
-from deepeval.models import DeepEvalBaseLLM
-from deepeval.metrics.multimodal_metrics.multimodal_faithfulness.schema import *
-from deepeval.metrics.indicator import metric_progress_indicator
-
-
-class MultimodalFaithfulnessMetric(BaseMultimodalMetric):
-
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
-        MLLMTestCaseParams.RETRIEVAL_CONTEXT,
-    ]
-
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, DeepEvalBaseLLM]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        truths_extraction_limit: Optional[int] = None,
-    ):
-        self.threshold = 1 if strict_mode else threshold
-        self.model, self.using_native_model = initialize_multimodal_model(model)
-        self.evaluation_model = self.model.get_model_name()
-        self.include_reason = include_reason
-        self.async_mode = async_mode
-        self.strict_mode = strict_mode
-        self.verbose_mode = verbose_mode
-
-        self.truths_extraction_limit = truths_extraction_limit
-        if self.truths_extraction_limit is not None:
-            self.truths_extraction_limit = max(self.truths_extraction_limit, 0)
-
-    def measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _log_metric_to_confident=_log_metric_to_confident,
-                    )
-                )
-            else:
-                self.truths = self._generate_truths(test_case.retrieval_context)
-                self.claims = self._generate_claims(test_case.actual_output)
-                self.verdicts = self._generate_verdicts()
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason()
-                self.success = self.score >= self.threshold
-                self.verbose_logs = construct_verbose_logs(
-                    self,
-                    steps=[
-                        f"Truths (limit={self.truths_extraction_limit}):\n{prettify_list(self.truths)}",
-                        f"Claims:\n{prettify_list(self.claims)}",
-                        f"Verdicts:\n{prettify_list(self.verdicts)}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            async_mode=True,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            self.truths, self.claims = await asyncio.gather(
-                self._a_generate_truths(test_case.retrieval_context),
-                self._a_generate_claims(test_case.actual_output),
-            )
-            self.verdicts = await self._a_generate_verdicts()
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason()
-            self.success = self.score >= self.threshold
-            self.verbose_logs = construct_verbose_logs(
-                self,
-                steps=[
-                    f"Truths (limit={self.truths_extraction_limit}):\n{prettify_list(self.truths)}",
-                    f"Claims:\n{prettify_list(self.claims)}",
-                    f"Verdicts:\n{prettify_list(self.verdicts)}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = MultimodalFaithfulnessTemplate.generate_reason(
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=MultimodalFaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodalFaithfulnessScoreReason = (
-                    await self.model.a_generate(
-                        prompt, schema=MultimodalFaithfulnessScoreReason
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _generate_reason(self) -> str:
-        if self.include_reason is False:
-            return None
-
-        contradictions = []
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() == "no":
-                contradictions.append(verdict.reason)
-
-        prompt: dict = MultimodalFaithfulnessTemplate.generate_reason(
-            contradictions=contradictions,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt, schema=MultimodalFaithfulnessScoreReason
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: MultimodalFaithfulnessScoreReason = self.model.generate(
-                    prompt, schema=MultimodalFaithfulnessScoreReason
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(self) -> List[FaithfulnessVerdict]:
-        if len(self.claims) == 0:
-            return []
-
-        verdicts: List[FaithfulnessVerdict] = []
-        prompt = MultimodalFaithfulnessTemplate.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(self) -> List[FaithfulnessVerdict]:
-        if len(self.claims) == 0:
-            return []
-
-        verdicts: List[FaithfulnessVerdict] = []
-        prompt = MultimodalFaithfulnessTemplate.generate_verdicts(
-            claims=self.claims, retrieval_context="\n\n".join(self.truths)
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    FaithfulnessVerdict(**item) for item in data["verdicts"]
-                ]
-                return verdicts
-
-    async def _a_generate_truths(
-        self, retrieval_context: List[Union[str, MLLMImage]]
-    ) -> List[str]:
-        prompt = MultimodalFaithfulnessTemplate.generate_truths(
-            excerpt=retrieval_context,
-            extraction_limit=self.truths_extraction_limit,
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = await self.model.a_generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
-
-    def _generate_truths(
-        self, retrieval_context: List[Union[str, MLLMImage]]
-    ) -> List[str]:
-        prompt = MultimodalFaithfulnessTemplate.generate_truths(
-            excerpt=retrieval_context,
-            extraction_limit=self.truths_extraction_limit,
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Truths)
-            self.evaluation_cost += cost
-            return res.truths
-        else:
-            try:
-                res: Truths = self.model.generate(prompt, schema=Truths)
-                return res.truths
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["truths"]
-
-    async def _a_generate_claims(
-        self, actual_output: List[Union[str, MLLMImage]]
-    ) -> List[str]:
-        prompt = MultimodalFaithfulnessTemplate.generate_claims(
-            excerpt=actual_output
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = await self.model.a_generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
-
-    def _generate_claims(
-        self, actual_output: List[Union[str, MLLMImage]]
-    ) -> List[str]:
-        prompt = MultimodalFaithfulnessTemplate.generate_claims(
-            excerpt=actual_output
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Claims)
-            self.evaluation_cost += cost
-            return res.claims
-        else:
-            try:
-                res: Claims = self.model.generate(prompt, schema=Claims)
-                return res.claims
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["claims"]
-
-    def _calculate_score(self) -> float:
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 1
-
-        faithfulness_count = 0
-        for verdict in self.verdicts:
-            if verdict.verdict.strip().lower() != "no":
-                faithfulness_count += 1
-
-        score = faithfulness_count / number_of_verdicts
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def is_successful(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Multimodal Faithfulness"

deepeval/metrics/multimodal_metrics/multimodal_faithfulness/template.py

@@ -1,175 +0,0 @@
-from typing import Union, List, Optional
-import textwrap
-
-from deepeval.test_case import MLLMImage
-
-
-class MultimodalFaithfulnessTemplate:
-    @staticmethod
-    def generate_claims(excerpt):
-        return (
-            [
-                textwrap.dedent(
-                    f"""Based on the given excerpt, which contains text and possibly images, please generate a comprehensive list of FACTUAL, undisputed truths, that can inferred from the provided text and images.
-
-                    Example:
-                    Example Excerpt: 
-                    "Einstein won the noble prize in 1968 for his discovery of the photoelectric effect."
-
-                    Example JSON: 
-                    {{
-                        "claims": [
-                            "Einstein won the noble prize for his discovery of the photoelectric effect.",
-                            "Einstein won the noble prize in 1968."
-                        ]  
-                    }}
-                    ===== END OF EXAMPLE ======
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the "claims" key as a list of strings. No words or explanation is needed.
-                    Only include claims that are factual, and the claims you extract should include the full context it was presented in, NOT cherry picked facts.
-                    You should NOT include any prior knowledge, and take the text at face value when extracting claims.
-                    **
-
-                    Text:
-                    """
-                )
-            ]
-            + excerpt
-            + [
-                textwrap.dedent(
-                    f"""
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def generate_truths(excerpt, extraction_limit: Optional[int] = None):
-        if extraction_limit is None:
-            limit = " FACTUAL, undisputed truths"
-        elif extraction_limit == 1:
-            limit = " the single most important FACTUAL, undisputed truth"
-        else:
-            limit = f" the {extraction_limit} most important FACTUAL, undisputed truths per document"
-        return (
-            [
-                textwrap.dedent(
-                    f"""Based on the given excerpt (text and images), please generate a comprehensive list of{limit}, that can inferred from the provided excerpt.
-
-                    Example:
-                    Example Excerpt: 
-                    "Einstein won the noble prize in 1968 for his discovery of the photoelectric effect."
-
-                    Example JSON: 
-                    {{
-                        "truths": [
-                            "Einstein won the noble prize for his discovery of the photoelectric effect.",
-                            "Einstein won the noble prize in 1968."
-                        ]  
-                    }}
-                    ===== END OF EXAMPLE ======
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the "truths" key as a list of strings. No words or explanation is needed.
-                    Only include truths that are factual.
-                    **
-
-                    Excerpt:
-                    """
-                )
-            ]
-            + excerpt
-            + [
-                textwrap.dedent(
-                    f"""
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def generate_verdicts(claims, retrieval_context):
-        return textwrap.dedent(
-            f"""Based on the given claims, which is a list of strings, generate a list of JSON objects to indicate whether EACH claim contradicts any facts in the retrieval context. The JSON will have 2 fields: 'verdict' and 'reason'.
-                The 'verdict' key should STRICTLY be either 'yes', 'no', or 'idk', which states whether the given claim agrees with the context. 
-                Provide a 'reason' ONLY if the answer is 'no' or 'idk'. 
-                The provided claim is drawn from the actual output. Try to provide a correction in the reason using the facts in the retrieval context.
-
-                **
-                IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON objects.
-                Example retrieval contexts: "Einstein won the Nobel Prize for his discovery of the photoelectric effect. Einstein won the Nobel Prize in 1968. Einstein is a German Scientist."
-                Example claims: ["Barack Obama is a caucasian male.", "Zurich is a city in London", "Einstein won the Nobel Prize for the discovery of the photoelectric effect which may have contributed to his fame.", "Einstein won the Nobel Prize in 1969 for his discovery of the photoelectric effect.", "Einstein was a German chef."]
-
-                Example:
-                {{
-                    "verdicts": [
-                        {{
-                            "reason": "The claim about Barack Obama is not directly addressed in the retrieval context, and so poses no contradiction.",
-                            "verdict": "idk"
-                        }},
-                        {{
-                            "reason": "The claim about Zurich being a city in London is incorrect but does not pose a contradiction to the retrieval context.",
-                            "verdict": "idk"
-                        }},
-                        {{
-                            "verdict": "yes"
-                        }},
-                        {{
-                            "reason": "The actual output claims Einstein won the Nobel Prize in 1969, which is untrue as the retrieval context states it is 1968 instead.",
-                            "verdict": "no"
-                        }},
-                        {{
-                            "reason": "The actual output claims Einstein is a German chef, which is not correct as the retrieval context states he was a German scientist instead.",
-                            "verdict": "no"
-                        }}
-                    ]  
-                }}
-                ===== END OF EXAMPLE ======
-
-                The length of 'verdicts' SHOULD BE STRICTLY EQUAL to that of claims.
-                You DON'T have to provide a reason if the answer is 'yes'.
-                ONLY provide a 'no' answer if the retrieval context DIRECTLY CONTRADICTS the claims. YOU SHOULD NEVER USE YOUR PRIOR KNOWLEDGE IN YOUR JUDGEMENT.
-                Claims made using vague, suggestive, speculative language such as 'may have', 'possibility due to', does NOT count as a contradiction.
-                Claims that is not backed up due to a lack of information/is not mentioned in the retrieval contexts MUST be answered 'idk', otherwise I WILL DIE.
-                **
-
-                Retrieval Contexts:
-                {retrieval_context}
-
-                Claims:
-                {claims}
-
-                JSON:
-                """
-        )
-
-    @staticmethod
-    def generate_reason(score, contradictions):
-        return textwrap.dedent(
-            f"""Below is a list of Contradictions. It is a list of strings explaining why the 'actual output' does not align with the information presented in the 'retrieval context'. Contradictions happen in the 'actual output', NOT the 'retrieval context'.
-            Given the faithfulness score, which is a 0-1 score indicating how faithful the `actual output` is to the retrieval context (higher the better), CONCISELY summarize the contradictions to justify the score. 
-
-            ** 
-            IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-            Example JSON:
-            {{
-                "reason": "The score is <faithfulness_score> because <your_reason>."
-            }}
-
-            If there are no contradictions, just say something positive with an upbeat encouraging tone (but don't overdo it otherwise it gets annoying).
-            Your reason MUST use information in `contradiction` in your reason.
-            Be sure in your reason, as if you know what the actual output is from the contradictions.
-            **
-
-            Faithfulness Score:
-            {score}
-
-            Contradictions:
-            {contradictions}
-
-            JSON:
-            """
-        )