deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_contextual_precision/multimodal_contextual_precision.py

@@ -1,301 +0,0 @@
-from typing import Optional, List, Union
-
-from deepeval.metrics import BaseMultimodalMetric
-from deepeval.test_case import MLLMTestCase
-from deepeval.metrics.multimodal_metrics.multimodal_contextual_precision.template import (
-    MultiModalContextualPrecisionTemplate,
-)
-from deepeval.utils import get_or_create_event_loop, prettify_list
-from deepeval.metrics.utils import (
-    construct_verbose_logs,
-    trimAndLoadJson,
-    check_mllm_test_case_params,
-    initialize_multimodal_model,
-)
-from deepeval.test_case import LLMTestCaseParams
-from deepeval.models import DeepEvalBaseMLLM
-import deepeval.metrics.multimodal_metrics.multimodal_contextual_precision.schema as mcpschema
-from deepeval.metrics.indicator import metric_progress_indicator
-
-
-class MultimodalContextualPrecisionMetric(BaseMultimodalMetric):
-
-    _required_params: List[LLMTestCaseParams] = [
-        LLMTestCaseParams.INPUT,
-        LLMTestCaseParams.ACTUAL_OUTPUT,
-        LLMTestCaseParams.RETRIEVAL_CONTEXT,
-        LLMTestCaseParams.EXPECTED_OUTPUT,
-    ]
-
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        model: Optional[Union[str, DeepEvalBaseMLLM]] = None,
-        include_reason: bool = True,
-        async_mode: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-    ):
-        self.threshold = 1 if strict_mode else threshold
-        self.include_reason = include_reason
-        self.model, self.using_native_model = initialize_multimodal_model(model)
-        self.evaluation_model = self.model.get_model_name()
-        self.async_mode = async_mode
-        self.strict_mode = strict_mode
-        self.verbose_mode = verbose_mode
-
-    def measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            if self.async_mode:
-                loop = get_or_create_event_loop()
-                loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _log_metric_to_confident=_log_metric_to_confident,
-                    )
-                )
-            else:
-                self.verdicts: List[mcpschema.ContextualPrecisionVerdict] = (
-                    self._generate_verdicts(
-                        test_case.input,
-                        test_case.expected_output,
-                        test_case.retrieval_context,
-                    )
-                )
-                self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
-                self.success = self.score >= self.threshold
-                self.verbose_logs = construct_verbose_logs(
-                    self,
-                    steps=[
-                        f"Verdicts:\n{prettify_list(self.verdicts)}",
-                        f"Score: {self.score}\nReason: {self.reason}",
-                    ],
-                )
-
-                return self.score
-
-    async def a_measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-
-        self.evaluation_cost = 0 if self.using_native_model else None
-        with metric_progress_indicator(
-            self,
-            async_mode=True,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-        ):
-            self.verdicts: List[mcpschema.ContextualPrecisionVerdict] = (
-                await self._a_generate_verdicts(
-                    test_case.input,
-                    test_case.expected_output,
-                    test_case.retrieval_context,
-                )
-            )
-            self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
-            self.success = self.score >= self.threshold
-            self.verbose_logs = construct_verbose_logs(
-                self,
-                steps=[
-                    f"Verdicts:\n{prettify_list(self.verdicts)}",
-                    f"Score: {self.score}\nReason: {self.reason}",
-                ],
-            )
-
-            return self.score
-
-    async def _a_generate_reason(self, input: str) -> Optional[str]:
-        if self.include_reason is False:
-            return None
-
-        retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reason": verdict.reason}
-            for verdict in self.verdicts
-        ]
-        prompt = MultiModalContextualPrecisionTemplate.generate_reason(
-            input=input,
-            verdicts=retrieval_contexts_verdicts,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt,
-                schema=mcpschema.MultimodelContextualPrecisionScoreReason,
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: mcpschema.MultimodelContextualPrecisionScoreReason = (
-                    await self.model.a_generate(
-                        prompt,
-                        schema=mcpschema.MultimodelContextualPrecisionScoreReason,
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    def _generate_reason(self, input: str) -> Optional[str]:
-        if self.include_reason is False:
-            return None
-
-        retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reason": verdict.reason}
-            for verdict in self.verdicts
-        ]
-        prompt = MultiModalContextualPrecisionTemplate.generate_reason(
-            input=input,
-            verdicts=retrieval_contexts_verdicts,
-            score=format(self.score, ".2f"),
-        )
-
-        if self.using_native_model:
-            res, cost = self.model.generate(
-                prompt,
-                schema=mcpschema.MultimodelContextualPrecisionScoreReason,
-            )
-            self.evaluation_cost += cost
-            return res.reason
-        else:
-            try:
-                res: mcpschema.MultimodelContextualPrecisionScoreReason = (
-                    self.model.generate(
-                        prompt,
-                        schema=mcpschema.MultimodelContextualPrecisionScoreReason,
-                    )
-                )
-                return res.reason
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                return data["reason"]
-
-    async def _a_generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[mcpschema.ContextualPrecisionVerdict]:
-        prompt = MultiModalContextualPrecisionTemplate.generate_verdicts(
-            input=input,
-            expected_output=expected_output,
-            retrieval_context=retrieval_context,
-        )
-        if self.using_native_model:
-            res, cost = await self.model.a_generate(
-                prompt, schema=mcpschema.Verdicts
-            )
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: mcpschema.Verdicts = await self.model.a_generate(
-                    prompt, schema=mcpschema.Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = await self.model.a_generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    mcpschema.ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _generate_verdicts(
-        self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[mcpschema.ContextualPrecisionVerdict]:
-        prompt = MultiModalContextualPrecisionTemplate.generate_verdicts(
-            input=input,
-            expected_output=expected_output,
-            retrieval_context=retrieval_context,
-        )
-        if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=mcpschema.Verdicts)
-            self.evaluation_cost += cost
-            verdicts = [item for item in res.verdicts]
-            return verdicts
-        else:
-            try:
-                res: mcpschema.Verdicts = self.model.generate(
-                    prompt, schema=mcpschema.Verdicts
-                )
-                verdicts = [item for item in res.verdicts]
-                return verdicts
-            except TypeError:
-                res = self.model.generate(prompt)
-                data = trimAndLoadJson(res, self)
-                verdicts = [
-                    mcpschema.ContextualPrecisionVerdict(**item)
-                    for item in data["verdicts"]
-                ]
-                return verdicts
-
-    def _calculate_score(self):
-        number_of_verdicts = len(self.verdicts)
-        if number_of_verdicts == 0:
-            return 0
-
-        # Convert verdicts to a binary list where 'yes' is 1 and others are 0
-        node_verdicts = [
-            1 if v.verdict.strip().lower() == "yes" else 0
-            for v in self.verdicts
-        ]
-
-        sum_weighted_precision_at_k = 0.0
-        relevant_nodes_count = 0
-        for k, is_relevant in enumerate(node_verdicts, start=1):
-            # If the item is relevant, update the counter and add the weighted precision at k to the sum
-            if is_relevant:
-                relevant_nodes_count += 1
-                precision_at_k = relevant_nodes_count / k
-                sum_weighted_precision_at_k += precision_at_k * is_relevant
-
-        if relevant_nodes_count == 0:
-            return 0
-        # Calculate weighted cumulative precision
-        score = sum_weighted_precision_at_k / relevant_nodes_count
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    def is_successful(self) -> bool:
-        if self.error is not None:
-            self.success = False
-        else:
-            try:
-                self.success = self.score >= self.threshold
-            except TypeError:
-                self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Multimodal Contextual Precision"

deepeval/metrics/multimodal_metrics/multimodal_contextual_precision/schema.py

@@ -1,15 +0,0 @@
-from typing import List, Optional
-from pydantic import BaseModel, Field
-
-
-class ContextualPrecisionVerdict(BaseModel):
-    verdict: str
-    reason: str
-
-
-class Verdicts(BaseModel):
-    verdicts: List[ContextualPrecisionVerdict]
-
-
-class MultimodelContextualPrecisionScoreReason(BaseModel):
-    reason: str

deepeval/metrics/multimodal_metrics/multimodal_contextual_precision/template.py

@@ -1,132 +0,0 @@
-from typing import Union, List
-import textwrap
-
-from deepeval.test_case import MLLMImage
-
-
-class MultiModalContextualPrecisionTemplate:
-    @staticmethod
-    def generate_verdicts(
-        input: List[Union[str, MLLMImage]],
-        expected_output: List[Union[str, MLLMImage]],
-        retrieval_context: List[Union[str, MLLMImage]],
-    ) -> List[Union[str, MLLMImage]]:
-        document_count_str = f" ({len(retrieval_context)} document{'s' if len(retrieval_context) > 1 else ''})"
-        return (
-            [
-                textwrap.dedent(
-                    f"""Given the input, expected output, and retrieval context, please generate a list of JSON objects to determine whether each node in the retrieval context was remotely useful in arriving at the expected output.
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the 'verdicts' key as a list of JSON. These JSON only contain the `verdict` key that outputs only 'yes' or 'no', and a `reason` key to justify the verdict. In your reason, you should aim to quote parts of the context (which can be text or an image).
-                    Example Retrieval Context: ["Einstein won the Nobel Prize for his discovery of the photoelectric effect", "He won the Nobel Prize in 1968.", "There was a cat."]
-                    Example Input: "Who won the Nobel Prize in 1968 and for what?"
-                    Example Expected Output: "Einstein won the Nobel Prize in 1968 for his discovery of the photoelectric effect."
-
-                    Example:
-                    {{
-                        "verdicts": [
-                            {{
-                                "reason": "It clearly addresses the question by stating that 'Einstein won the Nobel Prize for his discovery of the photoelectric effect.'",
-                                "verdict": "yes"
-                            }},
-                            {{
-                                "reason": "The text verifies that the prize was indeed won in 1968.",
-                                "verdict": "yes"
-                            }},
-                            {{
-                                "reason": "'There was a cat' is not at all relevant to the topic of winning a Nobel Prize.",
-                                "verdict": "no"
-                            }}
-                        ]  
-                    }}
-                    Since you are going to generate a verdict for each context, the number of 'verdicts' SHOULD BE STRICTLY EQUAL to that of the contexts.
-                    **
-
-                    Input:
-                    """
-                )
-            ]
-            + input
-            + [
-                textwrap.dedent(
-                    """
-                    Expected output:
-                    """
-                )
-            ]
-            + expected_output
-            + [
-                textwrap.dedent(
-                    f"""
-                    Retrieval Context{document_count_str}:
-                    """
-                )
-            ]
-            + MultiModalContextualPrecisionTemplate.id_retrieval_context(
-                retrieval_context
-            )
-            + [
-                textwrap.dedent(
-                    """
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def generate_reason(input, verdicts, score) -> List[Union[str, MLLMImage]]:
-        # given the input and retrieval context for this input, where the verdict is whether ... and the node is the ..., give a reason for the score
-        return (
-            [
-                textwrap.dedent(
-                    f"""Given the input, retrieval contexts, and contextual precision score, provide a CONCISE summarize for the score. Explain why it is not higher, but also why it is at its current score.
-                    The retrieval contexts is a list of JSON with three keys: `verdict`, `reason` (reason for the verdict) and `node`. `verdict` will be either 'yes' or 'no', which represents whether the corresponding 'node' in the retrieval context is relevant to the input. 
-                    Contextual precision represents if the relevant nodes are ranked higher than irrelevant nodes. Also note that retrieval contexts is given IN THE ORDER OF THEIR RANKINGS.
-
-                    **
-                    IMPORTANT: Please make sure to only return in JSON format, with the 'reason' key providing the reason.
-                    Example JSON:
-                    {{
-                        "reason": "The score is <contextual_precision_score> because <your_reason>."
-                    }}
-
-
-                    DO NOT mention 'verdict' in your reason, but instead phrase it as irrelevant nodes. The term 'verdict' are just here for you to understand the broader scope of things.
-                    Also DO NOT mention there are `reason` fields in the retrieval contexts you are presented with, instead just use the information in the `reason` field.
-                    In your reason, you MUST USE the `reason`, QUOTES in the 'reason', and the node RANK (starting from 1, eg. first node) to explain why the 'no' verdicts should be ranked lower than the 'yes' verdicts.
-                    When addressing nodes, make it explicit that it is nodes in retrieval context.
-                    If the score is 1, keep it short and say something positive with an upbeat tone (but don't overdo it otherwise it gets annoying).
-                    **
-
-                    Contextual Precision Score:
-                    {score}
-
-                    Input:
-                    """
-                )
-            ]
-            + input
-            + [
-                textwrap.dedent(
-                    f"""
-                    Retrieval Contexts:
-                    {verdicts}
-
-                    JSON:
-                    """
-                )
-            ]
-        )
-
-    @staticmethod
-    def id_retrieval_context(retrieval_context) -> List[Union[str, MLLMImage]]:
-        annotated_retrieval_context = []
-        for i, context in enumerate(retrieval_context):
-            if isinstance(context, str):
-                annotated_retrieval_context.append(f"Node {i + 1}: {context}")
-            elif isinstance(context, MLLMImage):
-                annotated_retrieval_context.append(f"Node {i + 1}:")
-                annotated_retrieval_context.append(context)
-        return annotated_retrieval_context