deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/metrics/multimodal_metrics/multimodal_tool_correctness/multimodal_tool_correctness.py

@@ -1,290 +0,0 @@
-from typing import List, Dict
-
-from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.utils import (
-    construct_verbose_logs,
-    check_mllm_test_case_params,
-)
-from deepeval.test_case import (
-    MLLMTestCase,
-    MLLMTestCaseParams,
-    ToolCallParams,
-    ToolCall,
-)
-from deepeval.metrics import BaseMultimodalMetric
-
-
-class MultimodalToolCorrectnessMetric(BaseMultimodalMetric):
-
-    _required_params: List[MLLMTestCaseParams] = [
-        MLLMTestCaseParams.INPUT,
-        MLLMTestCaseParams.ACTUAL_OUTPUT,
-        MLLMTestCaseParams.TOOLS_CALLED,
-        MLLMTestCaseParams.EXPECTED_TOOLS,
-    ]
-
-    def __init__(
-        self,
-        threshold: float = 0.5,
-        evaluation_params: List[ToolCallParams] = [],
-        include_reason: bool = True,
-        strict_mode: bool = False,
-        verbose_mode: bool = False,
-        should_exact_match: bool = False,
-        should_consider_ordering: bool = False,
-    ):
-        self.threshold = 1 if strict_mode else threshold
-        self.include_reason = include_reason
-        self.strict_mode = strict_mode
-        self.verbose_mode = verbose_mode
-        self.evaluation_params: List[ToolCallParams] = evaluation_params
-        self.should_exact_match = should_exact_match
-        self.should_consider_ordering = should_consider_ordering
-
-    def measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        check_mllm_test_case_params(
-            test_case, self._required_params, None, None, self
-        )
-        self.test_case = test_case
-        with metric_progress_indicator(
-            self, _show_indicator=_show_indicator, _in_component=_in_component
-        ):
-            self.tools_called: List[ToolCall] = test_case.tools_called
-            self.expected_tools: List[ToolCall] = test_case.expected_tools
-            self.score = self._calculate_score()
-            self.reason = self._generate_reason()
-            self.success = self.score >= self.threshold
-            expected_tools_formatted = (
-                "Expected Tools:\n[\n"
-                + ",\n".join(
-                    self.indent_multiline_string(
-                        repr(tool_call), indent_level=4
-                    )
-                    for tool_call in self.expected_tools
-                )
-                + "\n]"
-            )
-            tools_called_formatted = (
-                "Tools Called:\n[\n"
-                + ",\n".join(
-                    self.indent_multiline_string(
-                        repr(tool_call), indent_level=4
-                    )
-                    for tool_call in self.tools_called
-                )
-                + "\n]"
-            )
-            steps = [
-                f"{expected_tools_formatted}",
-                f"{tools_called_formatted}",
-            ]
-            steps.append(f"Score: {self.score}\nReason: {self.reason}")
-            self.verbose_logs = construct_verbose_logs(self, steps=steps)
-            return self.score
-
-    async def a_measure(
-        self,
-        test_case: MLLMTestCase,
-        _show_indicator: bool = True,
-        _in_component: bool = False,
-        _log_metric_to_confident: bool = True,
-    ) -> float:
-        return self.measure(
-            test_case,
-            _show_indicator=_show_indicator,
-            _in_component=_in_component,
-            _log_metric_to_confident=_log_metric_to_confident,
-        )
-
-    ##################################################
-    ### Tool Correctness (Tool) ######################
-    ##################################################
-
-    def _generate_reason(self):
-        tools_called_names = [
-            tool_called.name for tool_called in self.tools_called
-        ]
-        expected_tools_names = [
-            expected_tool.name for expected_tool in self.expected_tools
-        ]
-
-        if self.should_exact_match:
-            return f"{'Exact match' if self._calculate_exact_match_score() else 'Not an exact match'}: expected {expected_tools_names}, called {tools_called_names}. See details above."
-
-        elif self.should_consider_ordering:
-            lcs, weighted_length = self._compute_weighted_lcs()
-            score = weighted_length / len(expected_tools_names)
-            missing = set(expected_tools_names) - set(tools_called_names)
-            out_of_order = set(expected_tools_names) - set(
-                [tool.name for tool in lcs]
-            )
-            if score == 1:
-                return f"Correct ordering: all expected tools {expected_tools_names} were called in the correct order."
-            else:
-                issues = []
-                if missing:
-                    issues.append(f"missing tools {list(missing)}")
-                if out_of_order:
-                    issues.append(f"out-of-order tools {list(out_of_order)}")
-                return f"Incorrect tool usage: {' and '.join(issues)}; expected {expected_tools_names}, called {tools_called_names}. See more details above."
-        else:
-            used_expected = set(self.tools_called).intersection(
-                set(self.expected_tools)
-            )
-            missing = set(self.expected_tools) - used_expected
-            if self._calculate_non_exact_match_score() == 1:
-                return f"All expected tools {expected_tools_names} were called (order not considered)."
-            else:
-                return f"Incomplete tool usage: missing tools {list(missing)}; expected {expected_tools_names}, called {tools_called_names}. See more details above."
-
-    ##################################################
-    ### Score Helper Functions #######################
-    ##################################################
-
-    # Calculate score
-    def _calculate_score(self):
-        if self.should_exact_match:
-            score = self._calculate_exact_match_score()
-        elif self.should_consider_ordering:
-            _, weighted_length = self._compute_weighted_lcs()
-            score = weighted_length / len(self.expected_tools)
-        else:
-            score = self._calculate_non_exact_match_score()
-        return 0 if self.strict_mode and score < self.threshold else score
-
-    # Exact matching score
-    def _calculate_exact_match_score(self):
-        if len(self.tools_called) != len(self.expected_tools):
-            return 0.0
-        for i in range(len(self.tools_called)):
-            if self.tools_called[i].name != self.expected_tools[i].name:
-                return 0.0
-            if ToolCallParams.INPUT_PARAMETERS in self.evaluation_params:
-                if (
-                    self.tools_called[i].input_parameters
-                    != self.expected_tools[i].input_parameters
-                ):
-                    return 0.0
-            if ToolCallParams.OUTPUT in self.evaluation_params:
-                if self.tools_called[i].output != self.expected_tools[i].output:
-                    return 0.0
-        return 1.0
-
-    # Non exact matching score
-    def _calculate_non_exact_match_score(self):
-        total_score = 0.0
-        matched_called_tools = set()
-        for expected_tool in self.expected_tools:
-            best_score = 0.0
-            for called_tool in self.tools_called:
-                if called_tool in matched_called_tools:
-                    continue
-                if expected_tool.name == called_tool.name:
-                    match_score = 1.0
-                    if (
-                        ToolCallParams.INPUT_PARAMETERS
-                        in self.evaluation_params
-                    ):
-                        match_score *= self._compare_dicts(
-                            expected_tool.input_parameters,
-                            called_tool.input_parameters,
-                        )
-                    if (
-                        ToolCallParams.OUTPUT in self.evaluation_params
-                        and expected_tool.output != called_tool.output
-                    ):
-                        match_score = 0.0
-                    if match_score > best_score:
-                        best_score = match_score
-                        best_called_tool = called_tool
-            if best_score > 0:
-                total_score += best_score
-                matched_called_tools.add(best_called_tool)
-        return (
-            total_score / len(self.expected_tools)
-            if self.expected_tools
-            else 0.0
-        )
-
-    # Consider ordering score
-    def _compute_weighted_lcs(self):
-        m, n = len(self.expected_tools), len(self.tools_called)
-        dp = [[0.0] * (n + 1) for _ in range(m + 1)]
-        for i in range(1, m + 1):
-            for j in range(1, n + 1):
-                expected_tool, called_tool = (
-                    self.expected_tools[i - 1],
-                    self.tools_called[j - 1],
-                )
-                if expected_tool.name != called_tool.name:
-                    dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
-                    continue
-                score = 1.0
-                if ToolCallParams.INPUT_PARAMETERS in self.evaluation_params:
-                    score *= self._compare_dicts(
-                        expected_tool.input_parameters,
-                        called_tool.input_parameters,
-                    )
-                if (
-                    ToolCallParams.OUTPUT in self.evaluation_params
-                    and expected_tool.output != called_tool.output
-                ):
-                    score = 0.0
-                dp[i][j] = max(
-                    dp[i - 1][j],
-                    dp[i][j - 1],
-                    dp[i - 1][j - 1] + score if score > 0 else 0,
-                )
-        i, j, total_score = m, n, 0.0
-        lcs = []
-        while i > 0 and j > 0:
-            if dp[i][j] == dp[i - 1][j]:
-                i -= 1
-            elif dp[i][j] == dp[i][j - 1]:
-                j -= 1
-            else:
-                lcs.append(self.expected_tools[i - 1])
-                total_score += dp[i][j] - dp[i - 1][j - 1]
-                i, j = i - 1, j - 1
-        return lcs[::-1], total_score
-
-    # For matching input parameters
-    def _compare_dicts(self, dict1: Dict, dict2: Dict):
-        if self.should_exact_match:
-            return 1.0 if dict1 == dict2 else 0.0
-        match_score = 0
-        matched_keys = set(dict1.keys()).intersection(set(dict2.keys()))
-        total_keys = set(dict1.keys()).union(set(dict2.keys()))
-        for key in matched_keys:
-            if dict1[key] == dict2[key]:
-                match_score += 1 / len(total_keys)
-            elif isinstance(dict1[key], dict) and isinstance(dict2[key], dict):
-                match_score += self._compare_dicts(
-                    dict1[key], dict2[key]
-                ) / len(total_keys)
-        return match_score
-
-    ##################################################
-    ### Others #######################################
-    ##################################################
-
-    def is_successful(self) -> bool:
-        try:
-            self.success = self.score >= self.threshold
-        except:
-            self.success = False
-        return self.success
-
-    @property
-    def __name__(self):
-        return "Multi Modal Tool Correctness"
-
-    def indent_multiline_string(self, s, indent_level=4):
-        indent = " " * indent_level
-        return "\n".join(f"{indent}{line}" for line in s.splitlines())

deepeval/models/mlllms/__init__.py

@@ -1,4 +0,0 @@
-from .openai_model import MultimodalOpenAIModel
-from .ollama_model import MultimodalOllamaModel
-from .gemini_model import MultimodalGeminiModel
-from .azure_model import MultimodalAzureOpenAIMLLMModel

deepeval/models/mlllms/azure_model.py

@@ -1,343 +0,0 @@
-import base64
-from openai.types.chat.chat_completion import ChatCompletion
-from openai import AzureOpenAI, AsyncAzureOpenAI
-from typing import Optional, Tuple, Union, Dict, List
-from pydantic import BaseModel, SecretStr
-from io import BytesIO
-
-from deepeval.config.settings import get_settings
-from deepeval.models import DeepEvalBaseMLLM
-from deepeval.test_case import MLLMImage
-from deepeval.models.llms.openai_model import (
-    structured_outputs_models,
-    json_mode_models,
-    model_pricing,
-)
-from deepeval.models.retry_policy import (
-    create_retry_decorator,
-    sdk_retries_for,
-)
-
-from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import parse_model_name, require_secret_api_key
-from deepeval.constants import ProviderSlug as PS
-
-
-retry_azure = create_retry_decorator(PS.AZURE)
-
-
-class MultimodalAzureOpenAIMLLMModel(DeepEvalBaseMLLM):
-    def __init__(
-        self,
-        deployment_name: Optional[str] = None,
-        model_name: Optional[str] = None,
-        azure_openai_api_key: Optional[str] = None,
-        openai_api_version: Optional[str] = None,
-        azure_endpoint: Optional[str] = None,
-        temperature: float = 0,
-        generation_kwargs: Optional[Dict] = None,
-        **kwargs,
-    ):
-        settings = get_settings()
-        # fetch Azure deployment parameters
-        model_name = model_name or settings.AZURE_MODEL_NAME
-        self.deployment_name = deployment_name or settings.AZURE_DEPLOYMENT_NAME
-        if azure_openai_api_key is not None:
-            # keep it secret, keep it safe from serializings, logging and alike
-            self.azure_openai_api_key: SecretStr | None = SecretStr(
-                azure_openai_api_key
-            )
-        else:
-            self.azure_openai_api_key = settings.AZURE_OPENAI_API_KEY
-
-        self.openai_api_version = (
-            openai_api_version or settings.OPENAI_API_VERSION
-        )
-        self.azure_endpoint = (
-            azure_endpoint
-            or settings.AZURE_OPENAI_ENDPOINT
-            and str(settings.AZURE_OPENAI_ENDPOINT)
-        )
-        if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
-        self.temperature = temperature
-
-        # args and kwargs will be passed to the underlying model, in load_model function
-        self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
-        super().__init__(parse_model_name(model_name))
-
-    ###############################################
-    # Generate functions
-    ###############################################
-
-    @retry_azure
-    def generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> Tuple[Union[str, BaseModel], float]:
-        client = self.load_model(async_mode=False)
-        prompt = self.generate_prompt(multimodal_input)
-
-        if schema:
-            if self.model_name in structured_outputs_models:
-                messages = [{"role": "user", "content": prompt}]
-                completion = client.beta.chat.completions.parse(
-                    model=self.deployment_name,
-                    messages=messages,
-                    response_format=schema,
-                    temperature=self.temperature,
-                )
-                structured_output: BaseModel = completion.choices[
-                    0
-                ].message.parsed
-                cost = self.calculate_cost(
-                    completion.usage.prompt_tokens,
-                    completion.usage.completion_tokens,
-                )
-                return structured_output, cost
-            if self.model_name in json_mode_models:
-                messages = [{"role": "user", "content": prompt}]
-                completion = client.beta.chat.completions.parse(
-                    model=self.deployment_name,
-                    messages=messages,
-                    response_format={"type": "json_object"},
-                    temperature=self.temperature,
-                )
-                json_output = trim_and_load_json(
-                    completion.choices[0].message.content
-                )
-                cost = self.calculate_cost(
-                    completion.usage.prompt_tokens,
-                    completion.usage.completion_tokens,
-                )
-                return schema.model_validate(json_output), cost
-        print("Loading model client:")
-        print(client.base_url)
-        completion = client.chat.completions.create(
-            model=self.deployment_name,
-            messages=[{"role": "user", "content": prompt}],
-            temperature=self.temperature,
-            **self.generation_kwargs,
-        )
-        output = completion.choices[0].message.content
-        cost = self.calculate_cost(
-            completion.usage.prompt_tokens, completion.usage.completion_tokens
-        )
-        if schema:
-            json_output = trim_and_load_json(output)
-            return schema.model_validate(json_output), cost
-        else:
-            return output, cost
-
-    @retry_azure
-    async def a_generate(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        schema: Optional[BaseModel] = None,
-    ) -> Tuple[Union[str, BaseModel], float]:
-        client = self.load_model(async_mode=True)
-        prompt = self.generate_prompt(multimodal_input)
-
-        if schema:
-            if self.model_name in structured_outputs_models:
-                messages = [{"role": "user", "content": prompt}]
-                completion = await client.beta.chat.completions.parse(
-                    model=self.deployment_name,
-                    messages=messages,
-                    response_format=schema,
-                    temperature=self.temperature,
-                )
-                structured_output: BaseModel = completion.choices[
-                    0
-                ].message.parsed
-                cost = self.calculate_cost(
-                    completion.usage.prompt_tokens,
-                    completion.usage.completion_tokens,
-                )
-                return structured_output, cost
-            if self.model_name in json_mode_models:
-                messages = [{"role": "user", "content": prompt}]
-                completion = await client.beta.chat.completions.parse(
-                    model=self.deployment_name,
-                    messages=messages,
-                    response_format={"type": "json_object"},
-                    temperature=self.temperature,
-                    **self.generation_kwargs,
-                )
-                json_output = trim_and_load_json(
-                    completion.choices[0].message.content
-                )
-                cost = self.calculate_cost(
-                    completion.usage.prompt_tokens,
-                    completion.usage.completion_tokens,
-                )
-                return schema.model_validate(json_output), cost
-
-        completion = await client.chat.completions.create(
-            model=self.deployment_name,
-            messages=[{"role": "user", "content": prompt}],
-            temperature=self.temperature,
-            **self.generation_kwargs,
-        )
-        output = completion.choices[0].message.content
-        cost = self.calculate_cost(
-            completion.usage.prompt_tokens,
-            completion.usage.completion_tokens,
-        )
-        if schema:
-            json_output = trim_and_load_json(output)
-            return schema.model_validate(json_output), cost
-        else:
-            return output, cost
-
-    ###############################################
-    # Other generate functions
-    ###############################################
-
-    @retry_azure
-    def generate_raw_response(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        top_logprobs: int = 5,
-    ) -> Tuple[ChatCompletion, float]:
-        client = self.load_model(async_mode=False)
-        prompt = self.generate_prompt(multimodal_input)
-        messages = [{"role": "user", "content": prompt}]
-
-        # Generate completion
-        completion = client.chat.completions.create(
-            model=self.deployment_name,
-            messages=messages,
-            temperature=self.temperature,
-            logprobs=True,
-            top_logprobs=top_logprobs,
-            **self.generation_kwargs,
-        )
-        # Cost calculation
-        input_tokens = completion.usage.prompt_tokens
-        output_tokens = completion.usage.completion_tokens
-        cost = self.calculate_cost(input_tokens, output_tokens)
-
-        return completion, cost
-
-    @retry_azure
-    async def a_generate_raw_response(
-        self,
-        multimodal_input: List[Union[str, MLLMImage]],
-        top_logprobs: int = 5,
-    ) -> Tuple[ChatCompletion, float]:
-        client = self.load_model(async_mode=True)
-        prompt = self.generate_prompt(multimodal_input)
-        messages = [{"role": "user", "content": prompt}]
-
-        # Generate completion
-        completion = await client.chat.completions.create(
-            model=self.deployment_name,
-            messages=messages,
-            temperature=self.temperature,
-            logprobs=True,
-            top_logprobs=top_logprobs,
-            **self.generation_kwargs,
-        )
-        # Cost calculation
-        input_tokens = completion.usage.prompt_tokens
-        output_tokens = completion.usage.completion_tokens
-        cost = self.calculate_cost(input_tokens, output_tokens)
-
-        return completion, cost
-
-    ###############################################
-    # Utilities
-    ###############################################
-
-    def generate_prompt(
-        self, multimodal_input: List[Union[str, MLLMImage]] = []
-    ):
-        """Convert multimodal input into the proper message format for Azure OpenAI."""
-        prompt = []
-        for ele in multimodal_input:
-            if isinstance(ele, str):
-                prompt.append({"type": "text", "text": ele})
-            elif isinstance(ele, MLLMImage):
-                if ele.local:
-                    import PIL.Image
-
-                    image = PIL.Image.open(ele.url)
-                    visual_dict = {
-                        "type": "image_url",
-                        "image_url": {
-                            "url": f"data:image/jpeg;base64,{self.encode_pil_image(image)}"
-                        },
-                    }
-                else:
-                    visual_dict = {
-                        "type": "image_url",
-                        "image_url": {"url": ele.url},
-                    }
-                prompt.append(visual_dict)
-        return prompt
-
-    def encode_pil_image(self, pil_image):
-        """Encode a PIL image to base64 string."""
-        image_buffer = BytesIO()
-        if pil_image.mode in ("RGBA", "LA", "P"):
-            pil_image = pil_image.convert("RGB")
-        pil_image.save(image_buffer, format="JPEG")
-        image_bytes = image_buffer.getvalue()
-        base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
-        return base64_encoded_image
-
-    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing["gpt-4.1"])
-        input_cost = input_tokens * pricing["input"]
-        output_cost = output_tokens * pricing["output"]
-        return input_cost + output_cost
-
-    ###############################################
-    # Model
-    ###############################################
-
-    def get_model_name(self):
-        return f"Azure OpenAI ({self.model_name})"
-
-    def load_model(self, async_mode: bool = False):
-        if not async_mode:
-            return self._build_client(AzureOpenAI)
-        return self._build_client(AsyncAzureOpenAI)
-
-    def _client_kwargs(self) -> Dict:
-        """
-        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
-        If the user opts into SDK retries for 'azure' via DEEPEVAL_SDK_RETRY_PROVIDERS,
-        leave their retry settings as is.
-        """
-        kwargs = dict(self.kwargs or {})
-        if not sdk_retries_for(PS.AZURE):
-            kwargs["max_retries"] = 0
-        return kwargs
-
-    def _build_client(self, cls):
-        api_key = require_secret_api_key(
-            self.azure_openai_api_key,
-            provider_label="AzureOpenAI",
-            env_var_name="AZURE_OPENAI_API_KEY",
-            param_hint="`azure_openai_api_key` to MultimodalAzureOpenAIMLLMModel(...)",
-        )
-
-        kw = dict(
-            api_key=api_key,
-            api_version=self.openai_api_version,
-            azure_endpoint=self.azure_endpoint,
-            azure_deployment=self.deployment_name,
-            **self._client_kwargs(),
-        )
-        try:
-            return cls(**kw)
-        except TypeError as e:
-            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
-            if "max_retries" in str(e):
-                kw.pop("max_retries", None)
-                return cls(**kw)
-            raise