deepeval 3.7.5__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/models/llms/portkey_model.py

@@ -3,10 +3,13 @@ import requests
 from typing import Any, Dict, List, Optional, Union
 from pydantic import AnyUrl, SecretStr
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.utils import (
     require_secret_api_key,
 )
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.utils import require_param
 
@@ -29,16 +32,9 @@ class PortkeyModel(DeepEvalBaseLLM):
         settings = get_settings()
         model = model or settings.PORTKEY_MODEL_NAME
 
-        self.name = require_param(
-            model,
-            provider_label="Portkey",
-            env_var_name="PORTKEY_MODEL_NAME",
-            param_hint="model",
-        )
-
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.PORTKEY_API_KEY
 
@@ -47,6 +43,16 @@ class PortkeyModel(DeepEvalBaseLLM):
         elif settings.PORTKEY_BASE_URL is not None:
             base_url = str(settings.PORTKEY_BASE_URL).rstrip("/")
 
+        provider = provider or settings.PORTKEY_PROVIDER_NAME
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_MODEL_NAME",
+            param_hint="model",
+        )
+
         self.base_url = require_param(
             base_url,
             provider_label="Portkey",
@@ -54,7 +60,6 @@ class PortkeyModel(DeepEvalBaseLLM):
             param_hint="base_url",
         )
 
-        provider = provider or settings.PORTKEY_PROVIDER_NAME
         self.provider = require_param(
             provider,
             provider_label="Portkey",
@@ -64,6 +69,7 @@ class PortkeyModel(DeepEvalBaseLLM):
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
+        super().__init__(model)
 
     def _headers(self) -> Dict[str, str]:
         api_key = require_secret_api_key(
@@ -82,18 +88,51 @@ class PortkeyModel(DeepEvalBaseLLM):
         return headers
 
     def _payload(self, prompt: str) -> Dict[str, Any]:
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
         payload = {
             "model": self.name,
-            "messages": [{"role": "user", "content": prompt}],
+            "messages": [{"role": "user", "content": content}],
         }
         if self.generation_kwargs:
             payload.update(self.generation_kwargs)
         return payload
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
+
     def _extract_content(self, data: Dict[str, Any]) -> str:
         choices: Union[List[Dict[str, Any]], None] = data.get("choices")
         if not choices:
-            raise ValueError("Portkey response did not include any choices.")
+            raise DeepEvalError("Portkey response did not include any choices.")
         message = choices[0].get("message", {})
         content: Union[str, List[Dict[str, Any]], None] = message.get("content")
         if isinstance(content, str):
@@ -109,7 +148,7 @@ class PortkeyModel(DeepEvalBaseLLM):
                 f"{self.base_url}/chat/completions",
                 json=self._payload(prompt),
                 headers=self._headers(),
-                timeout=60,
+                timeout=_request_timeout_seconds(),
             )
             response.raise_for_status()
         except requests.HTTPError as error:
@@ -118,11 +157,11 @@ class PortkeyModel(DeepEvalBaseLLM):
                 body = response.json()
             except Exception:
                 body = response.text
-            raise ValueError(
+            raise DeepEvalError(
                 f"Portkey request failed with status {response.status_code}: {body}"
             ) from error
         except requests.RequestException as error:
-            raise ValueError(f"Portkey request failed: {error}") from error
+            raise DeepEvalError(f"Portkey request failed: {error}") from error
         return self._extract_content(response.json())
 
     async def a_generate(self, prompt: str) -> str:
@@ -132,11 +171,11 @@ class PortkeyModel(DeepEvalBaseLLM):
                 f"{self.base_url}/chat/completions",
                 json=self._payload(prompt),
                 headers=self._headers(),
-                timeout=60,
+                timeout=_request_timeout_seconds(),
             ) as response:
                 if response.status >= 400:
                     body = await response.text()
-                    raise ValueError(
+                    raise DeepEvalError(
                         f"Portkey request failed with status {response.status}: {body}"
                     )
                 data = await response.json()
@@ -147,3 +186,6 @@ class PortkeyModel(DeepEvalBaseLLM):
 
     def get_model_name(self):
         return f"{self.name} (Portkey)"
+
+    def supports_multimodal(self):
+        return True

deepeval/models/llms/utils.py

@@ -1,8 +1,11 @@
-from typing import Dict, List, Optional
+from typing import Dict
 import re
 import json
 import asyncio
 
+from deepeval.errors import DeepEvalError
+
+
 MULTIMODAL_MODELS = ["GPTModel", "AzureModel", "GeminiModel", "OllamaModel"]
 
 
@@ -20,7 +23,7 @@ def trim_and_load_json(
         return json.loads(jsonStr)
     except json.JSONDecodeError:
         error_str = "Evaluation LLM outputted an invalid JSON. Please use a better evaluation model."
-        raise ValueError(error_str)
+        raise DeepEvalError(error_str)
     except Exception as e:
         raise Exception(f"An unexpected error occurred: {str(e)}")
 

deepeval/models/utils.py

@@ -8,7 +8,7 @@ from deepeval.errors import DeepEvalError
 logger = logging.getLogger(__name__)
 
 
-def parse_model_name(model_name: Optional[str] = None) -> str:
+def parse_model_name(model_name: Optional[str] = None) -> Optional[str]:
     """Extract base model name from provider-prefixed format.
 
     This function is useful for extracting the actual model name from a
@@ -32,9 +32,9 @@ def parse_model_name(model_name: Optional[str] = None) -> str:
     if model_name is None:
         return None
 
-    if "/" in model_name:
-        _, parsed_model_name = model_name.split("/", 1)
-        return parsed_model_name
+    # if "/" in model_name:
+    #     _, parsed_model_name = model_name.split("/", 1)
+    #     return parsed_model_name
     return model_name
 
 
@@ -80,6 +80,62 @@ def require_secret_api_key(
     return api_key
 
 
+def require_costs(
+    model_data,
+    model_name: str,
+    input_token_envvar: str,
+    output_token_envvar: str,
+    cost_per_input_token: Optional[float] = None,
+    cost_per_output_token: Optional[float] = None,
+) -> Tuple[Optional[float], Optional[float]]:
+    """
+    Validates and returns the cost parameters (input and output tokens) for a model.
+
+    Arguments:
+    - model_data: The model's data object, which should contain `input_price` and `output_price`.
+    - model_name: The model name used for error messaging.
+    - cost_per_input_token: The input token cost provided during model initialization (optional).
+    - cost_per_output_token: The output token cost provided during model initialization (optional).
+    - input_token_envvar: The environment variable name for input cost.
+    - output_token_envvar: The environment variable name for output cost.
+
+    Returns:
+    - A tuple of validated values (input_cost, output_cost). If the values are provided, they are returned.
+      If not provided, they are fetched from settings or environment variables.
+    """
+
+    def validate_cost(
+        value: Optional[float], envvar_name: str
+    ) -> Optional[float]:
+        """Helper function to validate the cost values."""
+        if value is not None and value < 0:
+            raise DeepEvalError(f"{envvar_name} must be >= 0.")
+        return value
+
+    # Validate provided token costs
+    cost_per_input_token = validate_cost(
+        cost_per_input_token, input_token_envvar
+    )
+    cost_per_output_token = validate_cost(
+        cost_per_output_token, output_token_envvar
+    )
+
+    # If model data doesn't have pricing, use provided values or environment variables
+    if model_data.input_price is None or model_data.output_price is None:
+        if cost_per_input_token is None or cost_per_output_token is None:
+            raise DeepEvalError(
+                f"No pricing available for `{model_name}`. "
+                f"Please provide both `cost_per_input_token` and `cost_per_output_token` when initializing `{model_name}`, "
+                f"or set {input_token_envvar} and {output_token_envvar} environment variables."
+            )
+
+        # Return the validated cost values as a tuple
+        return cost_per_input_token, cost_per_output_token
+
+    # If no custom cost values are provided, return model's default cost values
+    return model_data.input_price, model_data.output_price
+
+
 def normalize_kwargs_and_extract_aliases(
     provider_label: str,
     kwargs: Dict[str, Any],

deepeval/simulator/conversation_simulator.py

@@ -20,6 +20,7 @@ from deepeval.simulator.template import (
     ConversationSimulatorTemplate,
 )
 from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.utils import MULTIMODAL_SUPPORTED_MODELS
 from deepeval.simulator.schema import (
     SimulatedInput,
     ConversationCompletion,
@@ -94,6 +95,26 @@ class ConversationSimulator:
                     )
                 )
             else:
+                multimodal = any(
+                    [golden.multimodal for golden in conversational_goldens]
+                )
+                if multimodal:
+                    if (
+                        not self.simulator_model
+                        or not self.simulator_model.supports_multimodal()
+                    ):
+                        if (
+                            self.simulator_model
+                            and type(self.simulator_model)
+                            in MULTIMODAL_SUPPORTED_MODELS
+                        ):
+                            raise ValueError(
+                                f"The evaluation model {self.simulator_model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {self.simulator_model.__class__.__name__} provider includes {', '.join(self.simulator_model.__class__.valid_multimodal_models)}."
+                            )
+                        else:
+                            raise ValueError(
+                                f"The evaluation model {self.simulator_model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+                            )
                 conversational_test_cases: List[ConversationalTestCase] = []
                 for conversation_index, golden in enumerate(
                     conversational_goldens
@@ -124,6 +145,28 @@ class ConversationSimulator:
         progress: Optional[Progress] = None,
         pbar_id: Optional[int] = None,
     ) -> List[ConversationalTestCase]:
+
+        multimodal = any(
+            [golden.multimodal for golden in conversational_goldens]
+        )
+        if multimodal:
+            if (
+                not self.simulator_model
+                or not self.simulator_model.supports_multimodal()
+            ):
+                if (
+                    self.simulator_model
+                    and type(self.simulator_model)
+                    in MULTIMODAL_SUPPORTED_MODELS
+                ):
+                    raise ValueError(
+                        f"The evaluation model {self.simulator_model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {self.simulator_model.__class__.__name__} provider includes {', '.join(self.simulator_model.__class__.valid_multimodal_models)}."
+                    )
+                else:
+                    raise ValueError(
+                        f"The evaluation model {self.simulator_model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+                    )
+
         self.simulation_cost = 0 if self.using_native_model else None
 
         async def simulate_conversations(

deepeval/simulator/template.py

@@ -7,6 +7,13 @@ from deepeval.test_case import Turn
 
 
 class ConversationSimulatorTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
 
     @staticmethod
     def simulate_first_user_turn(
@@ -23,6 +30,8 @@ class ConversationSimulatorTemplate:
             3. Avoid providing excessive details upfront; the goal is to initiate the conversation and build rapport, not to solve it in the first message.
             4. The message should be concise, ideally no more than 1-3 sentences.
 
+            {ConversationSimulatorTemplate.multimodal_rules}
+
             IMPORTANT: The output must be formatted as a JSON object with a single key `simulated_input`, where the value is the generated opening message in {language}.
 
             Example Language: english
@@ -61,6 +70,8 @@ class ConversationSimulatorTemplate:
             3. Keep the tone consistent with the previous user inputs.
             4. The generated user input should be concise, ideally no more than 1-2 sentences.
 
+            {ConversationSimulatorTemplate.multimodal_rules}
+
             IMPORTANT: The output must be formatted as a JSON object with a single key `simulated_input`, 
             where the value is the generated user input in {language}.
 
@@ -101,6 +112,8 @@ class ConversationSimulatorTemplate:
             2. If the expected outcome has been met, mark the conversation as complete.
             3. If not, mark it as incomplete and briefly describe what remains to be done.
 
+            {ConversationSimulatorTemplate.multimodal_rules}
+
             IMPORTANT: The output must be formatted as a JSON object with two keys:
             `is_complete` (a boolean) and `reason` (a string).
 

deepeval/test_case/api.py

@@ -12,7 +12,6 @@ from deepeval.test_case import (
     ConversationalTestCase,
     Turn,
 )
-from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 from deepeval.constants import PYTEST_RUN_TEST_NAME
 
 
@@ -33,7 +32,6 @@ def create_api_test_case(
     trace: Optional[TraceApi] = None,
     index: Optional[int] = None,
 ) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
-    from deepeval.utils import convert_to_multi_modal_array
 
     if isinstance(test_case, ConversationalTestCase):
         order = (
@@ -61,8 +59,10 @@ def create_api_test_case(
             context=test_case.context,
             tags=test_case.tags,
             comments=test_case.comments,
+            imagesMapping=test_case._get_images_mapping(),
             additionalMetadata=test_case.additional_metadata,
         )
+
         api_test_case.turns = [
             create_api_turn(
                 turn=turn,
@@ -86,48 +86,27 @@ def create_api_test_case(
             name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
         metrics_data = []
 
-        if isinstance(test_case, LLMTestCase) and test_case.multimodal is False:
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input=test_case.input,
-                actualOutput=test_case.actual_output,
-                expectedOutput=test_case.expected_output,
-                context=test_case.context,
-                retrievalContext=test_case.retrieval_context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                tags=test_case.tags,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-                trace=trace,
-            )
-        elif isinstance(test_case, LLMTestCase) and test_case.multimodal:
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input=test_case.input,
-                actualOutput=test_case.actual_output,
-                expectedOutput=test_case.expected_output,
-                retrievalContext=test_case.retrieval_context,
-                context=test_case.context,
-                imagesMapping=_MLLM_IMAGE_REGISTRY,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-            )
+        api_test_case = LLMApiTestCase(
+            name=name,
+            input=test_case.input,
+            actualOutput=test_case.actual_output,
+            expectedOutput=test_case.expected_output,
+            retrievalContext=test_case.retrieval_context,
+            context=test_case.context,
+            imagesMapping=test_case._get_images_mapping(),
+            toolsCalled=test_case.tools_called,
+            expectedTools=test_case.expected_tools,
+            tokenCost=test_case.token_cost,
+            completionTime=test_case.completion_time,
+            success=success,
+            metricsData=metrics_data,
+            runDuration=None,
+            evaluationCost=None,
+            order=order,
+            additionalMetadata=test_case.additional_metadata,
+            comments=test_case.comments,
+            tags=test_case.tags,
+            trace=trace,
+        )
         # llm_test_case_lookup_map[instance_id] = api_test_case
         return api_test_case

deepeval/test_case/arena_test_case.py

@@ -1,7 +1,7 @@
 from typing import List, Dict, Optional, Union
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pydantic import BaseModel
-
+import re
 from deepeval.test_case import (
     LLMTestCase,
 )
@@ -19,6 +19,7 @@ class Contestant(BaseModel):
 @dataclass
 class ArenaTestCase:
     contestants: List[Contestant]
+    multimodal: bool = field(default=False)
 
     def __post_init__(self):
         contestant_names = [contestant.name for contestant in self.contestants]
@@ -38,6 +39,10 @@ class ArenaTestCase:
                     "All contestants must have the same 'expected_output'."
                 )
 
+        for contestant in self.contestants:
+            if contestant.test_case.multimodal:
+                self.multimodal = True
+
 
 class Arena:
     test_cases: List[ArenaTestCase]

deepeval/test_case/conversational_test_case.py

@@ -1,3 +1,4 @@
+import re
 from pydantic import (
     BaseModel,
     Field,
@@ -17,6 +18,7 @@ from deepeval.test_case.mcp import (
     MCPToolCall,
     validate_mcp_servers,
 )
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 
 
 class TurnParams(Enum):
@@ -170,12 +172,28 @@ class ConversationalTestCase(BaseModel):
             return self
 
         pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
-        self.multimodal = any(
-            [
-                re.search(pattern, turn.content) is not None
-                for turn in self.turns
-            ]
-        )
+        if self.scenario:
+            if re.search(pattern, self.scenario) is not None:
+                self.multimodal = True
+                return self
+        if self.expected_outcome:
+            if re.search(pattern, self.expected_outcome) is not None:
+                self.multimodal = True
+                return self
+        if self.user_description:
+            if re.search(pattern, self.user_description) is not None:
+                self.multimodal = True
+                return self
+        if self.turns:
+            for turn in self.turns:
+                if re.search(pattern, turn.content) is not None:
+                    self.multimodal = True
+                    return self
+                if turn.retrieval_context is not None:
+                    self.multimodal = any(
+                        re.search(pattern, context) is not None
+                        for context in turn.retrieval_context
+                    )
 
         return self
 
@@ -215,3 +233,34 @@ class ConversationalTestCase(BaseModel):
         data["turns"] = copied_turns
 
         return data
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.scenario)
+        extract_ids_from_string(self.expected_outcome)
+        extract_ids_from_list(self.context)
+        extract_ids_from_string(self.user_description)
+        for turn in self.turns:
+            extract_ids_from_string(turn.content)
+            extract_ids_from_list(turn.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/test_case/llm_test_case.py

@@ -60,19 +60,34 @@ class MLLMImage:
             if self.local:
                 path = self.process_url(self.url)
                 self.filename = os.path.basename(path)
-                self.mimeType = (
-                    mimetypes.guess_type(path)[0] or "application/octet-stream"
-                )
-                with open(path, "rb") as f:
-                    raw = f.read()
-                self.dataBase64 = base64.b64encode(raw).decode("ascii")
+                self.mimeType = mimetypes.guess_type(path)[0] or "image/jpeg"
+
+                if not os.path.exists(path):
+                    raise FileNotFoundError(f"Image file not found: {path}")
+
+                self._load_base64(path)
             else:
+                if not self.url.startswith(("http://", "https://")):
+                    raise ValueError(
+                        f"Invalid remote URL format: {self.url}. URL must start with http:// or https://"
+                    )
                 self.filename = None
                 self.mimeType = None
                 self.dataBase64 = None
 
         _MLLM_IMAGE_REGISTRY[self._id] = self
 
+    def _load_base64(self, path: str):
+        with open(path, "rb") as f:
+            raw = f.read()
+        self.dataBase64 = base64.b64encode(raw).decode("ascii")
+
+    def ensure_images_loaded(self):
+        if self.local and self.dataBase64 is None:
+            path = self.process_url(self.url)
+            self._load_base64(path)
+        return self
+
     def _placeholder(self) -> str:
         return f"[DEEPEVAL:IMAGE:{self._id}]"
 
@@ -376,6 +391,16 @@ class LLMTestCase(BaseModel):
             if isinstance(self.input, str)
             else self.multimodal
         )
+        if self.retrieval_context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.retrieval_context
+            )
+        if self.context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.context
+            )
 
         self.multimodal = auto_detect
         return self
@@ -486,3 +511,32 @@ class LLMTestCase(BaseModel):
                 )
 
         return data
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.input)
+        extract_ids_from_string(self.actual_output)
+        extract_ids_from_string(self.expected_output)
+        extract_ids_from_list(self.context)
+        extract_ids_from_list(self.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/test_run/api.py

@@ -126,6 +126,9 @@ class ConversationalApiTestCase(BaseModel):
     additional_metadata: Optional[Dict] = Field(
         None, alias="additionalMetadata"
     )
+    images_mapping: Optional[Dict[str, MLLMImage]] = Field(
+        None, alias="imagesMapping"
+    )
     tags: Optional[List[str]] = Field(None)
 
     def update_metric_data(self, metrics_data: MetricData):