deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/test_case/conversational_test_case.py

@@ -1,3 +1,4 @@
+import re
 from pydantic import (
     BaseModel,
     Field,
@@ -9,7 +10,7 @@ from typing import List, Optional, Dict, Literal
 from copy import deepcopy
 from enum import Enum
 
-from deepeval.test_case import ToolCall
+from deepeval.test_case import ToolCall, MLLMImage
 from deepeval.test_case.mcp import (
     MCPServer,
     MCPPromptCall,
@@ -17,6 +18,7 @@ from deepeval.test_case.mcp import (
     MCPToolCall,
     validate_mcp_servers,
 )
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 
 
 class TurnParams(Enum):
@@ -156,11 +158,45 @@ class ConversationalTestCase(BaseModel):
     comments: Optional[str] = Field(default=None)
     tags: Optional[List[str]] = Field(default=None)
     mcp_servers: Optional[List[MCPServer]] = Field(default=None)
+    multimodal: bool = False
 
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        if self.scenario:
+            if re.search(pattern, self.scenario) is not None:
+                self.multimodal = True
+                return self
+        if self.expected_outcome:
+            if re.search(pattern, self.expected_outcome) is not None:
+                self.multimodal = True
+                return self
+        if self.user_description:
+            if re.search(pattern, self.user_description) is not None:
+                self.multimodal = True
+                return self
+        if self.turns:
+            for turn in self.turns:
+                if re.search(pattern, turn.content) is not None:
+                    self.multimodal = True
+                    return self
+                if turn.retrieval_context is not None:
+                    self.multimodal = any(
+                        re.search(pattern, context) is not None
+                        for context in turn.retrieval_context
+                    )
+
+        return self
+
     @model_validator(mode="before")
     def validate_input(cls, data):
         turns = data.get("turns")
@@ -197,3 +233,34 @@ class ConversationalTestCase(BaseModel):
         data["turns"] = copied_turns
 
         return data
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.scenario)
+        extract_ids_from_string(self.expected_outcome)
+        extract_ids_from_list(self.context)
+        extract_ids_from_string(self.user_description)
+        for turn in self.turns:
+            extract_ids_from_string(turn.content)
+            extract_ids_from_list(turn.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/test_case/llm_test_case.py

@@ -9,7 +9,12 @@ from typing import List, Optional, Dict, Any
 from enum import Enum
 import json
 import uuid
-
+import re
+import os
+import mimetypes
+import base64
+from dataclasses import dataclass, field
+from urllib.parse import urlparse, unquote
 from deepeval.utils import make_model_config
 
 from deepeval.test_case.mcp import (
@@ -20,6 +25,143 @@ from deepeval.test_case.mcp import (
     validate_mcp_servers,
 )
 
+_MLLM_IMAGE_REGISTRY: Dict[str, "MLLMImage"] = {}
+
+
+@dataclass
+class MLLMImage:
+    dataBase64: Optional[str] = None
+    mimeType: Optional[str] = None
+    url: Optional[str] = None
+    local: Optional[bool] = None
+    filename: Optional[str] = None
+    _id: str = field(default_factory=lambda: uuid.uuid4().hex)
+
+    def __post_init__(self):
+
+        if not self.url and not self.dataBase64:
+            raise ValueError(
+                "You must provide either a 'url' or both 'dataBase64' and 'mimeType' to create an MLLMImage."
+            )
+
+        if self.dataBase64 is not None:
+            if self.mimeType is None:
+                raise ValueError(
+                    "mimeType must be provided when initializing from Base64 data."
+                )
+        else:
+            is_local = self.is_local_path(self.url)
+            if self.local is not None:
+                assert self.local == is_local, "Local path mismatch"
+            else:
+                self.local = is_local
+
+            # compute filename, mime_type, and Base64 data
+            if self.local:
+                path = self.process_url(self.url)
+                self.filename = os.path.basename(path)
+                self.mimeType = mimetypes.guess_type(path)[0] or "image/jpeg"
+
+                if not os.path.exists(path):
+                    raise FileNotFoundError(f"Image file not found: {path}")
+
+                self._load_base64(path)
+            else:
+                if not self.url.startswith(("http://", "https://")):
+                    raise ValueError(
+                        f"Invalid remote URL format: {self.url}. URL must start with http:// or https://"
+                    )
+                self.filename = None
+                self.mimeType = None
+                self.dataBase64 = None
+
+        _MLLM_IMAGE_REGISTRY[self._id] = self
+
+    def _load_base64(self, path: str):
+        with open(path, "rb") as f:
+            raw = f.read()
+        self.dataBase64 = base64.b64encode(raw).decode("ascii")
+
+    def ensure_images_loaded(self):
+        if self.local and self.dataBase64 is None:
+            path = self.process_url(self.url)
+            self._load_base64(path)
+        return self
+
+    def _placeholder(self) -> str:
+        return f"[DEEPEVAL:IMAGE:{self._id}]"
+
+    def __str__(self) -> str:
+        return self._placeholder()
+
+    def __repr__(self) -> str:
+        return self._placeholder()
+
+    def __format__(self, format_spec: str) -> str:
+        return self._placeholder()
+
+    @staticmethod
+    def process_url(url: str) -> str:
+        if os.path.exists(url):
+            return url
+        parsed = urlparse(url)
+        if parsed.scheme == "file":
+            raw_path = (
+                f"//{parsed.netloc}{parsed.path}"
+                if parsed.netloc
+                else parsed.path
+            )
+            path = unquote(raw_path)
+            return path
+        return url
+
+    @staticmethod
+    def is_local_path(url: str) -> bool:
+        if os.path.exists(url):
+            return True
+        parsed = urlparse(url)
+        if parsed.scheme == "file":
+            raw_path = (
+                f"//{parsed.netloc}{parsed.path}"
+                if parsed.netloc
+                else parsed.path
+            )
+            path = unquote(raw_path)
+            return os.path.exists(path)
+        return False
+
+    def parse_multimodal_string(s: str):
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        matches = list(re.finditer(pattern, s))
+
+        result = []
+        last_end = 0
+
+        for m in matches:
+            start, end = m.span()
+
+            if start > last_end:
+                result.append(s[last_end:start])
+
+            img_id = m.group(1)
+
+            if img_id not in _MLLM_IMAGE_REGISTRY:
+                MLLMImage(url=img_id, _id=img_id)
+
+            result.append(_MLLM_IMAGE_REGISTRY[img_id])
+            last_end = end
+
+        if last_end < len(s):
+            result.append(s[last_end:])
+
+        return result
+
+    def as_data_uri(self) -> Optional[str]:
+        """Return the image as a data URI string, if Base64 data is available."""
+        if not self.dataBase64 or not self.mimeType:
+            return None
+        return f"data:{self.mimeType};base64,{self.dataBase64}"
+
 
 class LLMTestCaseParams(Enum):
     INPUT = "input"
@@ -208,6 +350,7 @@ class LLMTestCase(BaseModel):
         serialization_alias="completionTime",
         validation_alias=AliasChoices("completionTime", "completion_time"),
     )
+    multimodal: bool = Field(default=False)
     name: Optional[str] = Field(default=None)
     tags: Optional[List[str]] = Field(default=None)
     mcp_servers: Optional[List[MCPServer]] = Field(default=None)
@@ -229,6 +372,39 @@ class LLMTestCase(BaseModel):
         default_factory=lambda: str(uuid.uuid4())
     )
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+
+        auto_detect = (
+            any(
+                [
+                    re.search(pattern, self.input or "") is not None,
+                    re.search(pattern, self.actual_output or "") is not None,
+                ]
+            )
+            if isinstance(self.input, str)
+            else self.multimodal
+        )
+        if self.retrieval_context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.retrieval_context
+            )
+        if self.context is not None:
+            auto_detect = auto_detect or any(
+                re.search(pattern, context) is not None
+                for context in self.context
+            )
+
+        self.multimodal = auto_detect
+        return self
+
     @model_validator(mode="before")
     def validate_input(cls, data):
         input = data.get("input")
@@ -335,3 +511,32 @@ class LLMTestCase(BaseModel):
                 )
 
         return data
+
+    def _get_images_mapping(self) -> Dict[str, MLLMImage]:
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        image_ids = set()
+
+        def extract_ids_from_string(s: Optional[str]) -> None:
+            """Helper to extract image IDs from a string."""
+            if s is not None and isinstance(s, str):
+                matches = re.findall(pattern, s)
+                image_ids.update(matches)
+
+        def extract_ids_from_list(lst: Optional[List[str]]) -> None:
+            """Helper to extract image IDs from a list of strings."""
+            if lst is not None:
+                for item in lst:
+                    extract_ids_from_string(item)
+
+        extract_ids_from_string(self.input)
+        extract_ids_from_string(self.actual_output)
+        extract_ids_from_string(self.expected_output)
+        extract_ids_from_list(self.context)
+        extract_ids_from_list(self.retrieval_context)
+
+        images_mapping = {}
+        for img_id in image_ids:
+            if img_id in _MLLM_IMAGE_REGISTRY:
+                images_mapping[img_id] = _MLLM_IMAGE_REGISTRY[img_id]
+
+        return images_mapping if len(images_mapping) > 0 else None

deepeval/test_case/utils.py

@@ -1,24 +1,20 @@
 from typing import Union, List
 
-from deepeval.test_case import LLMTestCase, MLLMTestCase, ConversationalTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 
 
 def check_valid_test_cases_type(
-    test_cases: Union[
-        List[Union[LLMTestCase, MLLMTestCase]], List[ConversationalTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
 ):
     llm_test_case_count = 0
     conversational_test_case_count = 0
     for test_case in test_cases:
-        if isinstance(test_case, LLMTestCase) or isinstance(
-            test_case, MLLMTestCase
-        ):
+        if isinstance(test_case, LLMTestCase):
             llm_test_case_count += 1
         else:
             conversational_test_case_count += 1
 
     if llm_test_case_count > 0 and conversational_test_case_count > 0:
         raise ValueError(
-            "You cannot supply a mixture of `LLMTestCase`/`MLLMTestCase`(s) and `ConversationalTestCase`(s) as the list of test cases."
+            "You cannot supply a mixture of `LLMTestCase`(s) and `ConversationalTestCase`(s) as the list of test cases."
         )

deepeval/test_run/api.py

@@ -18,20 +18,21 @@ class LLMApiTestCase(BaseModel):
     token_cost: Optional[float] = Field(None, alias="tokenCost")
     completion_time: Optional[float] = Field(None, alias="completionTime")
     tags: Optional[List[str]] = Field(None)
-    multimodal_input: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalInput"
-    )
-    multimodal_input_actual_output: Optional[List[Union[str, MLLMImage]]] = (
-        Field(None, alias="multimodalActualOutput")
-    )
-    multimodal_expected_output: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalExpectedOutput"
-    )
-    multimodal_retrieval_context: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalRetrievalContext"
-    )
-    multimodal_context: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalContext"
+    # multimodal_input: Optional[str] = Field(None, alias="multimodalInput")
+    # multimodal_input_actual_output: Optional[str] = Field(
+    #     None, alias="multimodalActualOutput"
+    # )
+    # multimodal_expected_output: Optional[str] = Field(
+    #     None, alias="multimodalExpectedOutput"
+    # )
+    # multimodal_retrieval_context: Optional[List[str]] = Field(
+    #     None, alias="multimodalRetrievalContext"
+    # )
+    # multimodal_context: Optional[List[str]] = Field(
+    #     None, alias="multimodalContext"
+    # )
+    images_mapping: Optional[Dict[str, MLLMImage]] = Field(
+        None, alias="imagesMapping"
     )
 
     # make these optional, not all test cases in a conversation will be evaluated
@@ -125,6 +126,9 @@ class ConversationalApiTestCase(BaseModel):
     additional_metadata: Optional[Dict] = Field(
         None, alias="additionalMetadata"
     )
+    images_mapping: Optional[Dict[str, MLLMImage]] = Field(
+        None, alias="imagesMapping"
+    )
     tags: Optional[List[str]] = Field(None)
 
     def update_metric_data(self, metrics_data: MetricData):

deepeval/test_run/test_run.py

@@ -21,7 +21,7 @@ from deepeval.test_run.api import (
 )
 from deepeval.tracing.utils import make_json_serializable
 from deepeval.tracing.api import SpanApiType, span_api_type_literals
-from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.utils import (
     delete_file_if_exists,
     get_is_running_deepeval,
@@ -182,7 +182,7 @@ class TestRun(BaseModel):
 
     def set_dataset_properties(
         self,
-        test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+        test_case: Union[LLMTestCase, ConversationalTestCase],
     ):
         if self.dataset_alias is None:
             self.dataset_alias = test_case._dataset_alias
@@ -538,7 +538,7 @@ class TestRunManager:
     def update_test_run(
         self,
         api_test_case: Union[LLMApiTestCase, ConversationalApiTestCase],
-        test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+        test_case: Union[LLMTestCase, ConversationalTestCase],
     ):
         if (
             api_test_case.metrics_data is not None

deepeval/tracing/patchers.py

@@ -1,6 +1,7 @@
 import functools
 
-from anthropic import Anthropic
+from typing import TYPE_CHECKING
+
 from openai import OpenAI
 
 from deepeval.tracing.context import update_current_span, update_llm_span
@@ -8,6 +9,10 @@ from deepeval.tracing.context import current_span_context
 from deepeval.tracing.types import LlmSpan
 
 
+if TYPE_CHECKING:
+    from anthropic import Anthropic
+
+
 def patch_openai_client(client: OpenAI):
 
     original_methods = {}
@@ -61,7 +66,7 @@ def patch_openai_client(client: OpenAI):
                     output = None
                     try:
                         output = response.choices[0].message.content
-                    except Exception as e:
+                    except Exception:
                         pass
 
                     # extract input output token counts
@@ -70,7 +75,7 @@ def patch_openai_client(client: OpenAI):
                     try:
                         input_token_count = response.usage.prompt_tokens
                         output_token_count = response.usage.completion_tokens
-                    except Exception as e:
+                    except Exception:
                         pass
 
                     update_current_span(
@@ -86,7 +91,7 @@ def patch_openai_client(client: OpenAI):
             setattr(current_obj, method_name, wrapped_method)
 
 
-def patch_anthropic_client(client: Anthropic):
+def patch_anthropic_client(client: "Anthropic"):
     """
     Patch an Anthropic client instance to add tracing capabilities.
 

deepeval/tracing/tracing.py

@@ -19,7 +19,6 @@ import random
 import atexit
 import queue
 import uuid
-from anthropic import Anthropic
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
@@ -74,6 +73,7 @@ from deepeval.tracing.trace_test_manager import trace_testing_manager
 
 if TYPE_CHECKING:
     from deepeval.dataset.golden import Golden
+    from anthropic import Anthropic
 
 EVAL_DUMMY_SPAN_NAME = "evals_iterator"
 
@@ -154,7 +154,7 @@ class TraceManager:
         environment: Optional[str] = None,
         sampling_rate: Optional[float] = None,
         confident_api_key: Optional[str] = None,
-        anthropic_client: Optional[Anthropic] = None,
+        anthropic_client: Optional["Anthropic"] = None,
         openai_client: Optional[OpenAI] = None,
         tracing_enabled: Optional[bool] = None,
     ) -> None:

deepeval/utils.py

@@ -14,6 +14,7 @@ import logging
 
 from contextvars import ContextVar
 from enum import Enum
+from importlib import import_module
 from typing import Any, Dict, List, Optional, Protocol, Sequence, Union
 from collections.abc import Iterable
 from dataclasses import asdict, is_dataclass
@@ -537,6 +538,25 @@ def shorten(
     return stext[:cut] + suffix
 
 
+def convert_to_multi_modal_array(input: Union[str, List[str]]):
+    from deepeval.test_case import MLLMImage
+
+    if isinstance(input, str):
+        return MLLMImage.parse_multimodal_string(input)
+    elif isinstance(input, list):
+        new_list = []
+        for context in input:
+            parsed_array = MLLMImage.parse_multimodal_string(context)
+            new_list.extend(parsed_array)
+        return new_list
+
+
+def check_if_multimodal(input: str):
+    pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+    matches = list(re.finditer(pattern, input))
+    return bool(matches)
+
+
 def format_turn(
     turn: TurnLike,
     *,
@@ -829,7 +849,22 @@ def require_param(
     env_var_name: str,
     param_hint: str,
 ) -> Any:
+    """
+    Ensures that a required parameter is provided. If the parameter is `None`, raises a
+    `DeepEvalError` with a helpful message indicating the missing parameter and how to resolve it.
 
+    Args:
+        param (Optional[Any]): The parameter to validate.
+        provider_label (str): A label for the provider to be used in the error message.
+        env_var_name (str): The name of the environment variable where the parameter can be set.
+        param_hint (str): A hint for the parameter, usually the name of the argument.
+
+    Raises:
+        DeepEvalError: If the `param` is `None`, indicating that a required parameter is missing.
+
+    Returns:
+        Any: The value of `param` if it is provided.
+    """
     if param is None:
         raise DeepEvalError(
             f"{provider_label} is missing a required parameter. "
@@ -838,3 +873,33 @@ def require_param(
         )
 
     return param
+
+
+def require_dependency(
+    module_name: str,
+    *,
+    provider_label: str,
+    install_hint: Optional[str] = None,
+) -> Any:
+    """
+    Imports an optional dependency module or raises a `DeepEvalError` if the module is not found.
+    The error message includes a suggestion on how to install the missing module.
+
+    Args:
+        module_name (str): The name of the module to import.
+        provider_label (str): A label for the provider to be used in the error message.
+        install_hint (Optional[str]): A hint on how to install the missing module, usually a pip command.
+
+    Raises:
+        DeepEvalError: If the module cannot be imported, indicating that the dependency is missing.
+
+    Returns:
+        Any: The imported module if successful.
+    """
+    try:
+        return import_module(module_name)
+    except ImportError as exc:
+        hint = install_hint or f"Install it with `pip install {module_name}`."
+        raise DeepEvalError(
+            f"{provider_label} requires the `{module_name}` package. {hint}"
+        ) from exc

{deepeval-3.7.4.dist-info → deepeval-3.7.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.7.4
+Version: 3.7.6
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -13,13 +13,10 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: aiohttp
-Requires-Dist: anthropic
 Requires-Dist: click (>=8.0.0,<8.3.0)
-Requires-Dist: google-genai (>=1.9.0,<2.0.0)
 Requires-Dist: grpcio (>=1.67.1,<2.0.0)
 Requires-Dist: jinja2
 Requires-Dist: nest_asyncio
-Requires-Dist: ollama
 Requires-Dist: openai
 Requires-Dist: opentelemetry-api (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.24.0,<2.0.0)