deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/test_case/api.py

@@ -10,9 +10,9 @@ from deepeval.test_run.api import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
     Turn,
 )
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 from deepeval.constants import PYTEST_RUN_TEST_NAME
 
 
@@ -29,10 +29,12 @@ def create_api_turn(turn: Turn, index: int) -> TurnApi:
 
 
 def create_api_test_case(
-    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
     trace: Optional[TraceApi] = None,
     index: Optional[int] = None,
 ) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
+    from deepeval.utils import convert_to_multi_modal_array
+
     if isinstance(test_case, ConversationalTestCase):
         order = (
             test_case._dataset_rank
@@ -84,7 +86,7 @@ def create_api_test_case(
             name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
         metrics_data = []
 
-        if isinstance(test_case, LLMTestCase):
+        if isinstance(test_case, LLMTestCase) and test_case.multimodal is False:
             api_test_case = LLMApiTestCase(
                 name=name,
                 input=test_case.input,
@@ -106,15 +108,15 @@ def create_api_test_case(
                 comments=test_case.comments,
                 trace=trace,
             )
-        elif isinstance(test_case, MLLMTestCase):
+        elif isinstance(test_case, LLMTestCase) and test_case.multimodal:
             api_test_case = LLMApiTestCase(
                 name=name,
-                input="",
-                multimodalInput=test_case.input,
-                multimodalActualOutput=test_case.actual_output,
-                multimodalExpectedOutput=test_case.expected_output,
-                multimodalRetrievalContext=test_case.retrieval_context,
-                multimodalContext=test_case.context,
+                input=test_case.input,
+                actualOutput=test_case.actual_output,
+                expectedOutput=test_case.expected_output,
+                retrievalContext=test_case.retrieval_context,
+                context=test_case.context,
+                imagesMapping=_MLLM_IMAGE_REGISTRY,
                 toolsCalled=test_case.tools_called,
                 expectedTools=test_case.expected_tools,
                 tokenCost=test_case.token_cost,

deepeval/test_case/conversational_test_case.py

@@ -9,7 +9,7 @@ from typing import List, Optional, Dict, Literal
 from copy import deepcopy
 from enum import Enum
 
-from deepeval.test_case import ToolCall
+from deepeval.test_case import ToolCall, MLLMImage
 from deepeval.test_case.mcp import (
     MCPServer,
     MCPPromptCall,
@@ -156,11 +156,29 @@ class ConversationalTestCase(BaseModel):
     comments: Optional[str] = Field(default=None)
     tags: Optional[List[str]] = Field(default=None)
     mcp_servers: Optional[List[MCPServer]] = Field(default=None)
+    multimodal: bool = False
 
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        self.multimodal = any(
+            [
+                re.search(pattern, turn.content) is not None
+                for turn in self.turns
+            ]
+        )
+
+        return self
+
     @model_validator(mode="before")
     def validate_input(cls, data):
         turns = data.get("turns")

deepeval/test_case/llm_test_case.py

@@ -9,7 +9,12 @@ from typing import List, Optional, Dict, Any
 from enum import Enum
 import json
 import uuid
-
+import re
+import os
+import mimetypes
+import base64
+from dataclasses import dataclass, field
+from urllib.parse import urlparse, unquote
 from deepeval.utils import make_model_config
 
 from deepeval.test_case.mcp import (
@@ -20,6 +25,128 @@ from deepeval.test_case.mcp import (
     validate_mcp_servers,
 )
 
+_MLLM_IMAGE_REGISTRY: Dict[str, "MLLMImage"] = {}
+
+
+@dataclass
+class MLLMImage:
+    dataBase64: Optional[str] = None
+    mimeType: Optional[str] = None
+    url: Optional[str] = None
+    local: Optional[bool] = None
+    filename: Optional[str] = None
+    _id: str = field(default_factory=lambda: uuid.uuid4().hex)
+
+    def __post_init__(self):
+
+        if not self.url and not self.dataBase64:
+            raise ValueError(
+                "You must provide either a 'url' or both 'dataBase64' and 'mimeType' to create an MLLMImage."
+            )
+
+        if self.dataBase64 is not None:
+            if self.mimeType is None:
+                raise ValueError(
+                    "mimeType must be provided when initializing from Base64 data."
+                )
+        else:
+            is_local = self.is_local_path(self.url)
+            if self.local is not None:
+                assert self.local == is_local, "Local path mismatch"
+            else:
+                self.local = is_local
+
+            # compute filename, mime_type, and Base64 data
+            if self.local:
+                path = self.process_url(self.url)
+                self.filename = os.path.basename(path)
+                self.mimeType = (
+                    mimetypes.guess_type(path)[0] or "application/octet-stream"
+                )
+                with open(path, "rb") as f:
+                    raw = f.read()
+                self.dataBase64 = base64.b64encode(raw).decode("ascii")
+            else:
+                self.filename = None
+                self.mimeType = None
+                self.dataBase64 = None
+
+        _MLLM_IMAGE_REGISTRY[self._id] = self
+
+    def _placeholder(self) -> str:
+        return f"[DEEPEVAL:IMAGE:{self._id}]"
+
+    def __str__(self) -> str:
+        return self._placeholder()
+
+    def __repr__(self) -> str:
+        return self._placeholder()
+
+    def __format__(self, format_spec: str) -> str:
+        return self._placeholder()
+
+    @staticmethod
+    def process_url(url: str) -> str:
+        if os.path.exists(url):
+            return url
+        parsed = urlparse(url)
+        if parsed.scheme == "file":
+            raw_path = (
+                f"//{parsed.netloc}{parsed.path}"
+                if parsed.netloc
+                else parsed.path
+            )
+            path = unquote(raw_path)
+            return path
+        return url
+
+    @staticmethod
+    def is_local_path(url: str) -> bool:
+        if os.path.exists(url):
+            return True
+        parsed = urlparse(url)
+        if parsed.scheme == "file":
+            raw_path = (
+                f"//{parsed.netloc}{parsed.path}"
+                if parsed.netloc
+                else parsed.path
+            )
+            path = unquote(raw_path)
+            return os.path.exists(path)
+        return False
+
+    def parse_multimodal_string(s: str):
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        matches = list(re.finditer(pattern, s))
+
+        result = []
+        last_end = 0
+
+        for m in matches:
+            start, end = m.span()
+
+            if start > last_end:
+                result.append(s[last_end:start])
+
+            img_id = m.group(1)
+
+            if img_id not in _MLLM_IMAGE_REGISTRY:
+                MLLMImage(url=img_id, _id=img_id)
+
+            result.append(_MLLM_IMAGE_REGISTRY[img_id])
+            last_end = end
+
+        if last_end < len(s):
+            result.append(s[last_end:])
+
+        return result
+
+    def as_data_uri(self) -> Optional[str]:
+        """Return the image as a data URI string, if Base64 data is available."""
+        if not self.dataBase64 or not self.mimeType:
+            return None
+        return f"data:{self.mimeType};base64,{self.dataBase64}"
+
 
 class LLMTestCaseParams(Enum):
     INPUT = "input"
@@ -208,6 +335,7 @@ class LLMTestCase(BaseModel):
         serialization_alias="completionTime",
         validation_alias=AliasChoices("completionTime", "completion_time"),
     )
+    multimodal: bool = Field(default=False)
     name: Optional[str] = Field(default=None)
     tags: Optional[List[str]] = Field(default=None)
     mcp_servers: Optional[List[MCPServer]] = Field(default=None)
@@ -229,6 +357,29 @@ class LLMTestCase(BaseModel):
         default_factory=lambda: str(uuid.uuid4())
     )
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+
+        auto_detect = (
+            any(
+                [
+                    re.search(pattern, self.input or "") is not None,
+                    re.search(pattern, self.actual_output or "") is not None,
+                ]
+            )
+            if isinstance(self.input, str)
+            else self.multimodal
+        )
+
+        self.multimodal = auto_detect
+        return self
+
     @model_validator(mode="before")
     def validate_input(cls, data):
         input = data.get("input")

deepeval/test_case/utils.py

@@ -1,24 +1,20 @@
 from typing import Union, List
 
-from deepeval.test_case import LLMTestCase, MLLMTestCase, ConversationalTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 
 
 def check_valid_test_cases_type(
-    test_cases: Union[
-        List[Union[LLMTestCase, MLLMTestCase]], List[ConversationalTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
 ):
     llm_test_case_count = 0
     conversational_test_case_count = 0
     for test_case in test_cases:
-        if isinstance(test_case, LLMTestCase) or isinstance(
-            test_case, MLLMTestCase
-        ):
+        if isinstance(test_case, LLMTestCase):
             llm_test_case_count += 1
         else:
             conversational_test_case_count += 1
 
     if llm_test_case_count > 0 and conversational_test_case_count > 0:
         raise ValueError(
-            "You cannot supply a mixture of `LLMTestCase`/`MLLMTestCase`(s) and `ConversationalTestCase`(s) as the list of test cases."
+            "You cannot supply a mixture of `LLMTestCase`(s) and `ConversationalTestCase`(s) as the list of test cases."
         )

deepeval/test_run/api.py

@@ -18,20 +18,21 @@ class LLMApiTestCase(BaseModel):
     token_cost: Optional[float] = Field(None, alias="tokenCost")
     completion_time: Optional[float] = Field(None, alias="completionTime")
     tags: Optional[List[str]] = Field(None)
-    multimodal_input: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalInput"
-    )
-    multimodal_input_actual_output: Optional[List[Union[str, MLLMImage]]] = (
-        Field(None, alias="multimodalActualOutput")
-    )
-    multimodal_expected_output: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalExpectedOutput"
-    )
-    multimodal_retrieval_context: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalRetrievalContext"
-    )
-    multimodal_context: Optional[List[Union[str, MLLMImage]]] = Field(
-        None, alias="multimodalContext"
+    # multimodal_input: Optional[str] = Field(None, alias="multimodalInput")
+    # multimodal_input_actual_output: Optional[str] = Field(
+    #     None, alias="multimodalActualOutput"
+    # )
+    # multimodal_expected_output: Optional[str] = Field(
+    #     None, alias="multimodalExpectedOutput"
+    # )
+    # multimodal_retrieval_context: Optional[List[str]] = Field(
+    #     None, alias="multimodalRetrievalContext"
+    # )
+    # multimodal_context: Optional[List[str]] = Field(
+    #     None, alias="multimodalContext"
+    # )
+    images_mapping: Optional[Dict[str, MLLMImage]] = Field(
+        None, alias="imagesMapping"
     )
 
     # make these optional, not all test cases in a conversation will be evaluated

deepeval/test_run/cache.py

@@ -90,6 +90,8 @@ class CachedTestRun(BaseModel):
             # Pydantic version below 2.0
             body = self.dict(by_alias=True, exclude_none=True)
         json.dump(body, f, cls=CustomEncoder)
+        f.flush()
+        os.fsync(f.fileno())
         return self
 
     # load from file (this happens initially during a test run)

deepeval/test_run/test_run.py

@@ -21,7 +21,7 @@ from deepeval.test_run.api import (
 )
 from deepeval.tracing.utils import make_json_serializable
 from deepeval.tracing.api import SpanApiType, span_api_type_literals
-from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 from deepeval.utils import (
     delete_file_if_exists,
     get_is_running_deepeval,
@@ -182,7 +182,7 @@ class TestRun(BaseModel):
 
     def set_dataset_properties(
         self,
-        test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+        test_case: Union[LLMTestCase, ConversationalTestCase],
     ):
         if self.dataset_alias is None:
             self.dataset_alias = test_case._dataset_alias
@@ -406,9 +406,10 @@ class TestRun(BaseModel):
         try:
             body = self.model_dump(by_alias=True, exclude_none=True)
         except AttributeError:
-            # Pydantic version below 2.0
             body = self.dict(by_alias=True, exclude_none=True)
         json.dump(body, f, cls=TestRunEncoder)
+        f.flush()
+        os.fsync(f.fileno())
         return self
 
     @classmethod
@@ -515,6 +516,8 @@ class TestRunManager:
                             )
                         wrapper_data = {save_under_key: test_run_data}
                         json.dump(wrapper_data, file, cls=TestRunEncoder)
+                        file.flush()
+                        os.fsync(file.fileno())
                     else:
                         self.test_run.save(file)
             except portalocker.exceptions.LockException:
@@ -527,13 +530,15 @@ class TestRunManager:
                     LATEST_TEST_RUN_FILE_PATH, mode="w"
                 ) as file:
                     json.dump({LATEST_TEST_RUN_LINK_KEY: link}, file)
+                    file.flush()
+                    os.fsync(file.fileno())
             except portalocker.exceptions.LockException:
                 pass
 
     def update_test_run(
         self,
         api_test_case: Union[LLMApiTestCase, ConversationalApiTestCase],
-        test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+        test_case: Union[LLMTestCase, ConversationalTestCase],
     ):
         if (
             api_test_case.metrics_data is not None

deepeval/tracing/patchers.py

@@ -1,6 +1,7 @@
 import functools
 
-from anthropic import Anthropic
+from typing import TYPE_CHECKING
+
 from openai import OpenAI
 
 from deepeval.tracing.context import update_current_span, update_llm_span
@@ -8,6 +9,10 @@ from deepeval.tracing.context import current_span_context
 from deepeval.tracing.types import LlmSpan
 
 
+if TYPE_CHECKING:
+    from anthropic import Anthropic
+
+
 def patch_openai_client(client: OpenAI):
 
     original_methods = {}
@@ -61,7 +66,7 @@ def patch_openai_client(client: OpenAI):
                     output = None
                     try:
                         output = response.choices[0].message.content
-                    except Exception as e:
+                    except Exception:
                         pass
 
                     # extract input output token counts
@@ -70,7 +75,7 @@ def patch_openai_client(client: OpenAI):
                     try:
                         input_token_count = response.usage.prompt_tokens
                         output_token_count = response.usage.completion_tokens
-                    except Exception as e:
+                    except Exception:
                         pass
 
                     update_current_span(
@@ -86,7 +91,7 @@ def patch_openai_client(client: OpenAI):
             setattr(current_obj, method_name, wrapped_method)
 
 
-def patch_anthropic_client(client: Anthropic):
+def patch_anthropic_client(client: "Anthropic"):
     """
     Patch an Anthropic client instance to add tracing capabilities.
 

deepeval/tracing/tracing.py

@@ -19,7 +19,6 @@ import random
 import atexit
 import queue
 import uuid
-from anthropic import Anthropic
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
@@ -74,6 +73,7 @@ from deepeval.tracing.trace_test_manager import trace_testing_manager
 
 if TYPE_CHECKING:
     from deepeval.dataset.golden import Golden
+    from anthropic import Anthropic
 
 EVAL_DUMMY_SPAN_NAME = "evals_iterator"
 
@@ -154,7 +154,7 @@ class TraceManager:
         environment: Optional[str] = None,
         sampling_rate: Optional[float] = None,
         confident_api_key: Optional[str] = None,
-        anthropic_client: Optional[Anthropic] = None,
+        anthropic_client: Optional["Anthropic"] = None,
         openai_client: Optional[OpenAI] = None,
         tracing_enabled: Optional[bool] = None,
     ) -> None:

deepeval/utils.py

@@ -14,6 +14,7 @@ import logging
 
 from contextvars import ContextVar
 from enum import Enum
+from importlib import import_module
 from typing import Any, Dict, List, Optional, Protocol, Sequence, Union
 from collections.abc import Iterable
 from dataclasses import asdict, is_dataclass
@@ -21,6 +22,7 @@ from pydantic import BaseModel
 from rich.progress import Progress
 from rich.console import Console, Theme
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.config.utils import (
     get_env_bool,
@@ -536,6 +538,25 @@ def shorten(
     return stext[:cut] + suffix
 
 
+def convert_to_multi_modal_array(input: Union[str, List[str]]):
+    from deepeval.test_case import MLLMImage
+
+    if isinstance(input, str):
+        return MLLMImage.parse_multimodal_string(input)
+    elif isinstance(input, list):
+        new_list = []
+        for context in input:
+            parsed_array = MLLMImage.parse_multimodal_string(context)
+            new_list.extend(parsed_array)
+        return new_list
+
+
+def check_if_multimodal(input: str):
+    pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+    matches = list(re.finditer(pattern, input))
+    return bool(matches)
+
+
 def format_turn(
     turn: TurnLike,
     *,
@@ -814,3 +835,71 @@ def format_error_text(
 
 def is_read_only_env():
     return get_settings().DEEPEVAL_FILE_SYSTEM == "READ_ONLY"
+
+
+##############
+# validation #
+##############
+
+
+def require_param(
+    param: Optional[Any] = None,
+    *,
+    provider_label: str,
+    env_var_name: str,
+    param_hint: str,
+) -> Any:
+    """
+    Ensures that a required parameter is provided. If the parameter is `None`, raises a
+    `DeepEvalError` with a helpful message indicating the missing parameter and how to resolve it.
+
+    Args:
+        param (Optional[Any]): The parameter to validate.
+        provider_label (str): A label for the provider to be used in the error message.
+        env_var_name (str): The name of the environment variable where the parameter can be set.
+        param_hint (str): A hint for the parameter, usually the name of the argument.
+
+    Raises:
+        DeepEvalError: If the `param` is `None`, indicating that a required parameter is missing.
+
+    Returns:
+        Any: The value of `param` if it is provided.
+    """
+    if param is None:
+        raise DeepEvalError(
+            f"{provider_label} is missing a required parameter. "
+            f"Set {env_var_name} in your environment or pass "
+            f"{param_hint}."
+        )
+
+    return param
+
+
+def require_dependency(
+    module_name: str,
+    *,
+    provider_label: str,
+    install_hint: Optional[str] = None,
+) -> Any:
+    """
+    Imports an optional dependency module or raises a `DeepEvalError` if the module is not found.
+    The error message includes a suggestion on how to install the missing module.
+
+    Args:
+        module_name (str): The name of the module to import.
+        provider_label (str): A label for the provider to be used in the error message.
+        install_hint (Optional[str]): A hint on how to install the missing module, usually a pip command.
+
+    Raises:
+        DeepEvalError: If the module cannot be imported, indicating that the dependency is missing.
+
+    Returns:
+        Any: The imported module if successful.
+    """
+    try:
+        return import_module(module_name)
+    except ImportError as exc:
+        hint = install_hint or f"Install it with `pip install {module_name}`."
+        raise DeepEvalError(
+            f"{provider_label} requires the `{module_name}` package. {hint}"
+        ) from exc

{deepeval-3.7.3.dist-info → deepeval-3.7.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.7.3
+Version: 3.7.5
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -13,13 +13,10 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: aiohttp
-Requires-Dist: anthropic
 Requires-Dist: click (>=8.0.0,<8.3.0)
-Requires-Dist: google-genai (>=1.9.0,<2.0.0)
 Requires-Dist: grpcio (>=1.67.1,<2.0.0)
 Requires-Dist: jinja2
 Requires-Dist: nest_asyncio
-Requires-Dist: ollama
 Requires-Dist: openai
 Requires-Dist: opentelemetry-api (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.24.0,<2.0.0)