deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/{optimization → optimizer}/utils.py

@@ -2,11 +2,14 @@ from __future__ import annotations
 import inspect
 import random
 import re
+import statistics
 from typing import (
     Any,
     Callable,
     List,
     Optional,
+    Protocol,
+    Sequence,
     Tuple,
     TYPE_CHECKING,
     Union,
@@ -17,11 +20,13 @@ from typing import (
 from deepeval.errors import DeepEvalError
 from deepeval.metrics.base_metric import BaseMetric, BaseConversationalMetric
 from deepeval.prompt.prompt import Prompt
-from deepeval.prompt.api import PromptType, PromptMessage
-from deepeval.optimization.types import (
+from deepeval.prompt.api import PromptMessage
+from deepeval.optimizer.types import (
+    ModelCallback,
     ModuleId,
     PromptConfigurationId,
     PromptConfiguration,
+    PromptConfigSnapshot,
     OptimizationReport,
 )
 
@@ -54,7 +59,7 @@ def split_goldens(
         pareto_size: Number of items to allocate to the Pareto set bound between [0, len(goldens)].
         random_state: A shared `random.Random` instance that provides the source
             of randomness. For reproducible runs, pass the same object used by
-            the GEPA loop constructed from `GEPAConfig.random_seed`
+            the GEPA loop constructed from `GEPA.random_seed`
 
     Returns:
         (d_feedback, d_pareto)
@@ -151,87 +156,22 @@ def normalize_seed_prompts(
     return mapping
 
 
-def build_model_callback_kwargs(
-    *,
-    # scoring context
-    golden: Optional[Union["Golden", "ConversationalGolden"]] = None,
-    # rewriter context
-    feedback_text: Optional[str] = None,
-    # shared
-    prompt: Optional[Prompt] = None,
-    prompt_type: Optional[str] = None,
-    prompt_text: Optional[str] = None,
-    prompt_messages: Optional[List["PromptMessage"]] = None,
-) -> Dict[str, Any]:
-    """
-    Build a superset of kwargs for GEPA model callbacks.
-
-    All keys are present in the dict so callbacks can declare any subset of:
-
-        hook: str           # injected by (a_)invoke_model_callback
-        prompt: Prompt
-        prompt_type: str
-        prompt_text: str
-        prompt_messages: List[PromptMessage]
-        golden: Golden | ConversationalGolden
-        feedback_text: str
-
-    Non applicable fields are set to None.
-    """
-    return {
-        # scoring context
-        "golden": golden,
-        # rewriter context
-        "feedback_text": feedback_text,
-        # shared
-        "prompt": prompt,
-        "prompt_text": prompt_text,
-        "prompt_messages": prompt_messages,
-    }
-
-
 def invoke_model_callback(
     *,
-    hook: str,
-    model_callback: Callable[
-        ...,
-        Union[
-            str,
-            Dict,
-            Tuple[Union[str, Dict], float],
-        ],
-    ],
-    candidate_kwargs: Dict[str, Any],
-) -> Union[
-    str,
-    Dict,
-    Tuple[Union[str, Dict], float],
-]:
+    model_callback: ModelCallback,
+    prompt: Prompt,
+    golden: Union["Golden", "ConversationalGolden"],
+) -> str:
     """
     Call a user provided model_callback in a synchronous context.
 
-    - Filters kwargs to only those the callback accepts.
-    - Injects `hook` if the callback declares it.
-    - Raises if the callback returns an awaitable; callers must use async
-      helpers for async callbacks.
+    Raises if the callback returns an awaitable.
     """
-    sig = inspect.signature(model_callback)
-    supported = set(sig.parameters.keys())
-
-    filtered = {
-        key: value
-        for key, value in candidate_kwargs.items()
-        if key in supported
-    }
-
-    if "hook" in supported:
-        filtered["hook"] = hook
-
-    result = model_callback(**filtered)
+    result = model_callback(prompt, golden)
     if inspect.isawaitable(result):
         raise DeepEvalError(
             "model_callback returned an awaitable from a synchronous context. "
-            "Either declare the callback as `async def` and use async GEPA, or call "
+            "Either declare the callback as `async def` and use async optimization, or call "
             "`model.generate(...)` instead of `model.a_generate(...)` inside a sync callback."
         )
     return result
@@ -239,41 +179,16 @@ def invoke_model_callback(
 
 async def a_invoke_model_callback(
     *,
-    hook: str,
-    model_callback: Callable[
-        ...,
-        Union[
-            str,
-            Dict,
-            Tuple[Union[str, Dict], float],
-        ],
-    ],
-    candidate_kwargs: Dict[str, Any],
-) -> Union[
-    str,
-    Dict,
-    Tuple[Union[str, Dict], float],
-]:
+    model_callback: ModelCallback,
+    prompt: Prompt,
+    golden: Union["Golden", "ConversationalGolden"],
+) -> str:
     """
     Call a user provided model_callback in an async context.
 
-    - Filters kwargs to only those the callback accepts.
-    - Injects `hook` if the callback declares it.
-    - Supports both sync and async callbacks.
+    Supports both sync and async callbacks.
     """
-    sig = inspect.signature(model_callback)
-    supported = set(sig.parameters.keys())
-
-    filtered = {
-        key: value
-        for key, value in candidate_kwargs.items()
-        if key in supported
-    }
-
-    if "hook" in supported:
-        filtered["hook"] = hook
-
-    result = model_callback(**filtered)
+    result = model_callback(prompt, golden)
     if inspect.isawaitable(result):
         return await result
     return result
@@ -288,58 +203,17 @@ def build_prompt_config_snapshots(
     prompt_configurations_by_id: Dict[
         PromptConfigurationId, "PromptConfiguration"
     ],
-) -> Dict[PromptConfigurationId, Dict[str, Any]]:
+) -> Dict[PromptConfigurationId, PromptConfigSnapshot]:
     """
-    Build a serializable snapshot of all prompt configurations.
-
-    Shape matches the docs for `prompt_configurations`:
-
-    {
-      "<config_id>": {
-        "parent": "<parent_id or None>",
-        "prompts": {
-          "<module_id>": {
-            "type": "TEXT",
-            "text_template": "...",
-          }
-          # or
-          "<module_id>": {
-            "type": "LIST",
-            "messages": [
-              {"role": "system", "content": "..."},
-              ...
-            ],
-          },
-        },
-      },
-      ...
-    }
+    Build snapshots of all prompt configurations.
     """
-    snapshots: Dict[PromptConfigurationId, Dict[str, Any]] = {}
+    snapshots: Dict[PromptConfigurationId, PromptConfigSnapshot] = {}
 
     for cfg_id, cfg in prompt_configurations_by_id.items():
-        prompts_snapshot: Dict[str, Any] = {}
-
-        for module_id, prompt in cfg.prompts.items():
-            if prompt.type is PromptType.LIST:
-                messages = [
-                    {"role": msg.role, "content": (msg.content or "")}
-                    for msg in (prompt.messages_template or [])
-                ]
-                prompts_snapshot[module_id] = {
-                    "type": "LIST",
-                    "messages": messages,
-                }
-            else:
-                prompts_snapshot[module_id] = {
-                    "type": "TEXT",
-                    "text_template": (prompt.text_template or ""),
-                }
-
-        snapshots[cfg_id] = {
-            "parent": cfg.parent,
-            "prompts": prompts_snapshot,
-        }
+        snapshots[cfg_id] = PromptConfigSnapshot(
+            parent=cfg.parent,
+            prompts=dict(cfg.prompts),
+        )
 
     return snapshots
 
@@ -494,17 +368,8 @@ def validate_sequence_of(
 def validate_callback(
     *,
     component: str,
-    model_callback: Optional[
-        Callable[
-            ...,
-            Union[
-                str,
-                Dict,
-                Tuple[Union[str, Dict], float],
-            ],
-        ]
-    ],
-) -> Callable[..., Union[str, Dict, Tuple[Union[str, Dict], float]]]:
+    model_callback: Optional[ModelCallback],
+) -> ModelCallback:
     """
     Ensure that `model_callback` is provided.
 
@@ -596,3 +461,20 @@ def validate_int_in_range(
         )
 
     return value
+
+
+##############
+# Aggregates #
+##############
+
+
+class Aggregator(Protocol):
+    def __call__(self, scores: Sequence[float]) -> float: ...
+
+
+def mean_of_all(scores: Sequence[float]) -> float:
+    return statistics.fmean(scores) if scores else 0.0
+
+
+def median_of_all(scores: Sequence[float]) -> float:
+    return statistics.median(scores) if scores else 0.0

deepeval/prompt/prompt.py

@@ -4,7 +4,7 @@ import json
 import os
 
 from enum import Enum
-from typing import Optional, List, Dict, Type, Literal, TYPE_CHECKING
+from typing import Optional, List, Dict, Type, Literal
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 from rich.console import Console
 from pydantic import BaseModel, ValidationError
@@ -34,10 +34,6 @@ from deepeval.prompt.utils import (
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.constants import HIDDEN_DIR
 
-
-if TYPE_CHECKING:
-    from deepeval.optimization.types import OptimizationReport
-
 logger = logging.getLogger(__name__)
 
 portalocker = None
@@ -117,6 +113,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[Type[BaseModel]] = None,
+        interpolation_type: Optional[PromptInterpolationType] = None,
     ):
         if text_template and messages_template:
             raise TypeError(
@@ -129,7 +126,9 @@ class Prompt:
         self.output_type: Optional[OutputType] = output_type
         self.output_schema: Optional[Type[BaseModel]] = output_schema
         self.label: Optional[str] = None
-        self.interpolation_type: Optional[PromptInterpolationType] = None
+        self.interpolation_type: PromptInterpolationType = (
+            interpolation_type or PromptInterpolationType.FSTRING
+        )
 
         self._version = None
         self._prompt_version_id: Optional[str] = None
@@ -145,9 +144,6 @@ class Prompt:
         elif messages_template:
             self.type = PromptType.LIST
 
-        # updated after optimization runs
-        self.optimization_report: Optional["OptimizationReport"] = None
-
     def __del__(self):
         """Cleanup polling tasks when instance is destroyed"""
         try:

deepeval/test_case/__init__.py

@@ -3,13 +3,13 @@ from .llm_test_case import (
     LLMTestCaseParams,
     ToolCall,
     ToolCallParams,
+    MLLMImage,
 )
 from .conversational_test_case import (
     ConversationalTestCase,
     Turn,
     TurnParams,
 )
-from .mllm_test_case import MLLMTestCase, MLLMTestCaseParams, MLLMImage
 from .arena_test_case import ArenaTestCase, Contestant
 from .mcp import (
     MCPServer,
@@ -31,8 +31,6 @@ __all__ = [
     "MCPPromptCall",
     "MCPResourceCall",
     "MCPToolCall",
-    "MLLMTestCase",
-    "MLLMTestCaseParams",
     "MLLMImage",
     "ArenaTestCase",
     "Contestant",

deepeval/test_case/api.py

@@ -10,9 +10,9 @@ from deepeval.test_run.api import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
     Turn,
 )
+from deepeval.test_case.llm_test_case import _MLLM_IMAGE_REGISTRY
 from deepeval.constants import PYTEST_RUN_TEST_NAME
 
 
@@ -29,10 +29,12 @@ def create_api_turn(turn: Turn, index: int) -> TurnApi:
 
 
 def create_api_test_case(
-    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
     trace: Optional[TraceApi] = None,
     index: Optional[int] = None,
 ) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
+    from deepeval.utils import convert_to_multi_modal_array
+
     if isinstance(test_case, ConversationalTestCase):
         order = (
             test_case._dataset_rank
@@ -84,7 +86,7 @@ def create_api_test_case(
             name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
         metrics_data = []
 
-        if isinstance(test_case, LLMTestCase):
+        if isinstance(test_case, LLMTestCase) and test_case.multimodal is False:
             api_test_case = LLMApiTestCase(
                 name=name,
                 input=test_case.input,
@@ -106,15 +108,15 @@ def create_api_test_case(
                 comments=test_case.comments,
                 trace=trace,
             )
-        elif isinstance(test_case, MLLMTestCase):
+        elif isinstance(test_case, LLMTestCase) and test_case.multimodal:
             api_test_case = LLMApiTestCase(
                 name=name,
-                input="",
-                multimodalInput=test_case.input,
-                multimodalActualOutput=test_case.actual_output,
-                multimodalExpectedOutput=test_case.expected_output,
-                multimodalRetrievalContext=test_case.retrieval_context,
-                multimodalContext=test_case.context,
+                input=test_case.input,
+                actualOutput=test_case.actual_output,
+                expectedOutput=test_case.expected_output,
+                retrievalContext=test_case.retrieval_context,
+                context=test_case.context,
+                imagesMapping=_MLLM_IMAGE_REGISTRY,
                 toolsCalled=test_case.tools_called,
                 expectedTools=test_case.expected_tools,
                 tokenCost=test_case.token_cost,

deepeval/test_case/conversational_test_case.py

@@ -9,7 +9,7 @@ from typing import List, Optional, Dict, Literal
 from copy import deepcopy
 from enum import Enum
 
-from deepeval.test_case import ToolCall
+from deepeval.test_case import ToolCall, MLLMImage
 from deepeval.test_case.mcp import (
     MCPServer,
     MCPPromptCall,
@@ -156,11 +156,29 @@ class ConversationalTestCase(BaseModel):
     comments: Optional[str] = Field(default=None)
     tags: Optional[List[str]] = Field(default=None)
     mcp_servers: Optional[List[MCPServer]] = Field(default=None)
+    multimodal: bool = False
 
     _dataset_rank: Optional[int] = PrivateAttr(default=None)
     _dataset_alias: Optional[str] = PrivateAttr(default=None)
     _dataset_id: Optional[str] = PrivateAttr(default=None)
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        self.multimodal = any(
+            [
+                re.search(pattern, turn.content) is not None
+                for turn in self.turns
+            ]
+        )
+
+        return self
+
     @model_validator(mode="before")
     def validate_input(cls, data):
         turns = data.get("turns")

deepeval/test_case/llm_test_case.py

@@ -9,7 +9,12 @@ from typing import List, Optional, Dict, Any
 from enum import Enum
 import json
 import uuid
-
+import re
+import os
+import mimetypes
+import base64
+from dataclasses import dataclass, field
+from urllib.parse import urlparse, unquote
 from deepeval.utils import make_model_config
 
 from deepeval.test_case.mcp import (
@@ -20,6 +25,128 @@ from deepeval.test_case.mcp import (
     validate_mcp_servers,
 )
 
+_MLLM_IMAGE_REGISTRY: Dict[str, "MLLMImage"] = {}
+
+
+@dataclass
+class MLLMImage:
+    dataBase64: Optional[str] = None
+    mimeType: Optional[str] = None
+    url: Optional[str] = None
+    local: Optional[bool] = None
+    filename: Optional[str] = None
+    _id: str = field(default_factory=lambda: uuid.uuid4().hex)
+
+    def __post_init__(self):
+
+        if not self.url and not self.dataBase64:
+            raise ValueError(
+                "You must provide either a 'url' or both 'dataBase64' and 'mimeType' to create an MLLMImage."
+            )
+
+        if self.dataBase64 is not None:
+            if self.mimeType is None:
+                raise ValueError(
+                    "mimeType must be provided when initializing from Base64 data."
+                )
+        else:
+            is_local = self.is_local_path(self.url)
+            if self.local is not None:
+                assert self.local == is_local, "Local path mismatch"
+            else:
+                self.local = is_local
+
+            # compute filename, mime_type, and Base64 data
+            if self.local:
+                path = self.process_url(self.url)
+                self.filename = os.path.basename(path)
+                self.mimeType = (
+                    mimetypes.guess_type(path)[0] or "application/octet-stream"
+                )
+                with open(path, "rb") as f:
+                    raw = f.read()
+                self.dataBase64 = base64.b64encode(raw).decode("ascii")
+            else:
+                self.filename = None
+                self.mimeType = None
+                self.dataBase64 = None
+
+        _MLLM_IMAGE_REGISTRY[self._id] = self
+
+    def _placeholder(self) -> str:
+        return f"[DEEPEVAL:IMAGE:{self._id}]"
+
+    def __str__(self) -> str:
+        return self._placeholder()
+
+    def __repr__(self) -> str:
+        return self._placeholder()
+
+    def __format__(self, format_spec: str) -> str:
+        return self._placeholder()
+
+    @staticmethod
+    def process_url(url: str) -> str:
+        if os.path.exists(url):
+            return url
+        parsed = urlparse(url)
+        if parsed.scheme == "file":
+            raw_path = (
+                f"//{parsed.netloc}{parsed.path}"
+                if parsed.netloc
+                else parsed.path
+            )
+            path = unquote(raw_path)
+            return path
+        return url
+
+    @staticmethod
+    def is_local_path(url: str) -> bool:
+        if os.path.exists(url):
+            return True
+        parsed = urlparse(url)
+        if parsed.scheme == "file":
+            raw_path = (
+                f"//{parsed.netloc}{parsed.path}"
+                if parsed.netloc
+                else parsed.path
+            )
+            path = unquote(raw_path)
+            return os.path.exists(path)
+        return False
+
+    def parse_multimodal_string(s: str):
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+        matches = list(re.finditer(pattern, s))
+
+        result = []
+        last_end = 0
+
+        for m in matches:
+            start, end = m.span()
+
+            if start > last_end:
+                result.append(s[last_end:start])
+
+            img_id = m.group(1)
+
+            if img_id not in _MLLM_IMAGE_REGISTRY:
+                MLLMImage(url=img_id, _id=img_id)
+
+            result.append(_MLLM_IMAGE_REGISTRY[img_id])
+            last_end = end
+
+        if last_end < len(s):
+            result.append(s[last_end:])
+
+        return result
+
+    def as_data_uri(self) -> Optional[str]:
+        """Return the image as a data URI string, if Base64 data is available."""
+        if not self.dataBase64 or not self.mimeType:
+            return None
+        return f"data:{self.mimeType};base64,{self.dataBase64}"
+
 
 class LLMTestCaseParams(Enum):
     INPUT = "input"
@@ -208,6 +335,7 @@ class LLMTestCase(BaseModel):
         serialization_alias="completionTime",
         validation_alias=AliasChoices("completionTime", "completion_time"),
     )
+    multimodal: bool = Field(default=False)
     name: Optional[str] = Field(default=None)
     tags: Optional[List[str]] = Field(default=None)
     mcp_servers: Optional[List[MCPServer]] = Field(default=None)
@@ -229,6 +357,29 @@ class LLMTestCase(BaseModel):
         default_factory=lambda: str(uuid.uuid4())
     )
 
+    @model_validator(mode="after")
+    def set_is_multimodal(self):
+        import re
+
+        if self.multimodal is True:
+            return self
+
+        pattern = r"\[DEEPEVAL:IMAGE:(.*?)\]"
+
+        auto_detect = (
+            any(
+                [
+                    re.search(pattern, self.input or "") is not None,
+                    re.search(pattern, self.actual_output or "") is not None,
+                ]
+            )
+            if isinstance(self.input, str)
+            else self.multimodal
+        )
+
+        self.multimodal = auto_detect
+        return self
+
     @model_validator(mode="before")
     def validate_input(cls, data):
         input = data.get("input")

deepeval/test_case/utils.py

@@ -1,24 +1,20 @@
 from typing import Union, List
 
-from deepeval.test_case import LLMTestCase, MLLMTestCase, ConversationalTestCase
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
 
 
 def check_valid_test_cases_type(
-    test_cases: Union[
-        List[Union[LLMTestCase, MLLMTestCase]], List[ConversationalTestCase]
-    ],
+    test_cases: Union[List[LLMTestCase], List[ConversationalTestCase]],
 ):
     llm_test_case_count = 0
     conversational_test_case_count = 0
     for test_case in test_cases:
-        if isinstance(test_case, LLMTestCase) or isinstance(
-            test_case, MLLMTestCase
-        ):
+        if isinstance(test_case, LLMTestCase):
             llm_test_case_count += 1
         else:
             conversational_test_case_count += 1
 
     if llm_test_case_count > 0 and conversational_test_case_count > 0:
         raise ValueError(
-            "You cannot supply a mixture of `LLMTestCase`/`MLLMTestCase`(s) and `ConversationalTestCase`(s) as the list of test cases."
+            "You cannot supply a mixture of `LLMTestCase`(s) and `ConversationalTestCase`(s) as the list of test cases."
         )