deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/{optimization → optimizer}/utils.py

@@ -2,11 +2,14 @@ from __future__ import annotations
 import inspect
 import random
 import re
+import statistics
 from typing import (
     Any,
     Callable,
     List,
     Optional,
+    Protocol,
+    Sequence,
     Tuple,
     TYPE_CHECKING,
     Union,
@@ -17,11 +20,13 @@ from typing import (
 from deepeval.errors import DeepEvalError
 from deepeval.metrics.base_metric import BaseMetric, BaseConversationalMetric
 from deepeval.prompt.prompt import Prompt
-from deepeval.prompt.api import PromptType, PromptMessage
-from deepeval.optimization.types import (
+from deepeval.prompt.api import PromptMessage
+from deepeval.optimizer.types import (
+    ModelCallback,
     ModuleId,
     PromptConfigurationId,
     PromptConfiguration,
+    PromptConfigSnapshot,
     OptimizationReport,
 )
 
@@ -54,7 +59,7 @@ def split_goldens(
         pareto_size: Number of items to allocate to the Pareto set bound between [0, len(goldens)].
         random_state: A shared `random.Random` instance that provides the source
             of randomness. For reproducible runs, pass the same object used by
-            the GEPA loop constructed from `GEPAConfig.random_seed`
+            the GEPA loop constructed from `GEPA.random_seed`
 
     Returns:
         (d_feedback, d_pareto)
@@ -151,87 +156,22 @@ def normalize_seed_prompts(
     return mapping
 
 
-def build_model_callback_kwargs(
-    *,
-    # scoring context
-    golden: Optional[Union["Golden", "ConversationalGolden"]] = None,
-    # rewriter context
-    feedback_text: Optional[str] = None,
-    # shared
-    prompt: Optional[Prompt] = None,
-    prompt_type: Optional[str] = None,
-    prompt_text: Optional[str] = None,
-    prompt_messages: Optional[List["PromptMessage"]] = None,
-) -> Dict[str, Any]:
-    """
-    Build a superset of kwargs for GEPA model callbacks.
-
-    All keys are present in the dict so callbacks can declare any subset of:
-
-        hook: str           # injected by (a_)invoke_model_callback
-        prompt: Prompt
-        prompt_type: str
-        prompt_text: str
-        prompt_messages: List[PromptMessage]
-        golden: Golden | ConversationalGolden
-        feedback_text: str
-
-    Non applicable fields are set to None.
-    """
-    return {
-        # scoring context
-        "golden": golden,
-        # rewriter context
-        "feedback_text": feedback_text,
-        # shared
-        "prompt": prompt,
-        "prompt_text": prompt_text,
-        "prompt_messages": prompt_messages,
-    }
-
-
 def invoke_model_callback(
     *,
-    hook: str,
-    model_callback: Callable[
-        ...,
-        Union[
-            str,
-            Dict,
-            Tuple[Union[str, Dict], float],
-        ],
-    ],
-    candidate_kwargs: Dict[str, Any],
-) -> Union[
-    str,
-    Dict,
-    Tuple[Union[str, Dict], float],
-]:
+    model_callback: ModelCallback,
+    prompt: Prompt,
+    golden: Union["Golden", "ConversationalGolden"],
+) -> str:
     """
     Call a user provided model_callback in a synchronous context.
 
-    - Filters kwargs to only those the callback accepts.
-    - Injects `hook` if the callback declares it.
-    - Raises if the callback returns an awaitable; callers must use async
-      helpers for async callbacks.
+    Raises if the callback returns an awaitable.
     """
-    sig = inspect.signature(model_callback)
-    supported = set(sig.parameters.keys())
-
-    filtered = {
-        key: value
-        for key, value in candidate_kwargs.items()
-        if key in supported
-    }
-
-    if "hook" in supported:
-        filtered["hook"] = hook
-
-    result = model_callback(**filtered)
+    result = model_callback(prompt, golden)
     if inspect.isawaitable(result):
         raise DeepEvalError(
             "model_callback returned an awaitable from a synchronous context. "
-            "Either declare the callback as `async def` and use async GEPA, or call "
+            "Either declare the callback as `async def` and use async optimization, or call "
             "`model.generate(...)` instead of `model.a_generate(...)` inside a sync callback."
         )
     return result
@@ -239,41 +179,16 @@ def invoke_model_callback(
 
 async def a_invoke_model_callback(
     *,
-    hook: str,
-    model_callback: Callable[
-        ...,
-        Union[
-            str,
-            Dict,
-            Tuple[Union[str, Dict], float],
-        ],
-    ],
-    candidate_kwargs: Dict[str, Any],
-) -> Union[
-    str,
-    Dict,
-    Tuple[Union[str, Dict], float],
-]:
+    model_callback: ModelCallback,
+    prompt: Prompt,
+    golden: Union["Golden", "ConversationalGolden"],
+) -> str:
     """
     Call a user provided model_callback in an async context.
 
-    - Filters kwargs to only those the callback accepts.
-    - Injects `hook` if the callback declares it.
-    - Supports both sync and async callbacks.
+    Supports both sync and async callbacks.
     """
-    sig = inspect.signature(model_callback)
-    supported = set(sig.parameters.keys())
-
-    filtered = {
-        key: value
-        for key, value in candidate_kwargs.items()
-        if key in supported
-    }
-
-    if "hook" in supported:
-        filtered["hook"] = hook
-
-    result = model_callback(**filtered)
+    result = model_callback(prompt, golden)
     if inspect.isawaitable(result):
         return await result
     return result
@@ -288,58 +203,17 @@ def build_prompt_config_snapshots(
     prompt_configurations_by_id: Dict[
         PromptConfigurationId, "PromptConfiguration"
     ],
-) -> Dict[PromptConfigurationId, Dict[str, Any]]:
+) -> Dict[PromptConfigurationId, PromptConfigSnapshot]:
     """
-    Build a serializable snapshot of all prompt configurations.
-
-    Shape matches the docs for `prompt_configurations`:
-
-    {
-      "<config_id>": {
-        "parent": "<parent_id or None>",
-        "prompts": {
-          "<module_id>": {
-            "type": "TEXT",
-            "text_template": "...",
-          }
-          # or
-          "<module_id>": {
-            "type": "LIST",
-            "messages": [
-              {"role": "system", "content": "..."},
-              ...
-            ],
-          },
-        },
-      },
-      ...
-    }
+    Build snapshots of all prompt configurations.
     """
-    snapshots: Dict[PromptConfigurationId, Dict[str, Any]] = {}
+    snapshots: Dict[PromptConfigurationId, PromptConfigSnapshot] = {}
 
     for cfg_id, cfg in prompt_configurations_by_id.items():
-        prompts_snapshot: Dict[str, Any] = {}
-
-        for module_id, prompt in cfg.prompts.items():
-            if prompt.type is PromptType.LIST:
-                messages = [
-                    {"role": msg.role, "content": (msg.content or "")}
-                    for msg in (prompt.messages_template or [])
-                ]
-                prompts_snapshot[module_id] = {
-                    "type": "LIST",
-                    "messages": messages,
-                }
-            else:
-                prompts_snapshot[module_id] = {
-                    "type": "TEXT",
-                    "text_template": (prompt.text_template or ""),
-                }
-
-        snapshots[cfg_id] = {
-            "parent": cfg.parent,
-            "prompts": prompts_snapshot,
-        }
+        snapshots[cfg_id] = PromptConfigSnapshot(
+            parent=cfg.parent,
+            prompts=dict(cfg.prompts),
+        )
 
     return snapshots
 
@@ -494,17 +368,8 @@ def validate_sequence_of(
 def validate_callback(
     *,
     component: str,
-    model_callback: Optional[
-        Callable[
-            ...,
-            Union[
-                str,
-                Dict,
-                Tuple[Union[str, Dict], float],
-            ],
-        ]
-    ],
-) -> Callable[..., Union[str, Dict, Tuple[Union[str, Dict], float]]]:
+    model_callback: Optional[ModelCallback],
+) -> ModelCallback:
     """
     Ensure that `model_callback` is provided.
 
@@ -596,3 +461,20 @@ def validate_int_in_range(
         )
 
     return value
+
+
+##############
+# Aggregates #
+##############
+
+
+class Aggregator(Protocol):
+    def __call__(self, scores: Sequence[float]) -> float: ...
+
+
+def mean_of_all(scores: Sequence[float]) -> float:
+    return statistics.fmean(scores) if scores else 0.0
+
+
+def median_of_all(scores: Sequence[float]) -> float:
+    return statistics.median(scores) if scores else 0.0

deepeval/prompt/prompt.py

@@ -4,7 +4,7 @@ import json
 import os
 
 from enum import Enum
-from typing import Optional, List, Dict, Type, Literal, TYPE_CHECKING
+from typing import Optional, List, Dict, Type, Literal
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
 from rich.console import Console
 from pydantic import BaseModel, ValidationError
@@ -34,10 +34,6 @@ from deepeval.prompt.utils import (
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.constants import HIDDEN_DIR
 
-
-if TYPE_CHECKING:
-    from deepeval.optimization.types import OptimizationReport
-
 logger = logging.getLogger(__name__)
 
 portalocker = None
@@ -117,6 +113,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[Type[BaseModel]] = None,
+        interpolation_type: Optional[PromptInterpolationType] = None,
     ):
         if text_template and messages_template:
             raise TypeError(
@@ -129,7 +126,9 @@ class Prompt:
         self.output_type: Optional[OutputType] = output_type
         self.output_schema: Optional[Type[BaseModel]] = output_schema
         self.label: Optional[str] = None
-        self.interpolation_type: Optional[PromptInterpolationType] = None
+        self.interpolation_type: PromptInterpolationType = (
+            interpolation_type or PromptInterpolationType.FSTRING
+        )
 
         self._version = None
         self._prompt_version_id: Optional[str] = None
@@ -145,9 +144,6 @@ class Prompt:
         elif messages_template:
             self.type = PromptType.LIST
 
-        # updated after optimization runs
-        self.optimization_report: Optional["OptimizationReport"] = None
-
     def __del__(self):
         """Cleanup polling tasks when instance is destroyed"""
         try:

deepeval/simulator/conversation_simulator.py

@@ -20,6 +20,7 @@ from deepeval.simulator.template import (
     ConversationSimulatorTemplate,
 )
 from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.utils import MULTIMODAL_SUPPORTED_MODELS
 from deepeval.simulator.schema import (
     SimulatedInput,
     ConversationCompletion,
@@ -94,6 +95,26 @@ class ConversationSimulator:
                     )
                 )
             else:
+                multimodal = any(
+                    [golden.multimodal for golden in conversational_goldens]
+                )
+                if multimodal:
+                    if (
+                        not self.simulator_model
+                        or not self.simulator_model.supports_multimodal()
+                    ):
+                        if (
+                            self.simulator_model
+                            and type(self.simulator_model)
+                            in MULTIMODAL_SUPPORTED_MODELS
+                        ):
+                            raise ValueError(
+                                f"The evaluation model {self.simulator_model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {self.simulator_model.__class__.__name__} provider includes {', '.join(self.simulator_model.__class__.valid_multimodal_models)}."
+                            )
+                        else:
+                            raise ValueError(
+                                f"The evaluation model {self.simulator_model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+                            )
                 conversational_test_cases: List[ConversationalTestCase] = []
                 for conversation_index, golden in enumerate(
                     conversational_goldens
@@ -124,6 +145,28 @@ class ConversationSimulator:
         progress: Optional[Progress] = None,
         pbar_id: Optional[int] = None,
     ) -> List[ConversationalTestCase]:
+
+        multimodal = any(
+            [golden.multimodal for golden in conversational_goldens]
+        )
+        if multimodal:
+            if (
+                not self.simulator_model
+                or not self.simulator_model.supports_multimodal()
+            ):
+                if (
+                    self.simulator_model
+                    and type(self.simulator_model)
+                    in MULTIMODAL_SUPPORTED_MODELS
+                ):
+                    raise ValueError(
+                        f"The evaluation model {self.simulator_model.name} does not support multimodal evaluations at the moment. Available multi-modal models for the {self.simulator_model.__class__.__name__} provider includes {', '.join(self.simulator_model.__class__.valid_multimodal_models)}."
+                    )
+                else:
+                    raise ValueError(
+                        f"The evaluation model {self.simulator_model.name} does not support multimodal inputs, please use one of the following evaluation models: {', '.join([cls.__name__ for cls in MULTIMODAL_SUPPORTED_MODELS])}"
+                    )
+
         self.simulation_cost = 0 if self.using_native_model else None
 
         async def simulate_conversations(

deepeval/simulator/template.py

@@ -7,6 +7,13 @@ from deepeval.test_case import Turn
 
 
 class ConversationSimulatorTemplate:
+    multimodal_rules = """
+        --- MULTIMODAL INPUT RULES ---
+        - Treat image content as factual evidence.
+        - Only reference visual details that are explicitly and clearly visible.
+        - Do not infer or guess objects, text, or details not visibly present.
+        - If an image is unclear or ambiguous, mark uncertainty explicitly.
+    """
 
     @staticmethod
     def simulate_first_user_turn(
@@ -23,6 +30,8 @@ class ConversationSimulatorTemplate:
             3. Avoid providing excessive details upfront; the goal is to initiate the conversation and build rapport, not to solve it in the first message.
             4. The message should be concise, ideally no more than 1-3 sentences.
 
+            {ConversationSimulatorTemplate.multimodal_rules}
+
             IMPORTANT: The output must be formatted as a JSON object with a single key `simulated_input`, where the value is the generated opening message in {language}.
 
             Example Language: english
@@ -61,6 +70,8 @@ class ConversationSimulatorTemplate:
             3. Keep the tone consistent with the previous user inputs.
             4. The generated user input should be concise, ideally no more than 1-2 sentences.
 
+            {ConversationSimulatorTemplate.multimodal_rules}
+
             IMPORTANT: The output must be formatted as a JSON object with a single key `simulated_input`, 
             where the value is the generated user input in {language}.
 
@@ -101,6 +112,8 @@ class ConversationSimulatorTemplate:
             2. If the expected outcome has been met, mark the conversation as complete.
             3. If not, mark it as incomplete and briefly describe what remains to be done.
 
+            {ConversationSimulatorTemplate.multimodal_rules}
+
             IMPORTANT: The output must be formatted as a JSON object with two keys:
             `is_complete` (a boolean) and `reason` (a string).
 

deepeval/test_case/__init__.py

@@ -3,13 +3,13 @@ from .llm_test_case import (
     LLMTestCaseParams,
     ToolCall,
     ToolCallParams,
+    MLLMImage,
 )
 from .conversational_test_case import (
     ConversationalTestCase,
     Turn,
     TurnParams,
 )
-from .mllm_test_case import MLLMTestCase, MLLMTestCaseParams, MLLMImage
 from .arena_test_case import ArenaTestCase, Contestant
 from .mcp import (
     MCPServer,
@@ -31,8 +31,6 @@ __all__ = [
     "MCPPromptCall",
     "MCPResourceCall",
     "MCPToolCall",
-    "MLLMTestCase",
-    "MLLMTestCaseParams",
     "MLLMImage",
     "ArenaTestCase",
     "Contestant",

deepeval/test_case/api.py

@@ -10,7 +10,6 @@ from deepeval.test_run.api import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
     Turn,
 )
 from deepeval.constants import PYTEST_RUN_TEST_NAME
@@ -29,10 +28,11 @@ def create_api_turn(turn: Turn, index: int) -> TurnApi:
 
 
 def create_api_test_case(
-    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    test_case: Union[LLMTestCase, ConversationalTestCase],
     trace: Optional[TraceApi] = None,
     index: Optional[int] = None,
 ) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
+
     if isinstance(test_case, ConversationalTestCase):
         order = (
             test_case._dataset_rank
@@ -59,8 +59,10 @@ def create_api_test_case(
             context=test_case.context,
             tags=test_case.tags,
             comments=test_case.comments,
+            imagesMapping=test_case._get_images_mapping(),
             additionalMetadata=test_case.additional_metadata,
         )
+
         api_test_case.turns = [
             create_api_turn(
                 turn=turn,
@@ -84,48 +86,27 @@ def create_api_test_case(
             name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
         metrics_data = []
 
-        if isinstance(test_case, LLMTestCase):
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input=test_case.input,
-                actualOutput=test_case.actual_output,
-                expectedOutput=test_case.expected_output,
-                context=test_case.context,
-                retrievalContext=test_case.retrieval_context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                tags=test_case.tags,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-                trace=trace,
-            )
-        elif isinstance(test_case, MLLMTestCase):
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input="",
-                multimodalInput=test_case.input,
-                multimodalActualOutput=test_case.actual_output,
-                multimodalExpectedOutput=test_case.expected_output,
-                multimodalRetrievalContext=test_case.retrieval_context,
-                multimodalContext=test_case.context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-            )
+        api_test_case = LLMApiTestCase(
+            name=name,
+            input=test_case.input,
+            actualOutput=test_case.actual_output,
+            expectedOutput=test_case.expected_output,
+            retrievalContext=test_case.retrieval_context,
+            context=test_case.context,
+            imagesMapping=test_case._get_images_mapping(),
+            toolsCalled=test_case.tools_called,
+            expectedTools=test_case.expected_tools,
+            tokenCost=test_case.token_cost,
+            completionTime=test_case.completion_time,
+            success=success,
+            metricsData=metrics_data,
+            runDuration=None,
+            evaluationCost=None,
+            order=order,
+            additionalMetadata=test_case.additional_metadata,
+            comments=test_case.comments,
+            tags=test_case.tags,
+            trace=trace,
+        )
         # llm_test_case_lookup_map[instance_id] = api_test_case
         return api_test_case

deepeval/test_case/arena_test_case.py

@@ -1,7 +1,7 @@
 from typing import List, Dict, Optional, Union
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pydantic import BaseModel
-
+import re
 from deepeval.test_case import (
     LLMTestCase,
 )
@@ -19,6 +19,7 @@ class Contestant(BaseModel):
 @dataclass
 class ArenaTestCase:
     contestants: List[Contestant]
+    multimodal: bool = field(default=False)
 
     def __post_init__(self):
         contestant_names = [contestant.name for contestant in self.contestants]
@@ -38,6 +39,10 @@ class ArenaTestCase:
                     "All contestants must have the same 'expected_output'."
                 )
 
+        for contestant in self.contestants:
+            if contestant.test_case.multimodal:
+                self.multimodal = True
+
 
 class Arena:
     test_cases: List[ArenaTestCase]