deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/llms/openai_model.py

@@ -1,17 +1,23 @@
+import base64
 from openai.types.chat.chat_completion import ChatCompletion
-from typing import Optional, Tuple, Union, Dict
+from typing import Optional, Tuple, Union, Dict, List
+from deepeval.test_case import MLLMImage
 from pydantic import BaseModel, SecretStr
-
+from io import BytesIO
 from openai import (
     OpenAI,
     AsyncOpenAI,
 )
-
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.models.utils import (
+    parse_model_name,
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
@@ -20,6 +26,7 @@ from deepeval.models.retry_policy import (
 
 retry_openai = create_retry_decorator(PS.OPENAI)
 
+
 valid_gpt_models = [
     "gpt-3.5-turbo",
     "gpt-3.5-turbo-0125",
@@ -82,6 +89,15 @@ unsupported_log_probs_gpt_models = [
     "gpt-5-chat-latest",
 ]
 
+unsupported_log_probs_multimodal_gpt_models = [
+    "o1",
+    "o1-preview",
+    "o1-2024-12-17",
+    "o1-preview-2024-09-12",
+    "gpt-4.5-preview-2025-02-27",
+    "o4-mini",
+]
+
 structured_outputs_models = [
     "gpt-4o",
     "gpt-4o-2024-05-13",
@@ -214,20 +230,42 @@ def _request_timeout_seconds() -> float:
     return timeout if timeout > 0 else 30.0
 
 
+_ALIAS_MAP = {
+    "api_key": ["_openai_api_key"],
+}
+
+
 class GPTModel(DeepEvalBaseLLM):
+    valid_multimodal_models = [
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4.1",
+        "gpt-4.1-mini",
+        "gpt-5",
+    ]
+
     def __init__(
         self,
         model: Optional[str] = None,
-        _openai_api_key: Optional[str] = None,
+        api_key: Optional[str] = None,
         base_url: Optional[str] = None,
+        temperature: float = 0,
         cost_per_input_token: Optional[float] = None,
         cost_per_output_token: Optional[float] = None,
-        temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "GPTModel",
+            kwargs,
+            _ALIAS_MAP,
+        )
+
+        # re-map depricated keywords to re-named positional args
+        if api_key is None and "api_key" in alias_values:
+            api_key = alias_values["api_key"]
+
         settings = get_settings()
-        model_name = None
         model = model or settings.OPENAI_MODEL_NAME
         cost_per_input_token = (
             cost_per_input_token
@@ -240,51 +278,50 @@ class GPTModel(DeepEvalBaseLLM):
             else settings.OPENAI_COST_PER_OUTPUT_TOKEN
         )
 
+        if model is None:
+            model = default_gpt_model
+
         if isinstance(model, str):
-            model_name = parse_model_name(model)
-            if model_name not in valid_gpt_models:
+            model = parse_model_name(model)
+            if model not in valid_gpt_models:
                 raise ValueError(
                     f"Invalid model. Available GPT models: {', '.join(model for model in valid_gpt_models)}"
                 )
-        elif model is None:
-            model_name = default_gpt_model
 
-        if model_name not in model_pricing:
+        if model not in model_pricing:
             if cost_per_input_token is None or cost_per_output_token is None:
                 raise ValueError(
-                    f"No pricing available for `{model_name}`. "
+                    f"No pricing available for `{model}`. "
                     "Please provide both `cost_per_input_token` and `cost_per_output_token` when initializing `GPTModel`, "
                     "or set them via the CLI:\n"
                     "    deepeval set-openai --model=[...] --cost_per_input_token=[...] --cost_per_output_token=[...]"
                 )
             else:
-                model_pricing[model_name] = {
+                model_pricing[model] = {
                     "input": float(cost_per_input_token),
                     "output": float(cost_per_output_token),
                 }
 
-        elif model is None:
-            model_name = default_gpt_model
-
-        if _openai_api_key is not None:
+        if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self._openai_api_key: SecretStr | None = SecretStr(_openai_api_key)
+            self.api_key: SecretStr | None = SecretStr(api_key)
         else:
-            self._openai_api_key = get_settings().OPENAI_API_KEY
+            self.api_key = get_settings().OPENAI_API_KEY
 
         self.base_url = base_url
         # args and kwargs will be passed to the underlying model, in load_model function
 
         # Auto-adjust temperature for models that require it
-        if model_name in models_requiring_temperature_1:
+        if model in models_requiring_temperature_1:
             temperature = 1
 
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
-        self.kwargs = kwargs
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Generate functions
@@ -295,10 +332,15 @@ class GPTModel(DeepEvalBaseLLM):
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
         client = self.load_model(async_mode=False)
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            prompt = self.generate_prompt(prompt)
+
         if schema:
-            if self.model_name in structured_outputs_models:
+            if self.name in structured_outputs_models:
                 completion = client.beta.chat.completions.parse(
-                    model=self.model_name,
+                    model=self.name,
                     messages=[
                         {"role": "user", "content": prompt},
                     ],
@@ -314,9 +356,9 @@ class GPTModel(DeepEvalBaseLLM):
                     completion.usage.completion_tokens,
                 )
                 return structured_output, cost
-            if self.model_name in json_mode_models:
+            if self.name in json_mode_models:
                 completion = client.beta.chat.completions.parse(
-                    model=self.model_name,
+                    model=self.name,
                     messages=[
                         {"role": "user", "content": prompt},
                     ],
@@ -334,7 +376,7 @@ class GPTModel(DeepEvalBaseLLM):
                 return schema.model_validate(json_output), cost
 
         completion = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             **self.generation_kwargs,
@@ -354,10 +396,15 @@ class GPTModel(DeepEvalBaseLLM):
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, BaseModel], float]:
         client = self.load_model(async_mode=True)
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            prompt = self.generate_prompt(prompt)
+
         if schema:
-            if self.model_name in structured_outputs_models:
+            if self.name in structured_outputs_models:
                 completion = await client.beta.chat.completions.parse(
-                    model=self.model_name,
+                    model=self.name,
                     messages=[
                         {"role": "user", "content": prompt},
                     ],
@@ -373,9 +420,9 @@ class GPTModel(DeepEvalBaseLLM):
                     completion.usage.completion_tokens,
                 )
                 return structured_output, cost
-            if self.model_name in json_mode_models:
+            if self.name in json_mode_models:
                 completion = await client.beta.chat.completions.parse(
-                    model=self.model_name,
+                    model=self.name,
                     messages=[
                         {"role": "user", "content": prompt},
                     ],
@@ -393,7 +440,7 @@ class GPTModel(DeepEvalBaseLLM):
                 return schema.model_validate(json_output), cost
 
         completion = await client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             **self.generation_kwargs,
@@ -420,8 +467,11 @@ class GPTModel(DeepEvalBaseLLM):
     ) -> Tuple[ChatCompletion, float]:
         # Generate completion
         client = self.load_model(async_mode=False)
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            prompt = self.generate_prompt(prompt)
         completion = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             logprobs=True,
@@ -443,8 +493,11 @@ class GPTModel(DeepEvalBaseLLM):
     ) -> Tuple[ChatCompletion, float]:
         # Generate completion
         client = self.load_model(async_mode=True)
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            prompt = self.generate_prompt(prompt)
         completion = await client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             logprobs=True,
@@ -463,8 +516,11 @@ class GPTModel(DeepEvalBaseLLM):
         self, prompt: str, n: int, temperature: float
     ) -> Tuple[list[str], float]:
         client = self.load_model(async_mode=False)
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            prompt = self.generate_prompt(prompt)
         response = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             n=n,
             temperature=temperature,
@@ -479,7 +535,7 @@ class GPTModel(DeepEvalBaseLLM):
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
         # TODO: consider loggin a warning instead of defaulting to whole model pricing
-        pricing = model_pricing.get(self.model_name, model_pricing)
+        pricing = model_pricing.get(self.name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
         return input_cost + output_cost
@@ -488,8 +544,40 @@ class GPTModel(DeepEvalBaseLLM):
     # Model #
     #########
 
-    def get_model_name(self):
-        return self.model_name
+    def generate_prompt(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        prompt = []
+        for ele in multimodal_input:
+            if isinstance(ele, str):
+                prompt.append({"type": "text", "text": ele})
+            elif isinstance(ele, MLLMImage):
+                if ele.local:
+                    import PIL.Image
+
+                    image = PIL.Image.open(ele.url)
+                    visual_dict = {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{self.encode_pil_image(image)}"
+                        },
+                    }
+                else:
+                    visual_dict = {
+                        "type": "image_url",
+                        "image_url": {"url": ele.url},
+                    }
+                prompt.append(visual_dict)
+        return prompt
+
+    def encode_pil_image(self, pil_image):
+        image_buffer = BytesIO()
+        if pil_image.mode in ("RGBA", "LA", "P"):
+            pil_image = pil_image.convert("RGB")
+        pil_image.save(image_buffer, format="JPEG")
+        image_bytes = image_buffer.getvalue()
+        base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
+        return base64_encoded_image
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
@@ -512,10 +600,10 @@ class GPTModel(DeepEvalBaseLLM):
 
     def _build_client(self, cls):
         api_key = require_secret_api_key(
-            self._openai_api_key,
+            self.api_key,
             provider_label="OpenAI",
             env_var_name="OPENAI_API_KEY",
-            param_hint="`_openai_api_key` to GPTModel(...)",
+            param_hint="`api_key` to GPTModel(...)",
         )
 
         kw = dict(
@@ -531,3 +619,11 @@ class GPTModel(DeepEvalBaseLLM):
                 kw.pop("max_retries", None)
                 return cls(**kw)
             raise
+
+    def supports_multimodal(self):
+        if self.name in GPTModel.valid_multimodal_models:
+            return True
+        return False
+
+    def get_model_name(self):
+        return f"{self.name}"

deepeval/models/llms/portkey_model.py

@@ -4,11 +4,18 @@ from typing import Any, Dict, List, Optional, Union
 from pydantic import AnyUrl, SecretStr
 
 from deepeval.config.settings import get_settings
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.utils import require_param
 
 
+def _request_timeout_seconds() -> float:
+    timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+    return timeout if timeout > 0 else 30.0
+
+
 class PortkeyModel(DeepEvalBaseLLM):
     def __init__(
         self,
@@ -16,11 +23,13 @@ class PortkeyModel(DeepEvalBaseLLM):
         api_key: Optional[str] = None,
         base_url: Optional[AnyUrl] = None,
         provider: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **kwargs,
     ):
         settings = get_settings()
         model = model or settings.PORTKEY_MODEL_NAME
 
-        self.model = require_param(
+        self.name = require_param(
             model,
             provider_label="Portkey",
             env_var_name="PORTKEY_MODEL_NAME",
@@ -52,6 +61,9 @@ class PortkeyModel(DeepEvalBaseLLM):
             env_var_name="PORTKEY_PROVIDER_NAME",
             param_hint="provider",
         )
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = kwargs
+        self.generation_kwargs = generation_kwargs or {}
 
     def _headers(self) -> Dict[str, str]:
         api_key = require_secret_api_key(
@@ -70,10 +82,13 @@ class PortkeyModel(DeepEvalBaseLLM):
         return headers
 
     def _payload(self, prompt: str) -> Dict[str, Any]:
-        return {
-            "model": self.model,
+        payload = {
+            "model": self.name,
             "messages": [{"role": "user", "content": prompt}],
         }
+        if self.generation_kwargs:
+            payload.update(self.generation_kwargs)
+        return payload
 
     def _extract_content(self, data: Dict[str, Any]) -> str:
         choices: Union[List[Dict[str, Any]], None] = data.get("choices")
@@ -88,6 +103,7 @@ class PortkeyModel(DeepEvalBaseLLM):
         return ""
 
     def generate(self, prompt: str) -> str:
+
         try:
             response = requests.post(
                 f"{self.base_url}/chat/completions",
@@ -110,6 +126,7 @@ class PortkeyModel(DeepEvalBaseLLM):
         return self._extract_content(response.json())
 
     async def a_generate(self, prompt: str) -> str:
+
         async with aiohttp.ClientSession() as session:
             async with session.post(
                 f"{self.base_url}/chat/completions",
@@ -125,8 +142,8 @@ class PortkeyModel(DeepEvalBaseLLM):
                 data = await response.json()
                 return self._extract_content(data)
 
-    def get_model_name(self) -> str:
-        return f"Portkey ({self.model})"
-
     def load_model(self):
         return None
+
+    def get_model_name(self):
+        return f"{self.name} (Portkey)"

deepeval/models/llms/utils.py

@@ -1,8 +1,10 @@
-from typing import Dict
+from typing import Dict, List, Optional
 import re
 import json
 import asyncio
 
+MULTIMODAL_MODELS = ["GPTModel", "AzureModel", "GeminiModel", "OllamaModel"]
+
 
 def trim_and_load_json(
     input_string: str,
@@ -38,7 +40,7 @@ def safe_asyncio_run(coro):
                 return loop.run_until_complete(future)
             else:
                 return loop.run_until_complete(coro)
-        except Exception as inner_e:
+        except Exception:
             raise
-    except Exception as e:
+    except Exception:
         raise

deepeval/models/retry_policy.py

@@ -55,6 +55,7 @@ from tenacity.stop import stop_base
 from tenacity.wait import wait_base
 from contextvars import ContextVar, copy_context
 
+from deepeval.utils import require_dependency
 from deepeval.constants import (
     ProviderSlug as PS,
     slugify,
@@ -829,25 +830,23 @@ try:
 except Exception:  # botocore not present (aiobotocore optional)
     BEDROCK_ERROR_POLICY = None
 
-
 ####################
 # Anthropic Policy #
 ####################
 
 try:
-    from anthropic import (
-        AuthenticationError,
-        RateLimitError,
-        APIConnectionError,
-        APITimeoutError,
-        APIStatusError,
+
+    module = require_dependency(
+        "anthropic",
+        provider_label="retry_policy",
+        install_hint="Install it with `pip install anthropic`.",
     )
 
     ANTHROPIC_ERROR_POLICY = ErrorPolicy(
-        auth_excs=(AuthenticationError,),
-        rate_limit_excs=(RateLimitError,),
-        network_excs=(APIConnectionError, APITimeoutError),
-        http_excs=(APIStatusError,),
+        auth_excs=(module.AuthenticationError,),
+        rate_limit_excs=(module.RateLimitError,),
+        network_excs=(module.APIConnectionError, module.APITimeoutError),
+        http_excs=(module.APIStatusError,),
         non_retryable_codes=frozenset(),  # update if we learn of hard quota codes
         message_markers={},
     )
@@ -868,7 +867,11 @@ except Exception:  # Anthropic optional
 # and gate retries using message markers (code sniffing).
 # See: https://github.com/googleapis/python-genai?tab=readme-ov-file#error-handling
 try:
-    from google.genai import errors as gerrors
+    module = require_dependency(
+        "google.genai",
+        provider_label="retry_policy",
+        install_hint="Install it with `pip install google-genai`.",
+    )
 
     _HTTPX_NET_EXCS = _httpx_net_excs()
     _REQUESTS_EXCS = _requests_net_excs()
@@ -887,9 +890,9 @@ try:
     GOOGLE_ERROR_POLICY = ErrorPolicy(
         auth_excs=(),  # we will classify 401/403 via markers below (see non-retryable codes)
         rate_limit_excs=(
-            gerrors.ClientError,
+            module.gerrors.ClientError,
         ),  # includes 429; markers decide retry vs not
-        network_excs=(gerrors.ServerError,)
+        network_excs=(module.gerrors.ServerError,)
         + _HTTPX_NET_EXCS
         + _REQUESTS_EXCS,  # treat 5xx as transient
         http_excs=(),  # no reliable .status_code on exceptions; handled above

deepeval/models/utils.py

@@ -1,9 +1,13 @@
-from typing import Optional
+import logging
+from typing import Any, Dict, Optional, Tuple
 from pydantic import SecretStr
 
 from deepeval.errors import DeepEvalError
 
 
+logger = logging.getLogger(__name__)
+
+
 def parse_model_name(model_name: Optional[str] = None) -> str:
     """Extract base model name from provider-prefixed format.
 
@@ -74,3 +78,44 @@ def require_secret_api_key(
         )
 
     return api_key
+
+
+def normalize_kwargs_and_extract_aliases(
+    provider_label: str,
+    kwargs: Dict[str, Any],
+    alias_map: Dict[str, list],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    """
+    Normalize legacy keyword argument names according to alias_map.
+
+    alias_map is of the form: {new_name: [old_name1, old_name2, ...]}
+
+    - Returns (normalized_kwargs, extracted_values)
+      where:
+        - normalized_kwargs has all legacy keys removed (to prevent forwarding
+          to downstream SDK clients).
+        - extracted_values maps new_name -> value for any alias that was used.
+
+    - Logs a warning for each legacy keyword used, so callers know they should
+      migrate to the new name.
+    """
+    normalized = dict(kwargs)
+    extracted: Dict[str, Any] = {}
+
+    for new_name, old_names in alias_map.items():
+        for old_name in old_names:
+            if old_name in normalized:
+                value = normalized.pop(old_name)
+
+                logger.warning(
+                    "%s keyword '%s' is deprecated; please use '%s' instead.",
+                    provider_label,
+                    old_name,
+                    new_name,
+                )
+
+                # Only preserve the first alias value we see for a given new_name
+                if new_name not in extracted:
+                    extracted[new_name] = value
+
+    return normalized, extracted

deepeval/optimizer/__init__.py

@@ -0,0 +1,5 @@
+from deepeval.optimizer.prompt_optimizer import PromptOptimizer
+
+__all__ = [
+    "PromptOptimizer",
+]

deepeval/optimizer/algorithms/__init__.py

@@ -0,0 +1,6 @@
+from .gepa import GEPA
+from .miprov2 import MIPROV2
+from .copro import COPRO
+from .simba import SIMBA
+
+__all__ = ["GEPA", "MIPROV2", "COPRO", "SIMBA"]

deepeval/optimizer/algorithms/base.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from typing import Union, List, Dict, Tuple
+
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.prompt.prompt import Prompt
+from deepeval.dataset.golden import Golden, ConversationalGolden
+
+
+class BaseAlgorithm(ABC):
+    name: str
+    optimizer_model: DeepEvalBaseLLM
+    scorer: BaseScorer
+
+    @abstractmethod
+    def execute(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Tuple[Prompt, Dict]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_execute(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Tuple[Prompt, Dict]:
+        raise NotImplementedError

deepeval/optimizer/algorithms/configs.py

@@ -0,0 +1,18 @@
+# Internal GEPA constants - not exposed to users
+GEPA_MIN_DELTA: float = 0.0
+GEPA_TIE_TOLERANCE: float = 1e-9
+GEPA_REWRITE_INSTRUCTION_MAX_CHARS: int = 4096
+
+# Internal MIPROV2 constants - not exposed to users
+MIPROV2_MIN_DELTA: float = 0.0
+MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS: int = 4096
+MIPROV2_DEFAULT_NUM_CANDIDATES: int = 10
+MIPROV2_DEFAULT_NUM_TRIALS: int = 20
+MIPROV2_DEFAULT_MINIBATCH_SIZE: int = 25
+MIPROV2_DEFAULT_MINIBATCH_FULL_EVAL_STEPS: int = 10
+MIPROV2_DEFAULT_MAX_BOOTSTRAPPED_DEMOS: int = 4
+MIPROV2_DEFAULT_MAX_LABELED_DEMOS: int = 4
+MIPROV2_DEFAULT_NUM_DEMO_SETS: int = 5
+
+# Internal SIMBA constants - not exposed to users
+SIMBA_DEMO_INPUT_MAX_CHARS: int = 256

deepeval/optimizer/algorithms/copro/__init__.py

@@ -0,0 +1,5 @@
+from .copro import COPRO
+
+__all__ = [
+    "COPRO",
+]