deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/models/llms/portkey_model.py

@@ -3,12 +3,22 @@ import requests
 from typing import Any, Dict, List, Optional, Union
 from pydantic import AnyUrl, SecretStr
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.utils import require_param
 
 
+def _request_timeout_seconds() -> float:
+    timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+    return timeout if timeout > 0 else 30.0
+
+
 class PortkeyModel(DeepEvalBaseLLM):
     def __init__(
         self,
@@ -16,20 +26,15 @@ class PortkeyModel(DeepEvalBaseLLM):
         api_key: Optional[str] = None,
         base_url: Optional[AnyUrl] = None,
         provider: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **kwargs,
     ):
         settings = get_settings()
         model = model or settings.PORTKEY_MODEL_NAME
 
-        self.model = require_param(
-            model,
-            provider_label="Portkey",
-            env_var_name="PORTKEY_MODEL_NAME",
-            param_hint="model",
-        )
-
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.PORTKEY_API_KEY
 
@@ -38,6 +43,16 @@ class PortkeyModel(DeepEvalBaseLLM):
         elif settings.PORTKEY_BASE_URL is not None:
             base_url = str(settings.PORTKEY_BASE_URL).rstrip("/")
 
+        provider = provider or settings.PORTKEY_PROVIDER_NAME
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_MODEL_NAME",
+            param_hint="model",
+        )
+
         self.base_url = require_param(
             base_url,
             provider_label="Portkey",
@@ -45,13 +60,16 @@ class PortkeyModel(DeepEvalBaseLLM):
             param_hint="base_url",
         )
 
-        provider = provider or settings.PORTKEY_PROVIDER_NAME
         self.provider = require_param(
             provider,
             provider_label="Portkey",
             env_var_name="PORTKEY_PROVIDER_NAME",
             param_hint="provider",
         )
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = kwargs
+        self.generation_kwargs = generation_kwargs or {}
+        super().__init__(model)
 
     def _headers(self) -> Dict[str, str]:
         api_key = require_secret_api_key(
@@ -70,15 +88,51 @@ class PortkeyModel(DeepEvalBaseLLM):
         return headers
 
     def _payload(self, prompt: str) -> Dict[str, Any]:
-        return {
-            "model": self.model,
-            "messages": [{"role": "user", "content": prompt}],
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+        payload = {
+            "model": self.name,
+            "messages": [{"role": "user", "content": content}],
         }
+        if self.generation_kwargs:
+            payload.update(self.generation_kwargs)
+        return payload
+
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
 
     def _extract_content(self, data: Dict[str, Any]) -> str:
         choices: Union[List[Dict[str, Any]], None] = data.get("choices")
         if not choices:
-            raise ValueError("Portkey response did not include any choices.")
+            raise DeepEvalError("Portkey response did not include any choices.")
         message = choices[0].get("message", {})
         content: Union[str, List[Dict[str, Any]], None] = message.get("content")
         if isinstance(content, str):
@@ -88,12 +142,13 @@ class PortkeyModel(DeepEvalBaseLLM):
         return ""
 
     def generate(self, prompt: str) -> str:
+
         try:
             response = requests.post(
                 f"{self.base_url}/chat/completions",
                 json=self._payload(prompt),
                 headers=self._headers(),
-                timeout=60,
+                timeout=_request_timeout_seconds(),
             )
             response.raise_for_status()
         except requests.HTTPError as error:
@@ -102,31 +157,35 @@ class PortkeyModel(DeepEvalBaseLLM):
                 body = response.json()
             except Exception:
                 body = response.text
-            raise ValueError(
+            raise DeepEvalError(
                 f"Portkey request failed with status {response.status_code}: {body}"
             ) from error
         except requests.RequestException as error:
-            raise ValueError(f"Portkey request failed: {error}") from error
+            raise DeepEvalError(f"Portkey request failed: {error}") from error
         return self._extract_content(response.json())
 
     async def a_generate(self, prompt: str) -> str:
+
         async with aiohttp.ClientSession() as session:
             async with session.post(
                 f"{self.base_url}/chat/completions",
                 json=self._payload(prompt),
                 headers=self._headers(),
-                timeout=60,
+                timeout=_request_timeout_seconds(),
             ) as response:
                 if response.status >= 400:
                     body = await response.text()
-                    raise ValueError(
+                    raise DeepEvalError(
                         f"Portkey request failed with status {response.status}: {body}"
                     )
                 data = await response.json()
                 return self._extract_content(data)
 
-    def get_model_name(self) -> str:
-        return f"Portkey ({self.model})"
-
     def load_model(self):
         return None
+
+    def get_model_name(self):
+        return f"{self.name} (Portkey)"
+
+    def supports_multimodal(self):
+        return True

deepeval/models/llms/utils.py

@@ -3,6 +3,11 @@ import re
 import json
 import asyncio
 
+from deepeval.errors import DeepEvalError
+
+
+MULTIMODAL_MODELS = ["GPTModel", "AzureModel", "GeminiModel", "OllamaModel"]
+
 
 def trim_and_load_json(
     input_string: str,
@@ -18,7 +23,7 @@ def trim_and_load_json(
         return json.loads(jsonStr)
     except json.JSONDecodeError:
         error_str = "Evaluation LLM outputted an invalid JSON. Please use a better evaluation model."
-        raise ValueError(error_str)
+        raise DeepEvalError(error_str)
     except Exception as e:
         raise Exception(f"An unexpected error occurred: {str(e)}")
 
@@ -38,7 +43,7 @@ def safe_asyncio_run(coro):
                 return loop.run_until_complete(future)
             else:
                 return loop.run_until_complete(coro)
-        except Exception as inner_e:
+        except Exception:
             raise
-    except Exception as e:
+    except Exception:
         raise

deepeval/models/retry_policy.py

@@ -55,6 +55,7 @@ from tenacity.stop import stop_base
 from tenacity.wait import wait_base
 from contextvars import ContextVar, copy_context
 
+from deepeval.utils import require_dependency
 from deepeval.constants import (
     ProviderSlug as PS,
     slugify,
@@ -829,25 +830,23 @@ try:
 except Exception:  # botocore not present (aiobotocore optional)
     BEDROCK_ERROR_POLICY = None
 
-
 ####################
 # Anthropic Policy #
 ####################
 
 try:
-    from anthropic import (
-        AuthenticationError,
-        RateLimitError,
-        APIConnectionError,
-        APITimeoutError,
-        APIStatusError,
+
+    module = require_dependency(
+        "anthropic",
+        provider_label="retry_policy",
+        install_hint="Install it with `pip install anthropic`.",
     )
 
     ANTHROPIC_ERROR_POLICY = ErrorPolicy(
-        auth_excs=(AuthenticationError,),
-        rate_limit_excs=(RateLimitError,),
-        network_excs=(APIConnectionError, APITimeoutError),
-        http_excs=(APIStatusError,),
+        auth_excs=(module.AuthenticationError,),
+        rate_limit_excs=(module.RateLimitError,),
+        network_excs=(module.APIConnectionError, module.APITimeoutError),
+        http_excs=(module.APIStatusError,),
         non_retryable_codes=frozenset(),  # update if we learn of hard quota codes
         message_markers={},
     )
@@ -868,7 +867,11 @@ except Exception:  # Anthropic optional
 # and gate retries using message markers (code sniffing).
 # See: https://github.com/googleapis/python-genai?tab=readme-ov-file#error-handling
 try:
-    from google.genai import errors as gerrors
+    module = require_dependency(
+        "google.genai",
+        provider_label="retry_policy",
+        install_hint="Install it with `pip install google-genai`.",
+    )
 
     _HTTPX_NET_EXCS = _httpx_net_excs()
     _REQUESTS_EXCS = _requests_net_excs()
@@ -887,9 +890,9 @@ try:
     GOOGLE_ERROR_POLICY = ErrorPolicy(
         auth_excs=(),  # we will classify 401/403 via markers below (see non-retryable codes)
         rate_limit_excs=(
-            gerrors.ClientError,
+            module.gerrors.ClientError,
         ),  # includes 429; markers decide retry vs not
-        network_excs=(gerrors.ServerError,)
+        network_excs=(module.gerrors.ServerError,)
         + _HTTPX_NET_EXCS
         + _REQUESTS_EXCS,  # treat 5xx as transient
         http_excs=(),  # no reliable .status_code on exceptions; handled above

deepeval/models/utils.py

@@ -1,10 +1,14 @@
-from typing import Optional
+import logging
+from typing import Any, Dict, Optional, Tuple
 from pydantic import SecretStr
 
 from deepeval.errors import DeepEvalError
 
 
-def parse_model_name(model_name: Optional[str] = None) -> str:
+logger = logging.getLogger(__name__)
+
+
+def parse_model_name(model_name: Optional[str] = None) -> Optional[str]:
     """Extract base model name from provider-prefixed format.
 
     This function is useful for extracting the actual model name from a
@@ -28,9 +32,9 @@ def parse_model_name(model_name: Optional[str] = None) -> str:
     if model_name is None:
         return None
 
-    if "/" in model_name:
-        _, parsed_model_name = model_name.split("/", 1)
-        return parsed_model_name
+    # if "/" in model_name:
+    #     _, parsed_model_name = model_name.split("/", 1)
+    #     return parsed_model_name
     return model_name
 
 
@@ -74,3 +78,100 @@ def require_secret_api_key(
         )
 
     return api_key
+
+
+def require_costs(
+    model_data,
+    model_name: str,
+    input_token_envvar: str,
+    output_token_envvar: str,
+    cost_per_input_token: Optional[float] = None,
+    cost_per_output_token: Optional[float] = None,
+) -> Tuple[Optional[float], Optional[float]]:
+    """
+    Validates and returns the cost parameters (input and output tokens) for a model.
+
+    Arguments:
+    - model_data: The model's data object, which should contain `input_price` and `output_price`.
+    - model_name: The model name used for error messaging.
+    - cost_per_input_token: The input token cost provided during model initialization (optional).
+    - cost_per_output_token: The output token cost provided during model initialization (optional).
+    - input_token_envvar: The environment variable name for input cost.
+    - output_token_envvar: The environment variable name for output cost.
+
+    Returns:
+    - A tuple of validated values (input_cost, output_cost). If the values are provided, they are returned.
+      If not provided, they are fetched from settings or environment variables.
+    """
+
+    def validate_cost(
+        value: Optional[float], envvar_name: str
+    ) -> Optional[float]:
+        """Helper function to validate the cost values."""
+        if value is not None and value < 0:
+            raise DeepEvalError(f"{envvar_name} must be >= 0.")
+        return value
+
+    # Validate provided token costs
+    cost_per_input_token = validate_cost(
+        cost_per_input_token, input_token_envvar
+    )
+    cost_per_output_token = validate_cost(
+        cost_per_output_token, output_token_envvar
+    )
+
+    # If model data doesn't have pricing, use provided values or environment variables
+    if model_data.input_price is None or model_data.output_price is None:
+        if cost_per_input_token is None or cost_per_output_token is None:
+            raise DeepEvalError(
+                f"No pricing available for `{model_name}`. "
+                f"Please provide both `cost_per_input_token` and `cost_per_output_token` when initializing `{model_name}`, "
+                f"or set {input_token_envvar} and {output_token_envvar} environment variables."
+            )
+
+        # Return the validated cost values as a tuple
+        return cost_per_input_token, cost_per_output_token
+
+    # If no custom cost values are provided, return model's default cost values
+    return model_data.input_price, model_data.output_price
+
+
+def normalize_kwargs_and_extract_aliases(
+    provider_label: str,
+    kwargs: Dict[str, Any],
+    alias_map: Dict[str, list],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    """
+    Normalize legacy keyword argument names according to alias_map.
+
+    alias_map is of the form: {new_name: [old_name1, old_name2, ...]}
+
+    - Returns (normalized_kwargs, extracted_values)
+      where:
+        - normalized_kwargs has all legacy keys removed (to prevent forwarding
+          to downstream SDK clients).
+        - extracted_values maps new_name -> value for any alias that was used.
+
+    - Logs a warning for each legacy keyword used, so callers know they should
+      migrate to the new name.
+    """
+    normalized = dict(kwargs)
+    extracted: Dict[str, Any] = {}
+
+    for new_name, old_names in alias_map.items():
+        for old_name in old_names:
+            if old_name in normalized:
+                value = normalized.pop(old_name)
+
+                logger.warning(
+                    "%s keyword '%s' is deprecated; please use '%s' instead.",
+                    provider_label,
+                    old_name,
+                    new_name,
+                )
+
+                # Only preserve the first alias value we see for a given new_name
+                if new_name not in extracted:
+                    extracted[new_name] = value
+
+    return normalized, extracted

deepeval/optimizer/__init__.py

@@ -0,0 +1,5 @@
+from deepeval.optimizer.prompt_optimizer import PromptOptimizer
+
+__all__ = [
+    "PromptOptimizer",
+]

deepeval/optimizer/algorithms/__init__.py

@@ -0,0 +1,6 @@
+from .gepa import GEPA
+from .miprov2 import MIPROV2
+from .copro import COPRO
+from .simba import SIMBA
+
+__all__ = ["GEPA", "MIPROV2", "COPRO", "SIMBA"]

deepeval/optimizer/algorithms/base.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from typing import Union, List, Dict, Tuple
+
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.prompt.prompt import Prompt
+from deepeval.dataset.golden import Golden, ConversationalGolden
+
+
+class BaseAlgorithm(ABC):
+    name: str
+    optimizer_model: DeepEvalBaseLLM
+    scorer: BaseScorer
+
+    @abstractmethod
+    def execute(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Tuple[Prompt, Dict]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_execute(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Tuple[Prompt, Dict]:
+        raise NotImplementedError

deepeval/optimizer/algorithms/configs.py

@@ -0,0 +1,18 @@
+# Internal GEPA constants - not exposed to users
+GEPA_MIN_DELTA: float = 0.0
+GEPA_TIE_TOLERANCE: float = 1e-9
+GEPA_REWRITE_INSTRUCTION_MAX_CHARS: int = 4096
+
+# Internal MIPROV2 constants - not exposed to users
+MIPROV2_MIN_DELTA: float = 0.0
+MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS: int = 4096
+MIPROV2_DEFAULT_NUM_CANDIDATES: int = 10
+MIPROV2_DEFAULT_NUM_TRIALS: int = 20
+MIPROV2_DEFAULT_MINIBATCH_SIZE: int = 25
+MIPROV2_DEFAULT_MINIBATCH_FULL_EVAL_STEPS: int = 10
+MIPROV2_DEFAULT_MAX_BOOTSTRAPPED_DEMOS: int = 4
+MIPROV2_DEFAULT_MAX_LABELED_DEMOS: int = 4
+MIPROV2_DEFAULT_NUM_DEMO_SETS: int = 5
+
+# Internal SIMBA constants - not exposed to users
+SIMBA_DEMO_INPUT_MAX_CHARS: int = 256

deepeval/optimizer/algorithms/copro/__init__.py

@@ -0,0 +1,5 @@
+from .copro import COPRO
+
+__all__ = [
+    "COPRO",
+]