deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/llms/portkey_model.py

@@ -0,0 +1,149 @@
+import aiohttp
+import requests
+from typing import Any, Dict, List, Optional, Union
+from pydantic import AnyUrl, SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.utils import require_param
+
+
+def _request_timeout_seconds() -> float:
+    timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+    return timeout if timeout > 0 else 30.0
+
+
+class PortkeyModel(DeepEvalBaseLLM):
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[AnyUrl] = None,
+        provider: Optional[str] = None,
+        generation_kwargs: Optional[Dict] = None,
+        **kwargs,
+    ):
+        settings = get_settings()
+        model = model or settings.PORTKEY_MODEL_NAME
+
+        self.name = require_param(
+            model,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_MODEL_NAME",
+            param_hint="model",
+        )
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.PORTKEY_API_KEY
+
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.PORTKEY_BASE_URL is not None:
+            base_url = str(settings.PORTKEY_BASE_URL).rstrip("/")
+
+        self.base_url = require_param(
+            base_url,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_BASE_URL",
+            param_hint="base_url",
+        )
+
+        provider = provider or settings.PORTKEY_PROVIDER_NAME
+        self.provider = require_param(
+            provider,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_PROVIDER_NAME",
+            param_hint="provider",
+        )
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = kwargs
+        self.generation_kwargs = generation_kwargs or {}
+
+    def _headers(self) -> Dict[str, str]:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_API_KEY",
+            param_hint="`api_key` to PortkeyModel(...)",
+        )
+
+        headers = {
+            "Content-Type": "application/json",
+            "x-portkey-api-key": api_key,
+        }
+        if self.provider:
+            headers["x-portkey-provider"] = self.provider
+        return headers
+
+    def _payload(self, prompt: str) -> Dict[str, Any]:
+        payload = {
+            "model": self.name,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+        if self.generation_kwargs:
+            payload.update(self.generation_kwargs)
+        return payload
+
+    def _extract_content(self, data: Dict[str, Any]) -> str:
+        choices: Union[List[Dict[str, Any]], None] = data.get("choices")
+        if not choices:
+            raise ValueError("Portkey response did not include any choices.")
+        message = choices[0].get("message", {})
+        content: Union[str, List[Dict[str, Any]], None] = message.get("content")
+        if isinstance(content, str):
+            return content
+        if isinstance(content, list):
+            return "".join(part.get("text", "") for part in content)
+        return ""
+
+    def generate(self, prompt: str) -> str:
+
+        try:
+            response = requests.post(
+                f"{self.base_url}/chat/completions",
+                json=self._payload(prompt),
+                headers=self._headers(),
+                timeout=60,
+            )
+            response.raise_for_status()
+        except requests.HTTPError as error:
+            body: Union[str, Dict[str, Any]]
+            try:
+                body = response.json()
+            except Exception:
+                body = response.text
+            raise ValueError(
+                f"Portkey request failed with status {response.status_code}: {body}"
+            ) from error
+        except requests.RequestException as error:
+            raise ValueError(f"Portkey request failed: {error}") from error
+        return self._extract_content(response.json())
+
+    async def a_generate(self, prompt: str) -> str:
+
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                f"{self.base_url}/chat/completions",
+                json=self._payload(prompt),
+                headers=self._headers(),
+                timeout=60,
+            ) as response:
+                if response.status >= 400:
+                    body = await response.text()
+                    raise ValueError(
+                        f"Portkey request failed with status {response.status}: {body}"
+                    )
+                data = await response.json()
+                return self._extract_content(data)
+
+    def load_model(self):
+        return None
+
+    def get_model_name(self):
+        return f"{self.name} (Portkey)"

deepeval/models/llms/utils.py

@@ -1,8 +1,10 @@
-from typing import Dict
+from typing import Dict, List, Optional
 import re
 import json
 import asyncio
 
+MULTIMODAL_MODELS = ["GPTModel", "AzureModel", "GeminiModel", "OllamaModel"]
+
 
 def trim_and_load_json(
     input_string: str,
@@ -38,7 +40,7 @@ def safe_asyncio_run(coro):
                 return loop.run_until_complete(future)
             else:
                 return loop.run_until_complete(coro)
-        except Exception as inner_e:
+        except Exception:
             raise
-    except Exception as e:
+    except Exception:
         raise

deepeval/models/retry_policy.py

@@ -55,6 +55,7 @@ from tenacity.stop import stop_base
 from tenacity.wait import wait_base
 from contextvars import ContextVar, copy_context
 
+from deepeval.utils import require_dependency
 from deepeval.constants import (
     ProviderSlug as PS,
     slugify,
@@ -829,25 +830,23 @@ try:
 except Exception:  # botocore not present (aiobotocore optional)
     BEDROCK_ERROR_POLICY = None
 
-
 ####################
 # Anthropic Policy #
 ####################
 
 try:
-    from anthropic import (
-        AuthenticationError,
-        RateLimitError,
-        APIConnectionError,
-        APITimeoutError,
-        APIStatusError,
+
+    module = require_dependency(
+        "anthropic",
+        provider_label="retry_policy",
+        install_hint="Install it with `pip install anthropic`.",
     )
 
     ANTHROPIC_ERROR_POLICY = ErrorPolicy(
-        auth_excs=(AuthenticationError,),
-        rate_limit_excs=(RateLimitError,),
-        network_excs=(APIConnectionError, APITimeoutError),
-        http_excs=(APIStatusError,),
+        auth_excs=(module.AuthenticationError,),
+        rate_limit_excs=(module.RateLimitError,),
+        network_excs=(module.APIConnectionError, module.APITimeoutError),
+        http_excs=(module.APIStatusError,),
         non_retryable_codes=frozenset(),  # update if we learn of hard quota codes
         message_markers={},
     )
@@ -868,7 +867,11 @@ except Exception:  # Anthropic optional
 # and gate retries using message markers (code sniffing).
 # See: https://github.com/googleapis/python-genai?tab=readme-ov-file#error-handling
 try:
-    from google.genai import errors as gerrors
+    module = require_dependency(
+        "google.genai",
+        provider_label="retry_policy",
+        install_hint="Install it with `pip install google-genai`.",
+    )
 
     _HTTPX_NET_EXCS = _httpx_net_excs()
     _REQUESTS_EXCS = _requests_net_excs()
@@ -887,9 +890,9 @@ try:
     GOOGLE_ERROR_POLICY = ErrorPolicy(
         auth_excs=(),  # we will classify 401/403 via markers below (see non-retryable codes)
         rate_limit_excs=(
-            gerrors.ClientError,
+            module.gerrors.ClientError,
         ),  # includes 429; markers decide retry vs not
-        network_excs=(gerrors.ServerError,)
+        network_excs=(module.gerrors.ServerError,)
         + _HTTPX_NET_EXCS
         + _REQUESTS_EXCS,  # treat 5xx as transient
         http_excs=(),  # no reliable .status_code on exceptions; handled above

deepeval/models/utils.py

@@ -1,4 +1,11 @@
-from typing import Optional
+import logging
+from typing import Any, Dict, Optional, Tuple
+from pydantic import SecretStr
+
+from deepeval.errors import DeepEvalError
+
+
+logger = logging.getLogger(__name__)
 
 
 def parse_model_name(model_name: Optional[str] = None) -> str:
@@ -25,7 +32,90 @@ def parse_model_name(model_name: Optional[str] = None) -> str:
     if model_name is None:
         return None
 
-    # if "/" in model_name:
-    #     _, parsed_model_name = model_name.split("/", 1)
-    #     return parsed_model_name
+    if "/" in model_name:
+        _, parsed_model_name = model_name.split("/", 1)
+        return parsed_model_name
     return model_name
+
+
+def require_secret_api_key(
+    secret: Optional[SecretStr],
+    *,
+    provider_label: str,
+    env_var_name: str,
+    param_hint: str,
+) -> str:
+    """
+    Normalize and validate a provider API key stored as a SecretStr.
+
+    Args:
+        secret:
+            The SecretStr coming from Settings or an explicit constructor arg.
+        provider_label:
+            Human readable provider name for error messages, such as Anthropic, or OpenAI etc
+        env_var_name:
+            The environment variable backing this key
+        param_hint:
+            A short hint telling users how to pass the key explicitly
+
+    Returns:
+        The underlying API key string.
+
+    Raises:
+        DeepEvalError: if the key is missing or empty.
+    """
+    if secret is None:
+        raise DeepEvalError(
+            f"{provider_label} API key is not configured. "
+            f"Set {env_var_name} in your environment or pass "
+            f"{param_hint}."
+        )
+
+    api_key = secret.get_secret_value()
+    if not api_key:
+        raise DeepEvalError(
+            f"{provider_label} API key is empty. Please configure a valid key."
+        )
+
+    return api_key
+
+
+def normalize_kwargs_and_extract_aliases(
+    provider_label: str,
+    kwargs: Dict[str, Any],
+    alias_map: Dict[str, list],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    """
+    Normalize legacy keyword argument names according to alias_map.
+
+    alias_map is of the form: {new_name: [old_name1, old_name2, ...]}
+
+    - Returns (normalized_kwargs, extracted_values)
+      where:
+        - normalized_kwargs has all legacy keys removed (to prevent forwarding
+          to downstream SDK clients).
+        - extracted_values maps new_name -> value for any alias that was used.
+
+    - Logs a warning for each legacy keyword used, so callers know they should
+      migrate to the new name.
+    """
+    normalized = dict(kwargs)
+    extracted: Dict[str, Any] = {}
+
+    for new_name, old_names in alias_map.items():
+        for old_name in old_names:
+            if old_name in normalized:
+                value = normalized.pop(old_name)
+
+                logger.warning(
+                    "%s keyword '%s' is deprecated; please use '%s' instead.",
+                    provider_label,
+                    old_name,
+                    new_name,
+                )
+
+                # Only preserve the first alias value we see for a given new_name
+                if new_name not in extracted:
+                    extracted[new_name] = value
+
+    return normalized, extracted

deepeval/optimizer/__init__.py

@@ -0,0 +1,5 @@
+from deepeval.optimizer.prompt_optimizer import PromptOptimizer
+
+__all__ = [
+    "PromptOptimizer",
+]

deepeval/optimizer/algorithms/__init__.py

@@ -0,0 +1,6 @@
+from .gepa import GEPA
+from .miprov2 import MIPROV2
+from .copro import COPRO
+from .simba import SIMBA
+
+__all__ = ["GEPA", "MIPROV2", "COPRO", "SIMBA"]

deepeval/optimizer/algorithms/base.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from typing import Union, List, Dict, Tuple
+
+from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.optimizer.scorer.base import BaseScorer
+from deepeval.prompt.prompt import Prompt
+from deepeval.dataset.golden import Golden, ConversationalGolden
+
+
+class BaseAlgorithm(ABC):
+    name: str
+    optimizer_model: DeepEvalBaseLLM
+    scorer: BaseScorer
+
+    @abstractmethod
+    def execute(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Tuple[Prompt, Dict]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def a_execute(
+        self,
+        prompt: Prompt,
+        goldens: Union[List[Golden], List[ConversationalGolden]],
+    ) -> Tuple[Prompt, Dict]:
+        raise NotImplementedError

deepeval/optimizer/algorithms/configs.py

@@ -0,0 +1,18 @@
+# Internal GEPA constants - not exposed to users
+GEPA_MIN_DELTA: float = 0.0
+GEPA_TIE_TOLERANCE: float = 1e-9
+GEPA_REWRITE_INSTRUCTION_MAX_CHARS: int = 4096
+
+# Internal MIPROV2 constants - not exposed to users
+MIPROV2_MIN_DELTA: float = 0.0
+MIPROV2_REWRITE_INSTRUCTION_MAX_CHARS: int = 4096
+MIPROV2_DEFAULT_NUM_CANDIDATES: int = 10
+MIPROV2_DEFAULT_NUM_TRIALS: int = 20
+MIPROV2_DEFAULT_MINIBATCH_SIZE: int = 25
+MIPROV2_DEFAULT_MINIBATCH_FULL_EVAL_STEPS: int = 10
+MIPROV2_DEFAULT_MAX_BOOTSTRAPPED_DEMOS: int = 4
+MIPROV2_DEFAULT_MAX_LABELED_DEMOS: int = 4
+MIPROV2_DEFAULT_NUM_DEMO_SETS: int = 5
+
+# Internal SIMBA constants - not exposed to users
+SIMBA_DEMO_INPUT_MAX_CHARS: int = 256

deepeval/optimizer/algorithms/copro/__init__.py

@@ -0,0 +1,5 @@
+from .copro import COPRO
+
+__all__ = [
+    "COPRO",
+]