deepeval 3.7.2__py3-none-any.whl → 3.7.4__py3-none-any.whl

deepeval/models/mlllms/gemini_model.py

@@ -1,16 +1,16 @@
-from typing import Optional, List, Union
 import requests
-from pydantic import BaseModel
+from typing import Optional, List, Union
+from pydantic import BaseModel, SecretStr
 from google.genai import types
 from google import genai
 
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
-from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
 
 
@@ -60,77 +60,31 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         *args,
         **kwargs,
     ):
+        settings = get_settings()
         model_name = (
             model_name
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GEMINI_MODEL_NAME)
+            or settings.GEMINI_MODEL_NAME
             or default_multimodal_gemini_model
         )
 
-        # Get API key from key handler if not provided
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_API_KEY
-        )
-        self.project = project or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_PROJECT
-        )
-        self.location = location or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_LOCATION
-        )
-        self.use_vertexai = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_GENAI_USE_VERTEXAI
-        )
-
-        super().__init__(model_name, *args, **kwargs)
-        self.model = self.load_model(*args, **kwargs)
-
-    def should_use_vertexai(self):
-        """Checks if the model should use Vertex AI for generation.
-
-        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
-        environment variable. If not set, it checks for the presence of the
-        project and location.
-
-        Returns:
-            True if the model should use Vertex AI, False otherwise
-        """
-        if self.use_vertexai is not None:
-            return self.use_vertexai.lower() == "yes"
-
-        if self.project and self.location:
-            return True
+        # Get API key from settings if not provided
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self.api_key: SecretStr | None = SecretStr(api_key)
         else:
-            return False
-
-    def load_model(self, *args, **kwargs):
-        """Creates a client.
-        With Gen AI SDK, model is set at inference time, so there is no
-        model to load and initialize.
-        This method name is kept for compatibility with other LLMs.
-
-        Returns:
-            A GenerativeModel instance configured for evaluation.
-        """
-        if self.should_use_vertexai():
-            if not self.project or not self.location:
-                raise ValueError(
-                    "When using Vertex AI API, both project and location are required."
-                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
-                    "or set them in your DeepEval configuration."
-                )
+            self.api_key = settings.GOOGLE_API_KEY
 
-            # Create client for Vertex AI
-            self.client = genai.Client(
-                vertexai=True, project=self.project, location=self.location
-            )
-        else:
-            if not self.api_key:
-                raise ValueError(
-                    "Google API key is required. Either provide it directly, set GOOGLE_API_KEY environment variable, "
-                    "or set it in your DeepEval configuration."
-                )
+        self.project = project or settings.GOOGLE_CLOUD_PROJECT
+        self.location = (
+            location
+            or settings.GOOGLE_CLOUD_LOCATION is not None
+            and str(settings.GOOGLE_CLOUD_LOCATION)
+        )
+        self.use_vertexai = settings.GOOGLE_GENAI_USE_VERTEXAI
 
-            # Create client for Gemini API
-            self.client = genai.Client(api_key=self.api_key)
+        # Keep any extra kwargs for the underlying genai.Client
+        self.args = args
+        self.kwargs = kwargs
 
         # Configure default model generation settings
         self.model_safety_settings = [
@@ -152,9 +106,28 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             ),
         ]
         self.model_temperature = 0.0
-        return self.client.models
 
-    # TODO: Refactor genete prompt to minimize the work done on retry
+        super().__init__(model_name, *args, **kwargs)
+
+    def should_use_vertexai(self):
+        """Checks if the model should use Vertex AI for generation.
+
+        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
+        environment variable. If not set, it checks for the presence of the
+        project and location.
+
+        Returns:
+            True if the model should use Vertex AI, False otherwise
+        """
+        if self.use_vertexai is not None:
+            return self.use_vertexai.lower() == "yes"
+
+        if self.project and self.location:
+            return True
+        else:
+            return False
+
+    # TODO: Refactor generate prompt to minimize the work done on retry
     @retry_gemini
     def generate_prompt(
         self, multimodal_input: List[Union[str, MLLMImage]] = []
@@ -214,10 +187,11 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         Returns:
             Generated text response
         """
+        client = self.load_model()
         prompt = self.generate_prompt(multimodal_input)
 
         if schema is not None:
-            response = self.client.models.generate_content(
+            response = client.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -229,7 +203,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.parsed, 0
         else:
-            response = self.client.models.generate_content(
+            response = client.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -254,10 +228,11 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         Returns:
             Generated text response
         """
+        client = self.load_model()
         prompt = self.generate_prompt(multimodal_input)
 
         if schema is not None:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -269,7 +244,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.parsed, 0
         else:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -279,6 +254,60 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.text, 0
 
+    #########
+    # Model #
+    #########
+
     def get_model_name(self) -> str:
         """Returns the name of the Gemini model being used."""
         return self.model_name
+
+    def load_model(self, *args, **kwargs):
+        """Creates and returns a GenAI client.
+
+        With the Gen AI SDK, the model is set at inference time, so we only
+        construct the client here. Kept for compatibility with other MLLMs.
+        """
+        return self._build_client(**kwargs)
+
+    def _client_kwargs(self, **override_kwargs) -> dict:
+        """
+        Return kwargs forwarded to genai.Client.
+
+        Start from the ctor kwargs captured on `self.kwargs`, then apply any
+        overrides passed via load_model(...).
+        """
+        client_kwargs = dict(self.kwargs or {})
+        if override_kwargs:
+            client_kwargs.update(override_kwargs)
+        return client_kwargs
+
+    def _build_client(self, **override_kwargs):
+        """Build and return a genai.Client for either Gemini API or Vertex AI."""
+        client_kwargs = self._client_kwargs(**override_kwargs)
+
+        if self.should_use_vertexai():
+            if not self.project or not self.location:
+                raise ValueError(
+                    "When using Vertex AI API, both project and location are required."
+                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
+                    "or set them in your DeepEval configuration."
+                )
+
+            # Create client for Vertex AI
+            return genai.Client(
+                vertexai=True,
+                project=self.project,
+                location=self.location,
+                **client_kwargs,
+            )
+
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Google Gemini",
+            env_var_name="GOOGLE_API_KEY",
+            param_hint="`api_key` to MultimodalGeminiModel(...)",
+        )
+
+        # Create client for Gemini API
+        return genai.Client(api_key=api_key, **client_kwargs)

deepeval/models/mlllms/ollama_model.py

@@ -8,7 +8,6 @@ import io
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-from deepeval.key_handler import KEY_FILE_HANDLER, ModelKeyValues
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
 from deepeval.config.settings import get_settings
@@ -19,14 +18,34 @@ retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class MultimodalOllamaModel(DeepEvalBaseMLLM):
-    def __init__(self, **kwargs):
-        model_name = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        self.base_url = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_BASE_URL
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        host: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        Multimodal Ollama model.
+
+        - `model`: Ollama model name (e.g. "llava").
+        - `host`: Ollama base URL (e.g. "http://localhost:11434").
+        - extra **kwargs are passed through to the underlying Client.
+        """
+        settings = get_settings()
+
+        # Resolve host/base URL
+        self.base_url = (
+            host
+            or settings.LOCAL_MODEL_BASE_URL
+            and str(settings.LOCAL_MODEL_BASE_URL)
         )
-        self.kwargs = kwargs
+
+        # Resolve model name
+        model_name = model or settings.LOCAL_MODEL_NAME
+
+        # Client kwargs
+        self.kwargs = kwargs or {}
+
         super().__init__(model_name)
 
     @retry_ollama
@@ -132,13 +151,25 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
             print(f"Error converting image to base64: {e}")
             return None
 
+    ###############################################
+    # Model
+    ###############################################
+
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(Client)
         return self._build_client(AsyncClient)
 
+    def _client_kwargs(self) -> Dict:
+        """
+        Return client-init kwargs.
+        Ollama's Python client doesn't have built-in retry config like OpenAI,
+        so we just pass these through untouched.
+        """
+        return dict(self.kwargs or {})
+
     def _build_client(self, cls):
-        return cls(host=self.base_url, **self.kwargs)
+        return cls(host=self.base_url, **self._client_kwargs())
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"

deepeval/models/mlllms/openai_model.py

@@ -1,18 +1,20 @@
-from typing import Optional, Tuple, List, Union
+import base64
+from typing import Optional, Tuple, List, Union, Dict
 from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ParsedChatCompletion
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 from io import BytesIO
-import base64
 
+from deepeval.config.settings import get_settings
 from deepeval.models.llms.openai_model import (
     model_pricing,
     structured_outputs_models,
+    _request_timeout_seconds,
 )
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.test_case import MLLMImage
-from deepeval.models.utils import parse_model_name
+from deepeval.models.utils import parse_model_name, require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
@@ -60,17 +62,26 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         *args,
         **kwargs,
     ):
+        settings = get_settings()
         model_name = None
         if isinstance(model, str):
             model_name = parse_model_name(model)
             if model_name not in valid_multimodal_gpt_models:
                 raise ValueError(
-                    f"Invalid model. Available Multimodal GPT models: {', '.join(model for model in valid_multimodal_gpt_models)}"
+                    f"Invalid model. Available Multimodal GPT models: "
+                    f"{', '.join(model for model in valid_multimodal_gpt_models)}"
                 )
+        elif settings.OPENAI_MODEL_NAME is not None:
+            model_name = settings.OPENAI_MODEL_NAME
         elif model is None:
             model_name = default_multimodal_gpt_model
 
-        self._openai_api_key = _openai_api_key
+        if _openai_api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self._openai_api_key: SecretStr | None = SecretStr(_openai_api_key)
+        else:
+            self._openai_api_key = settings.OPENAI_API_KEY
+
         self.args = args
         self.kwargs = kwargs
 
@@ -86,7 +97,7 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         multimodal_input: List[Union[str, MLLMImage]],
         schema: Optional[BaseModel] = None,
     ) -> Tuple[str, float]:
-        client = OpenAI(api_key=self._openai_api_key)
+        client = self.load_model(async_mode=False)
         prompt = self.generate_prompt(multimodal_input)
 
         if schema:
@@ -123,7 +134,7 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         multimodal_input: List[Union[str, MLLMImage]],
         schema: Optional[BaseModel] = None,
     ) -> Tuple[str, float]:
-        client = AsyncOpenAI(api_key=self._openai_api_key)
+        client = self.load_model(async_mode=True)
         prompt = self.generate_prompt(multimodal_input)
 
         if schema:
@@ -247,12 +258,52 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
         return base64_encoded_image
 
-    def _client(self, async_mode: bool = False):
-        kw = {"api_key": self._openai_api_key}
-        if not sdk_retries_for(PS.OPENAI):
-            kw["max_retries"] = 0
-        Client = AsyncOpenAI if async_mode else OpenAI
-        return Client(**kw)
+    ###############################################
+    # Model
+    ###############################################
 
     def get_model_name(self):
         return self.model_name
+
+    def load_model(self, async_mode: bool = False):
+        Client = AsyncOpenAI if async_mode else OpenAI
+        return self._build_client(Client)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid
+        double retries. If the user opts into SDK retries for 'openai' via
+        DEEPEVAL_SDK_RETRY_PROVIDERS, leave their retry settings as is.
+        """
+        kwargs: Dict = {}
+        if not sdk_retries_for(PS.OPENAI):
+            kwargs["max_retries"] = 0
+
+        if not kwargs.get("timeout"):
+            kwargs["timeout"] = _request_timeout_seconds()
+        return kwargs
+
+    def _build_client(self, cls):
+        api_key = require_secret_api_key(
+            self._openai_api_key,
+            provider_label="OpenAI",
+            env_var_name="OPENAI_API_KEY",
+            param_hint="`_openai_api_key` to MultimodalOpenAIModel(...)",
+        )
+
+        kw = dict(
+            api_key=api_key,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise
+
+    def _client(self, async_mode: bool = False):
+        # Backwards-compat path for internal callers in this module
+        return self.load_model(async_mode=async_mode)

deepeval/models/utils.py

@@ -1,4 +1,7 @@
 from typing import Optional
+from pydantic import SecretStr
+
+from deepeval.errors import DeepEvalError
 
 
 def parse_model_name(model_name: Optional[str] = None) -> str:
@@ -25,7 +28,49 @@ def parse_model_name(model_name: Optional[str] = None) -> str:
     if model_name is None:
         return None
 
-    # if "/" in model_name:
-    #     _, parsed_model_name = model_name.split("/", 1)
-    #     return parsed_model_name
+    if "/" in model_name:
+        _, parsed_model_name = model_name.split("/", 1)
+        return parsed_model_name
     return model_name
+
+
+def require_secret_api_key(
+    secret: Optional[SecretStr],
+    *,
+    provider_label: str,
+    env_var_name: str,
+    param_hint: str,
+) -> str:
+    """
+    Normalize and validate a provider API key stored as a SecretStr.
+
+    Args:
+        secret:
+            The SecretStr coming from Settings or an explicit constructor arg.
+        provider_label:
+            Human readable provider name for error messages, such as Anthropic, or OpenAI etc
+        env_var_name:
+            The environment variable backing this key
+        param_hint:
+            A short hint telling users how to pass the key explicitly
+
+    Returns:
+        The underlying API key string.
+
+    Raises:
+        DeepEvalError: if the key is missing or empty.
+    """
+    if secret is None:
+        raise DeepEvalError(
+            f"{provider_label} API key is not configured. "
+            f"Set {env_var_name} in your environment or pass "
+            f"{param_hint}."
+        )
+
+    api_key = secret.get_secret_value()
+    if not api_key:
+        raise DeepEvalError(
+            f"{provider_label} API key is empty. Please configure a valid key."
+        )
+
+    return api_key

deepeval/optimization/__init__.py

@@ -0,0 +1,13 @@
+from deepeval.optimization.prompt_optimizer import PromptOptimizer
+from deepeval.optimization.configs import OptimizerDisplayConfig
+from deepeval.optimization.gepa.loop import (
+    GEPARunner as GEPARunner,
+    GEPAConfig as GEPAConfig,
+)
+
+__all__ = [
+    "GEPARunner",
+    "GEPAConfig",
+    "PromptOptimizer",
+    "OptimizerDisplayConfig",
+]

deepeval/optimization/adapters/__init__.py

@@ -0,0 +1,2 @@
+# nothing yet
+__all__ = []