deepeval 3.7.3__py3-none-any.whl → 3.7.4__py3-none-any.whl

deepeval/models/llms/gemini_model.py

@@ -1,15 +1,17 @@
-from pydantic import BaseModel
+import json
+
+from pydantic import BaseModel, SecretStr
 from google.genai import types, Client
 from typing import Optional, Dict
 
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 from google.oauth2 import service_account
-import json
 
 default_gemini_model = "gemini-1.5-pro"
 
@@ -57,31 +59,32 @@ class GeminiModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+
+        settings = get_settings()
+
         model_name = (
-            model_name
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GEMINI_MODEL_NAME)
-            or default_gemini_model
+            model_name or settings.GEMINI_MODEL_NAME or default_gemini_model
         )
 
-        # Get API key from key handler if not provided
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_API_KEY
-        )
-        self.project = project or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_PROJECT
-        )
-        self.location = location or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_LOCATION
-        )
-        self.use_vertexai = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_GENAI_USE_VERTEXAI
+        # Get API key from settings if not provided
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.GOOGLE_API_KEY
+
+        self.project = project or settings.GOOGLE_CLOUD_PROJECT
+        self.location = (
+            location
+            or settings.GOOGLE_CLOUD_LOCATION is not None
+            and str(settings.GOOGLE_CLOUD_LOCATION)
         )
+        self.use_vertexai = settings.GOOGLE_GENAI_USE_VERTEXAI
+
         if service_account_key:
             self.service_account_key = service_account_key
         else:
-            service_account_key_data = KEY_FILE_HANDLER.fetch_data(
-                ModelKeyValues.GOOGLE_SERVICE_ACCOUNT_KEY
-            )
+            service_account_key_data = settings.GOOGLE_SERVICE_ACCOUNT_KEY
             if service_account_key_data is None:
                 self.service_account_key = None
             elif isinstance(service_account_key_data, str):
@@ -90,69 +93,10 @@ class GeminiModel(DeepEvalBaseLLM):
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
+
+        # Raw kwargs destined for the underlying Client
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name, **kwargs)
-
-    def should_use_vertexai(self):
-        """Checks if the model should use Vertex AI for generation.
-
-        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
-        environment variable. If not set, it checks for the presence of the
-        project and location.
-
-        Returns:
-            True if the model should use Vertex AI, False otherwise
-        """
-        if self.use_vertexai is not None:
-            return self.use_vertexai.lower() == "yes"
-        if self.project and self.location:
-            return True
-        else:
-            return False
-
-    def load_model(self, *args, **kwargs):
-        """Creates a client.
-        With Gen AI SDK, model is set at inference time, so there is no
-        model to load and initialize.
-        This method name is kept for compatibility with other LLMs.
-
-        Returns:
-            A GenerativeModel instance configured for evaluation.
-        """
-        if self.should_use_vertexai():
-            if not self.project or not self.location:
-                raise ValueError(
-                    "When using Vertex AI API, both project and location are required."
-                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
-                    "or set them in your DeepEval configuration."
-                )
-
-            # Create client for Vertex AI
-            self.client = Client(
-                vertexai=True,
-                project=self.project,
-                location=self.location,
-                credentials=(
-                    service_account.Credentials.from_service_account_info(
-                        self.service_account_key,
-                        scopes=[
-                            "https://www.googleapis.com/auth/cloud-platform"
-                        ],
-                    )
-                    if self.service_account_key
-                    else None
-                ),
-                **self.kwargs,
-            )
-        else:
-            if not self.api_key:
-                raise ValueError(
-                    "Google API key is required. Either provide it directly, set GOOGLE_API_KEY environment variable, "
-                    "or set it in your DeepEval configuration."
-                )
-            # Create client for Gemini API
-            self.client = Client(api_key=self.api_key, **self.kwargs)
 
         # Configure default model generation settings
         self.model_safety_settings = [
@@ -173,7 +117,29 @@ class GeminiModel(DeepEvalBaseLLM):
                 threshold=types.HarmBlockThreshold.BLOCK_NONE,
             ),
         ]
-        return self.client.models
+
+        super().__init__(model_name, **kwargs)
+
+    def should_use_vertexai(self) -> bool:
+        """Checks if the model should use Vertex AI for generation.
+
+        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
+        environment variable. If not set, it checks for the presence of the
+        project and location.
+
+        Returns:
+            True if the model should use Vertex AI, False otherwise
+        """
+        if self.use_vertexai is not None:
+            return self.use_vertexai.lower() == "yes"
+        if self.project and self.location:
+            return True
+        else:
+            return False
+
+    ###############################################
+    # Generate functions
+    ###############################################
 
     @retry_gemini
     def generate(self, prompt: str, schema: Optional[BaseModel] = None) -> str:
@@ -186,8 +152,10 @@ class GeminiModel(DeepEvalBaseLLM):
         Returns:
             Generated text response or structured output as Pydantic model
         """
+        client = self.load_model()
+
         if schema is not None:
-            response = self.client.models.generate_content(
+            response = client.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -200,7 +168,7 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.parsed, 0
         else:
-            response = self.client.models.generate_content(
+            response = client.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -224,8 +192,10 @@ class GeminiModel(DeepEvalBaseLLM):
         Returns:
             Generated text response or structured output as Pydantic model
         """
+        client = self.load_model()
+
         if schema is not None:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -238,7 +208,7 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.parsed, 0
         else:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -249,6 +219,69 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.text, 0
 
+    #########
+    # Model #
+    #########
+
     def get_model_name(self) -> str:
         """Returns the name of the Gemini model being used."""
         return self.model_name
+
+    def load_model(self, *args, **kwargs):
+        """Creates a client.
+        With Gen AI SDK, model is set at inference time, so there is no
+        model to load and initialize.
+        This method name is kept for compatibility with other LLMs.
+
+        Returns:
+            A GenerativeModel instance configured for evaluation.
+        """
+        return self._build_client(**kwargs)
+
+    def _client_kwargs(self, **override_kwargs) -> Dict:
+        """Merge ctor kwargs with any overrides passed at load_model time."""
+        client_kwargs = dict(self.kwargs or {})
+        if override_kwargs:
+            client_kwargs.update(override_kwargs)
+        return client_kwargs
+
+    def _build_client(self, **override_kwargs) -> Client:
+        client_kwargs = self._client_kwargs(**override_kwargs)
+
+        if self.should_use_vertexai():
+            if not self.project or not self.location:
+                raise ValueError(
+                    "When using Vertex AI API, both project and location are required. "
+                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and "
+                    "GOOGLE_CLOUD_LOCATION in your DeepEval configuration."
+                )
+
+            credentials = (
+                service_account.Credentials.from_service_account_info(
+                    self.service_account_key,
+                    scopes=[
+                        "https://www.googleapis.com/auth/cloud-platform",
+                    ],
+                )
+                if self.service_account_key
+                else None
+            )
+
+            client = Client(
+                vertexai=True,
+                project=self.project,
+                location=self.location,
+                credentials=credentials,
+                **client_kwargs,
+            )
+        else:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="Google Gemini",
+                env_var_name="GOOGLE_API_KEY",
+                param_hint="`api_key` to GeminiModel(...)",
+            )
+
+            client = Client(api_key=api_key, **client_kwargs)
+
+        return client

deepeval/models/llms/grok_model.py

@@ -1,14 +1,13 @@
-import os
-
 from typing import Optional, Tuple, Union, Dict
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
@@ -62,27 +61,28 @@ class GrokModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GROK_MODEL_NAME
-        )
+        settings = get_settings()
+
+        model_name = model or settings.GROK_MODEL_NAME
+
         if model_name not in model_pricing:
             raise ValueError(
                 f"Invalid model. Available Grok models: {', '.join(model_pricing.keys())}"
             )
-        temperature_from_key = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.TEMPERATURE
-        )
+        temperature_from_key = settings.TEMPERATURE
         if temperature_from_key is None:
             self.temperature = temperature
         else:
             self.temperature = float(temperature_from_key)
         if self.temperature < 0:
             raise ValueError("Temperature must be >= 0.")
-        self.api_key = (
-            api_key
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GROK_API_KEY)
-            or os.getenv("GROK_API_KEY")
-        )
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.GROK_API_KEY
+
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
         super().__init__(model_name)
@@ -226,7 +226,14 @@ class GrokModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
-        kw = dict(api_key=self.api_key, **self._client_kwargs())
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Grok",
+            env_var_name="GROK_API_KEY",
+            param_hint="`api_key` to GrokModel(...)",
+        )
+
+        kw = dict(api_key=api_key, **self._client_kwargs())
         try:
             return cls(**kw)
         except TypeError as e:

deepeval/models/llms/kimi_model.py

@@ -1,13 +1,14 @@
 from typing import Optional, Tuple, Union, Dict
 from openai import OpenAI, AsyncOpenAI
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
@@ -79,25 +80,28 @@ class KimiModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.MOONSHOT_MODEL_NAME
-        )
+        settings = get_settings()
+
+        model_name = model or settings.MOONSHOT_MODEL_NAME
         if model_name not in model_pricing:
             raise ValueError(
                 f"Invalid model. Available Moonshot models: {', '.join(model_pricing.keys())}"
             )
-        temperature_from_key = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.TEMPERATURE
-        )
+
+        temperature_from_key = settings.TEMPERATURE
         if temperature_from_key is None:
             self.temperature = temperature
         else:
             self.temperature = float(temperature_from_key)
         if self.temperature < 0:
             raise ValueError("Temperature must be >= 0.")
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.MOONSHOT_API_KEY
-        )
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.MOONSHOT_API_KEY
+
         self.base_url = "https://api.moonshot.cn/v1"
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
@@ -218,8 +222,15 @@ class KimiModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Kimi",
+            env_var_name="MOONSHOT_API_KEY",
+            param_hint="`api_key` to KimiModel(...)",
+        )
+
         kw = dict(
-            api_key=self.api_key,
+            api_key=api_key,
             base_url=self.base_url,
             **self._client_kwargs(),
         )

deepeval/models/llms/litellm_model.py

@@ -1,6 +1,6 @@
-from typing import Optional, Tuple, Union, Dict, List, Any
-from pydantic import BaseModel
 import logging
+from typing import Optional, Tuple, Union, Dict, List, Any
+from pydantic import BaseModel, SecretStr
 from tenacity import (
     retry,
     stop_after_attempt,
@@ -8,11 +8,11 @@ from tenacity import (
     wait_exponential_jitter,
     RetryCallState,
 )
-import os
 
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 
 
 def log_retry_error(retry_state: RetryCallState):
@@ -44,33 +44,41 @@ class LiteLLMModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        from litellm import completion, acompletion, get_llm_provider
 
+        settings = get_settings()
         # Get model name from parameter or key file
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LITELLM_MODEL_NAME
-        )
+        model_name = model or settings.LITELLM_MODEL_NAME
         if not model_name:
             raise ValueError(
                 "Model name must be provided either through parameter or set-litellm command"
             )
 
-        # Get API key from parameter, key file, or environment variable
-        self.api_key = (
-            api_key
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LITELLM_API_KEY)
-            or os.getenv("LITELLM_PROXY_API_KEY")
-            or os.getenv("OPENAI_API_KEY")
-            or os.getenv("ANTHROPIC_API_KEY")
-            or os.getenv("GOOGLE_API_KEY")
-        )
+        # Get API key from parameter, or settings
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = (
+                settings.LITELLM_API_KEY
+                or settings.LITELLM_PROXY_API_KEY
+                or settings.OPENAI_API_KEY
+                or settings.ANTHROPIC_API_KEY
+                or settings.GOOGLE_API_KEY
+            )
 
         # Get API base from parameter, key file, or environment variable
         self.api_base = (
             api_base
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LITELLM_API_BASE)
-            or os.getenv("LITELLM_API_BASE")
-            or os.getenv("LITELLM_PROXY_API_BASE")
+            or (
+                str(settings.LITELLM_API_BASE)
+                if settings.LITELLM_API_BASE is not None
+                else None
+            )
+            or (
+                str(settings.LITELLM_PROXY_API_BASE)
+                if settings.LITELLM_PROXY_API_BASE is not None
+                else None
+            )
         )
 
         if temperature < 0:
@@ -101,7 +109,13 @@ class LiteLLMModel(DeepEvalBaseLLM):
         }
 
         if self.api_key:
-            completion_params["api_key"] = self.api_key
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
+            completion_params["api_key"] = api_key
         if self.api_base:
             completion_params["api_base"] = self.api_base
 
@@ -150,7 +164,13 @@ class LiteLLMModel(DeepEvalBaseLLM):
         }
 
         if self.api_key:
-            completion_params["api_key"] = self.api_key
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
+            completion_params["api_key"] = api_key
         if self.api_base:
             completion_params["api_base"] = self.api_base
 
@@ -195,11 +215,17 @@ class LiteLLMModel(DeepEvalBaseLLM):
         from litellm import completion
 
         try:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
             completion_params = {
                 "model": self.model_name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": self.temperature,
-                "api_key": self.api_key,
+                "api_key": api_key,
                 "api_base": self.api_base,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
@@ -230,11 +256,17 @@ class LiteLLMModel(DeepEvalBaseLLM):
         from litellm import acompletion
 
         try:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
             completion_params = {
                 "model": self.model_name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": self.temperature,
-                "api_key": self.api_key,
+                "api_key": api_key,
                 "api_base": self.api_base,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
@@ -263,12 +295,18 @@ class LiteLLMModel(DeepEvalBaseLLM):
         from litellm import completion
 
         try:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
             completion_params = {
                 "model": self.model_name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": temperature,
                 "n": n,
-                "api_key": self.api_key,
+                "api_key": api_key,
                 "api_base": self.api_base,
             }
             completion_params.update(self.kwargs)

deepeval/models/llms/local_model.py

@@ -1,15 +1,16 @@
 from typing import Optional, Tuple, Union, Dict
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ChatCompletion
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models import DeepEvalBaseLLM
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.constants import ProviderSlug as PS
 
 
@@ -28,18 +29,21 @@ class LocalModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        self.local_model_api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_API_KEY
-        )
-        self.base_url = base_url or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_BASE_URL
-        )
-        self.format = format or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_FORMAT
+        settings = get_settings()
+
+        model_name = model or settings.LOCAL_MODEL_NAME
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.local_model_api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.local_model_api_key = settings.LOCAL_MODEL_API_KEY
+
+        self.base_url = (
+            base_url
+            or settings.LOCAL_MODEL_BASE_URL
+            and str(settings.LOCAL_MODEL_BASE_URL)
         )
+        self.format = format or settings.LOCAL_MODEL_FORMAT
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
@@ -94,10 +98,7 @@ class LocalModel(DeepEvalBaseLLM):
     ###############################################
 
     def get_model_name(self):
-        model_name = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        return f"{model_name} (Local Model)"
+        return f"{self.model_name} (Local Model)"
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
@@ -115,8 +116,15 @@ class LocalModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
+        local_model_api_key = require_secret_api_key(
+            self.local_model_api_key,
+            provider_label="Local",
+            env_var_name="LOCAL_MODEL_API_KEY",
+            param_hint="`api_key` to LocalModel(...)",
+        )
+
         kw = dict(
-            api_key=self.local_model_api_key,
+            api_key=local_model_api_key,
             base_url=self.base_url,
             **self._client_kwargs(),
         )