deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/llms/kimi_model.py

@@ -1,13 +1,16 @@
 from typing import Optional, Tuple, Union, Dict
 from openai import OpenAI, AsyncOpenAI
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
@@ -73,35 +76,39 @@ model_pricing = {
 class KimiModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        api_key: Optional[str] = None,
         model: Optional[str] = None,
+        api_key: Optional[str] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.MOONSHOT_MODEL_NAME
-        )
-        if model_name not in model_pricing:
+        settings = get_settings()
+
+        model = model or settings.MOONSHOT_MODEL_NAME
+        if model not in model_pricing:
             raise ValueError(
                 f"Invalid model. Available Moonshot models: {', '.join(model_pricing.keys())}"
             )
-        temperature_from_key = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.TEMPERATURE
-        )
+
+        temperature_from_key = settings.TEMPERATURE
         if temperature_from_key is None:
             self.temperature = temperature
         else:
             self.temperature = float(temperature_from_key)
         if self.temperature < 0:
             raise ValueError("Temperature must be >= 0.")
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.MOONSHOT_API_KEY
-        )
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.MOONSHOT_API_KEY
+
         self.base_url = "https://api.moonshot.cn/v1"
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -111,10 +118,11 @@ class KimiModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=False)
-        if schema and self.model_name in json_mode_models:
+        if schema and self.name in json_mode_models:
             completion = client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
@@ -130,7 +138,7 @@ class KimiModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
 
         completion = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             **self.generation_kwargs,
         )
@@ -149,10 +157,11 @@ class KimiModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=True)
-        if schema and self.model_name in json_mode_models:
+        if schema and self.name in json_mode_models:
             completion = await client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
@@ -168,7 +177,7 @@ class KimiModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
 
         completion = await client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             **self.generation_kwargs,
         )
@@ -192,7 +201,7 @@ class KimiModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
+        pricing = model_pricing.get(self.name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
         return input_cost + output_cost
@@ -218,8 +227,15 @@ class KimiModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Kimi",
+            env_var_name="MOONSHOT_API_KEY",
+            param_hint="`api_key` to KimiModel(...)",
+        )
+
         kw = dict(
-            api_key=self.api_key,
+            api_key=api_key,
             base_url=self.base_url,
             **self._client_kwargs(),
         )
@@ -233,4 +249,4 @@ class KimiModel(DeepEvalBaseLLM):
             raise
 
     def get_model_name(self):
-        return f"{self.model_name}"
+        return f"{self.name} (KIMI)"

deepeval/models/llms/litellm_model.py

@@ -1,6 +1,6 @@
-from typing import Optional, Tuple, Union, Dict, List, Any
-from pydantic import BaseModel
 import logging
+from typing import Optional, Tuple, Union, Dict, List, Any
+from pydantic import BaseModel, SecretStr
 from tenacity import (
     retry,
     stop_after_attempt,
@@ -8,11 +8,14 @@ from tenacity import (
     wait_exponential_jitter,
     RetryCallState,
 )
-import os
 
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import (
+    require_secret_api_key,
+    normalize_kwargs_and_extract_aliases,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 
 
 def log_retry_error(retry_state: RetryCallState):
@@ -27,6 +30,10 @@ retryable_exceptions = (
     Exception,  # LiteLLM handles specific exceptions internally
 )
 
+_ALIAS_MAP = {
+    "base_url": ["api_base"],
+}
+
 
 class LiteLLMModel(DeepEvalBaseLLM):
     EXP_BASE: int = 2
@@ -39,47 +46,66 @@ class LiteLLMModel(DeepEvalBaseLLM):
         self,
         model: Optional[str] = None,
         api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
+        base_url: Optional[str] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        from litellm import completion, acompletion, get_llm_provider
 
-        # Get model name from parameter or key file
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LITELLM_MODEL_NAME
+        normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
+            "LiteLLMModel",
+            kwargs,
+            _ALIAS_MAP,
         )
-        if not model_name:
+
+        # re-map depricated keywords to re-named positional args
+        if base_url is None and "base_url" in alias_values:
+            base_url = alias_values["base_url"]
+
+        settings = get_settings()
+        # Get model name from parameter or key file
+        model = model or settings.LITELLM_MODEL_NAME
+        if not model:
             raise ValueError(
                 "Model name must be provided either through parameter or set-litellm command"
             )
 
-        # Get API key from parameter, key file, or environment variable
-        self.api_key = (
-            api_key
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LITELLM_API_KEY)
-            or os.getenv("LITELLM_PROXY_API_KEY")
-            or os.getenv("OPENAI_API_KEY")
-            or os.getenv("ANTHROPIC_API_KEY")
-            or os.getenv("GOOGLE_API_KEY")
-        )
+        # Get API key from parameter, or settings
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = (
+                settings.LITELLM_API_KEY
+                or settings.LITELLM_PROXY_API_KEY
+                or settings.OPENAI_API_KEY
+                or settings.ANTHROPIC_API_KEY
+                or settings.GOOGLE_API_KEY
+            )
 
         # Get API base from parameter, key file, or environment variable
-        self.api_base = (
-            api_base
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LITELLM_API_BASE)
-            or os.getenv("LITELLM_API_BASE")
-            or os.getenv("LITELLM_PROXY_API_BASE")
+        self.base_url = (
+            base_url
+            or (
+                str(settings.LITELLM_API_BASE)
+                if settings.LITELLM_API_BASE is not None
+                else None
+            )
+            or (
+                str(settings.LITELLM_PROXY_API_BASE)
+                if settings.LITELLM_PROXY_API_BASE is not None
+                else None
+            )
         )
 
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
-        self.kwargs = kwargs
+        # Keep sanitized kwargs for client call to strip legacy keys
+        self.kwargs = normalized_kwargs
         self.generation_kwargs = generation_kwargs or {}
         self.evaluation_cost = 0.0  # Initialize cost to 0.0
-        super().__init__(model_name)
+        super().__init__(model)
 
     @retry(
         wait=wait_exponential_jitter(
@@ -92,18 +118,25 @@ class LiteLLMModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Union[str, Dict, Tuple[str, float]]:
+
         from litellm import completion
 
         completion_params = {
-            "model": self.model_name,
+            "model": self.name,
             "messages": [{"role": "user", "content": prompt}],
             "temperature": self.temperature,
         }
 
         if self.api_key:
-            completion_params["api_key"] = self.api_key
-        if self.api_base:
-            completion_params["api_base"] = self.api_base
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
+            completion_params["api_key"] = api_key
+        if self.base_url:
+            completion_params["api_base"] = self.base_url
 
         # Add schema if provided
         if schema:
@@ -141,18 +174,25 @@ class LiteLLMModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Union[str, Dict, Tuple[str, float]]:
+
         from litellm import acompletion
 
         completion_params = {
-            "model": self.model_name,
+            "model": self.name,
             "messages": [{"role": "user", "content": prompt}],
             "temperature": self.temperature,
         }
 
         if self.api_key:
-            completion_params["api_key"] = self.api_key
-        if self.api_base:
-            completion_params["api_base"] = self.api_base
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
+            completion_params["api_key"] = api_key
+        if self.base_url:
+            completion_params["api_base"] = self.base_url
 
         # Add schema if provided
         if schema:
@@ -195,12 +235,18 @@ class LiteLLMModel(DeepEvalBaseLLM):
         from litellm import completion
 
         try:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": self.temperature,
-                "api_key": self.api_key,
-                "api_base": self.api_base,
+                "api_key": api_key,
+                "api_base": self.base_url,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
             }
@@ -230,12 +276,18 @@ class LiteLLMModel(DeepEvalBaseLLM):
         from litellm import acompletion
 
         try:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": self.temperature,
-                "api_key": self.api_key,
-                "api_base": self.api_base,
+                "api_key": api_key,
+                "api_base": self.base_url,
                 "logprobs": True,
                 "top_logprobs": top_logprobs,
             }
@@ -263,13 +315,19 @@ class LiteLLMModel(DeepEvalBaseLLM):
         from litellm import completion
 
         try:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="LiteLLM",
+                env_var_name="LITELLM_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY",
+                param_hint="`api_key` to LiteLLMModel(...)",
+            )
             completion_params = {
-                "model": self.model_name,
+                "model": self.name,
                 "messages": [{"role": "user", "content": prompt}],
                 "temperature": temperature,
                 "n": n,
-                "api_key": self.api_key,
-                "api_base": self.api_base,
+                "api_key": api_key,
+                "api_base": self.base_url,
             }
             completion_params.update(self.kwargs)
 
@@ -315,8 +373,8 @@ class LiteLLMModel(DeepEvalBaseLLM):
     def get_model_name(self) -> str:
         from litellm import get_llm_provider
 
-        provider = get_llm_provider(self.model_name)
-        return f"{self.model_name} ({provider})"
+        provider = get_llm_provider(self.name)
+        return f"{self.name} ({provider})"
 
     def load_model(self, async_mode: bool = False):
         """

deepeval/models/llms/local_model.py

@@ -1,15 +1,18 @@
 from typing import Optional, Tuple, Union, Dict
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ChatCompletion
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.constants import ProviderSlug as PS
 
 
@@ -21,31 +24,35 @@ class LocalModel(DeepEvalBaseLLM):
     def __init__(
         self,
         model: Optional[str] = None,
-        base_url: Optional[str] = None,
         api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
         temperature: float = 0,
         format: Optional[str] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        self.local_model_api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_API_KEY
-        )
-        self.base_url = base_url or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_BASE_URL
-        )
-        self.format = format or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_FORMAT
+        settings = get_settings()
+
+        model = model or settings.LOCAL_MODEL_NAME
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.local_model_api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.local_model_api_key = settings.LOCAL_MODEL_API_KEY
+
+        self.base_url = (
+            base_url
+            or settings.LOCAL_MODEL_BASE_URL
+            and str(settings.LOCAL_MODEL_BASE_URL)
         )
+        self.format = format or settings.LOCAL_MODEL_FORMAT
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -55,9 +62,10 @@ class LocalModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=False)
         response: ChatCompletion = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             **self.generation_kwargs,
@@ -74,9 +82,10 @@ class LocalModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=True)
         response: ChatCompletion = await client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             temperature=self.temperature,
             **self.generation_kwargs,
@@ -94,10 +103,7 @@ class LocalModel(DeepEvalBaseLLM):
     ###############################################
 
     def get_model_name(self):
-        model_name = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        return f"{model_name} (Local Model)"
+        return f"{self.name} (Local Model)"
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
@@ -115,8 +121,15 @@ class LocalModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
+        local_model_api_key = require_secret_api_key(
+            self.local_model_api_key,
+            provider_label="Local",
+            env_var_name="LOCAL_MODEL_API_KEY",
+            param_hint="`api_key` to LocalModel(...)",
+        )
+
         kw = dict(
-            api_key=self.local_model_api_key,
+            api_key=local_model_api_key,
             base_url=self.base_url,
             **self._client_kwargs(),
         )