deepeval 3.7.3__py3-none-any.whl → 3.7.4__py3-none-any.whl

deepeval/models/llms/ollama_model.py

@@ -2,12 +2,11 @@ from ollama import Client, AsyncClient, ChatResponse
 from typing import Optional, Tuple, Union, Dict
 from pydantic import BaseModel
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-
 from deepeval.models import DeepEvalBaseLLM
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.constants import ProviderSlug as PS
 
 
@@ -23,17 +22,20 @@ class OllamaModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
+        settings = get_settings()
+        model_name = model or settings.LOCAL_MODEL_NAME
         self.base_url = (
             base_url
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_BASE_URL)
+            or (
+                settings.LOCAL_MODEL_BASE_URL
+                and str(settings.LOCAL_MODEL_BASE_URL)
+            )
             or "http://localhost:11434"
         )
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
+        # Raw kwargs destined for the underlying Ollama client
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
         super().__init__(model_name)
@@ -97,8 +99,16 @@ class OllamaModel(DeepEvalBaseLLM):
             return self._build_client(Client)
         return self._build_client(AsyncClient)
 
+    def _client_kwargs(self) -> Dict:
+        """Return kwargs forwarded to the underlying Ollama Client/AsyncClient."""
+        return dict(self.kwargs or {})
+
     def _build_client(self, cls):
-        return cls(host=self.base_url, **self.kwargs)
+        kw = dict(
+            host=self.base_url,
+            **self._client_kwargs(),
+        )
+        return cls(**kw)
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"

deepeval/models/llms/openai_model.py

@@ -1,7 +1,6 @@
 from openai.types.chat.chat_completion import ChatCompletion
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from typing import Optional, Tuple, Union, Dict
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 
 from openai import (
     OpenAI,
@@ -12,7 +11,7 @@ from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import parse_model_name
+from deepeval.models.utils import parse_model_name, require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
@@ -227,23 +226,18 @@ class GPTModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+        settings = get_settings()
         model_name = None
-        model = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.OPENAI_MODEL_NAME
-        )
+        model = model or settings.OPENAI_MODEL_NAME
         cost_per_input_token = (
             cost_per_input_token
             if cost_per_input_token is not None
-            else KEY_FILE_HANDLER.fetch_data(
-                ModelKeyValues.OPENAI_COST_PER_INPUT_TOKEN
-            )
+            else settings.OPENAI_COST_PER_INPUT_TOKEN
         )
         cost_per_output_token = (
             cost_per_output_token
             if cost_per_output_token is not None
-            else KEY_FILE_HANDLER.fetch_data(
-                ModelKeyValues.OPENAI_COST_PER_OUTPUT_TOKEN
-            )
+            else settings.OPENAI_COST_PER_OUTPUT_TOKEN
         )
 
         if isinstance(model, str):
@@ -272,7 +266,12 @@ class GPTModel(DeepEvalBaseLLM):
         elif model is None:
             model_name = default_gpt_model
 
-        self._openai_api_key = _openai_api_key
+        if _openai_api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self._openai_api_key: SecretStr | None = SecretStr(_openai_api_key)
+        else:
+            self._openai_api_key = get_settings().OPENAI_API_KEY
+
         self.base_url = base_url
         # args and kwargs will be passed to the underlying model, in load_model function
 
@@ -485,9 +484,9 @@ class GPTModel(DeepEvalBaseLLM):
         output_cost = output_tokens * pricing["output"]
         return input_cost + output_cost
 
-    ###############################################
-    # Model
-    ###############################################
+    #########
+    # Model #
+    #########
 
     def get_model_name(self):
         return self.model_name
@@ -512,9 +511,15 @@ class GPTModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
+        api_key = require_secret_api_key(
+            self._openai_api_key,
+            provider_label="OpenAI",
+            env_var_name="OPENAI_API_KEY",
+            param_hint="`_openai_api_key` to GPTModel(...)",
+        )
 
         kw = dict(
-            api_key=self._openai_api_key,
+            api_key=api_key,
             base_url=self.base_url,
             **self._client_kwargs(),
         )

deepeval/models/llms/portkey_model.py

@@ -0,0 +1,132 @@
+import aiohttp
+import requests
+from typing import Any, Dict, List, Optional, Union
+from pydantic import AnyUrl, SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import require_secret_api_key
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.utils import require_param
+
+
+class PortkeyModel(DeepEvalBaseLLM):
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[AnyUrl] = None,
+        provider: Optional[str] = None,
+    ):
+        settings = get_settings()
+        model = model or settings.PORTKEY_MODEL_NAME
+
+        self.model = require_param(
+            model,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_MODEL_NAME",
+            param_hint="model",
+        )
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.PORTKEY_API_KEY
+
+        if base_url is not None:
+            base_url = str(base_url).rstrip("/")
+        elif settings.PORTKEY_BASE_URL is not None:
+            base_url = str(settings.PORTKEY_BASE_URL).rstrip("/")
+
+        self.base_url = require_param(
+            base_url,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_BASE_URL",
+            param_hint="base_url",
+        )
+
+        provider = provider or settings.PORTKEY_PROVIDER_NAME
+        self.provider = require_param(
+            provider,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_PROVIDER_NAME",
+            param_hint="provider",
+        )
+
+    def _headers(self) -> Dict[str, str]:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Portkey",
+            env_var_name="PORTKEY_API_KEY",
+            param_hint="`api_key` to PortkeyModel(...)",
+        )
+
+        headers = {
+            "Content-Type": "application/json",
+            "x-portkey-api-key": api_key,
+        }
+        if self.provider:
+            headers["x-portkey-provider"] = self.provider
+        return headers
+
+    def _payload(self, prompt: str) -> Dict[str, Any]:
+        return {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+
+    def _extract_content(self, data: Dict[str, Any]) -> str:
+        choices: Union[List[Dict[str, Any]], None] = data.get("choices")
+        if not choices:
+            raise ValueError("Portkey response did not include any choices.")
+        message = choices[0].get("message", {})
+        content: Union[str, List[Dict[str, Any]], None] = message.get("content")
+        if isinstance(content, str):
+            return content
+        if isinstance(content, list):
+            return "".join(part.get("text", "") for part in content)
+        return ""
+
+    def generate(self, prompt: str) -> str:
+        try:
+            response = requests.post(
+                f"{self.base_url}/chat/completions",
+                json=self._payload(prompt),
+                headers=self._headers(),
+                timeout=60,
+            )
+            response.raise_for_status()
+        except requests.HTTPError as error:
+            body: Union[str, Dict[str, Any]]
+            try:
+                body = response.json()
+            except Exception:
+                body = response.text
+            raise ValueError(
+                f"Portkey request failed with status {response.status_code}: {body}"
+            ) from error
+        except requests.RequestException as error:
+            raise ValueError(f"Portkey request failed: {error}") from error
+        return self._extract_content(response.json())
+
+    async def a_generate(self, prompt: str) -> str:
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                f"{self.base_url}/chat/completions",
+                json=self._payload(prompt),
+                headers=self._headers(),
+                timeout=60,
+            ) as response:
+                if response.status >= 400:
+                    body = await response.text()
+                    raise ValueError(
+                        f"Portkey request failed with status {response.status}: {body}"
+                    )
+                data = await response.json()
+                return self._extract_content(data)
+
+    def get_model_name(self) -> str:
+        return f"Portkey ({self.model})"
+
+    def load_model(self):
+        return None

deepeval/models/mlllms/azure_model.py

@@ -1,12 +1,12 @@
+import base64
 from openai.types.chat.chat_completion import ChatCompletion
 from openai import AzureOpenAI, AsyncAzureOpenAI
 from typing import Optional, Tuple, Union, Dict, List
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 from io import BytesIO
-import base64
 
+from deepeval.config.settings import get_settings
 from deepeval.models import DeepEvalBaseMLLM
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.test_case import MLLMImage
 from deepeval.models.llms.openai_model import (
     structured_outputs_models,
@@ -19,7 +19,7 @@ from deepeval.models.retry_policy import (
 )
 
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import parse_model_name
+from deepeval.models.utils import parse_model_name, require_secret_api_key
 from deepeval.constants import ProviderSlug as PS
 
 
@@ -38,23 +38,25 @@ class MultimodalAzureOpenAIMLLMModel(DeepEvalBaseMLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+        settings = get_settings()
         # fetch Azure deployment parameters
-        model_name = model_name or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.AZURE_MODEL_NAME
-        )
-        self.deployment_name = deployment_name or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.AZURE_DEPLOYMENT_NAME
-        )
-        self.azure_openai_api_key = (
-            azure_openai_api_key
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.AZURE_OPENAI_API_KEY)
-        )
+        model_name = model_name or settings.AZURE_MODEL_NAME
+        self.deployment_name = deployment_name or settings.AZURE_DEPLOYMENT_NAME
+        if azure_openai_api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.azure_openai_api_key: SecretStr | None = SecretStr(
+                azure_openai_api_key
+            )
+        else:
+            self.azure_openai_api_key = settings.AZURE_OPENAI_API_KEY
+
         self.openai_api_version = (
-            openai_api_version
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.OPENAI_API_VERSION)
+            openai_api_version or settings.OPENAI_API_VERSION
         )
-        self.azure_endpoint = azure_endpoint or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.AZURE_OPENAI_ENDPOINT
+        self.azure_endpoint = (
+            azure_endpoint
+            or settings.AZURE_OPENAI_ENDPOINT
+            and str(settings.AZURE_OPENAI_ENDPOINT)
         )
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
@@ -317,8 +319,15 @@ class MultimodalAzureOpenAIMLLMModel(DeepEvalBaseMLLM):
         return kwargs
 
     def _build_client(self, cls):
+        api_key = require_secret_api_key(
+            self.azure_openai_api_key,
+            provider_label="AzureOpenAI",
+            env_var_name="AZURE_OPENAI_API_KEY",
+            param_hint="`azure_openai_api_key` to MultimodalAzureOpenAIMLLMModel(...)",
+        )
+
         kw = dict(
-            api_key=self.azure_openai_api_key,
+            api_key=api_key,
             api_version=self.openai_api_version,
             azure_endpoint=self.azure_endpoint,
             azure_deployment=self.deployment_name,

deepeval/models/mlllms/gemini_model.py

@@ -1,16 +1,16 @@
-from typing import Optional, List, Union
 import requests
-from pydantic import BaseModel
+from typing import Optional, List, Union
+from pydantic import BaseModel, SecretStr
 from google.genai import types
 from google import genai
 
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
-from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
 
 
@@ -60,77 +60,31 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         *args,
         **kwargs,
     ):
+        settings = get_settings()
         model_name = (
             model_name
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GEMINI_MODEL_NAME)
+            or settings.GEMINI_MODEL_NAME
             or default_multimodal_gemini_model
         )
 
-        # Get API key from key handler if not provided
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_API_KEY
-        )
-        self.project = project or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_PROJECT
-        )
-        self.location = location or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_LOCATION
-        )
-        self.use_vertexai = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_GENAI_USE_VERTEXAI
-        )
-
-        super().__init__(model_name, *args, **kwargs)
-        self.model = self.load_model(*args, **kwargs)
-
-    def should_use_vertexai(self):
-        """Checks if the model should use Vertex AI for generation.
-
-        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
-        environment variable. If not set, it checks for the presence of the
-        project and location.
-
-        Returns:
-            True if the model should use Vertex AI, False otherwise
-        """
-        if self.use_vertexai is not None:
-            return self.use_vertexai.lower() == "yes"
-
-        if self.project and self.location:
-            return True
+        # Get API key from settings if not provided
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self.api_key: SecretStr | None = SecretStr(api_key)
         else:
-            return False
-
-    def load_model(self, *args, **kwargs):
-        """Creates a client.
-        With Gen AI SDK, model is set at inference time, so there is no
-        model to load and initialize.
-        This method name is kept for compatibility with other LLMs.
-
-        Returns:
-            A GenerativeModel instance configured for evaluation.
-        """
-        if self.should_use_vertexai():
-            if not self.project or not self.location:
-                raise ValueError(
-                    "When using Vertex AI API, both project and location are required."
-                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
-                    "or set them in your DeepEval configuration."
-                )
+            self.api_key = settings.GOOGLE_API_KEY
 
-            # Create client for Vertex AI
-            self.client = genai.Client(
-                vertexai=True, project=self.project, location=self.location
-            )
-        else:
-            if not self.api_key:
-                raise ValueError(
-                    "Google API key is required. Either provide it directly, set GOOGLE_API_KEY environment variable, "
-                    "or set it in your DeepEval configuration."
-                )
+        self.project = project or settings.GOOGLE_CLOUD_PROJECT
+        self.location = (
+            location
+            or settings.GOOGLE_CLOUD_LOCATION is not None
+            and str(settings.GOOGLE_CLOUD_LOCATION)
+        )
+        self.use_vertexai = settings.GOOGLE_GENAI_USE_VERTEXAI
 
-            # Create client for Gemini API
-            self.client = genai.Client(api_key=self.api_key)
+        # Keep any extra kwargs for the underlying genai.Client
+        self.args = args
+        self.kwargs = kwargs
 
         # Configure default model generation settings
         self.model_safety_settings = [
@@ -152,9 +106,28 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             ),
         ]
         self.model_temperature = 0.0
-        return self.client.models
 
-    # TODO: Refactor genete prompt to minimize the work done on retry
+        super().__init__(model_name, *args, **kwargs)
+
+    def should_use_vertexai(self):
+        """Checks if the model should use Vertex AI for generation.
+
+        This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
+        environment variable. If not set, it checks for the presence of the
+        project and location.
+
+        Returns:
+            True if the model should use Vertex AI, False otherwise
+        """
+        if self.use_vertexai is not None:
+            return self.use_vertexai.lower() == "yes"
+
+        if self.project and self.location:
+            return True
+        else:
+            return False
+
+    # TODO: Refactor generate prompt to minimize the work done on retry
     @retry_gemini
     def generate_prompt(
         self, multimodal_input: List[Union[str, MLLMImage]] = []
@@ -214,10 +187,11 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         Returns:
             Generated text response
         """
+        client = self.load_model()
         prompt = self.generate_prompt(multimodal_input)
 
         if schema is not None:
-            response = self.client.models.generate_content(
+            response = client.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -229,7 +203,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.parsed, 0
         else:
-            response = self.client.models.generate_content(
+            response = client.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -254,10 +228,11 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         Returns:
             Generated text response
         """
+        client = self.load_model()
         prompt = self.generate_prompt(multimodal_input)
 
         if schema is not None:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -269,7 +244,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.parsed, 0
         else:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=prompt,
                 config=types.GenerateContentConfig(
@@ -279,6 +254,60 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.text, 0
 
+    #########
+    # Model #
+    #########
+
     def get_model_name(self) -> str:
         """Returns the name of the Gemini model being used."""
         return self.model_name
+
+    def load_model(self, *args, **kwargs):
+        """Creates and returns a GenAI client.
+
+        With the Gen AI SDK, the model is set at inference time, so we only
+        construct the client here. Kept for compatibility with other MLLMs.
+        """
+        return self._build_client(**kwargs)
+
+    def _client_kwargs(self, **override_kwargs) -> dict:
+        """
+        Return kwargs forwarded to genai.Client.
+
+        Start from the ctor kwargs captured on `self.kwargs`, then apply any
+        overrides passed via load_model(...).
+        """
+        client_kwargs = dict(self.kwargs or {})
+        if override_kwargs:
+            client_kwargs.update(override_kwargs)
+        return client_kwargs
+
+    def _build_client(self, **override_kwargs):
+        """Build and return a genai.Client for either Gemini API or Vertex AI."""
+        client_kwargs = self._client_kwargs(**override_kwargs)
+
+        if self.should_use_vertexai():
+            if not self.project or not self.location:
+                raise ValueError(
+                    "When using Vertex AI API, both project and location are required."
+                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
+                    "or set them in your DeepEval configuration."
+                )
+
+            # Create client for Vertex AI
+            return genai.Client(
+                vertexai=True,
+                project=self.project,
+                location=self.location,
+                **client_kwargs,
+            )
+
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Google Gemini",
+            env_var_name="GOOGLE_API_KEY",
+            param_hint="`api_key` to MultimodalGeminiModel(...)",
+        )
+
+        # Create client for Gemini API
+        return genai.Client(api_key=api_key, **client_kwargs)

deepeval/models/mlllms/ollama_model.py

@@ -8,7 +8,6 @@ import io
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-from deepeval.key_handler import KEY_FILE_HANDLER, ModelKeyValues
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
 from deepeval.config.settings import get_settings
@@ -19,14 +18,34 @@ retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class MultimodalOllamaModel(DeepEvalBaseMLLM):
-    def __init__(self, **kwargs):
-        model_name = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_NAME
-        )
-        self.base_url = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.LOCAL_MODEL_BASE_URL
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        host: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        Multimodal Ollama model.
+
+        - `model`: Ollama model name (e.g. "llava").
+        - `host`: Ollama base URL (e.g. "http://localhost:11434").
+        - extra **kwargs are passed through to the underlying Client.
+        """
+        settings = get_settings()
+
+        # Resolve host/base URL
+        self.base_url = (
+            host
+            or settings.LOCAL_MODEL_BASE_URL
+            and str(settings.LOCAL_MODEL_BASE_URL)
         )
-        self.kwargs = kwargs
+
+        # Resolve model name
+        model_name = model or settings.LOCAL_MODEL_NAME
+
+        # Client kwargs
+        self.kwargs = kwargs or {}
+
         super().__init__(model_name)
 
     @retry_ollama
@@ -132,13 +151,25 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
             print(f"Error converting image to base64: {e}")
             return None
 
+    ###############################################
+    # Model
+    ###############################################
+
     def load_model(self, async_mode: bool = False):
         if not async_mode:
             return self._build_client(Client)
         return self._build_client(AsyncClient)
 
+    def _client_kwargs(self) -> Dict:
+        """
+        Return client-init kwargs.
+        Ollama's Python client doesn't have built-in retry config like OpenAI,
+        so we just pass these through untouched.
+        """
+        return dict(self.kwargs or {})
+
     def _build_client(self, cls):
-        return cls(host=self.base_url, **self.kwargs)
+        return cls(host=self.base_url, **self._client_kwargs())
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"