deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/llms/gemini_model.py

@@ -1,15 +1,32 @@
-from pydantic import BaseModel
-from google.genai import types, Client
-from typing import Optional, Dict
+import json
+import requests
+from pydantic import BaseModel, SecretStr
+from typing import TYPE_CHECKING, Optional, Dict, List, Union
 
+from deepeval.test_case import MLLMImage
+from deepeval.config.settings import get_settings
+from deepeval.models.utils import require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
+from deepeval.utils import (
+    convert_to_multi_modal_array,
+    check_if_multimodal,
+    require_dependency,
+)
 from deepeval.models.base_model import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-from google.oauth2 import service_account
-import json
+
+valid_multimodal_models = [
+    "gemini-2.5-pro",
+    "gemini-2.5-flash",
+    "gemini-1.5-pro",
+    "gemini-1.5-flash",
+    # TODO: Add more models later
+]
+
+if TYPE_CHECKING:
+    from google.genai import Client
 
 default_gemini_model = "gemini-1.5-pro"
 
@@ -26,7 +43,7 @@ class GeminiModel(DeepEvalBaseLLM):
     To use Vertex AI API, set project and location attributes.
 
     Attributes:
-        model_name: Name of the Gemini model to use
+        model: Name of the Gemini model to use
         api_key: Google API key for authentication
         project: Google Cloud project ID
         location: Google Cloud location
@@ -37,7 +54,7 @@ class GeminiModel(DeepEvalBaseLLM):
 
         # Initialize the model
         model = GeminiModel(
-            model_name="gemini-1.5-pro-001",
+            model="gemini-1.5-pro-001",
             api_key="your-api-key"
         )
 
@@ -48,40 +65,39 @@ class GeminiModel(DeepEvalBaseLLM):
 
     def __init__(
         self,
-        model_name: Optional[str] = None,
+        model: Optional[str] = None,
         api_key: Optional[str] = None,
+        temperature: float = 0,
         project: Optional[str] = None,
         location: Optional[str] = None,
         service_account_key: Optional[Dict[str, str]] = None,
-        temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = (
-            model_name
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GEMINI_MODEL_NAME)
-            or default_gemini_model
-        )
 
-        # Get API key from key handler if not provided
-        self.api_key = api_key or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_API_KEY
-        )
-        self.project = project or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_PROJECT
-        )
-        self.location = location or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_CLOUD_LOCATION
-        )
-        self.use_vertexai = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GOOGLE_GENAI_USE_VERTEXAI
+        settings = get_settings()
+
+        model = model or settings.GEMINI_MODEL_NAME or default_gemini_model
+
+        # Get API key from settings if not provided
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and aolike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.GOOGLE_API_KEY
+
+        self.project = project or settings.GOOGLE_CLOUD_PROJECT
+        self.location = (
+            location
+            or settings.GOOGLE_CLOUD_LOCATION is not None
+            and str(settings.GOOGLE_CLOUD_LOCATION)
         )
+        self.use_vertexai = settings.GOOGLE_GENAI_USE_VERTEXAI
+
         if service_account_key:
             self.service_account_key = service_account_key
         else:
-            service_account_key_data = KEY_FILE_HANDLER.fetch_data(
-                ModelKeyValues.GOOGLE_SERVICE_ACCOUNT_KEY
-            )
+            service_account_key_data = settings.GOOGLE_SERVICE_ACCOUNT_KEY
             if service_account_key_data is None:
                 self.service_account_key = None
             elif isinstance(service_account_key_data, str):
@@ -90,11 +106,35 @@ class GeminiModel(DeepEvalBaseLLM):
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
+
+        # Raw kwargs destined for the underlying Client
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name, **kwargs)
 
-    def should_use_vertexai(self):
+        self._module = self._require_module()
+        # Configure default model generation settings
+        self.model_safety_settings = [
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
+            ),
+        ]
+
+        super().__init__(model)
+
+    def should_use_vertexai(self) -> bool:
         """Checks if the model should use Vertex AI for generation.
 
         This is determined first by the value of `GOOGLE_GENAI_USE_VERTEXAI`
@@ -111,69 +151,53 @@ class GeminiModel(DeepEvalBaseLLM):
         else:
             return False
 
-    def load_model(self, *args, **kwargs):
-        """Creates a client.
-        With Gen AI SDK, model is set at inference time, so there is no
-        model to load and initialize.
-        This method name is kept for compatibility with other LLMs.
+    @retry_gemini
+    def generate_prompt(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ) -> List[Union[str, MLLMImage]]:
+        """Converts DeepEval multimodal input into GenAI SDK compatible format.
+
+        Args:
+            multimodal_input: List of strings and MLLMImage objects
 
         Returns:
-            A GenerativeModel instance configured for evaluation.
+            List of strings and PIL Image objects ready for model input
+
+        Raises:
+            ValueError: If an invalid input type is provided
         """
-        if self.should_use_vertexai():
-            if not self.project or not self.location:
-                raise ValueError(
-                    "When using Vertex AI API, both project and location are required."
-                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION environment variables, "
-                    "or set them in your DeepEval configuration."
-                )
+        prompt = []
+        settings = get_settings()
 
-            # Create client for Vertex AI
-            self.client = Client(
-                vertexai=True,
-                project=self.project,
-                location=self.location,
-                credentials=(
-                    service_account.Credentials.from_service_account_info(
-                        self.service_account_key,
-                        scopes=[
-                            "https://www.googleapis.com/auth/cloud-platform"
-                        ],
+        for ele in multimodal_input:
+            if isinstance(ele, str):
+                prompt.append(ele)
+            elif isinstance(ele, MLLMImage):
+                if ele.local:
+                    with open(ele.url, "rb") as f:
+                        image_data = f.read()
+                else:
+                    response = requests.get(
+                        ele.url,
+                        timeout=(
+                            settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                            settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                        ),
                     )
-                    if self.service_account_key
-                    else None
-                ),
-                **self.kwargs,
-            )
-        else:
-            if not self.api_key:
-                raise ValueError(
-                    "Google API key is required. Either provide it directly, set GOOGLE_API_KEY environment variable, "
-                    "or set it in your DeepEval configuration."
+                    response.raise_for_status()
+                    image_data = response.content
+
+                image_part = self._module.types.Part.from_bytes(
+                    data=image_data, mime_type="image/jpeg"
                 )
-            # Create client for Gemini API
-            self.client = Client(api_key=self.api_key, **self.kwargs)
+                prompt.append(image_part)
+            else:
+                raise ValueError(f"Invalid input type: {type(ele)}")
+        return prompt
 
-        # Configure default model generation settings
-        self.model_safety_settings = [
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
-            ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
-            ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
-            ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
-            ),
-        ]
-        return self.client.models
+    ###############################################
+    # Generate functions
+    ###############################################
 
     @retry_gemini
     def generate(self, prompt: str, schema: Optional[BaseModel] = None) -> str:
@@ -186,11 +210,18 @@ class GeminiModel(DeepEvalBaseLLM):
         Returns:
             Generated text response or structured output as Pydantic model
         """
+        client = self.load_model()
+
+        if check_if_multimodal(prompt):
+
+            prompt = convert_to_multi_modal_array(prompt)
+            prompt = self.generate_prompt(prompt)
+
         if schema is not None:
-            response = self.client.models.generate_content(
-                model=self.model_name,
+            response = client.models.generate_content(
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     response_mime_type="application/json",
                     response_schema=schema,
                     safety_settings=self.model_safety_settings,
@@ -200,10 +231,10 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.parsed, 0
         else:
-            response = self.client.models.generate_content(
-                model=self.model_name,
+            response = client.models.generate_content(
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     safety_settings=self.model_safety_settings,
                     temperature=self.temperature,
                     **self.generation_kwargs,
@@ -224,11 +255,17 @@ class GeminiModel(DeepEvalBaseLLM):
         Returns:
             Generated text response or structured output as Pydantic model
         """
+        client = self.load_model()
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(prompt)
+            prompt = self.generate_prompt(prompt)
+
         if schema is not None:
-            response = await self.client.aio.models.generate_content(
-                model=self.model_name,
+            response = await client.aio.models.generate_content(
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     response_mime_type="application/json",
                     response_schema=schema,
                     safety_settings=self.model_safety_settings,
@@ -238,10 +275,10 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.parsed, 0
         else:
-            response = await self.client.aio.models.generate_content(
-                model=self.model_name,
+            response = await client.aio.models.generate_content(
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     safety_settings=self.model_safety_settings,
                     temperature=self.temperature,
                     **self.generation_kwargs,
@@ -249,6 +286,88 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.text, 0
 
-    def get_model_name(self) -> str:
-        """Returns the name of the Gemini model being used."""
-        return self.model_name
+    #########
+    # Model #
+    #########
+
+    def load_model(self):
+        """Creates a client.
+        With Gen AI SDK, model is set at inference time, so there is no
+        model to load and initialize.
+        This method name is kept for compatibility with other LLMs.
+
+        Returns:
+            A GenerativeModel instance configured for evaluation.
+        """
+        return self._build_client()
+
+    def _require_oauth2(self):
+        return require_dependency(
+            "google.oauth2",
+            provider_label="GeminiModel",
+            install_hint="Install it with `pip install google-auth`.",
+        )
+
+    def _require_module(self):
+        return require_dependency(
+            "google.genai",
+            provider_label="GeminiModel",
+            install_hint="Install it with `pip install google-genai`.",
+        )
+
+    def _client_kwargs(self, **override_kwargs) -> Dict:
+        """Merge ctor kwargs with any overrides passed at load_model time."""
+        client_kwargs = dict(self.kwargs or {})
+        if override_kwargs:
+            client_kwargs.update(override_kwargs)
+        return client_kwargs
+
+    def _build_client(self) -> "Client":
+        client_kwargs = self._client_kwargs(**self.kwargs)
+
+        if self.should_use_vertexai():
+            if not self.project or not self.location:
+                raise ValueError(
+                    "When using Vertex AI API, both project and location are required. "
+                    "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and "
+                    "GOOGLE_CLOUD_LOCATION in your DeepEval configuration."
+                )
+
+            oauth2 = self._require_oauth2()
+            credentials = (
+                oauth2.service_account.Credentials.from_service_account_info(
+                    self.service_account_key,
+                    scopes=[
+                        "https://www.googleapis.com/auth/cloud-platform",
+                    ],
+                )
+                if self.service_account_key
+                else None
+            )
+
+            client = self._module.Client(
+                vertexai=True,
+                project=self.project,
+                location=self.location,
+                credentials=credentials,
+                **client_kwargs,
+            )
+        else:
+            api_key = require_secret_api_key(
+                self.api_key,
+                provider_label="Google Gemini",
+                env_var_name="GOOGLE_API_KEY",
+                param_hint="`api_key` to GeminiModel(...)",
+            )
+
+            client = self._module.Client(api_key=api_key, **client_kwargs)
+
+        return client
+
+    def supports_multimodal(self):
+        if self.name in valid_multimodal_models:
+            return True
+        return False
+
+    def get_model_name(self):
+        return f"{self.name} (Gemini)"

deepeval/models/llms/grok_model.py

@@ -1,18 +1,18 @@
-import os
-
 from typing import Optional, Tuple, Union, Dict
-from pydantic import BaseModel
+from pydantic import BaseModel, SecretStr
 
+from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
-from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
-
 # consistent retry rules
 retry_grok = create_retry_decorator(PS.GROK)
 
@@ -62,30 +62,33 @@ class GrokModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
-        model_name = model or KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.GROK_MODEL_NAME
-        )
-        if model_name not in model_pricing:
+
+        settings = get_settings()
+
+        model = model or settings.GROK_MODEL_NAME
+
+        if model not in model_pricing:
             raise ValueError(
                 f"Invalid model. Available Grok models: {', '.join(model_pricing.keys())}"
             )
-        temperature_from_key = KEY_FILE_HANDLER.fetch_data(
-            ModelKeyValues.TEMPERATURE
-        )
+        temperature_from_key = settings.TEMPERATURE
         if temperature_from_key is None:
             self.temperature = temperature
         else:
             self.temperature = float(temperature_from_key)
         if self.temperature < 0:
             raise ValueError("Temperature must be >= 0.")
-        self.api_key = (
-            api_key
-            or KEY_FILE_HANDLER.fetch_data(ModelKeyValues.GROK_API_KEY)
-            or os.getenv("GROK_API_KEY")
-        )
+
+        if api_key is not None:
+            # keep it secret, keep it safe from serializings, logging and alike
+            self.api_key: SecretStr | None = SecretStr(api_key)
+        else:
+            self.api_key = settings.GROK_API_KEY
+
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -95,6 +98,7 @@ class GrokModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         try:
             from xai_sdk.chat import user
         except ImportError:
@@ -103,13 +107,13 @@ class GrokModel(DeepEvalBaseLLM):
             )
         client = self.load_model(async_mode=False)
         chat = client.chat.create(
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
         chat.append(user(prompt))
 
-        if schema and self.model_name in structured_outputs_models:
+        if schema and self.name in structured_outputs_models:
             response, structured_output = chat.parse(schema)
             cost = self.calculate_cost(
                 response.usage.prompt_tokens,
@@ -133,6 +137,7 @@ class GrokModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         try:
             from xai_sdk.chat import user
         except ImportError:
@@ -141,13 +146,13 @@ class GrokModel(DeepEvalBaseLLM):
             )
         client = self.load_model(async_mode=True)
         chat = client.chat.create(
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
         chat.append(user(prompt))
 
-        if schema and self.model_name in structured_outputs_models:
+        if schema and self.name in structured_outputs_models:
             response, structured_output = await chat.parse(schema)
             cost = self.calculate_cost(
                 response.usage.prompt_tokens,
@@ -176,7 +181,7 @@ class GrokModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
+        pricing = model_pricing.get(self.name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
         return input_cost + output_cost
@@ -198,9 +203,6 @@ class GrokModel(DeepEvalBaseLLM):
                 "xai_sdk is required to use GrokModel. Please install it with: pip install xai-sdk"
             )
 
-    def get_model_name(self):
-        return f"{self.model_name}"
-
     def _client_kwargs(self) -> Dict:
         """
         If Tenacity is managing retries, disable gRPC channel retries to avoid double retry.
@@ -226,7 +228,14 @@ class GrokModel(DeepEvalBaseLLM):
         return kwargs
 
     def _build_client(self, cls):
-        kw = dict(api_key=self.api_key, **self._client_kwargs())
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Grok",
+            env_var_name="GROK_API_KEY",
+            param_hint="`api_key` to GrokModel(...)",
+        )
+
+        kw = dict(api_key=api_key, **self._client_kwargs())
         try:
             return cls(**kw)
         except TypeError as e:
@@ -235,3 +244,6 @@ class GrokModel(DeepEvalBaseLLM):
                 kw.pop("channel_options", None)
                 return cls(**kw)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Grok)"