deepeval 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/models/llms/gemini_model.py

@@ -1,17 +1,32 @@
 import json
-
+import requests
 from pydantic import BaseModel, SecretStr
-from google.genai import types, Client
-from typing import Optional, Dict
+from typing import TYPE_CHECKING, Optional, Dict, List, Union
 
+from deepeval.test_case import MLLMImage
 from deepeval.config.settings import get_settings
 from deepeval.models.utils import require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
+from deepeval.utils import (
+    convert_to_multi_modal_array,
+    check_if_multimodal,
+    require_dependency,
+)
 from deepeval.models.base_model import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-from google.oauth2 import service_account
+
+valid_multimodal_models = [
+    "gemini-2.5-pro",
+    "gemini-2.5-flash",
+    "gemini-1.5-pro",
+    "gemini-1.5-flash",
+    # TODO: Add more models later
+]
+
+if TYPE_CHECKING:
+    from google.genai import Client
 
 default_gemini_model = "gemini-1.5-pro"
 
@@ -28,7 +43,7 @@ class GeminiModel(DeepEvalBaseLLM):
     To use Vertex AI API, set project and location attributes.
 
     Attributes:
-        model_name: Name of the Gemini model to use
+        model: Name of the Gemini model to use
         api_key: Google API key for authentication
         project: Google Cloud project ID
         location: Google Cloud location
@@ -39,7 +54,7 @@ class GeminiModel(DeepEvalBaseLLM):
 
         # Initialize the model
         model = GeminiModel(
-            model_name="gemini-1.5-pro-001",
+            model="gemini-1.5-pro-001",
             api_key="your-api-key"
         )
 
@@ -50,21 +65,19 @@ class GeminiModel(DeepEvalBaseLLM):
 
     def __init__(
         self,
-        model_name: Optional[str] = None,
+        model: Optional[str] = None,
         api_key: Optional[str] = None,
+        temperature: float = 0,
         project: Optional[str] = None,
         location: Optional[str] = None,
         service_account_key: Optional[Dict[str, str]] = None,
-        temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
 
         settings = get_settings()
 
-        model_name = (
-            model_name or settings.GEMINI_MODEL_NAME or default_gemini_model
-        )
+        model = model or settings.GEMINI_MODEL_NAME or default_gemini_model
 
         # Get API key from settings if not provided
         if api_key is not None:
@@ -98,27 +111,28 @@ class GeminiModel(DeepEvalBaseLLM):
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
 
+        self._module = self._require_module()
         # Configure default model generation settings
         self.model_safety_settings = [
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
         ]
 
-        super().__init__(model_name, **kwargs)
+        super().__init__(model)
 
     def should_use_vertexai(self) -> bool:
         """Checks if the model should use Vertex AI for generation.
@@ -137,6 +151,50 @@ class GeminiModel(DeepEvalBaseLLM):
         else:
             return False
 
+    @retry_gemini
+    def generate_prompt(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ) -> List[Union[str, MLLMImage]]:
+        """Converts DeepEval multimodal input into GenAI SDK compatible format.
+
+        Args:
+            multimodal_input: List of strings and MLLMImage objects
+
+        Returns:
+            List of strings and PIL Image objects ready for model input
+
+        Raises:
+            ValueError: If an invalid input type is provided
+        """
+        prompt = []
+        settings = get_settings()
+
+        for ele in multimodal_input:
+            if isinstance(ele, str):
+                prompt.append(ele)
+            elif isinstance(ele, MLLMImage):
+                if ele.local:
+                    with open(ele.url, "rb") as f:
+                        image_data = f.read()
+                else:
+                    response = requests.get(
+                        ele.url,
+                        timeout=(
+                            settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                            settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                        ),
+                    )
+                    response.raise_for_status()
+                    image_data = response.content
+
+                image_part = self._module.types.Part.from_bytes(
+                    data=image_data, mime_type="image/jpeg"
+                )
+                prompt.append(image_part)
+            else:
+                raise ValueError(f"Invalid input type: {type(ele)}")
+        return prompt
+
     ###############################################
     # Generate functions
     ###############################################
@@ -154,11 +212,16 @@ class GeminiModel(DeepEvalBaseLLM):
         """
         client = self.load_model()
 
+        if check_if_multimodal(prompt):
+
+            prompt = convert_to_multi_modal_array(prompt)
+            prompt = self.generate_prompt(prompt)
+
         if schema is not None:
             response = client.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     response_mime_type="application/json",
                     response_schema=schema,
                     safety_settings=self.model_safety_settings,
@@ -169,9 +232,9 @@ class GeminiModel(DeepEvalBaseLLM):
             return response.parsed, 0
         else:
             response = client.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     safety_settings=self.model_safety_settings,
                     temperature=self.temperature,
                     **self.generation_kwargs,
@@ -194,11 +257,15 @@ class GeminiModel(DeepEvalBaseLLM):
         """
         client = self.load_model()
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(prompt)
+            prompt = self.generate_prompt(prompt)
+
         if schema is not None:
             response = await client.aio.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     response_mime_type="application/json",
                     response_schema=schema,
                     safety_settings=self.model_safety_settings,
@@ -209,9 +276,9 @@ class GeminiModel(DeepEvalBaseLLM):
             return response.parsed, 0
         else:
             response = await client.aio.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     safety_settings=self.model_safety_settings,
                     temperature=self.temperature,
                     **self.generation_kwargs,
@@ -223,11 +290,7 @@ class GeminiModel(DeepEvalBaseLLM):
     # Model #
     #########
 
-    def get_model_name(self) -> str:
-        """Returns the name of the Gemini model being used."""
-        return self.model_name
-
-    def load_model(self, *args, **kwargs):
+    def load_model(self):
         """Creates a client.
         With Gen AI SDK, model is set at inference time, so there is no
         model to load and initialize.
@@ -236,7 +299,21 @@ class GeminiModel(DeepEvalBaseLLM):
         Returns:
             A GenerativeModel instance configured for evaluation.
         """
-        return self._build_client(**kwargs)
+        return self._build_client()
+
+    def _require_oauth2(self):
+        return require_dependency(
+            "google.oauth2",
+            provider_label="GeminiModel",
+            install_hint="Install it with `pip install google-auth`.",
+        )
+
+    def _require_module(self):
+        return require_dependency(
+            "google.genai",
+            provider_label="GeminiModel",
+            install_hint="Install it with `pip install google-genai`.",
+        )
 
     def _client_kwargs(self, **override_kwargs) -> Dict:
         """Merge ctor kwargs with any overrides passed at load_model time."""
@@ -245,8 +322,8 @@ class GeminiModel(DeepEvalBaseLLM):
             client_kwargs.update(override_kwargs)
         return client_kwargs
 
-    def _build_client(self, **override_kwargs) -> Client:
-        client_kwargs = self._client_kwargs(**override_kwargs)
+    def _build_client(self) -> "Client":
+        client_kwargs = self._client_kwargs(**self.kwargs)
 
         if self.should_use_vertexai():
             if not self.project or not self.location:
@@ -256,8 +333,9 @@ class GeminiModel(DeepEvalBaseLLM):
                     "GOOGLE_CLOUD_LOCATION in your DeepEval configuration."
                 )
 
+            oauth2 = self._require_oauth2()
             credentials = (
-                service_account.Credentials.from_service_account_info(
+                oauth2.service_account.Credentials.from_service_account_info(
                     self.service_account_key,
                     scopes=[
                         "https://www.googleapis.com/auth/cloud-platform",
@@ -267,7 +345,7 @@ class GeminiModel(DeepEvalBaseLLM):
                 else None
             )
 
-            client = Client(
+            client = self._module.Client(
                 vertexai=True,
                 project=self.project,
                 location=self.location,
@@ -282,6 +360,14 @@ class GeminiModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to GeminiModel(...)",
             )
 
-            client = Client(api_key=api_key, **client_kwargs)
+            client = self._module.Client(api_key=api_key, **client_kwargs)
 
         return client
+
+    def supports_multimodal(self):
+        if self.name in valid_multimodal_models:
+            return True
+        return False
+
+    def get_model_name(self):
+        return f"{self.name} (Gemini)"

deepeval/models/llms/grok_model.py

@@ -7,11 +7,12 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
-
 # consistent retry rules
 retry_grok = create_retry_decorator(PS.GROK)
 
@@ -61,11 +62,12 @@ class GrokModel(DeepEvalBaseLLM):
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+
         settings = get_settings()
 
-        model_name = model or settings.GROK_MODEL_NAME
+        model = model or settings.GROK_MODEL_NAME
 
-        if model_name not in model_pricing:
+        if model not in model_pricing:
             raise ValueError(
                 f"Invalid model. Available Grok models: {', '.join(model_pricing.keys())}"
             )
@@ -83,9 +85,10 @@ class GrokModel(DeepEvalBaseLLM):
         else:
             self.api_key = settings.GROK_API_KEY
 
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -95,6 +98,7 @@ class GrokModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         try:
             from xai_sdk.chat import user
         except ImportError:
@@ -103,13 +107,13 @@ class GrokModel(DeepEvalBaseLLM):
             )
         client = self.load_model(async_mode=False)
         chat = client.chat.create(
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
         chat.append(user(prompt))
 
-        if schema and self.model_name in structured_outputs_models:
+        if schema and self.name in structured_outputs_models:
             response, structured_output = chat.parse(schema)
             cost = self.calculate_cost(
                 response.usage.prompt_tokens,
@@ -133,6 +137,7 @@ class GrokModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         try:
             from xai_sdk.chat import user
         except ImportError:
@@ -141,13 +146,13 @@ class GrokModel(DeepEvalBaseLLM):
             )
         client = self.load_model(async_mode=True)
         chat = client.chat.create(
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
         chat.append(user(prompt))
 
-        if schema and self.model_name in structured_outputs_models:
+        if schema and self.name in structured_outputs_models:
             response, structured_output = await chat.parse(schema)
             cost = self.calculate_cost(
                 response.usage.prompt_tokens,
@@ -176,7 +181,7 @@ class GrokModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
+        pricing = model_pricing.get(self.name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
         return input_cost + output_cost
@@ -198,9 +203,6 @@ class GrokModel(DeepEvalBaseLLM):
                 "xai_sdk is required to use GrokModel. Please install it with: pip install xai-sdk"
             )
 
-    def get_model_name(self):
-        return f"{self.model_name}"
-
     def _client_kwargs(self) -> Dict:
         """
         If Tenacity is managing retries, disable gRPC channel retries to avoid double retry.
@@ -242,3 +244,6 @@ class GrokModel(DeepEvalBaseLLM):
                 kw.pop("channel_options", None)
                 return cls(**kw)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Grok)"

deepeval/models/llms/kimi_model.py

@@ -8,7 +8,9 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
 
@@ -74,16 +76,16 @@ model_pricing = {
 class KimiModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        api_key: Optional[str] = None,
         model: Optional[str] = None,
+        api_key: Optional[str] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
         settings = get_settings()
 
-        model_name = model or settings.MOONSHOT_MODEL_NAME
-        if model_name not in model_pricing:
+        model = model or settings.MOONSHOT_MODEL_NAME
+        if model not in model_pricing:
             raise ValueError(
                 f"Invalid model. Available Moonshot models: {', '.join(model_pricing.keys())}"
             )
@@ -103,9 +105,10 @@ class KimiModel(DeepEvalBaseLLM):
             self.api_key = settings.MOONSHOT_API_KEY
 
         self.base_url = "https://api.moonshot.cn/v1"
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
         self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -115,10 +118,11 @@ class KimiModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=False)
-        if schema and self.model_name in json_mode_models:
+        if schema and self.name in json_mode_models:
             completion = client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
@@ -134,7 +138,7 @@ class KimiModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
 
         completion = client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             **self.generation_kwargs,
         )
@@ -153,10 +157,11 @@ class KimiModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
+
         client = self.load_model(async_mode=True)
-        if schema and self.model_name in json_mode_models:
+        if schema and self.name in json_mode_models:
             completion = await client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
@@ -172,7 +177,7 @@ class KimiModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
 
         completion = await client.chat.completions.create(
-            model=self.model_name,
+            model=self.name,
             messages=[{"role": "user", "content": prompt}],
             **self.generation_kwargs,
         )
@@ -196,7 +201,7 @@ class KimiModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
+        pricing = model_pricing.get(self.name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
         return input_cost + output_cost
@@ -244,4 +249,4 @@ class KimiModel(DeepEvalBaseLLM):
             raise
 
     def get_model_name(self):
-        return f"{self.model_name}"
+        return f"{self.name} (KIMI)"