deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/models/llms/deepseek_model.py

@@ -2,66 +2,101 @@ from typing import Optional, Tuple, Union, Dict
 from openai import OpenAI, AsyncOpenAI
 from pydantic import BaseModel, SecretStr
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_costs,
+    require_secret_api_key,
+)
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.constants import ProviderSlug as PS
+from deepeval.models.llms.constants import DEEPSEEK_MODELS_DATA
+from deepeval.utils import require_param
 
 
 # consistent retry rules
 retry_deepseek = create_retry_decorator(PS.DEEPSEEK)
 
-model_pricing = {
-    "deepseek-chat": {
-        "input": 0.27 / 1e6,
-        "output": 1.10 / 1e6,
-    },
-    "deepseek-reasoner": {
-        "input": 0.55 / 1e6,
-        "output": 2.19 / 1e6,
-    },
-}
-
 
 class DeepSeekModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        api_key: Optional[str] = None,
         model: Optional[str] = None,
-        temperature: float = 0,
+        api_key: Optional[str] = None,
+        temperature: Optional[float] = None,
+        cost_per_input_token: Optional[float] = None,
+        cost_per_output_token: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
         settings = get_settings()
 
-        model_name = model or settings.DEEPSEEK_MODEL_NAME
-        if model_name not in model_pricing:
-            raise ValueError(
-                f"Invalid model. Available DeepSeek models: {', '.join(model_pricing.keys())}"
-            )
-        temperature_from_key = settings.TEMPERATURE
-        if temperature_from_key is None:
-            self.temperature = temperature
+        model = model or settings.DEEPSEEK_MODEL_NAME
+
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
         else:
-            self.temperature = float(temperature_from_key)
-        if self.temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            temperature = 0.0
+
+        cost_per_input_token = (
+            cost_per_input_token
+            if cost_per_input_token is not None
+            else settings.DEEPSEEK_COST_PER_INPUT_TOKEN
+        )
+        cost_per_output_token = (
+            cost_per_output_token
+            if cost_per_output_token is not None
+            else settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN
+        )
 
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.DEEPSEEK_API_KEY
 
         self.base_url = "https://api.deepseek.com"
+
+        # validation
+        model = require_param(
+            model,
+            provider_label="DeepSeekModel",
+            env_var_name="DEEPSEEK_MODEL_NAME",
+            param_hint="model",
+        )
+
+        if temperature < 0:
+            raise DeepEvalError("Temperature must be >= 0.")
+
+        self.model_data = DEEPSEEK_MODELS_DATA.get(model)
+        self.temperature = temperature
+
+        cost_per_input_token, cost_per_output_token = require_costs(
+            self.model_data,
+            model,
+            "DEEPSEEK_COST_PER_INPUT_TOKEN",
+            "DEEPSEEK_COST_PER_OUTPUT_TOKEN",
+            cost_per_input_token,
+            cost_per_output_token,
+        )
+        self.model_data.input_price = cost_per_input_token
+        self.model_data.output_price = cost_per_output_token
+
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -70,11 +105,12 @@ class DeepSeekModel(DeepEvalBaseLLM):
     @retry_deepseek
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
         client = self.load_model(async_mode=False)
         if schema:
             completion = client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
@@ -90,7 +126,7 @@ class DeepSeekModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
         else:
             completion = client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 **self.generation_kwargs,
             )
@@ -104,11 +140,12 @@ class DeepSeekModel(DeepEvalBaseLLM):
     @retry_deepseek
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
         client = self.load_model(async_mode=True)
         if schema:
             completion = await client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
@@ -124,7 +161,7 @@ class DeepSeekModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
         else:
             completion = await client.chat.completions.create(
-                model=self.model_name,
+                model=self.name,
                 messages=[{"role": "user", "content": prompt}],
                 **self.generation_kwargs,
             )
@@ -144,11 +181,29 @@ class DeepSeekModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
-        input_cost = input_tokens * pricing["input"]
-        output_cost = output_tokens * pricing["output"]
+        input_cost = input_tokens * self.model_data.input_price
+        output_cost = output_tokens * self.model_data.output_price
         return input_cost + output_cost
 
+    ###############################################
+    # Capabilities
+    ###############################################
+
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        return self.model_data.supports_json
+
     ###############################################
     # Model
     ###############################################
@@ -158,9 +213,6 @@ class DeepSeekModel(DeepEvalBaseLLM):
             return self._build_client(OpenAI)
         return self._build_client(AsyncOpenAI)
 
-    def get_model_name(self):
-        return f"{self.model_name}"
-
     def _client_kwargs(self) -> Dict:
         kwargs = dict(self.kwargs or {})
         # if we are managing retries with Tenacity, force SDK retries off to avoid double retries.
@@ -190,3 +242,6 @@ class DeepSeekModel(DeepEvalBaseLLM):
                 kw.pop("max_retries", None)
                 return cls(**kw)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Deepseek)"

deepeval/models/llms/gemini_model.py

@@ -1,17 +1,26 @@
 import json
-
+import base64
 from pydantic import BaseModel, SecretStr
-from google.genai import types, Client
-from typing import Optional, Dict
+from typing import TYPE_CHECKING, Optional, Dict, List, Union, Tuple
 
+from deepeval.errors import DeepEvalError
+from deepeval.test_case import MLLMImage
 from deepeval.config.settings import get_settings
 from deepeval.models.utils import require_secret_api_key
 from deepeval.models.retry_policy import (
     create_retry_decorator,
 )
+from deepeval.utils import (
+    convert_to_multi_modal_array,
+    check_if_multimodal,
+    require_dependency,
+)
 from deepeval.models.base_model import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-from google.oauth2 import service_account
+from deepeval.models.llms.constants import GEMINI_MODELS_DATA
+
+if TYPE_CHECKING:
+    from google.genai import Client
 
 default_gemini_model = "gemini-1.5-pro"
 
@@ -28,7 +37,7 @@ class GeminiModel(DeepEvalBaseLLM):
     To use Vertex AI API, set project and location attributes.
 
     Attributes:
-        model_name: Name of the Gemini model to use
+        model: Name of the Gemini model to use
         api_key: Google API key for authentication
         project: Google Cloud project ID
         location: Google Cloud location
@@ -39,7 +48,7 @@ class GeminiModel(DeepEvalBaseLLM):
 
         # Initialize the model
         model = GeminiModel(
-            model_name="gemini-1.5-pro-001",
+            model="gemini-1.5-pro-001",
             api_key="your-api-key"
         )
 
@@ -50,75 +59,89 @@ class GeminiModel(DeepEvalBaseLLM):
 
     def __init__(
         self,
-        model_name: Optional[str] = None,
+        model: Optional[str] = None,
         api_key: Optional[str] = None,
+        temperature: Optional[float] = None,
         project: Optional[str] = None,
         location: Optional[str] = None,
-        service_account_key: Optional[Dict[str, str]] = None,
-        temperature: float = 0,
+        service_account_key: Optional[Union[str, Dict[str, str]]] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
 
         settings = get_settings()
 
-        model_name = (
-            model_name or settings.GEMINI_MODEL_NAME or default_gemini_model
-        )
+        model = model or settings.GEMINI_MODEL_NAME or default_gemini_model
+        self.model_data = GEMINI_MODELS_DATA.get(model)
 
         # Get API key from settings if not provided
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and aolike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.GOOGLE_API_KEY
 
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
         self.project = project or settings.GOOGLE_CLOUD_PROJECT
-        self.location = (
-            location
-            or settings.GOOGLE_CLOUD_LOCATION is not None
-            and str(settings.GOOGLE_CLOUD_LOCATION)
+        location = (
+            location if location is not None else settings.GOOGLE_CLOUD_LOCATION
         )
+        self.location = str(location).strip() if location is not None else None
         self.use_vertexai = settings.GOOGLE_GENAI_USE_VERTEXAI
 
-        if service_account_key:
-            self.service_account_key = service_account_key
+        self.service_account_key: Optional[SecretStr] = None
+        if service_account_key is None:
+            self.service_account_key = settings.GOOGLE_SERVICE_ACCOUNT_KEY
+        elif isinstance(service_account_key, dict):
+            self.service_account_key = SecretStr(
+                json.dumps(service_account_key)
+            )
         else:
-            service_account_key_data = settings.GOOGLE_SERVICE_ACCOUNT_KEY
-            if service_account_key_data is None:
-                self.service_account_key = None
-            elif isinstance(service_account_key_data, str):
-                self.service_account_key = json.loads(service_account_key_data)
+            str_value = str(service_account_key).strip()
+            self.service_account_key = (
+                SecretStr(str_value) if str_value else None
+            )
 
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
+
         self.temperature = temperature
 
         # Raw kwargs destined for the underlying Client
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
+        self.kwargs.pop("temperature", None)
 
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
+        self._module = self._require_module()
         # Configure default model generation settings
         self.model_safety_settings = [
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
-            types.SafetySetting(
-                category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-                threshold=types.HarmBlockThreshold.BLOCK_NONE,
+            self._module.types.SafetySetting(
+                category=self._module.types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                threshold=self._module.types.HarmBlockThreshold.BLOCK_NONE,
             ),
         ]
 
-        super().__init__(model_name, **kwargs)
+        super().__init__(model)
 
     def should_use_vertexai(self) -> bool:
         """Checks if the model should use Vertex AI for generation.
@@ -131,18 +154,73 @@ class GeminiModel(DeepEvalBaseLLM):
             True if the model should use Vertex AI, False otherwise
         """
         if self.use_vertexai is not None:
-            return self.use_vertexai.lower() == "yes"
+            return self.use_vertexai
         if self.project and self.location:
             return True
         else:
             return False
 
+    @retry_gemini
+    def generate_content(
+        self, multimodal_input: Optional[List[Union[str, MLLMImage]]] = None
+    ):
+        multimodal_input = (
+            multimodal_input if multimodal_input is not None else []
+        )
+        content = []
+
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append(element)
+            elif isinstance(element, MLLMImage):
+                # Gemini doesn't support direct external URLs
+                # Must convert all images to bytes
+                if element.url and not element.local:
+                    import requests
+
+                    settings = get_settings()
+
+                    response = requests.get(
+                        element.url,
+                        timeout=(
+                            settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                            settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                        ),
+                    )
+                    response.raise_for_status()
+                    image_data = response.content
+                    mime_type = response.headers.get(
+                        "content-type", element.mimeType or "image/jpeg"
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    try:
+                        image_data = base64.b64decode(element.dataBase64)
+                    except Exception:
+                        raise ValueError(
+                            f"Invalid base64 data in MLLMImage: {element._id}"
+                        )
+
+                    mime_type = element.mimeType or "image/jpeg"
+
+                # Create Part from bytes
+                image_part = self._module.types.Part.from_bytes(
+                    data=image_data, mime_type=mime_type
+                )
+                content.append(image_part)
+            else:
+                raise DeepEvalError(f"Invalid input type: {type(element)}")
+
+        return content
+
     ###############################################
     # Generate functions
     ###############################################
 
     @retry_gemini
-    def generate(self, prompt: str, schema: Optional[BaseModel] = None) -> str:
+    def generate(
+        self, prompt: str, schema: Optional[BaseModel] = None
+    ) -> Tuple[Union[str, BaseModel], float]:
         """Generates text from a prompt.
 
         Args:
@@ -154,11 +232,15 @@ class GeminiModel(DeepEvalBaseLLM):
         """
         client = self.load_model()
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(prompt)
+            prompt = self.generate_content(prompt)
+
         if schema is not None:
             response = client.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     response_mime_type="application/json",
                     response_schema=schema,
                     safety_settings=self.model_safety_settings,
@@ -169,9 +251,9 @@ class GeminiModel(DeepEvalBaseLLM):
             return response.parsed, 0
         else:
             response = client.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     safety_settings=self.model_safety_settings,
                     temperature=self.temperature,
                     **self.generation_kwargs,
@@ -182,7 +264,7 @@ class GeminiModel(DeepEvalBaseLLM):
     @retry_gemini
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> str:
+    ) -> Tuple[Union[str, BaseModel], float]:
         """Asynchronously generates text from a prompt.
 
         Args:
@@ -194,11 +276,15 @@ class GeminiModel(DeepEvalBaseLLM):
         """
         client = self.load_model()
 
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(prompt)
+            prompt = self.generate_content(prompt)
+
         if schema is not None:
             response = await client.aio.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     response_mime_type="application/json",
                     response_schema=schema,
                     safety_settings=self.model_safety_settings,
@@ -209,9 +295,9 @@ class GeminiModel(DeepEvalBaseLLM):
             return response.parsed, 0
         else:
             response = await client.aio.models.generate_content(
-                model=self.model_name,
+                model=self.name,
                 contents=prompt,
-                config=types.GenerateContentConfig(
+                config=self._module.types.GenerateContentConfig(
                     safety_settings=self.model_safety_settings,
                     temperature=self.temperature,
                     **self.generation_kwargs,
@@ -219,15 +305,37 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.text, 0
 
+    #########################
+    # Capabilities          #
+    #########################
+
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        """
+        OpenAI models that natively enforce typed structured outputs.
+         Used by generate(...) when a schema is provided.
+        """
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        """
+        OpenAI models that enforce JSON mode
+        """
+        return self.model_data.supports_json
+
     #########
     # Model #
     #########
 
-    def get_model_name(self) -> str:
-        """Returns the name of the Gemini model being used."""
-        return self.model_name
-
-    def load_model(self, *args, **kwargs):
+    def load_model(self):
         """Creates a client.
         With Gen AI SDK, model is set at inference time, so there is no
         model to load and initialize.
@@ -236,7 +344,21 @@ class GeminiModel(DeepEvalBaseLLM):
         Returns:
             A GenerativeModel instance configured for evaluation.
         """
-        return self._build_client(**kwargs)
+        return self._build_client()
+
+    def _require_oauth2(self):
+        return require_dependency(
+            "google.oauth2",
+            provider_label="GeminiModel",
+            install_hint="Install it with `pip install google-auth`.",
+        )
+
+    def _require_module(self):
+        return require_dependency(
+            "google.genai",
+            provider_label="GeminiModel",
+            install_hint="Install it with `pip install google-genai`.",
+        )
 
     def _client_kwargs(self, **override_kwargs) -> Dict:
         """Merge ctor kwargs with any overrides passed at load_model time."""
@@ -245,29 +367,49 @@ class GeminiModel(DeepEvalBaseLLM):
             client_kwargs.update(override_kwargs)
         return client_kwargs
 
-    def _build_client(self, **override_kwargs) -> Client:
-        client_kwargs = self._client_kwargs(**override_kwargs)
+    def _build_client(self) -> "Client":
+        client_kwargs = self._client_kwargs(**self.kwargs)
 
         if self.should_use_vertexai():
+            service_account_key_json = require_secret_api_key(
+                self.service_account_key,
+                provider_label="Google Gemini",
+                env_var_name="GOOGLE_SERVICE_ACCOUNT_KEY",
+                param_hint="`service_account_key` to GeminiModel(...)",
+            )
+
+            try:
+                service_account_key = json.loads(service_account_key_json)
+            except Exception as e:
+                raise DeepEvalError(
+                    "GOOGLE_SERVICE_ACCOUNT_KEY must be valid JSON for a Google service account."
+                ) from e
+
+            if not isinstance(service_account_key, dict):
+                raise DeepEvalError(
+                    "GOOGLE_SERVICE_ACCOUNT_KEY must decode to a JSON object."
+                )
+
             if not self.project or not self.location:
-                raise ValueError(
+                raise DeepEvalError(
                     "When using Vertex AI API, both project and location are required. "
                     "Either provide them as arguments or set GOOGLE_CLOUD_PROJECT and "
                     "GOOGLE_CLOUD_LOCATION in your DeepEval configuration."
                 )
 
+            oauth2 = self._require_oauth2()
             credentials = (
-                service_account.Credentials.from_service_account_info(
-                    self.service_account_key,
+                oauth2.service_account.Credentials.from_service_account_info(
+                    service_account_key,
                     scopes=[
                         "https://www.googleapis.com/auth/cloud-platform",
                     ],
                 )
-                if self.service_account_key
+                if service_account_key
                 else None
             )
 
-            client = Client(
+            client = self._module.Client(
                 vertexai=True,
                 project=self.project,
                 location=self.location,
@@ -282,6 +424,9 @@ class GeminiModel(DeepEvalBaseLLM):
                 param_hint="`api_key` to GeminiModel(...)",
             )
 
-            client = Client(api_key=api_key, **client_kwargs)
+            client = self._module.Client(api_key=api_key, **client_kwargs)
 
         return client
+
+    def get_model_name(self):
+        return f"{self.name} (Gemini)"