deepeval 3.7.4__py3-none-any.whl → 3.7.6__py3-none-any.whl

deepeval/models/llms/grok_model.py

@@ -1,91 +1,101 @@
-from typing import Optional, Tuple, Union, Dict
+from typing import Optional, Tuple, Union, Dict, List
 from pydantic import BaseModel, SecretStr
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_costs,
+    require_secret_api_key,
+)
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.models.llms.constants import GROK_MODELS_DATA
+from deepeval.utils import require_param
 
 # consistent retry rules
 retry_grok = create_retry_decorator(PS.GROK)
 
 
-structured_outputs_models = [
-    "grok-4-0709",
-    "grok-3",
-    "grok-3-mini",
-    "grok-3-fast",
-    "grok-3-mini-fast",
-]
-
-model_pricing = {
-    "grok-4-0709": {
-        "input": 0.20 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-    "grok-3": {
-        "input": 1.00 / 1e6,
-        "output": 3.00 / 1e6,
-    },
-    "grok-3-mini": {
-        "input": 2.00 / 1e6,
-        "output": 5.00 / 1e6,
-    },
-    "grok-3-fast": {
-        "input": 0.60 / 1e6,
-        "output": 2.50 / 1e6,
-    },
-    "grok-3-mini-fast": {
-        "input": 30 / 1e6,
-        "output": 30 / 1e6,
-    },
-    "grok-2-vision-1212": {
-        "input": 1.00 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-}
-
-
 class GrokModel(DeepEvalBaseLLM):
     def __init__(
         self,
         model: Optional[str] = None,
         api_key: Optional[str] = None,
-        temperature: float = 0,
+        temperature: Optional[float] = None,
+        cost_per_input_token: Optional[float] = None,
+        cost_per_output_token: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+
         settings = get_settings()
 
-        model_name = model or settings.GROK_MODEL_NAME
+        model = model or settings.GROK_MODEL_NAME
 
-        if model_name not in model_pricing:
-            raise ValueError(
-                f"Invalid model. Available Grok models: {', '.join(model_pricing.keys())}"
-            )
-        temperature_from_key = settings.TEMPERATURE
-        if temperature_from_key is None:
-            self.temperature = temperature
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
         else:
-            self.temperature = float(temperature_from_key)
-        if self.temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            temperature = 0.0
+
+        cost_per_input_token = (
+            cost_per_input_token
+            if cost_per_input_token is not None
+            else settings.GROK_COST_PER_INPUT_TOKEN
+        )
+        cost_per_output_token = (
+            cost_per_output_token
+            if cost_per_output_token is not None
+            else settings.GROK_COST_PER_OUTPUT_TOKEN
+        )
 
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.GROK_API_KEY
 
+        model = require_param(
+            model,
+            provider_label="GrokModel",
+            env_var_name="GROK_MODEL_NAME",
+            param_hint="model",
+        )
+
+        # validation
+        if temperature < 0:
+            raise DeepEvalError("Temperature must be >= 0.")
+
+        self.model_data = GROK_MODELS_DATA.get(model)
+        self.temperature = temperature
+
+        cost_per_input_token, cost_per_output_token = require_costs(
+            self.model_data,
+            model,
+            "GROK_COST_PER_INPUT_TOKEN",
+            "GROK_COST_PER_OUTPUT_TOKEN",
+            cost_per_input_token,
+            cost_per_output_token,
+        )
+        self.model_data.input_price = cost_per_input_token
+        self.model_data.output_price = cost_per_output_token
+
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -94,22 +104,29 @@ class GrokModel(DeepEvalBaseLLM):
     @retry_grok
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
         try:
             from xai_sdk.chat import user
         except ImportError:
             raise ImportError(
                 "xai_sdk is required to use GrokModel. Please install it with: pip install xai-sdk"
             )
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         client = self.load_model(async_mode=False)
         chat = client.chat.create(
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
-        chat.append(user(prompt))
+        chat.append(user(content))
 
-        if schema and self.model_name in structured_outputs_models:
+        if schema and self.supports_structured_outputs() is True:
             response, structured_output = chat.parse(schema)
             cost = self.calculate_cost(
                 response.usage.prompt_tokens,
@@ -132,22 +149,30 @@ class GrokModel(DeepEvalBaseLLM):
     @retry_grok
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
         try:
             from xai_sdk.chat import user
         except ImportError:
             raise ImportError(
                 "xai_sdk is required to use GrokModel. Please install it with: pip install xai-sdk"
             )
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         client = self.load_model(async_mode=True)
         chat = client.chat.create(
-            model=self.model_name,
+            model=self.name,
             temperature=self.temperature,
             **self.generation_kwargs,
         )
-        chat.append(user(prompt))
+        chat.append(user(content))
 
-        if schema and self.model_name in structured_outputs_models:
+        if schema and self.supports_structured_outputs() is True:
             response, structured_output = await chat.parse(schema)
             cost = self.calculate_cost(
                 response.usage.prompt_tokens,
@@ -167,6 +192,34 @@ class GrokModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
+
     ###############################################
     # Utilities
     ###############################################
@@ -176,11 +229,29 @@ class GrokModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
-        input_cost = input_tokens * pricing["input"]
-        output_cost = output_tokens * pricing["output"]
+        input_cost = input_tokens * self.model_data.input_price
+        output_cost = output_tokens * self.model_data.output_price
         return input_cost + output_cost
 
+    ###############################################
+    # Capabilities
+    ###############################################
+
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        return self.model_data.supports_json
+
     ###############################################
     # Model
     ###############################################
@@ -198,9 +269,6 @@ class GrokModel(DeepEvalBaseLLM):
                 "xai_sdk is required to use GrokModel. Please install it with: pip install xai-sdk"
             )
 
-    def get_model_name(self):
-        return f"{self.model_name}"
-
     def _client_kwargs(self) -> Dict:
         """
         If Tenacity is managing retries, disable gRPC channel retries to avoid double retry.
@@ -242,3 +310,6 @@ class GrokModel(DeepEvalBaseLLM):
                 kw.pop("channel_options", None)
                 return cls(**kw)
             raise
+
+    def get_model_name(self):
+        return f"{self.name} (Grok)"

deepeval/models/llms/kimi_model.py

@@ -1,111 +1,101 @@
-from typing import Optional, Tuple, Union, Dict
+from typing import Optional, Tuple, Union, Dict, List
 from openai import OpenAI, AsyncOpenAI
 from pydantic import BaseModel, SecretStr
 
+from deepeval.errors import DeepEvalError
 from deepeval.config.settings import get_settings
 from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
 from deepeval.models.llms.utils import trim_and_load_json
-from deepeval.models.utils import require_secret_api_key
+from deepeval.models.utils import (
+    require_costs,
+    require_secret_api_key,
+)
+from deepeval.test_case import MLLMImage
+from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-
+from deepeval.models.llms.constants import KIMI_MODELS_DATA
+from deepeval.utils import require_param
 
 retry_kimi = create_retry_decorator(PS.KIMI)
 
-json_mode_models = [
-    "kimi-thinking-preview",
-    "kimi-k2-0711-preview",
-    "kimi-latest-128k",
-    "kimi-latest-32k",
-    "kimi-latest-8k",
-]
-
-model_pricing = {
-    "kimi-latest-8k": {
-        "input": 0.20 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-    "kimi-latest-32k": {
-        "input": 1.00 / 1e6,
-        "output": 3.00 / 1e6,
-    },
-    "kimi-latest-128k": {
-        "input": 2.00 / 1e6,
-        "output": 5.00 / 1e6,
-    },
-    "kimi-k2-0711-preview": {
-        "input": 0.60 / 1e6,
-        "output": 2.50 / 1e6,
-    },
-    "kimi-thinking-preview": {
-        "input": 30 / 1e6,
-        "output": 30 / 1e6,
-    },
-    "moonshot-v1-8k": {
-        "input": 1.00 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-    "moonshot-v1-32k": {
-        "input": 2.00 / 1e6,
-        "output": 3.00 / 1e6,
-    },
-    "moonshot-v1-128k": {
-        "input": 0.20 / 1e6,
-        "output": 5.00 / 1e6,
-    },
-    "moonshot-v1-8k-vision-preview": {
-        "input": 1.00 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-    "moonshot-v1-32k-vision-preview": {
-        "input": 2.00 / 1e6,
-        "output": 3.00 / 1e6,
-    },
-    "moonshot-v1-128k-vision-preview": {
-        "input": 0.20 / 1e6,
-        "output": 5.00 / 1e6,
-    },
-}
-
 
 class KimiModel(DeepEvalBaseLLM):
     def __init__(
         self,
-        api_key: Optional[str] = None,
         model: Optional[str] = None,
-        temperature: float = 0,
+        api_key: Optional[str] = None,
+        temperature: Optional[float] = None,
+        cost_per_input_token: Optional[float] = None,
+        cost_per_output_token: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
         settings = get_settings()
 
-        model_name = model or settings.MOONSHOT_MODEL_NAME
-        if model_name not in model_pricing:
-            raise ValueError(
-                f"Invalid model. Available Moonshot models: {', '.join(model_pricing.keys())}"
-            )
+        model = model or settings.MOONSHOT_MODEL_NAME
 
-        temperature_from_key = settings.TEMPERATURE
-        if temperature_from_key is None:
-            self.temperature = temperature
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
         else:
-            self.temperature = float(temperature_from_key)
-        if self.temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            temperature = 0.0
+
+        cost_per_input_token = (
+            cost_per_input_token
+            if cost_per_input_token is not None
+            else settings.MOONSHOT_COST_PER_INPUT_TOKEN
+        )
+        cost_per_output_token = (
+            cost_per_output_token
+            if cost_per_output_token is not None
+            else settings.MOONSHOT_COST_PER_OUTPUT_TOKEN
+        )
 
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
             self.api_key = settings.MOONSHOT_API_KEY
 
+        # validation
+        model = require_param(
+            model,
+            provider_label="KimiModel",
+            env_var_name="MOONSHOT_MODEL_NAME",
+            param_hint="model",
+        )
+
+        if temperature < 0:
+            raise DeepEvalError("Temperature must be >= 0.")
+
+        self.model_data = KIMI_MODELS_DATA.get(model)
+        self.temperature = temperature
+
+        cost_per_input_token, cost_per_output_token = require_costs(
+            self.model_data,
+            model,
+            "MOONSHOT_COST_PER_INPUT_TOKEN",
+            "MOONSHOT_COST_PER_OUTPUT_TOKEN",
+            cost_per_input_token,
+            cost_per_output_token,
+        )
+        self.model_data.input_price = float(cost_per_input_token)
+        self.model_data.output_price = float(cost_per_output_token)
+
         self.base_url = "https://api.moonshot.cn/v1"
+        # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = kwargs
-        self.generation_kwargs = generation_kwargs or {}
-        super().__init__(model_name)
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
+        super().__init__(model)
 
     ###############################################
     # Other generate functions
@@ -114,12 +104,19 @@ class KimiModel(DeepEvalBaseLLM):
     @retry_kimi
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         client = self.load_model(async_mode=False)
-        if schema and self.model_name in json_mode_models:
+        if schema and self.supports_json_mode() is True:
             completion = client.chat.completions.create(
-                model=self.model_name,
-                messages=[{"role": "user", "content": prompt}],
+                model=self.name,
+                messages=[{"role": "user", "content": content}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
                 **self.generation_kwargs,
@@ -134,8 +131,8 @@ class KimiModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
 
         completion = client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
+            model=self.name,
+            messages=[{"role": "user", "content": content}],
             **self.generation_kwargs,
         )
         output = completion.choices[0].message.content
@@ -152,12 +149,19 @@ class KimiModel(DeepEvalBaseLLM):
     @retry_kimi
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
+
+        if check_if_multimodal(prompt):
+            prompt = convert_to_multi_modal_array(input=prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
+
         client = self.load_model(async_mode=True)
-        if schema and self.model_name in json_mode_models:
+        if schema and self.supports_json_mode() is True:
             completion = await client.chat.completions.create(
-                model=self.model_name,
-                messages=[{"role": "user", "content": prompt}],
+                model=self.name,
+                messages=[{"role": "user", "content": content}],
                 response_format={"type": "json_object"},
                 temperature=self.temperature,
                 **self.generation_kwargs,
@@ -172,8 +176,8 @@ class KimiModel(DeepEvalBaseLLM):
             return schema.model_validate(json_output), cost
 
         completion = await client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": prompt}],
+            model=self.name,
+            messages=[{"role": "user", "content": content}],
             **self.generation_kwargs,
         )
         output = completion.choices[0].message.content
@@ -187,6 +191,34 @@ class KimiModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
+    def generate_content(
+        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    ):
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
+                else:
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
+
     ###############################################
     # Utilities
     ###############################################
@@ -196,11 +228,29 @@ class KimiModel(DeepEvalBaseLLM):
         input_tokens: int,
         output_tokens: int,
     ) -> float:
-        pricing = model_pricing.get(self.model_name, model_pricing)
-        input_cost = input_tokens * pricing["input"]
-        output_cost = output_tokens * pricing["output"]
+        input_cost = input_tokens * self.model_data.input_price
+        output_cost = output_tokens * self.model_data.output_price
         return input_cost + output_cost
 
+    ###############################################
+    # Capabilities
+    ###############################################
+
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        return self.model_data.supports_json
+
     ###############################################
     # Model
     ###############################################
@@ -244,4 +294,4 @@ class KimiModel(DeepEvalBaseLLM):
             raise
 
     def get_model_name(self):
-        return f"{self.model_name}"
+        return f"{self.name} (KIMI)"