deepeval 3.7.5__py3-none-any.whl → 3.7.7__py3-none-any.whl

deepeval/models/llms/openai_model.py

@@ -1,13 +1,13 @@
-import base64
 from openai.types.chat.chat_completion import ChatCompletion
 from typing import Optional, Tuple, Union, Dict, List
 from deepeval.test_case import MLLMImage
 from pydantic import BaseModel, SecretStr
-from io import BytesIO
 from openai import (
     OpenAI,
     AsyncOpenAI,
 )
+
+from deepeval.errors import DeepEvalError
 from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
@@ -15,6 +15,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models.utils import (
     parse_model_name,
+    require_costs,
     require_secret_api_key,
     normalize_kwargs_and_extract_aliases,
 )
@@ -22,208 +23,15 @@ from deepeval.models.retry_policy import (
     create_retry_decorator,
     sdk_retries_for,
 )
+from deepeval.models.llms.constants import (
+    OPENAI_MODELS_DATA,
+)
 
 
 retry_openai = create_retry_decorator(PS.OPENAI)
 
-
-valid_gpt_models = [
-    "gpt-3.5-turbo",
-    "gpt-3.5-turbo-0125",
-    "gpt-3.5-turbo-1106",
-    "gpt-4-0125-preview",
-    "gpt-4-1106-preview",
-    "gpt-4-turbo",
-    "gpt-4-turbo-2024-04-09",
-    "gpt-4-turbo-preview",
-    "gpt-4o",
-    "gpt-4o-2024-05-13",
-    "gpt-4o-2024-08-06",
-    "gpt-4o-2024-11-20",
-    "gpt-4o-mini",
-    "gpt-4o-mini-2024-07-18",
-    "gpt-4-32k",
-    "gpt-4-32k-0613",
-    "gpt-4.1",
-    "gpt-4.1-mini",
-    "gpt-4.1-nano",
-    "gpt-4.5-preview",
-    "o1",
-    "o1-preview",
-    "o1-2024-12-17",
-    "o1-preview-2024-09-12",
-    "o1-mini",
-    "o1-mini-2024-09-12",
-    "o3-mini",
-    "o3-mini-2025-01-31",
-    "o4-mini",
-    "o4-mini-2025-04-16",
-    "gpt-4.5-preview-2025-02-27",
-    "gpt-5",
-    "gpt-5-2025-08-07",
-    "gpt-5-mini",
-    "gpt-5-mini-2025-08-07",
-    "gpt-5-nano",
-    "gpt-5-nano-2025-08-07",
-    "gpt-5-chat-latest",
-]
-
-unsupported_log_probs_gpt_models = [
-    "o1",
-    "o1-preview",
-    "o1-2024-12-17",
-    "o1-preview-2024-09-12",
-    "o1-mini",
-    "o1-mini-2024-09-12",
-    "o3-mini",
-    "o3-mini-2025-01-31",
-    "o4-mini",
-    "o4-mini-2025-04-16",
-    "gpt-4.5-preview-2025-02-27",
-    "gpt-5",
-    "gpt-5-2025-08-07",
-    "gpt-5-mini",
-    "gpt-5-mini-2025-08-07",
-    "gpt-5-nano",
-    "gpt-5-nano-2025-08-07",
-    "gpt-5-chat-latest",
-]
-
-unsupported_log_probs_multimodal_gpt_models = [
-    "o1",
-    "o1-preview",
-    "o1-2024-12-17",
-    "o1-preview-2024-09-12",
-    "gpt-4.5-preview-2025-02-27",
-    "o4-mini",
-]
-
-structured_outputs_models = [
-    "gpt-4o",
-    "gpt-4o-2024-05-13",
-    "gpt-4o-2024-08-06",
-    "gpt-4o-2024-11-20",
-    "gpt-4o-mini",
-    "gpt-4o-mini-2024-07-18",
-    "gpt-4.1",
-    "gpt-4.1-mini",
-    "gpt-4.1-nano",
-    "o1",
-    "o1-preview",
-    "o1-2024-12-17",
-    "o3-mini",
-    "o3-mini-2025-01-31",
-    "o4-mini",
-    "o4-mini-2025-04-16",
-    "gpt-4.5-preview-2025-02-27",
-    "gpt-5",
-    "gpt-5-2025-08-07",
-    "gpt-5-mini",
-    "gpt-5-mini-2025-08-07",
-    "gpt-5-nano",
-    "gpt-5-nano-2025-08-07",
-]
-
-json_mode_models = [
-    "gpt-3.5-turbo",
-    "gpt-3.5-turbo-0125",
-    "gpt-3.5-turbo-1106",
-    "gpt-4-0125-preview",
-    "gpt-4-1106-preview",
-    "gpt-4-turbo",
-    "gpt-4-turbo-2024-04-09",
-    "gpt-4-turbo-preview",
-    "gpt-4-32k",
-    "gpt-4-32k-0613",
-]
-
-model_pricing = {
-    "gpt-4o-mini": {"input": 0.150 / 1e6, "output": 0.600 / 1e6},
-    "gpt-4o": {"input": 2.50 / 1e6, "output": 10.00 / 1e6},
-    "gpt-4-turbo": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
-    "gpt-4-turbo-preview": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
-    "gpt-4-0125-preview": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
-    "gpt-4-1106-preview": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
-    "gpt-4": {"input": 30.00 / 1e6, "output": 60.00 / 1e6},
-    "gpt-4-32k": {"input": 60.00 / 1e6, "output": 120.00 / 1e6},
-    "gpt-3.5-turbo-1106": {"input": 1.00 / 1e6, "output": 2.00 / 1e6},
-    "gpt-3.5-turbo": {"input": 0.50 / 1e6, "output": 1.50 / 1e6},
-    "gpt-3.5-turbo-16k": {"input": 3.00 / 1e6, "output": 4.00 / 1e6},
-    "gpt-3.5-turbo-0125": {"input": 0.50 / 1e6, "output": 1.50 / 1e6},
-    "gpt-3.5-turbo-instruct": {"input": 1.50 / 1e6, "output": 2.00 / 1e6},
-    "o1": {"input": 15.00 / 1e6, "output": 60.00 / 1e6},
-    "o1-preview": {"input": 15.00 / 1e6, "output": 60.00 / 1e6},
-    "o1-2024-12-17": {"input": 15.00 / 1e6, "output": 60.00 / 1e6},
-    "o3-mini": {"input": 1.10 / 1e6, "output": 4.40 / 1e6},
-    "o3-mini-2025-01-31": {"input": 1.10 / 1e6, "output": 4.40 / 1e6},
-    "o4-mini": {"input": 1.10 / 1e6, "output": 4.40 / 1e6},
-    "o4-mini-2025-04-16": {"input": 1.10 / 1e6, "output": 4.40 / 1e6},
-    "gpt-4.1": {
-        "input": 2.00 / 1e6,
-        "output": 8.00 / 1e6,
-    },
-    "gpt-4.1-mini": {
-        "input": 0.4 / 1e6,
-        "output": 1.60 / 1e6,
-    },
-    "gpt-4.1-nano": {
-        "input": 0.1 / 1e6,
-        "output": 0.4 / 1e6,
-    },
-    "gpt-4.5-preview": {
-        "input": 75.00 / 1e6,
-        "output": 150.00 / 1e6,
-    },
-    "gpt-5": {
-        "input": 1.25 / 1e6,
-        "output": 10.00 / 1e6,
-    },
-    "gpt-5-2025-08-07": {
-        "input": 1.25 / 1e6,
-        "output": 10.00 / 1e6,
-    },
-    "gpt-5-mini": {
-        "input": 0.25 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-    "gpt-5-mini-2025-08-07": {
-        "input": 0.25 / 1e6,
-        "output": 2.00 / 1e6,
-    },
-    "gpt-5-nano": {
-        "input": 0.05 / 1e6,
-        "output": 0.40 / 1e6,
-    },
-    "gpt-5-nano-2025-08-07": {
-        "input": 0.05 / 1e6,
-        "output": 0.40 / 1e6,
-    },
-    "gpt-5-chat-latest": {
-        "input": 1.25 / 1e6,
-        "output": 10.00 / 1e6,
-    },
-}
-
 default_gpt_model = "gpt-4.1"
 
-# Thinking models that require temperature=1
-models_requiring_temperature_1 = [
-    "o1",
-    "o1-2024-12-17",
-    "o1-mini",
-    "o1-mini-2024-09-12",
-    "o3-mini",
-    "o3-mini-2025-01-31",
-    "o4-mini",
-    "o4-mini-2025-04-16",
-    "gpt-5",
-    "gpt-5-2025-08-07",
-    "gpt-5-mini",
-    "gpt-5-mini-2025-08-07",
-    "gpt-5-nano",
-    "gpt-5-nano-2025-08-07",
-]
-
 
 def _request_timeout_seconds() -> float:
     timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
@@ -236,25 +44,20 @@ _ALIAS_MAP = {
 
 
 class GPTModel(DeepEvalBaseLLM):
-    valid_multimodal_models = [
-        "gpt-4o",
-        "gpt-4o-mini",
-        "gpt-4.1",
-        "gpt-4.1-mini",
-        "gpt-5",
-    ]
 
     def __init__(
         self,
         model: Optional[str] = None,
         api_key: Optional[str] = None,
         base_url: Optional[str] = None,
-        temperature: float = 0,
+        temperature: Optional[float] = None,
         cost_per_input_token: Optional[float] = None,
         cost_per_output_token: Optional[float] = None,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
     ):
+        settings = get_settings()
+
         normalized_kwargs, alias_values = normalize_kwargs_and_extract_aliases(
             "GPTModel",
             kwargs,
@@ -265,8 +68,10 @@ class GPTModel(DeepEvalBaseLLM):
         if api_key is None and "api_key" in alias_values:
             api_key = alias_values["api_key"]
 
-        settings = get_settings()
         model = model or settings.OPENAI_MODEL_NAME
+        if model is None:
+            model = default_gpt_model
+
         cost_per_input_token = (
             cost_per_input_token
             if cost_per_input_token is not None
@@ -278,71 +83,80 @@ class GPTModel(DeepEvalBaseLLM):
             else settings.OPENAI_COST_PER_OUTPUT_TOKEN
         )
 
-        if model is None:
-            model = default_gpt_model
-
-        if isinstance(model, str):
-            model = parse_model_name(model)
-            if model not in valid_gpt_models:
-                raise ValueError(
-                    f"Invalid model. Available GPT models: {', '.join(model for model in valid_gpt_models)}"
-                )
-
-        if model not in model_pricing:
-            if cost_per_input_token is None or cost_per_output_token is None:
-                raise ValueError(
-                    f"No pricing available for `{model}`. "
-                    "Please provide both `cost_per_input_token` and `cost_per_output_token` when initializing `GPTModel`, "
-                    "or set them via the CLI:\n"
-                    "    deepeval set-openai --model=[...] --cost_per_input_token=[...] --cost_per_output_token=[...]"
-                )
-            else:
-                model_pricing[model] = {
-                    "input": float(cost_per_input_token),
-                    "output": float(cost_per_output_token),
-                }
-
         if api_key is not None:
             # keep it secret, keep it safe from serializings, logging and alike
-            self.api_key: SecretStr | None = SecretStr(api_key)
+            self.api_key: Optional[SecretStr] = SecretStr(api_key)
         else:
-            self.api_key = get_settings().OPENAI_API_KEY
+            self.api_key = settings.OPENAI_API_KEY
 
-        self.base_url = base_url
+        self.base_url = (
+            str(base_url).rstrip("/") if base_url is not None else None
+        )
         # args and kwargs will be passed to the underlying model, in load_model function
 
-        # Auto-adjust temperature for models that require it
-        if model in models_requiring_temperature_1:
+        if temperature is not None:
+            temperature = float(temperature)
+        elif settings.TEMPERATURE is not None:
+            temperature = settings.TEMPERATURE
+        else:
+            temperature = 0.0
+
+        if isinstance(model, str):
+            model = parse_model_name(model)
+
+        self.model_data = OPENAI_MODELS_DATA.get(model)
+
+        # Auto-adjust temperature for known models that require it
+        if self.model_data.supports_temperature is False:
             temperature = 1
 
+        # validation
+        cost_per_input_token, cost_per_output_token = require_costs(
+            self.model_data,
+            model,
+            "OPENAI_COST_PER_INPUT_TOKEN",
+            "OPENAI_COST_PER_OUTPUT_TOKEN",
+            cost_per_input_token,
+            cost_per_output_token,
+        )
+        self.model_data.input_price = cost_per_input_token
+        self.model_data.output_price = cost_per_output_token
+
         if temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
+            raise DeepEvalError("Temperature must be >= 0.")
+
         self.temperature = temperature
         # Keep sanitized kwargs for client call to strip legacy keys
         self.kwargs = normalized_kwargs
-        self.generation_kwargs = generation_kwargs or {}
+        self.kwargs.pop("temperature", None)
+
+        self.generation_kwargs = dict(generation_kwargs or {})
+        self.generation_kwargs.pop("temperature", None)
+
         super().__init__(model)
 
-    ###############################################
-    # Generate functions
-    ###############################################
+    ######################
+    # Generate functions #
+    ######################
 
     @retry_openai
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, Dict], float]:
+    ) -> Tuple[Union[str, BaseModel], float]:
         client = self.load_model(async_mode=False)
 
         if check_if_multimodal(prompt):
             prompt = convert_to_multi_modal_array(input=prompt)
-            prompt = self.generate_prompt(prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
 
         if schema:
-            if self.name in structured_outputs_models:
+            if self.supports_structured_outputs() is True:
                 completion = client.beta.chat.completions.parse(
                     model=self.name,
                     messages=[
-                        {"role": "user", "content": prompt},
+                        {"role": "user", "content": content},
                     ],
                     response_format=schema,
                     temperature=self.temperature,
@@ -356,11 +170,11 @@ class GPTModel(DeepEvalBaseLLM):
                     completion.usage.completion_tokens,
                 )
                 return structured_output, cost
-            if self.name in json_mode_models:
+            if self.supports_json_mode() is True:
                 completion = client.beta.chat.completions.parse(
                     model=self.name,
                     messages=[
-                        {"role": "user", "content": prompt},
+                        {"role": "user", "content": content},
                     ],
                     response_format={"type": "json_object"},
                     temperature=self.temperature,
@@ -377,7 +191,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         completion = client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -399,14 +213,16 @@ class GPTModel(DeepEvalBaseLLM):
 
         if check_if_multimodal(prompt):
             prompt = convert_to_multi_modal_array(input=prompt)
-            prompt = self.generate_prompt(prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
 
         if schema:
-            if self.name in structured_outputs_models:
+            if self.supports_structured_outputs() is True:
                 completion = await client.beta.chat.completions.parse(
                     model=self.name,
                     messages=[
-                        {"role": "user", "content": prompt},
+                        {"role": "user", "content": content},
                     ],
                     response_format=schema,
                     temperature=self.temperature,
@@ -420,11 +236,11 @@ class GPTModel(DeepEvalBaseLLM):
                     completion.usage.completion_tokens,
                 )
                 return structured_output, cost
-            if self.name in json_mode_models:
+            if self.supports_json_mode() is True:
                 completion = await client.beta.chat.completions.parse(
                     model=self.name,
                     messages=[
-                        {"role": "user", "content": prompt},
+                        {"role": "user", "content": content},
                     ],
                     response_format={"type": "json_object"},
                     temperature=self.temperature,
@@ -441,7 +257,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         completion = await client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             **self.generation_kwargs,
         )
@@ -455,9 +271,9 @@ class GPTModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
-    ###############################################
-    # Other generate functions
-    ###############################################
+    ############################
+    # Other generate functions #
+    ############################
 
     @retry_openai
     def generate_raw_response(
@@ -466,13 +282,26 @@ class GPTModel(DeepEvalBaseLLM):
         top_logprobs: int = 5,
     ) -> Tuple[ChatCompletion, float]:
         # Generate completion
+        model_name = self.name
+        is_multimodal = check_if_multimodal(prompt)
+
+        # validate that this model supports logprobs
+        if self.supports_log_probs() is False:
+            raise DeepEvalError(
+                f"Model `{model_name}` does not support `logprobs` / `top_logprobs`. "
+                "Please use a different OpenAI model (for example `gpt-4.1` or `gpt-4o`) "
+                "when calling `generate_raw_response`."
+            )
+
         client = self.load_model(async_mode=False)
-        if check_if_multimodal(prompt):
+        if is_multimodal:
             prompt = convert_to_multi_modal_array(input=prompt)
-            prompt = self.generate_prompt(prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
         completion = client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             logprobs=True,
             top_logprobs=top_logprobs,
@@ -492,13 +321,26 @@ class GPTModel(DeepEvalBaseLLM):
         top_logprobs: int = 5,
     ) -> Tuple[ChatCompletion, float]:
         # Generate completion
+        model_name = self.name
+        is_multimodal = check_if_multimodal(prompt)
+
+        # validate that this model supports logprobs
+        if self.supports_log_probs() is False:
+            raise DeepEvalError(
+                f"Model `{model_name}` does not support `logprobs` / `top_logprobs`. "
+                "Please use a different OpenAI model (for example `gpt-4.1` or `gpt-4o`) "
+                "when calling `a_generate_raw_response`."
+            )
+
         client = self.load_model(async_mode=True)
-        if check_if_multimodal(prompt):
+        if is_multimodal:
             prompt = convert_to_multi_modal_array(input=prompt)
-            prompt = self.generate_prompt(prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
         completion = await client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             temperature=self.temperature,
             logprobs=True,
             top_logprobs=top_logprobs,
@@ -514,14 +356,16 @@ class GPTModel(DeepEvalBaseLLM):
     @retry_openai
     def generate_samples(
         self, prompt: str, n: int, temperature: float
-    ) -> Tuple[list[str], float]:
+    ) -> list[str]:
         client = self.load_model(async_mode=False)
         if check_if_multimodal(prompt):
             prompt = convert_to_multi_modal_array(input=prompt)
-            prompt = self.generate_prompt(prompt)
+            content = self.generate_content(prompt)
+        else:
+            content = [{"type": "text", "text": prompt}]
         response = client.chat.completions.create(
             model=self.name,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{"role": "user", "content": content}],
             n=n,
             temperature=temperature,
             **self.generation_kwargs,
@@ -529,55 +373,74 @@ class GPTModel(DeepEvalBaseLLM):
         completions = [choice.message.content for choice in response.choices]
         return completions
 
-    ###############################################
-    # Utilities
-    ###############################################
+    #############
+    # Utilities #
+    #############
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
-        # TODO: consider loggin a warning instead of defaulting to whole model pricing
-        pricing = model_pricing.get(self.name, model_pricing)
-        input_cost = input_tokens * pricing["input"]
-        output_cost = output_tokens * pricing["output"]
-        return input_cost + output_cost
+        if self.model_data.input_price and self.model_data.output_price:
+            input_cost = input_tokens * self.model_data.input_price
+            output_cost = output_tokens * self.model_data.output_price
+            return input_cost + output_cost
+
+    #########################
+    # Capabilities          #
+    #########################
+
+    def supports_log_probs(self) -> Union[bool, None]:
+        return self.model_data.supports_log_probs
+
+    def supports_temperature(self) -> Union[bool, None]:
+        return self.model_data.supports_temperature
+
+    def supports_multimodal(self) -> Union[bool, None]:
+        return self.model_data.supports_multimodal
+
+    def supports_structured_outputs(self) -> Union[bool, None]:
+        """
+        OpenAI models that natively enforce typed structured outputs.
+         Used by generate(...) when a schema is provided.
+        """
+        return self.model_data.supports_structured_outputs
+
+    def supports_json_mode(self) -> Union[bool, None]:
+        """
+        OpenAI models that enforce JSON mode
+        """
+        return self.model_data.supports_json
 
     #########
     # Model #
     #########
 
-    def generate_prompt(
-        self, multimodal_input: List[Union[str, MLLMImage]] = []
+    def generate_content(
+        self, multimodal_input: Optional[List[Union[str, MLLMImage]]] = None
     ):
-        prompt = []
-        for ele in multimodal_input:
-            if isinstance(ele, str):
-                prompt.append({"type": "text", "text": ele})
-            elif isinstance(ele, MLLMImage):
-                if ele.local:
-                    import PIL.Image
-
-                    image = PIL.Image.open(ele.url)
-                    visual_dict = {
-                        "type": "image_url",
-                        "image_url": {
-                            "url": f"data:image/jpeg;base64,{self.encode_pil_image(image)}"
-                        },
-                    }
+        multimodal_input = [] if multimodal_input is None else multimodal_input
+        content = []
+        for element in multimodal_input:
+            if isinstance(element, str):
+                content.append({"type": "text", "text": element})
+            elif isinstance(element, MLLMImage):
+                if element.url and not element.local:
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": element.url},
+                        }
+                    )
                 else:
-                    visual_dict = {
-                        "type": "image_url",
-                        "image_url": {"url": ele.url},
-                    }
-                prompt.append(visual_dict)
-        return prompt
-
-    def encode_pil_image(self, pil_image):
-        image_buffer = BytesIO()
-        if pil_image.mode in ("RGBA", "LA", "P"):
-            pil_image = pil_image.convert("RGB")
-        pil_image.save(image_buffer, format="JPEG")
-        image_bytes = image_buffer.getvalue()
-        base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
-        return base64_encoded_image
+                    element.ensure_images_loaded()
+                    data_uri = (
+                        f"data:{element.mimeType};base64,{element.dataBase64}"
+                    )
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": data_uri},
+                        }
+                    )
+        return content
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
@@ -620,10 +483,5 @@ class GPTModel(DeepEvalBaseLLM):
                 return cls(**kw)
             raise
 
-    def supports_multimodal(self):
-        if self.name in GPTModel.valid_multimodal_models:
-            return True
-        return False
-
     def get_model_name(self):
         return f"{self.name}"