deepeval 3.5.2__py3-none-any.whl → 3.5.3__py3-none-any.whl

deepeval/models/llms/openai_model.py

@@ -1,5 +1,3 @@
-import logging
-
 from openai.types.chat.chat_completion import ChatCompletion
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from typing import Optional, Tuple, Union, Dict
@@ -10,27 +8,17 @@ from openai import (
     AsyncOpenAI,
 )
 
-from tenacity import retry, RetryCallState, before_sleep_log
-
+from deepeval.constants import ProviderSlug as PS
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models.utils import parse_model_name
 from deepeval.models.retry_policy import (
-    OPENAI_ERROR_POLICY,
-    default_wait,
-    default_stop,
-    retry_predicate,
+    create_retry_decorator,
+    sdk_retries_for,
 )
 
-logger = logging.getLogger("deepeval.openai_model")
-
-
-def log_retry_error(retry_state: RetryCallState):
-    exception = retry_state.outcome.exception()
-    logger.error(
-        f"OpenAI Error: {exception} Retrying: {retry_state.attempt_number} time(s)..."
-    )
 
+retry_openai = create_retry_decorator(PS.OPENAI)
 
 valid_gpt_models = [
     "gpt-3.5-turbo",
@@ -219,21 +207,6 @@ models_requiring_temperature_1 = [
     "gpt-5-chat-latest",
 ]
 
-_base_retry_rules_kw = dict(
-    wait=default_wait(),
-    stop=default_stop(),
-    retry=retry_predicate(OPENAI_ERROR_POLICY),
-    before_sleep=before_sleep_log(
-        logger, logging.INFO
-    ),  # <- logs only on retries
-    after=log_retry_error,
-)
-
-
-def _openai_client_kwargs():
-    # Avoid double-retry at SDK layer by disabling the SDK's own retries so tenacity is the single source of truth for retry logic.
-    return {"max_retries": 0}
-
 
 class GPTModel(DeepEvalBaseLLM):
     def __init__(
@@ -311,7 +284,7 @@ class GPTModel(DeepEvalBaseLLM):
     # Generate functions
     ###############################################
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -370,7 +343,7 @@ class GPTModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, BaseModel], float]:
@@ -434,7 +407,7 @@ class GPTModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     def generate_raw_response(
         self,
         prompt: str,
@@ -457,7 +430,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         return completion, cost
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     async def a_generate_raw_response(
         self,
         prompt: str,
@@ -480,7 +453,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         return completion, cost
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     def generate_samples(
         self, prompt: str, n: int, temperature: float
     ) -> Tuple[list[str], float]:
@@ -500,6 +473,7 @@ class GPTModel(DeepEvalBaseLLM):
     ###############################################
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        # TODO: consider loggin a warning instead of defaulting to whole model pricing
         pricing = model_pricing.get(self.model_name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
@@ -513,13 +487,32 @@ class GPTModel(DeepEvalBaseLLM):
         return self.model_name
 
     def load_model(self, async_mode: bool = False):
-        kwargs = {**self.kwargs, **_openai_client_kwargs()}
         if not async_mode:
-            return OpenAI(
-                api_key=self._openai_api_key,
-                base_url=self.base_url,
-                **kwargs,
-            )
-        return AsyncOpenAI(
-            api_key=self._openai_api_key, base_url=self.base_url, **kwargs
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'openai' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.OPENAI):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self._openai_api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
         )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/mlllms/gemini_model.py

@@ -4,12 +4,19 @@ from pydantic import BaseModel
 from google.genai import types
 from google import genai
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
 
 
 default_multimodal_gemini_model = "gemini-1.5-pro"
+# consistent retry rules
+retry_gemini = create_retry_decorator(PS.GOOGLE)
 
 
 class MultimodalGeminiModel(DeepEvalBaseMLLM):
@@ -147,6 +154,8 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         self.model_temperature = 0.0
         return self.client.models
 
+    # TODO: Refactor genete prompt to minimize the work done on retry
+    @retry_gemini
     def generate_prompt(
         self, multimodal_input: List[Union[str, MLLMImage]] = []
     ) -> List[Union[str, MLLMImage]]:
@@ -162,6 +171,8 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             ValueError: If an invalid input type is provided
         """
         prompt = []
+        settings = get_settings()
+
         for ele in multimodal_input:
             if isinstance(ele, str):
                 prompt.append(ele)
@@ -170,9 +181,14 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
                     with open(ele.url, "rb") as f:
                         image_data = f.read()
                 else:
-                    response = requests.get(ele.url)
-                    if response.status_code != 200:
-                        raise ValueError(f"Failed to download image: {ele.url}")
+                    response = requests.get(
+                        ele.url,
+                        timeout=(
+                            settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                            settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                        ),
+                    )
+                    response.raise_for_status()
                     image_data = response.content
 
                 image_part = types.Part.from_bytes(
@@ -183,6 +199,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
                 raise ValueError(f"Invalid input type: {type(ele)}")
         return prompt
 
+    @retry_gemini
     def generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -222,6 +239,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.text, 0
 
+    @retry_gemini
     async def a_generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],

deepeval/models/mlllms/ollama_model.py

@@ -5,23 +5,31 @@ import requests
 import base64
 import io
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
 from deepeval.key_handler import KEY_FILE_HANDLER, ModelKeyValues
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class MultimodalOllamaModel(DeepEvalBaseMLLM):
-    def __init__(
-        self,
-    ):
+    def __init__(self, **kwargs):
         model_name = KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.LOCAL_MODEL_NAME
         )
         self.base_url = KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.LOCAL_MODEL_BASE_URL
         )
+        self.kwargs = kwargs
         super().__init__(model_name)
 
+    @retry_ollama
     def generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -43,6 +51,7 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
             0,
         )
 
+    @retry_ollama
     async def a_generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -77,12 +86,14 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
                     }
                 )
             elif isinstance(ele, MLLMImage):
-                messages.append(
-                    {
-                        "role": "user",
-                        "images": [self.convert_to_base64(ele.url, ele.local)],
-                    }
-                )
+                img_b64 = self.convert_to_base64(ele.url, ele.local)
+                if img_b64 is not None:
+                    messages.append(
+                        {
+                            "role": "user",
+                            "images": [img_b64],
+                        }
+                    )
         return messages
 
     ###############################################
@@ -92,9 +103,17 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
     def convert_to_base64(self, image_source: str, is_local: bool) -> str:
         from PIL import Image
 
+        settings = get_settings()
         try:
             if not is_local:
-                response = requests.get(image_source, stream=True)
+                response = requests.get(
+                    image_source,
+                    stream=True,
+                    timeout=(
+                        settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                        settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                    ),
+                )
                 response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
                 image = Image.open(io.BytesIO(response.content))
             else:
@@ -105,15 +124,21 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
             img_str = base64.b64encode(buffered.getvalue()).decode()
             return img_str
 
+        except (requests.exceptions.RequestException, OSError) as e:
+            # Log, then rethrow so @retry_ollama can retry generate_messages() on network failures
+            print(f"Image fetch/encode failed: {e}")
+            raise
         except Exception as e:
             print(f"Error converting image to base64: {e}")
             return None
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return Client(host=self.base_url)
-        else:
-            return AsyncClient(host=self.base_url)
+            return self._build_client(Client)
+        return self._build_client(AsyncClient)
+
+    def _build_client(self, cls):
+        return cls(host=self.base_url, **self.kwargs)
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"

deepeval/models/mlllms/openai_model.py

@@ -3,15 +3,7 @@ from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ParsedChatCompletion
 from pydantic import BaseModel
 from io import BytesIO
-import logging
-import openai
 import base64
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    wait_exponential_jitter,
-    RetryCallState,
-)
 
 from deepeval.models.llms.openai_model import (
     model_pricing,
@@ -21,21 +13,14 @@ from deepeval.models import DeepEvalBaseMLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.test_case import MLLMImage
 from deepeval.models.utils import parse_model_name
-
-retryable_exceptions = (
-    openai.RateLimitError,
-    openai.APIConnectionError,
-    openai.APITimeoutError,
-    openai.LengthFinishReasonError,
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
 )
+from deepeval.constants import ProviderSlug as PS
 
 
-def log_retry_error(retry_state: RetryCallState):
-    exception = retry_state.outcome.exception()
-    logging.error(
-        f"OpenAI Error: {exception} Retrying: {retry_state.attempt_number} time(s)..."
-    )
-
+retry_openai = create_retry_decorator(PS.OPENAI)
 
 valid_multimodal_gpt_models = [
     "gpt-4o",
@@ -95,11 +80,7 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
     # Generate functions
     ###############################################
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     def generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -136,11 +117,7 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         else:
             return output, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     async def a_generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -181,17 +158,13 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
     # Other generate functions
     ###############################################
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     def generate_raw_response(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
         top_logprobs: int = 5,
     ) -> Tuple[ParsedChatCompletion, float]:
-        client = OpenAI(api_key=self._openai_api_key)
+        client = self._client()
         prompt = self.generate_prompt(multimodal_input)
         messages = [{"role": "user", "content": prompt}]
         completion = client.chat.completions.create(
@@ -206,17 +179,13 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         cost = self.calculate_cost(input_tokens, output_tokens)
         return completion, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     async def a_generate_raw_response(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
         top_logprobs: int = 5,
     ) -> Tuple[ParsedChatCompletion, float]:
-        client = AsyncOpenAI(api_key=self._openai_api_key)
+        client = self._client(async_mode=True)
         prompt = self.generate_prompt(multimodal_input)
         messages = [{"role": "user", "content": prompt}]
         completion = await client.chat.completions.create(
@@ -278,5 +247,12 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
         return base64_encoded_image
 
+    def _client(self, async_mode: bool = False):
+        kw = {"api_key": self._openai_api_key}
+        if not sdk_retries_for(PS.OPENAI):
+            kw["max_retries"] = 0
+        Client = AsyncOpenAI if async_mode else OpenAI
+        return Client(**kw)
+
     def get_model_name(self):
         return self.model_name