deepeval 3.5.1__py3-none-any.whl → 3.5.3__py3-none-any.whl

deepeval/models/mlllms/ollama_model.py

@@ -5,23 +5,31 @@ import requests
 import base64
 import io
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
 from deepeval.key_handler import KEY_FILE_HANDLER, ModelKeyValues
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class MultimodalOllamaModel(DeepEvalBaseMLLM):
-    def __init__(
-        self,
-    ):
+    def __init__(self, **kwargs):
         model_name = KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.LOCAL_MODEL_NAME
         )
         self.base_url = KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.LOCAL_MODEL_BASE_URL
         )
+        self.kwargs = kwargs
         super().__init__(model_name)
 
+    @retry_ollama
     def generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -43,6 +51,7 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
             0,
         )
 
+    @retry_ollama
     async def a_generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -77,12 +86,14 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
                     }
                 )
             elif isinstance(ele, MLLMImage):
-                messages.append(
-                    {
-                        "role": "user",
-                        "images": [self.convert_to_base64(ele.url, ele.local)],
-                    }
-                )
+                img_b64 = self.convert_to_base64(ele.url, ele.local)
+                if img_b64 is not None:
+                    messages.append(
+                        {
+                            "role": "user",
+                            "images": [img_b64],
+                        }
+                    )
         return messages
 
     ###############################################
@@ -92,9 +103,17 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
     def convert_to_base64(self, image_source: str, is_local: bool) -> str:
         from PIL import Image
 
+        settings = get_settings()
         try:
             if not is_local:
-                response = requests.get(image_source, stream=True)
+                response = requests.get(
+                    image_source,
+                    stream=True,
+                    timeout=(
+                        settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                        settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                    ),
+                )
                 response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
                 image = Image.open(io.BytesIO(response.content))
             else:
@@ -105,15 +124,21 @@ class MultimodalOllamaModel(DeepEvalBaseMLLM):
             img_str = base64.b64encode(buffered.getvalue()).decode()
             return img_str
 
+        except (requests.exceptions.RequestException, OSError) as e:
+            # Log, then rethrow so @retry_ollama can retry generate_messages() on network failures
+            print(f"Image fetch/encode failed: {e}")
+            raise
         except Exception as e:
             print(f"Error converting image to base64: {e}")
             return None
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return Client(host=self.base_url)
-        else:
-            return AsyncClient(host=self.base_url)
+            return self._build_client(Client)
+        return self._build_client(AsyncClient)
+
+    def _build_client(self, cls):
+        return cls(host=self.base_url, **self.kwargs)
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"

deepeval/models/mlllms/openai_model.py

@@ -3,15 +3,7 @@ from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ParsedChatCompletion
 from pydantic import BaseModel
 from io import BytesIO
-import logging
-import openai
 import base64
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    wait_exponential_jitter,
-    RetryCallState,
-)
 
 from deepeval.models.llms.openai_model import (
     model_pricing,
@@ -21,21 +13,14 @@ from deepeval.models import DeepEvalBaseMLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.test_case import MLLMImage
 from deepeval.models.utils import parse_model_name
-
-retryable_exceptions = (
-    openai.RateLimitError,
-    openai.APIConnectionError,
-    openai.APITimeoutError,
-    openai.LengthFinishReasonError,
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
 )
+from deepeval.constants import ProviderSlug as PS
 
 
-def log_retry_error(retry_state: RetryCallState):
-    exception = retry_state.outcome.exception()
-    logging.error(
-        f"OpenAI Error: {exception} Retrying: {retry_state.attempt_number} time(s)..."
-    )
-
+retry_openai = create_retry_decorator(PS.OPENAI)
 
 valid_multimodal_gpt_models = [
     "gpt-4o",
@@ -95,11 +80,7 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
     # Generate functions
     ###############################################
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     def generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -136,11 +117,7 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         else:
             return output, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     async def a_generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -181,17 +158,13 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
     # Other generate functions
     ###############################################
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     def generate_raw_response(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
         top_logprobs: int = 5,
     ) -> Tuple[ParsedChatCompletion, float]:
-        client = OpenAI(api_key=self._openai_api_key)
+        client = self._client()
         prompt = self.generate_prompt(multimodal_input)
         messages = [{"role": "user", "content": prompt}]
         completion = client.chat.completions.create(
@@ -206,17 +179,13 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         cost = self.calculate_cost(input_tokens, output_tokens)
         return completion, cost
 
-    @retry(
-        wait=wait_exponential_jitter(initial=1, exp_base=2, jitter=2, max=10),
-        retry=retry_if_exception_type(retryable_exceptions),
-        after=log_retry_error,
-    )
+    @retry_openai
     async def a_generate_raw_response(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
         top_logprobs: int = 5,
     ) -> Tuple[ParsedChatCompletion, float]:
-        client = AsyncOpenAI(api_key=self._openai_api_key)
+        client = self._client(async_mode=True)
         prompt = self.generate_prompt(multimodal_input)
         messages = [{"role": "user", "content": prompt}]
         completion = await client.chat.completions.create(
@@ -278,5 +247,12 @@ class MultimodalOpenAIModel(DeepEvalBaseMLLM):
         base64_encoded_image = base64.b64encode(image_bytes).decode("utf-8")
         return base64_encoded_image
 
+    def _client(self, async_mode: bool = False):
+        kw = {"api_key": self._openai_api_key}
+        if not sdk_retries_for(PS.OPENAI):
+            kw["max_retries"] = 0
+        Client = AsyncOpenAI if async_mode else OpenAI
+        return Client(**kw)
+
     def get_model_name(self):
         return self.model_name