deepeval 3.5.1__py3-none-any.whl → 3.5.3__py3-none-any.whl

deepeval/models/llms/deepseek_model.py

@@ -5,6 +5,15 @@ from pydantic import BaseModel
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models import DeepEvalBaseLLM
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
+from deepeval.constants import ProviderSlug as PS
+
+
+# consistent retry rules
+retry_deepseek = create_retry_decorator(PS.DEEPSEEK)
 
 model_pricing = {
     "deepseek-chat": {
@@ -55,6 +64,7 @@ class DeepSeekModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
+    @retry_deepseek
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -88,6 +98,7 @@ class DeepSeekModel(DeepEvalBaseLLM):
             )
             return output, cost
 
+    @retry_deepseek
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -141,13 +152,31 @@ class DeepSeekModel(DeepEvalBaseLLM):
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return OpenAI(
-                api_key=self.api_key, base_url=self.base_url, **self.kwargs
-            )
-        else:
-            return AsyncOpenAI(
-                api_key=self.api_key, base_url=self.base_url, **self.kwargs
-            )
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
 
     def get_model_name(self):
         return f"{self.model_name}"
+
+    def _client_kwargs(self) -> Dict:
+        kwargs = dict(self.kwargs or {})
+        # if we are managing retries with Tenacity, force SDK retries off to avoid double retries.
+        # if the user opts into SDK retries for "deepseek" via DEEPEVAL_SDK_RETRY_PROVIDERS, honor it.
+        if not sdk_retries_for(PS.DEEPSEEK):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self.api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # In case an older OpenAI client doesn’t accept max_retries, drop it and retry.
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/llms/gemini_model.py

@@ -3,11 +3,19 @@ from google.genai import types
 from typing import Optional, Dict
 from google import genai
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseLLM
+from deepeval.constants import ProviderSlug as PS
+
 
 default_gemini_model = "gemini-1.5-pro"
 
+# consistent retry rules
+retry_gemini = create_retry_decorator(PS.GOOGLE)
+
 
 class GeminiModel(DeepEvalBaseLLM):
     """Class that implements Google Gemini models for text-based evaluation.
@@ -145,6 +153,7 @@ class GeminiModel(DeepEvalBaseLLM):
         ]
         return self.client.models
 
+    @retry_gemini
     def generate(self, prompt: str, schema: Optional[BaseModel] = None) -> str:
         """Generates text from a prompt.
 
@@ -180,6 +189,7 @@ class GeminiModel(DeepEvalBaseLLM):
             )
             return response.text, 0
 
+    @retry_gemini
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> str:

deepeval/models/llms/grok_model.py

@@ -1,10 +1,20 @@
+import os
+
 from typing import Optional, Tuple, Union, Dict
 from pydantic import BaseModel
-import os
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models import DeepEvalBaseLLM
+from deepeval.constants import ProviderSlug as PS
+
+
+# consistent retry rules
+retry_grok = create_retry_decorator(PS.GROK)
 
 
 structured_outputs_models = [
@@ -81,6 +91,7 @@ class GrokModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
+    @retry_grok
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -118,6 +129,7 @@ class GrokModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
+    @retry_grok
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -178,9 +190,9 @@ class GrokModel(DeepEvalBaseLLM):
             from xai_sdk import Client, AsyncClient
 
             if not async_mode:
-                return Client(api_key=self.api_key, **self.kwargs)
+                return self._build_client(Client)
             else:
-                return AsyncClient(api_key=self.api_key, **self.kwargs)
+                return self._build_client(AsyncClient)
         except ImportError:
             raise ImportError(
                 "xai_sdk is required to use GrokModel. Please install it with: pip install xai-sdk"
@@ -188,3 +200,38 @@ class GrokModel(DeepEvalBaseLLM):
 
     def get_model_name(self):
         return f"{self.model_name}"
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, disable gRPC channel retries to avoid double retry.
+        If the user opts into SDK retries for 'grok' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave channel options as is
+        """
+        kwargs = dict(self.kwargs or {})
+        opts = list(kwargs.get("channel_options", []))
+        if not sdk_retries_for(PS.GROK):
+            # remove any explicit enable flag, then disable retries
+            opts = [
+                option
+                for option in opts
+                if not (
+                    isinstance(option, (tuple, list))
+                    and option
+                    and option[0] == "grpc.enable_retries"
+                )
+            ]
+            opts.append(("grpc.enable_retries", 0))
+        if opts:
+            kwargs["channel_options"] = opts
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(api_key=self.api_key, **self._client_kwargs())
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # fallback: older SDK version might not accept channel_options
+            if "channel_options" in str(e):
+                kw.pop("channel_options", None)
+                return cls(**kw)
+            raise

deepeval/models/llms/kimi_model.py

@@ -2,11 +2,18 @@ from typing import Optional, Tuple, Union, Dict
 from openai import OpenAI, AsyncOpenAI
 from pydantic import BaseModel
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models import DeepEvalBaseLLM
+from deepeval.constants import ProviderSlug as PS
 
 
+retry_kimi = create_retry_decorator(PS.KIMI)
+
 json_mode_models = [
     "kimi-thinking-preview",
     "kimi-k2-0711-preview",
@@ -100,6 +107,7 @@ class KimiModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
+    @retry_kimi
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -137,6 +145,7 @@ class KimiModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
+    @retry_kimi
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -194,13 +203,34 @@ class KimiModel(DeepEvalBaseLLM):
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return OpenAI(
-                api_key=self.api_key, base_url=self.base_url, **self.kwargs
-            )
-        else:
-            return AsyncOpenAI(
-                api_key=self.api_key, base_url=self.base_url, **self.kwargs
-            )
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'kimi' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.KIMI):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self.api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise
 
     def get_model_name(self):
         return f"{self.model_name}"

deepeval/models/llms/local_model.py

@@ -1,12 +1,20 @@
 from typing import Optional, Tuple, Union, Dict
 from pydantic import BaseModel
-
 from openai import OpenAI, AsyncOpenAI
 from openai.types.chat import ChatCompletion
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
+from deepeval.constants import ProviderSlug as PS
+
+
+# consistent retry rules
+retry_local = create_retry_decorator(PS.LOCAL)
 
 
 class LocalModel(DeepEvalBaseLLM):
@@ -43,6 +51,7 @@ class LocalModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
+    @retry_local
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -61,6 +70,7 @@ class LocalModel(DeepEvalBaseLLM):
         else:
             return res_content, 0.0
 
+    @retry_local
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -91,14 +101,30 @@ class LocalModel(DeepEvalBaseLLM):
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return OpenAI(
-                api_key=self.local_model_api_key,
-                base_url=self.base_url,
-                **self.kwargs,
-            )
-        else:
-            return AsyncOpenAI(
-                api_key=self.local_model_api_key,
-                base_url=self.base_url,
-                **self.kwargs,
-            )
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity manages retries, turn off OpenAI SDK retries to avoid double retrying.
+        If users opt into SDK retries via DEEPEVAL_SDK_RETRY_PROVIDERS=local, leave them enabled.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.LOCAL):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self.local_model_api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # Older OpenAI SDKs may not accept max_retries; drop and retry once.
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/llms/ollama_model.py

@@ -2,8 +2,16 @@ from ollama import Client, AsyncClient, ChatResponse
 from typing import Optional, Tuple, Union, Dict
 from pydantic import BaseModel
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
+
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class OllamaModel(DeepEvalBaseLLM):
@@ -34,6 +42,7 @@ class OllamaModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
+    @retry_ollama
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -56,6 +65,7 @@ class OllamaModel(DeepEvalBaseLLM):
             0,
         )
 
+    @retry_ollama
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[str, float]:
@@ -84,9 +94,11 @@ class OllamaModel(DeepEvalBaseLLM):
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return Client(host=self.base_url, **self.kwargs)
-        else:
-            return AsyncClient(host=self.base_url, **self.kwargs)
+            return self._build_client(Client)
+        return self._build_client(AsyncClient)
+
+    def _build_client(self, cls):
+        return cls(host=self.base_url, **self.kwargs)
 
     def get_model_name(self):
         return f"{self.model_name} (Ollama)"

deepeval/models/llms/openai_model.py

@@ -1,5 +1,3 @@
-import logging
-
 from openai.types.chat.chat_completion import ChatCompletion
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from typing import Optional, Tuple, Union, Dict
@@ -10,27 +8,17 @@ from openai import (
     AsyncOpenAI,
 )
 
-from tenacity import retry, RetryCallState, before_sleep_log
-
+from deepeval.constants import ProviderSlug as PS
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
 from deepeval.models.utils import parse_model_name
 from deepeval.models.retry_policy import (
-    OPENAI_ERROR_POLICY,
-    default_wait,
-    default_stop,
-    retry_predicate,
+    create_retry_decorator,
+    sdk_retries_for,
 )
 
-logger = logging.getLogger("deepeval.openai_model")
-
-
-def log_retry_error(retry_state: RetryCallState):
-    exception = retry_state.outcome.exception()
-    logger.error(
-        f"OpenAI Error: {exception} Retrying: {retry_state.attempt_number} time(s)..."
-    )
 
+retry_openai = create_retry_decorator(PS.OPENAI)
 
 valid_gpt_models = [
     "gpt-3.5-turbo",
@@ -219,21 +207,6 @@ models_requiring_temperature_1 = [
     "gpt-5-chat-latest",
 ]
 
-_base_retry_rules_kw = dict(
-    wait=default_wait(),
-    stop=default_stop(),
-    retry=retry_predicate(OPENAI_ERROR_POLICY),
-    before_sleep=before_sleep_log(
-        logger, logging.INFO
-    ),  # <- logs only on retries
-    after=log_retry_error,
-)
-
-
-def _openai_client_kwargs():
-    # Avoid double-retry at SDK layer by disabling the SDK's own retries so tenacity is the single source of truth for retry logic.
-    return {"max_retries": 0}
-
 
 class GPTModel(DeepEvalBaseLLM):
     def __init__(
@@ -311,7 +284,7 @@ class GPTModel(DeepEvalBaseLLM):
     # Generate functions
     ###############################################
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
@@ -370,7 +343,7 @@ class GPTModel(DeepEvalBaseLLM):
         else:
             return output, cost
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, BaseModel], float]:
@@ -434,7 +407,7 @@ class GPTModel(DeepEvalBaseLLM):
     # Other generate functions
     ###############################################
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     def generate_raw_response(
         self,
         prompt: str,
@@ -457,7 +430,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         return completion, cost
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     async def a_generate_raw_response(
         self,
         prompt: str,
@@ -480,7 +453,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         return completion, cost
 
-    @retry(**_base_retry_rules_kw)
+    @retry_openai
     def generate_samples(
         self, prompt: str, n: int, temperature: float
     ) -> Tuple[list[str], float]:
@@ -500,6 +473,7 @@ class GPTModel(DeepEvalBaseLLM):
     ###############################################
 
     def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+        # TODO: consider loggin a warning instead of defaulting to whole model pricing
         pricing = model_pricing.get(self.model_name, model_pricing)
         input_cost = input_tokens * pricing["input"]
         output_cost = output_tokens * pricing["output"]
@@ -513,13 +487,32 @@ class GPTModel(DeepEvalBaseLLM):
         return self.model_name
 
     def load_model(self, async_mode: bool = False):
-        kwargs = {**self.kwargs, **_openai_client_kwargs()}
         if not async_mode:
-            return OpenAI(
-                api_key=self._openai_api_key,
-                base_url=self.base_url,
-                **kwargs,
-            )
-        return AsyncOpenAI(
-            api_key=self._openai_api_key, base_url=self.base_url, **kwargs
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'openai' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.OPENAI):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self._openai_api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
         )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/mlllms/gemini_model.py

@@ -4,12 +4,19 @@ from pydantic import BaseModel
 from google.genai import types
 from google import genai
 
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseMLLM
 from deepeval.test_case import MLLMImage
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
 
 
 default_multimodal_gemini_model = "gemini-1.5-pro"
+# consistent retry rules
+retry_gemini = create_retry_decorator(PS.GOOGLE)
 
 
 class MultimodalGeminiModel(DeepEvalBaseMLLM):
@@ -147,6 +154,8 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
         self.model_temperature = 0.0
         return self.client.models
 
+    # TODO: Refactor genete prompt to minimize the work done on retry
+    @retry_gemini
     def generate_prompt(
         self, multimodal_input: List[Union[str, MLLMImage]] = []
     ) -> List[Union[str, MLLMImage]]:
@@ -162,6 +171,8 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             ValueError: If an invalid input type is provided
         """
         prompt = []
+        settings = get_settings()
+
         for ele in multimodal_input:
             if isinstance(ele, str):
                 prompt.append(ele)
@@ -170,9 +181,14 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
                     with open(ele.url, "rb") as f:
                         image_data = f.read()
                 else:
-                    response = requests.get(ele.url)
-                    if response.status_code != 200:
-                        raise ValueError(f"Failed to download image: {ele.url}")
+                    response = requests.get(
+                        ele.url,
+                        timeout=(
+                            settings.MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS,
+                            settings.MEDIA_IMAGE_READ_TIMEOUT_SECONDS,
+                        ),
+                    )
+                    response.raise_for_status()
                     image_data = response.content
 
                 image_part = types.Part.from_bytes(
@@ -183,6 +199,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
                 raise ValueError(f"Invalid input type: {type(ele)}")
         return prompt
 
+    @retry_gemini
     def generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],
@@ -222,6 +239,7 @@ class MultimodalGeminiModel(DeepEvalBaseMLLM):
             )
             return response.text, 0
 
+    @retry_gemini
     async def a_generate(
         self,
         multimodal_input: List[Union[str, MLLMImage]],