deepeval 3.5.2__py3-none-any.whl → 3.5.3__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.5.2"
+__version__: str = "3.5.3"

deepeval/config/settings.py

@@ -9,6 +9,7 @@ Central config for DeepEval.
   type coercion.
 """
 
+import logging
 import os
 import re
 
@@ -16,11 +17,17 @@ from dotenv import dotenv_values
 from pathlib import Path
 from pydantic import AnyUrl, SecretStr, field_validator, confloat
 from pydantic_settings import BaseSettings, SettingsConfigDict
-from typing import Any, Dict, Optional, NamedTuple
+from typing import Any, Dict, List, Optional, NamedTuple
 
-from deepeval.config.utils import parse_bool
+from deepeval.config.utils import (
+    parse_bool,
+    coerce_to_list,
+    dedupe_preserve_order,
+)
+from deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify
 
 
+logger = logging.getLogger(__name__)
 _SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
 
 
@@ -264,6 +271,13 @@ class Settings(BaseSettings):
     LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None
     LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None
 
+    #
+    # Retry Policy
+    #
+    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = None
+    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = None  # default -> INFO
+    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR
+
     #
     # Telemetry and Debug
     #
@@ -283,6 +297,12 @@ class Settings(BaseSettings):
     CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
     OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
 
+    #
+    # Network
+    #
+    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05
+    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0
+
     ##############
     # Validators #
     ##############
@@ -401,6 +421,78 @@ class Settings(BaseSettings):
             return None
         return s.upper()
 
+    @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="before")
+    @classmethod
+    def _coerce_to_list(cls, v):
+        # works with JSON list, comma/space/semicolon separated, or real lists
+        return coerce_to_list(v, lower=True)
+
+    @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="after")
+    @classmethod
+    def _validate_sdk_provider_list(cls, v):
+        if v is None:
+            return None
+
+        normalized: list[str] = []
+        star = False
+
+        for item in v:
+            s = str(item).strip()
+            if not s:
+                continue
+            if s == "*":
+                star = True
+                continue
+            s = slugify(s)
+            if s in SUPPORTED_PROVIDER_SLUGS:
+                normalized.append(s)
+            else:
+                if cls.DEEPEVAL_VERBOSE_MODE:
+                    logger.warning("Unknown provider slug %r dropped", item)
+
+        if star:
+            return ["*"]
+
+        # It is important to dedup after normalization to catch variants
+        normalized = dedupe_preserve_order(normalized)
+        return normalized or None
+
+    @field_validator(
+        "DEEPEVAL_RETRY_BEFORE_LOG_LEVEL",
+        "DEEPEVAL_RETRY_AFTER_LOG_LEVEL",
+        mode="before",
+    )
+    @classmethod
+    def _coerce_log_level(cls, v):
+        if v is None:
+            return None
+        if isinstance(v, (int, float)):
+            return int(v)
+
+        s = str(v).strip().upper()
+        if not s:
+            return None
+
+        import logging
+
+        # Accept standard names or numeric strings
+        name_to_level = {
+            "CRITICAL": logging.CRITICAL,
+            "ERROR": logging.ERROR,
+            "WARNING": logging.WARNING,
+            "INFO": logging.INFO,
+            "DEBUG": logging.DEBUG,
+            "NOTSET": logging.NOTSET,
+        }
+        if s.isdigit() or (s.startswith("-") and s[1:].isdigit()):
+            return int(s)
+        if s in name_to_level:
+            return name_to_level[s]
+        raise ValueError(
+            "Retry log level must be one of DEBUG, INFO, WARNING, ERROR, "
+            "CRITICAL, NOTSET, or a numeric logging level."
+        )
+
     #######################
     # Persistence support #
     #######################

deepeval/config/utils.py

@@ -1,8 +1,13 @@
+import json
 import os
-from typing import Any, Optional
+import re
+
+from typing import Any, Iterable, List, Optional
+
 
 _TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on", "enable", "enabled"})
 _FALSY = frozenset({"0", "false", "f", "no", "n", "off", "disable", "disabled"})
+_LIST_SEP_RE = re.compile(r"[,\s;]+")
 
 
 def parse_bool(value: Any, default: bool = False) -> bool:
@@ -84,3 +89,51 @@ def set_env_bool(key: str, value: Optional[bool] = False) -> None:
         - Use `get_env_bool` to read back and parse the value safely.
     """
     os.environ[key] = bool_to_env_str(bool(value))
+
+
+def coerce_to_list(
+    v,
+    *,
+    lower: bool = False,
+    allow_json: bool = True,
+    sep_re: re.Pattern = _LIST_SEP_RE,
+) -> Optional[List[str]]:
+    """
+    Coerce None / str / list / tuple / set into a clean List[str].
+    - Accepts JSON arrays ("[...]"") or delimited strings (comma/space/semicolon).
+    - Strips whitespace, drops empties, optionally lowercases.
+    """
+    if v is None:
+        return None
+    if isinstance(v, (list, tuple, set)):
+        items = list(v)
+    else:
+        s = str(v).strip()
+        if not s:
+            return None
+        if allow_json and s.startswith("[") and s.endswith("]"):
+            try:
+                parsed = json.loads(s)
+                items = parsed if isinstance(parsed, list) else [s]
+            except Exception:
+                items = sep_re.split(s)
+        else:
+            items = sep_re.split(s)
+
+    out: List[str] = []
+    for item in items:
+        s = str(item).strip()
+        if not s:
+            continue
+        out.append(s.lower() if lower else s)
+    return out or None
+
+
+def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
+    seen = set()
+    out: List[str] = []
+    for x in items:
+        if x not in seen:
+            seen.add(x)
+            out.append(x)
+    return out

deepeval/constants.py

@@ -1,3 +1,5 @@
+from enum import Enum
+
 KEY_FILE: str = ".deepeval"
 HIDDEN_DIR: str = ".deepeval"
 PYTEST_RUN_TEST_NAME: str = "CONFIDENT_AI_RUN_TEST_NAME"
@@ -11,3 +13,28 @@ CONFIDENT_TRACE_ENVIRONMENT = "CONFIDENT_TRACE_ENVIRONMENT"
 CONFIDENT_TRACING_ENABLED = "CONFIDENT_TRACING_ENABLED"
 CONFIDENT_OPEN_BROWSER = "CONFIDENT_OPEN_BROWSER"
 CONFIDENT_TEST_CASE_BATCH_SIZE = "CONFIDENT_TEST_CASE_BATCH_SIZE"
+
+
+class ProviderSlug(str, Enum):
+    OPENAI = "openai"
+    AZURE = "azure"
+    ANTHROPIC = "anthropic"
+    BEDROCK = "bedrock"
+    DEEPSEEK = "deepseek"
+    GOOGLE = "google"
+    GROK = "grok"
+    KIMI = "kimi"
+    LITELLM = "litellm"
+    LOCAL = "local"
+    OLLAMA = "ollama"
+
+
+def slugify(value: str | ProviderSlug) -> str:
+    return (
+        value.value
+        if isinstance(value, ProviderSlug)
+        else str(value).strip().lower()
+    )
+
+
+SUPPORTED_PROVIDER_SLUGS = frozenset(s.value for s in ProviderSlug)

deepeval/metrics/pii_leakage/pii_leakage.py

@@ -284,7 +284,7 @@ class PIILeakageMetric(BaseMetric):
                 no_privacy_count += 1
 
         score = no_privacy_count / number_of_verdicts
-        return 1 if self.strict_mode and score < 1 else score
+        return 0 if self.strict_mode and score < self.threshold else score
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/models/embedding_models/azure_embedding_model.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import Dict, List
 from openai import AzureOpenAI, AsyncAzureOpenAI
 from deepeval.key_handler import (
     EmbeddingKeyValues,
@@ -6,10 +6,18 @@ from deepeval.key_handler import (
     KEY_FILE_HANDLER,
 )
 from deepeval.models import DeepEvalBaseEmbeddingModel
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_azure = create_retry_decorator(PS.AZURE)
 
 
 class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
-    def __init__(self):
+    def __init__(self, **kwargs):
         self.azure_openai_api_key = KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.AZURE_OPENAI_API_KEY
         )
@@ -23,7 +31,9 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
             ModelKeyValues.AZURE_OPENAI_ENDPOINT
         )
         self.model_name = self.azure_embedding_deployment
+        self.kwargs = kwargs
 
+    @retry_azure
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
@@ -32,6 +42,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response.data[0].embedding
 
+    @retry_azure
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
@@ -40,6 +51,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return [item.embedding for item in response.data]
 
+    @retry_azure
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
@@ -48,6 +60,7 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response.data[0].embedding
 
+    @retry_azure
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
@@ -61,15 +74,33 @@ class AzureOpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
 
     def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return AzureOpenAI(
-                api_key=self.azure_openai_api_key,
-                api_version=self.openai_api_version,
-                azure_endpoint=self.azure_endpoint,
-                azure_deployment=self.azure_embedding_deployment,
-            )
-        return AsyncAzureOpenAI(
+            return self._build_client(AzureOpenAI)
+        return self._build_client(AsyncAzureOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'azure' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.AZURE):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
             api_key=self.azure_openai_api_key,
             api_version=self.openai_api_version,
             azure_endpoint=self.azure_endpoint,
             azure_deployment=self.azure_embedding_deployment,
+            **self._client_kwargs(),
         )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/embedding_models/local_embedding_model.py

@@ -1,12 +1,21 @@
-from openai import OpenAI
-from typing import List
+from openai import OpenAI, AsyncOpenAI
+from typing import Dict, List
 
 from deepeval.key_handler import EmbeddingKeyValues, KEY_FILE_HANDLER
 from deepeval.models import DeepEvalBaseEmbeddingModel
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
+from deepeval.constants import ProviderSlug as PS
+
+
+# consistent retry rules
+retry_local = create_retry_decorator(PS.LOCAL)
 
 
 class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, **kwargs):
         self.base_url = KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_BASE_URL
         )
@@ -16,13 +25,10 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
         self.api_key = KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_API_KEY
         )
-        self.args = args
         self.kwargs = kwargs
         super().__init__(model_name)
 
-    def load_model(self):
-        return OpenAI(base_url=self.base_url, api_key=self.api_key)
-
+    @retry_local
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
@@ -31,6 +37,7 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response.data[0].embedding
 
+    @retry_local
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embeddings.create(
@@ -39,21 +46,57 @@ class LocalEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return [data.embedding for data in response.data]
 
+    @retry_local
     async def a_embed_text(self, text: str) -> List[float]:
-        embedding_model = self.load_model()
+        embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
             model=self.model_name,
             input=[text],
         )
         return response.data[0].embedding
 
+    @retry_local
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
-        embedding_model = self.load_model()
+        embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embeddings.create(
             model=self.model_name,
             input=texts,
         )
         return [data.embedding for data in response.data]
 
+    ###############################################
+    # Model
+    ###############################################
+
     def get_model_name(self):
         return self.model_name
+
+    def load_model(self, async_mode: bool = False):
+        if not async_mode:
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
+
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity manages retries, turn off OpenAI SDK retries to avoid double retrying.
+        If users opt into SDK retries via DEEPEVAL_SDK_RETRY_PROVIDERS=local, leave them enabled.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.LOCAL):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self.api_key,
+            base_url=self.base_url,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # Older OpenAI SDKs may not accept max_retries; drop and retry once.
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise

deepeval/models/embedding_models/ollama_embedding_model.py

@@ -3,6 +3,13 @@ from typing import List
 
 from deepeval.key_handler import EmbeddingKeyValues, KEY_FILE_HANDLER
 from deepeval.models import DeepEvalBaseEmbeddingModel
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+)
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_ollama = create_retry_decorator(PS.OLLAMA)
 
 
 class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
@@ -13,6 +20,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         model_name = KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_MODEL_NAME
         )
+        # TODO: This is not being used. Clean it up in consistency PR
         self.api_key = KEY_FILE_HANDLER.fetch_data(
             EmbeddingKeyValues.LOCAL_EMBEDDING_API_KEY
         )
@@ -20,12 +28,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         self.kwargs = kwargs
         super().__init__(model_name)
 
-    def load_model(self, async_mode: bool = False):
-        if not async_mode:
-            return Client(host=self.base_url)
-
-        return AsyncClient(host=self.base_url)
-
+    @retry_ollama
     def embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
@@ -34,6 +37,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response["embeddings"][0]
 
+    @retry_ollama
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model()
         response = embedding_model.embed(
@@ -42,6 +46,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response["embeddings"]
 
+    @retry_ollama
     async def a_embed_text(self, text: str) -> List[float]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
@@ -50,6 +55,7 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response["embeddings"][0]
 
+    @retry_ollama
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         embedding_model = self.load_model(async_mode=True)
         response = await embedding_model.embed(
@@ -58,5 +64,17 @@ class OllamaEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response["embeddings"]
 
+    ###############################################
+    # Model
+    ###############################################
+
+    def load_model(self, async_mode: bool = False):
+        if not async_mode:
+            return self._build_client(Client)
+        return self._build_client(AsyncClient)
+
+    def _build_client(self, cls):
+        return cls(host=self.base_url, **self.kwargs)
+
     def get_model_name(self):
-        return self.model_name
+        return f"{self.model_name} (Ollama)"

deepeval/models/embedding_models/openai_embedding_model.py

@@ -1,6 +1,14 @@
-from typing import Optional, List
+from typing import Dict, Optional, List
 from openai import OpenAI, AsyncOpenAI
 from deepeval.models import DeepEvalBaseEmbeddingModel
+from deepeval.models.retry_policy import (
+    create_retry_decorator,
+    sdk_retries_for,
+)
+from deepeval.constants import ProviderSlug as PS
+
+
+retry_openai = create_retry_decorator(PS.OPENAI)
 
 valid_openai_embedding_models = [
     "text-embedding-3-small",
@@ -15,6 +23,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         self,
         model: Optional[str] = None,
         _openai_api_key: Optional[str] = None,
+        **kwargs,
     ):
         model_name = model if model else default_openai_embedding_model
         if model_name not in valid_openai_embedding_models:
@@ -23,7 +32,9 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
             )
         self._openai_api_key = _openai_api_key
         self.model_name = model_name
+        self.kwargs = kwargs
 
+    @retry_openai
     def embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
@@ -32,6 +43,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response.data[0].embedding
 
+    @retry_openai
     def embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=False)
         response = client.embeddings.create(
@@ -40,6 +52,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return [item.embedding for item in response.data]
 
+    @retry_openai
     async def a_embed_text(self, text: str) -> List[float]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
@@ -48,6 +61,7 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return response.data[0].embedding
 
+    @retry_openai
     async def a_embed_texts(self, texts: List[str]) -> List[List[float]]:
         client = self.load_model(async_mode=True)
         response = await client.embeddings.create(
@@ -56,11 +70,39 @@ class OpenAIEmbeddingModel(DeepEvalBaseEmbeddingModel):
         )
         return [item.embedding for item in response.data]
 
-    def get_model_name(self) -> str:
+    ###############################################
+    # Model
+    ###############################################
+
+    def get_model_name(self):
         return self.model_name
 
-    def load_model(self, async_mode: bool):
+    def load_model(self, async_mode: bool = False):
         if not async_mode:
-            return OpenAI(api_key=self._openai_api_key)
+            return self._build_client(OpenAI)
+        return self._build_client(AsyncOpenAI)
 
-        return AsyncOpenAI(api_key=self._openai_api_key)
+    def _client_kwargs(self) -> Dict:
+        """
+        If Tenacity is managing retries, force OpenAI SDK retries off to avoid double retries.
+        If the user opts into SDK retries for 'openai' via DEEPEVAL_SDK_RETRY_PROVIDERS,
+        leave their retry settings as is.
+        """
+        kwargs = dict(self.kwargs or {})
+        if not sdk_retries_for(PS.OPENAI):
+            kwargs["max_retries"] = 0
+        return kwargs
+
+    def _build_client(self, cls):
+        kw = dict(
+            api_key=self._openai_api_key,
+            **self._client_kwargs(),
+        )
+        try:
+            return cls(**kw)
+        except TypeError as e:
+            # older OpenAI SDKs may not accept max_retries, in that case remove and retry once
+            if "max_retries" in str(e):
+                kw.pop("max_retries", None)
+                return cls(**kw)
+            raise