deepeval 3.5.2__py3-none-any.whl → 3.5.4__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.5.2"
+__version__: str = "3.5.4"

deepeval/config/settings.py

@@ -9,6 +9,7 @@ Central config for DeepEval.
   type coercion.
 """
 
+import logging
 import os
 import re
 
@@ -16,11 +17,17 @@ from dotenv import dotenv_values
 from pathlib import Path
 from pydantic import AnyUrl, SecretStr, field_validator, confloat
 from pydantic_settings import BaseSettings, SettingsConfigDict
-from typing import Any, Dict, Optional, NamedTuple
+from typing import Any, Dict, List, Optional, NamedTuple
 
-from deepeval.config.utils import parse_bool
+from deepeval.config.utils import (
+    parse_bool,
+    coerce_to_list,
+    dedupe_preserve_order,
+)
+from deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify
 
 
+logger = logging.getLogger(__name__)
 _SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
 
 
@@ -264,6 +271,13 @@ class Settings(BaseSettings):
     LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None
     LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None
 
+    #
+    # Retry Policy
+    #
+    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = None
+    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = None  # default -> INFO
+    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR
+
     #
     # Telemetry and Debug
     #
@@ -283,6 +297,12 @@ class Settings(BaseSettings):
     CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
     OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
 
+    #
+    # Network
+    #
+    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05
+    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0
+
     ##############
     # Validators #
     ##############
@@ -401,6 +421,78 @@ class Settings(BaseSettings):
             return None
         return s.upper()
 
+    @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="before")
+    @classmethod
+    def _coerce_to_list(cls, v):
+        # works with JSON list, comma/space/semicolon separated, or real lists
+        return coerce_to_list(v, lower=True)
+
+    @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="after")
+    @classmethod
+    def _validate_sdk_provider_list(cls, v):
+        if v is None:
+            return None
+
+        normalized: list[str] = []
+        star = False
+
+        for item in v:
+            s = str(item).strip()
+            if not s:
+                continue
+            if s == "*":
+                star = True
+                continue
+            s = slugify(s)
+            if s in SUPPORTED_PROVIDER_SLUGS:
+                normalized.append(s)
+            else:
+                if cls.DEEPEVAL_VERBOSE_MODE:
+                    logger.warning("Unknown provider slug %r dropped", item)
+
+        if star:
+            return ["*"]
+
+        # It is important to dedup after normalization to catch variants
+        normalized = dedupe_preserve_order(normalized)
+        return normalized or None
+
+    @field_validator(
+        "DEEPEVAL_RETRY_BEFORE_LOG_LEVEL",
+        "DEEPEVAL_RETRY_AFTER_LOG_LEVEL",
+        mode="before",
+    )
+    @classmethod
+    def _coerce_log_level(cls, v):
+        if v is None:
+            return None
+        if isinstance(v, (int, float)):
+            return int(v)
+
+        s = str(v).strip().upper()
+        if not s:
+            return None
+
+        import logging
+
+        # Accept standard names or numeric strings
+        name_to_level = {
+            "CRITICAL": logging.CRITICAL,
+            "ERROR": logging.ERROR,
+            "WARNING": logging.WARNING,
+            "INFO": logging.INFO,
+            "DEBUG": logging.DEBUG,
+            "NOTSET": logging.NOTSET,
+        }
+        if s.isdigit() or (s.startswith("-") and s[1:].isdigit()):
+            return int(s)
+        if s in name_to_level:
+            return name_to_level[s]
+        raise ValueError(
+            "Retry log level must be one of DEBUG, INFO, WARNING, ERROR, "
+            "CRITICAL, NOTSET, or a numeric logging level."
+        )
+
     #######################
     # Persistence support #
     #######################

deepeval/config/utils.py

@@ -1,8 +1,13 @@
+import json
 import os
-from typing import Any, Optional
+import re
+
+from typing import Any, Iterable, List, Optional
+
 
 _TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on", "enable", "enabled"})
 _FALSY = frozenset({"0", "false", "f", "no", "n", "off", "disable", "disabled"})
+_LIST_SEP_RE = re.compile(r"[,\s;]+")
 
 
 def parse_bool(value: Any, default: bool = False) -> bool:
@@ -84,3 +89,51 @@ def set_env_bool(key: str, value: Optional[bool] = False) -> None:
         - Use `get_env_bool` to read back and parse the value safely.
     """
     os.environ[key] = bool_to_env_str(bool(value))
+
+
+def coerce_to_list(
+    v,
+    *,
+    lower: bool = False,
+    allow_json: bool = True,
+    sep_re: re.Pattern = _LIST_SEP_RE,
+) -> Optional[List[str]]:
+    """
+    Coerce None / str / list / tuple / set into a clean List[str].
+    - Accepts JSON arrays ("[...]"") or delimited strings (comma/space/semicolon).
+    - Strips whitespace, drops empties, optionally lowercases.
+    """
+    if v is None:
+        return None
+    if isinstance(v, (list, tuple, set)):
+        items = list(v)
+    else:
+        s = str(v).strip()
+        if not s:
+            return None
+        if allow_json and s.startswith("[") and s.endswith("]"):
+            try:
+                parsed = json.loads(s)
+                items = parsed if isinstance(parsed, list) else [s]
+            except Exception:
+                items = sep_re.split(s)
+        else:
+            items = sep_re.split(s)
+
+    out: List[str] = []
+    for item in items:
+        s = str(item).strip()
+        if not s:
+            continue
+        out.append(s.lower() if lower else s)
+    return out or None
+
+
+def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
+    seen = set()
+    out: List[str] = []
+    for x in items:
+        if x not in seen:
+            seen.add(x)
+            out.append(x)
+    return out

deepeval/constants.py

@@ -1,3 +1,5 @@
+from enum import Enum
+
 KEY_FILE: str = ".deepeval"
 HIDDEN_DIR: str = ".deepeval"
 PYTEST_RUN_TEST_NAME: str = "CONFIDENT_AI_RUN_TEST_NAME"
@@ -11,3 +13,28 @@ CONFIDENT_TRACE_ENVIRONMENT = "CONFIDENT_TRACE_ENVIRONMENT"
 CONFIDENT_TRACING_ENABLED = "CONFIDENT_TRACING_ENABLED"
 CONFIDENT_OPEN_BROWSER = "CONFIDENT_OPEN_BROWSER"
 CONFIDENT_TEST_CASE_BATCH_SIZE = "CONFIDENT_TEST_CASE_BATCH_SIZE"
+
+
+class ProviderSlug(str, Enum):
+    OPENAI = "openai"
+    AZURE = "azure"
+    ANTHROPIC = "anthropic"
+    BEDROCK = "bedrock"
+    DEEPSEEK = "deepseek"
+    GOOGLE = "google"
+    GROK = "grok"
+    KIMI = "kimi"
+    LITELLM = "litellm"
+    LOCAL = "local"
+    OLLAMA = "ollama"
+
+
+def slugify(value: str | ProviderSlug) -> str:
+    return (
+        value.value
+        if isinstance(value, ProviderSlug)
+        else str(value).strip().lower()
+    )
+
+
+SUPPORTED_PROVIDER_SLUGS = frozenset(s.value for s in ProviderSlug)

deepeval/integrations/pydantic_ai/__init__.py

@@ -1,3 +1,5 @@
+from .agent import DeepEvalPydanticAIAgent as Agent
 from .patcher import instrument as instrument_pydantic_ai
+from .otel import instrument_pydantic_ai as otel_instrument_pydantic_ai
 
-__all__ = ["instrument_pydantic_ai"]
+__all__ = ["instrument_pydantic_ai", "Agent", otel_instrument_pydantic_ai]

deepeval/integrations/pydantic_ai/agent.py

@@ -0,0 +1,339 @@
+import inspect
+from typing import Optional, List, Generic, TypeVar
+from contextvars import ContextVar
+from contextlib import asynccontextmanager
+
+from deepeval.prompt import Prompt
+from deepeval.tracing.types import AgentSpan
+from deepeval.tracing.tracing import Observer
+from deepeval.metrics.base_metric import BaseMetric
+from deepeval.tracing.context import current_span_context
+from deepeval.integrations.pydantic_ai.utils import extract_tools_called
+
+try:
+    from pydantic_ai.agent import Agent
+    from pydantic_ai.tools import AgentDepsT
+    from pydantic_ai.output import OutputDataT
+    from deepeval.integrations.pydantic_ai.utils import (
+        create_patched_tool,
+        update_trace_context,
+        patch_llm_model,
+    )
+
+    is_pydantic_ai_installed = True
+except:
+    is_pydantic_ai_installed = False
+
+
+def pydantic_ai_installed():
+    if not is_pydantic_ai_installed:
+        raise ImportError(
+            "Pydantic AI is not installed. Please install it with `pip install pydantic-ai`."
+        )
+
+
+_IS_RUN_SYNC = ContextVar("deepeval_is_run_sync", default=False)
+
+
+class DeepEvalPydanticAIAgent(
+    Agent[AgentDepsT, OutputDataT], Generic[AgentDepsT, OutputDataT]
+):
+
+    trace_name: Optional[str] = None
+    trace_tags: Optional[List[str]] = None
+    trace_metadata: Optional[dict] = None
+    trace_thread_id: Optional[str] = None
+    trace_user_id: Optional[str] = None
+    trace_metric_collection: Optional[str] = None
+    trace_metrics: Optional[List[BaseMetric]] = None
+
+    llm_prompt: Optional[Prompt] = None
+    llm_metrics: Optional[List[BaseMetric]] = None
+    llm_metric_collection: Optional[str] = None
+
+    agent_metrics: Optional[List[BaseMetric]] = None
+    agent_metric_collection: Optional[str] = None
+
+    def __init__(
+        self,
+        *args,
+        trace_name: Optional[str] = None,
+        trace_tags: Optional[List[str]] = None,
+        trace_metadata: Optional[dict] = None,
+        trace_thread_id: Optional[str] = None,
+        trace_user_id: Optional[str] = None,
+        trace_metric_collection: Optional[str] = None,
+        trace_metrics: Optional[List[BaseMetric]] = None,
+        llm_metric_collection: Optional[str] = None,
+        llm_metrics: Optional[List[BaseMetric]] = None,
+        llm_prompt: Optional[Prompt] = None,
+        agent_metric_collection: Optional[str] = None,
+        agent_metrics: Optional[List[BaseMetric]] = None,
+        **kwargs
+    ):
+        pydantic_ai_installed()
+
+        self.trace_name = trace_name
+        self.trace_tags = trace_tags
+        self.trace_metadata = trace_metadata
+        self.trace_thread_id = trace_thread_id
+        self.trace_user_id = trace_user_id
+        self.trace_metric_collection = trace_metric_collection
+        self.trace_metrics = trace_metrics
+
+        self.llm_metric_collection = llm_metric_collection
+        self.llm_metrics = llm_metrics
+        self.llm_prompt = llm_prompt
+
+        self.agent_metric_collection = agent_metric_collection
+        self.agent_metrics = agent_metrics
+
+        super().__init__(*args, **kwargs)
+
+        patch_llm_model(
+            self._model, llm_metric_collection, llm_metrics, llm_prompt
+        )  # TODO: Add dual patch guards
+
+    async def run(
+        self,
+        *args,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        user_id: Optional[str] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        **kwargs
+    ):
+        sig = inspect.signature(super().run)
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        input = bound.arguments.get("user_prompt", None)
+
+        agent_name = super().name if super().name is not None else "Agent"
+
+        with Observer(
+            span_type="agent" if not _IS_RUN_SYNC.get() else "custom",
+            func_name=agent_name if not _IS_RUN_SYNC.get() else "run",
+            function_kwargs={"input": input},
+            metrics=self.agent_metrics if not _IS_RUN_SYNC.get() else None,
+            metric_collection=(
+                self.agent_metric_collection if not _IS_RUN_SYNC.get() else None
+            ),
+        ) as observer:
+            result = await super().run(*args, **kwargs)
+            observer.result = result.output
+            update_trace_context(
+                trace_name=name if name is not None else self.trace_name,
+                trace_tags=tags if tags is not None else self.trace_tags,
+                trace_metadata=(
+                    metadata if metadata is not None else self.trace_metadata
+                ),
+                trace_thread_id=(
+                    thread_id if thread_id is not None else self.trace_thread_id
+                ),
+                trace_user_id=(
+                    user_id if user_id is not None else self.trace_user_id
+                ),
+                trace_metric_collection=(
+                    metric_collection
+                    if metric_collection is not None
+                    else self.trace_metric_collection
+                ),
+                trace_metrics=(
+                    metrics if metrics is not None else self.trace_metrics
+                ),
+                trace_input=input,
+                trace_output=result.output,
+            )
+
+            agent_span: AgentSpan = current_span_context.get()
+            try:
+                agent_span.tools_called = extract_tools_called(result)
+            except:
+                pass
+            # TODO: available tools
+            # TODO: agent handoffs
+
+        return result
+
+    def run_sync(
+        self,
+        *args,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metric_collection: Optional[str] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        **kwargs
+    ):
+        sig = inspect.signature(super().run_sync)
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+        input = bound.arguments.get("user_prompt", None)
+
+        token = _IS_RUN_SYNC.set(True)
+
+        agent_name = super().name if super().name is not None else "Agent"
+
+        with Observer(
+            span_type="agent",
+            func_name=agent_name,
+            function_kwargs={"input": input},
+            metrics=self.agent_metrics,
+            metric_collection=self.agent_metric_collection,
+        ) as observer:
+            try:
+                result = super().run_sync(*args, **kwargs)
+            finally:
+                _IS_RUN_SYNC.reset(token)
+
+            observer.result = result.output
+            update_trace_context(
+                trace_name=name if name is not None else self.trace_name,
+                trace_tags=tags if tags is not None else self.trace_tags,
+                trace_metadata=(
+                    metadata if metadata is not None else self.trace_metadata
+                ),
+                trace_thread_id=(
+                    thread_id if thread_id is not None else self.trace_thread_id
+                ),
+                trace_user_id=(
+                    user_id if user_id is not None else self.trace_user_id
+                ),
+                trace_metric_collection=(
+                    metric_collection
+                    if metric_collection is not None
+                    else self.trace_metric_collection
+                ),
+                trace_metrics=(
+                    metrics if metrics is not None else self.trace_metrics
+                ),
+                trace_input=input,
+                trace_output=result.output,
+            )
+
+            agent_span: AgentSpan = current_span_context.get()
+            try:
+                agent_span.tools_called = extract_tools_called(result)
+            except:
+                pass
+
+            # TODO: available tools
+            # TODO: agent handoffs
+
+        return result
+
+    @asynccontextmanager
+    async def run_stream(
+        self,
+        *args,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[dict] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metric_collection: Optional[str] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        **kwargs
+    ):
+        sig = inspect.signature(super().run_stream)
+        super_params = sig.parameters
+        super_kwargs = {k: v for k, v in kwargs.items() if k in super_params}
+        bound = sig.bind_partial(*args, **super_kwargs)
+        bound.apply_defaults()
+        input = bound.arguments.get("user_prompt", None)
+
+        agent_name = super().name if super().name is not None else "Agent"
+
+        with Observer(
+            span_type="agent",
+            func_name=agent_name,
+            function_kwargs={"input": input},
+            metrics=self.agent_metrics,
+            metric_collection=self.agent_metric_collection,
+        ) as observer:
+            final_result = None
+            async with super().run_stream(*args, **super_kwargs) as result:
+                try:
+                    yield result
+                finally:
+                    try:
+                        final_result = await result.get_output()
+                        observer.result = final_result
+                    except Exception:
+                        pass
+
+                    update_trace_context(
+                        trace_name=(
+                            name if name is not None else self.trace_name
+                        ),
+                        trace_tags=(
+                            tags if tags is not None else self.trace_tags
+                        ),
+                        trace_metadata=(
+                            metadata
+                            if metadata is not None
+                            else self.trace_metadata
+                        ),
+                        trace_thread_id=(
+                            thread_id
+                            if thread_id is not None
+                            else self.trace_thread_id
+                        ),
+                        trace_user_id=(
+                            user_id
+                            if user_id is not None
+                            else self.trace_user_id
+                        ),
+                        trace_metric_collection=(
+                            metric_collection
+                            if metric_collection is not None
+                            else self.trace_metric_collection
+                        ),
+                        trace_metrics=(
+                            metrics
+                            if metrics is not None
+                            else self.trace_metrics
+                        ),
+                        trace_input=input,
+                        trace_output=(
+                            final_result if final_result is not None else None
+                        ),
+                    )
+                    agent_span: AgentSpan = current_span_context.get()
+                    try:
+                        if final_result is not None:
+                            agent_span.tools_called = extract_tools_called(
+                                final_result
+                            )
+                    except:
+                        pass
+
+    def tool(
+        self,
+        *args,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+        **kwargs
+    ):
+        # Direct decoration: @agent.tool
+        if args and callable(args[0]):
+            patched_func = create_patched_tool(
+                args[0], metrics, metric_collection
+            )
+            new_args = (patched_func,) + args[1:]
+            return super(DeepEvalPydanticAIAgent, self).tool(
+                *new_args, **kwargs
+            )
+        # Decoration with args: @agent.tool(...)
+        super_tool = super(DeepEvalPydanticAIAgent, self).tool
+
+        def decorator(func):
+            patched_func = create_patched_tool(func, metrics, metric_collection)
+            return super_tool(*args, **kwargs)(patched_func)
+
+        return decorator