deepeval 3.5.1__tar.gz → 3.5.3__tar.gz

{deepeval-3.5.1 → deepeval-3.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.1
+Version: 3.5.3
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.5.3/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.5.3"

{deepeval-3.5.1 → deepeval-3.5.3}/deepeval/config/settings.py

@@ -9,6 +9,7 @@ Central config for DeepEval.
   type coercion.
 """
 
+import logging
 import os
 import re
 
@@ -16,11 +17,17 @@ from dotenv import dotenv_values
 from pathlib import Path
 from pydantic import AnyUrl, SecretStr, field_validator, confloat
 from pydantic_settings import BaseSettings, SettingsConfigDict
-from typing import Any, Dict, Optional, NamedTuple
+from typing import Any, Dict, List, Optional, NamedTuple
 
-from deepeval.config.utils import parse_bool
+from deepeval.config.utils import (
+    parse_bool,
+    coerce_to_list,
+    dedupe_preserve_order,
+)
+from deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify
 
 
+logger = logging.getLogger(__name__)
 _SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
 
 
@@ -264,6 +271,13 @@ class Settings(BaseSettings):
     LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None
     LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None
 
+    #
+    # Retry Policy
+    #
+    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = None
+    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = None  # default -> INFO
+    DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR
+
     #
     # Telemetry and Debug
     #
@@ -283,6 +297,12 @@ class Settings(BaseSettings):
     CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
     OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
 
+    #
+    # Network
+    #
+    MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05
+    MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0
+
     ##############
     # Validators #
     ##############
@@ -401,6 +421,78 @@ class Settings(BaseSettings):
             return None
         return s.upper()
 
+    @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="before")
+    @classmethod
+    def _coerce_to_list(cls, v):
+        # works with JSON list, comma/space/semicolon separated, or real lists
+        return coerce_to_list(v, lower=True)
+
+    @field_validator("DEEPEVAL_SDK_RETRY_PROVIDERS", mode="after")
+    @classmethod
+    def _validate_sdk_provider_list(cls, v):
+        if v is None:
+            return None
+
+        normalized: list[str] = []
+        star = False
+
+        for item in v:
+            s = str(item).strip()
+            if not s:
+                continue
+            if s == "*":
+                star = True
+                continue
+            s = slugify(s)
+            if s in SUPPORTED_PROVIDER_SLUGS:
+                normalized.append(s)
+            else:
+                if cls.DEEPEVAL_VERBOSE_MODE:
+                    logger.warning("Unknown provider slug %r dropped", item)
+
+        if star:
+            return ["*"]
+
+        # It is important to dedup after normalization to catch variants
+        normalized = dedupe_preserve_order(normalized)
+        return normalized or None
+
+    @field_validator(
+        "DEEPEVAL_RETRY_BEFORE_LOG_LEVEL",
+        "DEEPEVAL_RETRY_AFTER_LOG_LEVEL",
+        mode="before",
+    )
+    @classmethod
+    def _coerce_log_level(cls, v):
+        if v is None:
+            return None
+        if isinstance(v, (int, float)):
+            return int(v)
+
+        s = str(v).strip().upper()
+        if not s:
+            return None
+
+        import logging
+
+        # Accept standard names or numeric strings
+        name_to_level = {
+            "CRITICAL": logging.CRITICAL,
+            "ERROR": logging.ERROR,
+            "WARNING": logging.WARNING,
+            "INFO": logging.INFO,
+            "DEBUG": logging.DEBUG,
+            "NOTSET": logging.NOTSET,
+        }
+        if s.isdigit() or (s.startswith("-") and s[1:].isdigit()):
+            return int(s)
+        if s in name_to_level:
+            return name_to_level[s]
+        raise ValueError(
+            "Retry log level must be one of DEBUG, INFO, WARNING, ERROR, "
+            "CRITICAL, NOTSET, or a numeric logging level."
+        )
+
     #######################
     # Persistence support #
     #######################

{deepeval-3.5.1 → deepeval-3.5.3}/deepeval/config/utils.py

@@ -1,8 +1,13 @@
+import json
 import os
-from typing import Any, Optional
+import re
+
+from typing import Any, Iterable, List, Optional
+
 
 _TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on", "enable", "enabled"})
 _FALSY = frozenset({"0", "false", "f", "no", "n", "off", "disable", "disabled"})
+_LIST_SEP_RE = re.compile(r"[,\s;]+")
 
 
 def parse_bool(value: Any, default: bool = False) -> bool:
@@ -84,3 +89,51 @@ def set_env_bool(key: str, value: Optional[bool] = False) -> None:
         - Use `get_env_bool` to read back and parse the value safely.
     """
     os.environ[key] = bool_to_env_str(bool(value))
+
+
+def coerce_to_list(
+    v,
+    *,
+    lower: bool = False,
+    allow_json: bool = True,
+    sep_re: re.Pattern = _LIST_SEP_RE,
+) -> Optional[List[str]]:
+    """
+    Coerce None / str / list / tuple / set into a clean List[str].
+    - Accepts JSON arrays ("[...]"") or delimited strings (comma/space/semicolon).
+    - Strips whitespace, drops empties, optionally lowercases.
+    """
+    if v is None:
+        return None
+    if isinstance(v, (list, tuple, set)):
+        items = list(v)
+    else:
+        s = str(v).strip()
+        if not s:
+            return None
+        if allow_json and s.startswith("[") and s.endswith("]"):
+            try:
+                parsed = json.loads(s)
+                items = parsed if isinstance(parsed, list) else [s]
+            except Exception:
+                items = sep_re.split(s)
+        else:
+            items = sep_re.split(s)
+
+    out: List[str] = []
+    for item in items:
+        s = str(item).strip()
+        if not s:
+            continue
+        out.append(s.lower() if lower else s)
+    return out or None
+
+
+def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
+    seen = set()
+    out: List[str] = []
+    for x in items:
+        if x not in seen:
+            seen.add(x)
+            out.append(x)
+    return out

{deepeval-3.5.1 → deepeval-3.5.3}/deepeval/constants.py

@@ -1,3 +1,5 @@
+from enum import Enum
+
 KEY_FILE: str = ".deepeval"
 HIDDEN_DIR: str = ".deepeval"
 PYTEST_RUN_TEST_NAME: str = "CONFIDENT_AI_RUN_TEST_NAME"
@@ -11,3 +13,28 @@ CONFIDENT_TRACE_ENVIRONMENT = "CONFIDENT_TRACE_ENVIRONMENT"
 CONFIDENT_TRACING_ENABLED = "CONFIDENT_TRACING_ENABLED"
 CONFIDENT_OPEN_BROWSER = "CONFIDENT_OPEN_BROWSER"
 CONFIDENT_TEST_CASE_BATCH_SIZE = "CONFIDENT_TEST_CASE_BATCH_SIZE"
+
+
+class ProviderSlug(str, Enum):
+    OPENAI = "openai"
+    AZURE = "azure"
+    ANTHROPIC = "anthropic"
+    BEDROCK = "bedrock"
+    DEEPSEEK = "deepseek"
+    GOOGLE = "google"
+    GROK = "grok"
+    KIMI = "kimi"
+    LITELLM = "litellm"
+    LOCAL = "local"
+    OLLAMA = "ollama"
+
+
+def slugify(value: str | ProviderSlug) -> str:
+    return (
+        value.value
+        if isinstance(value, ProviderSlug)
+        else str(value).strip().lower()
+    )
+
+
+SUPPORTED_PROVIDER_SLUGS = frozenset(s.value for s in ProviderSlug)

deepeval-3.5.3/deepeval/integrations/langchain/__init__.py

@@ -0,0 +1,4 @@
+from .callback import CallbackHandler, tool
+
+
+__all__ = ["CallbackHandler", "tool"]

deepeval-3.5.3/deepeval/integrations/langchain/callback.py

@@ -0,0 +1,345 @@
+from typing import Any, Optional, List, Dict
+from uuid import UUID
+from time import perf_counter
+from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.types import (
+    LlmOutput,
+    LlmToolCall,
+)
+from deepeval.metrics import BaseMetric
+
+try:
+    from langchain_core.callbacks.base import BaseCallbackHandler
+    from langchain_core.outputs import LLMResult
+    from langchain_core.outputs import ChatGeneration
+    from langchain_core.messages import AIMessage
+
+    # contains langchain imports
+    from deepeval.integrations.langchain.utils import (
+        parse_prompts_to_messages,
+        extract_name,
+        safe_extract_model_name,
+        safe_extract_token_usage,
+        enter_current_context,
+        exit_current_context,
+    )
+    from deepeval.integrations.langchain.patch import tool
+
+    langchain_installed = True
+except:
+    langchain_installed = False
+
+
+def is_langchain_installed():
+    if not langchain_installed:
+        raise ImportError(
+            "LangChain is not installed. Please install it with `pip install langchain`."
+        )
+
+
+from deepeval.tracing import trace_manager
+from deepeval.tracing.types import (
+    LlmSpan,
+    RetrieverSpan,
+    TraceSpanStatus,
+    ToolSpan,
+)
+from deepeval.telemetry import capture_tracing_integration
+
+
+class CallbackHandler(BaseCallbackHandler):
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metrics: Optional[List[BaseMetric]] = None,
+        metric_collection: Optional[str] = None,
+    ):
+        is_langchain_installed()
+        with capture_tracing_integration("langchain.callback.CallbackHandler"):
+            trace = trace_manager.start_new_trace()
+
+            self.trace_uuid = trace.uuid
+
+            trace.name = name
+            trace.tags = tags
+            trace.metadata = metadata
+            trace.thread_id = thread_id
+            trace.user_id = user_id
+            self.metrics = metrics
+            self.metric_collection = metric_collection
+            current_trace_context.set(trace)
+            super().__init__()
+
+    def on_chain_start(
+        self,
+        serialized: dict[str, Any],
+        inputs: dict[str, Any],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        if parent_run_id is None:
+            uuid_str = str(run_id)
+            base_span = enter_current_context(
+                uuid_str=uuid_str,
+                span_type="custom",
+                func_name=extract_name(serialized, **kwargs),
+            )
+            base_span.input = inputs
+            current_trace_context.get().input = inputs
+            base_span.metrics = self.metrics
+            base_span.metric_collection = self.metric_collection
+
+    def on_chain_end(
+        self,
+        output: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> Any:
+        uuid_str = str(run_id)
+        base_span = trace_manager.get_span_by_uuid(uuid_str)
+        if base_span:
+            base_span.output = output
+            current_trace_context.get().output = output
+            exit_current_context(uuid_str=uuid_str)
+
+    def on_llm_start(
+        self,
+        serialized: dict[str, Any],
+        prompts: list[str],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        uuid_str = str(run_id)
+        input_messages = parse_prompts_to_messages(prompts, **kwargs)
+        model = safe_extract_model_name(metadata, **kwargs)
+
+        llm_span: LlmSpan = enter_current_context(
+            uuid_str=uuid_str,
+            span_type="llm",
+            func_name=extract_name(serialized, **kwargs),
+        )
+
+        llm_span.input = input_messages
+        llm_span.model = model
+        metrics = metadata.pop("metrics", None)
+        metric_collection = metadata.pop("metric_collection", None)
+        prompt = metadata.pop("prompt", None)
+        llm_span.metrics = metrics
+        llm_span.metric_collection = metric_collection
+        llm_span.prompt = prompt
+
+    def on_llm_end(
+        self,
+        response: LLMResult,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,  # un-logged kwargs
+    ) -> Any:
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+
+        output = ""
+        total_input_tokens = 0
+        total_output_tokens = 0
+        model = None
+
+        for generation in response.generations:
+            for gen in generation:
+                if isinstance(gen, ChatGeneration):
+                    if gen.message.response_metadata and isinstance(
+                        gen.message.response_metadata, dict
+                    ):
+                        # extract model name from response_metadata
+                        model = gen.message.response_metadata.get("model_name")
+
+                        # extract input and output token
+                        input_tokens, output_tokens = safe_extract_token_usage(
+                            gen.message.response_metadata
+                        )
+                        total_input_tokens += input_tokens
+                        total_output_tokens += output_tokens
+
+                    if isinstance(gen.message, AIMessage):
+                        ai_message = gen.message
+                        tool_calls = []
+                        for tool_call in ai_message.tool_calls:
+                            tool_calls.append(
+                                LlmToolCall(
+                                    name=tool_call["name"],
+                                    args=tool_call["args"],
+                                    id=tool_call["id"],
+                                )
+                            )
+                        output = LlmOutput(
+                            role="AI",
+                            content=ai_message.content,
+                            tool_calls=tool_calls,
+                        )
+
+        llm_span.model = model if model else llm_span.model
+        llm_span.input = llm_span.input
+        llm_span.output = output
+        llm_span.input_token_count = (
+            total_input_tokens if total_input_tokens > 0 else None
+        )
+        llm_span.output_token_count = (
+            total_output_tokens if total_output_tokens > 0 else None
+        )
+
+        exit_current_context(uuid_str=uuid_str)
+
+    def on_llm_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,
+    ) -> Any:
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+        llm_span.status = TraceSpanStatus.ERRORED
+        llm_span.error = str(error)
+        exit_current_context(uuid_str=uuid_str)
+
+    def on_llm_new_token(
+        self,
+        token: str,
+        *,
+        chunk,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        **kwargs: Any,
+    ):
+        uuid_str = str(run_id)
+        llm_span: LlmSpan = trace_manager.get_span_by_uuid(uuid_str)
+        if llm_span.token_intervals is None:
+            llm_span.token_intervals = {perf_counter(): token}
+        else:
+            llm_span.token_intervals[perf_counter()] = token
+
+    def on_tool_start(
+        self,
+        serialized: dict[str, Any],
+        input_str: str,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        inputs: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        uuid_str = str(run_id)
+
+        tool_span = enter_current_context(
+            uuid_str=uuid_str,
+            span_type="tool",
+            func_name=extract_name(
+                serialized, **kwargs
+            ),  # ignored when setting the input
+        )
+        tool_span.input = inputs
+
+    def on_tool_end(
+        self,
+        output: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,  # un-logged kwargs
+    ) -> Any:
+
+        uuid_str = str(run_id)
+        tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
+        tool_span.output = output
+        exit_current_context(uuid_str=uuid_str)
+
+    def on_tool_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,  # un-logged kwargs
+    ) -> Any:
+        uuid_str = str(run_id)
+        tool_span: ToolSpan = trace_manager.get_span_by_uuid(uuid_str)
+        tool_span.status = TraceSpanStatus.ERRORED
+        tool_span.error = str(error)
+        exit_current_context(uuid_str=uuid_str)
+
+    def on_retriever_start(
+        self,
+        serialized: dict[str, Any],
+        query: str,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[list[str]] = None,
+        metadata: Optional[dict[str, Any]] = None,
+        **kwargs: Any,  # un-logged kwargs
+    ) -> Any:
+        uuid_str = str(run_id)
+        retriever_span = enter_current_context(
+            uuid_str=uuid_str,
+            span_type="retriever",
+            func_name=extract_name(serialized, **kwargs),
+            observe_kwargs={
+                "embedder": metadata.get("ls_embedding_provider", "unknown"),
+            },
+        )
+        retriever_span.input = query
+
+    def on_retriever_end(
+        self,
+        output: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,  # un-logged kwargs
+    ) -> Any:
+        uuid_str = str(run_id)
+        retriever_span: RetrieverSpan = trace_manager.get_span_by_uuid(uuid_str)
+
+        # prepare output
+        output_list = []
+        if isinstance(output, list):
+            for item in output:
+                output_list.append(str(item))
+        else:
+            output_list.append(str(output))
+
+        retriever_span.output = output_list
+        exit_current_context(uuid_str=uuid_str)
+
+    def on_retriever_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        **kwargs: Any,  # un-logged kwargs
+    ) -> Any:
+        uuid_str = str(run_id)
+        retriever_span: RetrieverSpan = trace_manager.get_span_by_uuid(uuid_str)
+        retriever_span.status = TraceSpanStatus.ERRORED
+        retriever_span.error = str(error)
+        exit_current_context(uuid_str=uuid_str)

deepeval-3.5.3/deepeval/integrations/langchain/patch.py

@@ -0,0 +1,43 @@
+import functools
+from deepeval.metrics import BaseMetric
+from deepeval.tracing.context import current_span_context
+from typing import List, Optional, Callable
+from langchain_core.tools import tool as original_tool, BaseTool
+
+
+def tool(
+    *args,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+    **kwargs
+):
+    """
+    Patched version of langchain_core.tools.tool that prints inputs and outputs
+    """
+
+    # original_tool returns a decorator function, so we need to return a decorator
+    def decorator(func: Callable) -> BaseTool:
+        func = _patch_tool_decorator(func, metrics, metric_collection)
+        tool_instance = original_tool(*args, **kwargs)(func)
+        return tool_instance
+
+    return decorator
+
+
+def _patch_tool_decorator(
+    func: Callable,
+    metrics: Optional[List[BaseMetric]] = None,
+    metric_collection: Optional[str] = None,
+):
+    original_func = func
+
+    @functools.wraps(original_func)
+    def wrapper(*args, **kwargs):
+        current_span = current_span_context.get()
+        current_span.metrics = metrics
+        current_span.metric_collection = metric_collection
+        res = original_func(*args, **kwargs)
+        return res
+
+    tool = wrapper
+    return tool