deepeval 3.6.8__py3-none-any.whl → 3.7.0__py3-none-any.whl

deepeval/models/llms/amazon_bedrock_model.py

@@ -76,23 +76,26 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
-        payload = self.get_converse_request_body(prompt)
-        client = await self._ensure_client()
-        response = await client.converse(
-            modelId=self.model_id,
-            messages=payload["messages"],
-            inferenceConfig=payload["inferenceConfig"],
-        )
-        message = response["output"]["message"]["content"][0]["text"]
-        cost = self.calculate_cost(
-            response["usage"]["inputTokens"],
-            response["usage"]["outputTokens"],
-        )
-        if schema is None:
-            return message, cost
-        else:
-            json_output = trim_and_load_json(message)
-            return schema.model_validate(json_output), cost
+        try:
+            payload = self.get_converse_request_body(prompt)
+            client = await self._ensure_client()
+            response = await client.converse(
+                modelId=self.model_id,
+                messages=payload["messages"],
+                inferenceConfig=payload["inferenceConfig"],
+            )
+            message = response["output"]["message"]["content"][0]["text"]
+            cost = self.calculate_cost(
+                response["usage"]["inputTokens"],
+                response["usage"]["outputTokens"],
+            )
+            if schema is None:
+                return message, cost
+            else:
+                json_output = trim_and_load_json(message)
+                return schema.model_validate(json_output), cost
+        finally:
+            await self.close()
 
     ###############################################
     # Client management

deepeval/models/llms/openai_model.py

@@ -8,6 +8,7 @@ from openai import (
     AsyncOpenAI,
 )
 
+from deepeval.config.settings import get_settings
 from deepeval.constants import ProviderSlug as PS
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.utils import trim_and_load_json
@@ -209,6 +210,11 @@ models_requiring_temperature_1 = [
 ]
 
 
+def _request_timeout_seconds() -> float:
+    timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+    return timeout if timeout > 0 else 30.0
+
+
 class GPTModel(DeepEvalBaseLLM):
     def __init__(
         self,
@@ -387,7 +393,6 @@ class GPTModel(DeepEvalBaseLLM):
                 )
                 return schema.model_validate(json_output), cost
 
-        client: AsyncOpenAI
         completion = await client.chat.completions.create(
             model=self.model_name,
             messages=[{"role": "user", "content": prompt}],
@@ -501,9 +506,13 @@ class GPTModel(DeepEvalBaseLLM):
         kwargs = dict(self.kwargs or {})
         if not sdk_retries_for(PS.OPENAI):
             kwargs["max_retries"] = 0
+
+        if not kwargs.get("timeout"):
+            kwargs["timeout"] = _request_timeout_seconds()
         return kwargs
 
     def _build_client(self, cls):
+
         kw = dict(
             api_key=self._openai_api_key,
             base_url=self.base_url,

deepeval/models/retry_policy.py

@@ -39,6 +39,7 @@ import itertools
 import functools
 import threading
 import logging
+import time
 
 from dataclasses import dataclass, field
 from typing import Callable, Iterable, Mapping, Optional, Sequence, Tuple, Union
@@ -52,6 +53,7 @@ from tenacity import (
 )
 from tenacity.stop import stop_base
 from tenacity.wait import wait_base
+from contextvars import ContextVar, copy_context
 
 from deepeval.constants import (
     ProviderSlug as PS,
@@ -65,6 +67,81 @@ Provider = Union[str, PS]
 _MAX_TIMEOUT_THREADS = get_settings().DEEPEVAL_TIMEOUT_THREAD_LIMIT
 _TIMEOUT_SEMA = threading.BoundedSemaphore(_MAX_TIMEOUT_THREADS)
 _WORKER_ID = itertools.count(1)
+_OUTER_DEADLINE = ContextVar("deepeval_outer_deadline", default=None)
+
+
+def set_outer_deadline(seconds: float | None):
+    """Set (or clear) the outer task time budget.
+
+    Stores a deadline in a local context variable so nested code
+    can cooperatively respect a shared budget. Always pair this with
+    `reset_outer_deadline(token)` in a `finally` block.
+
+    Args:
+        seconds: Number of seconds from now to set as the deadline. If `None`,
+            `0`, or a non-positive value is provided, the deadline is cleared.
+
+    Returns:
+        contextvars.Token: The token returned by the underlying ContextVar `.set()`
+        call, which must be passed to `reset_outer_deadline` to restore the
+        previous value.
+    """
+    if seconds and seconds > 0:
+        return _OUTER_DEADLINE.set(time.monotonic() + seconds)
+    return _OUTER_DEADLINE.set(None)
+
+
+def reset_outer_deadline(token):
+    """Restore the previous outer deadline set by `set_outer_deadline`.
+
+    This should be called in a `finally` block to ensure the deadline
+    is restored even if an exception occurs.
+
+    Args:
+        token: The `contextvars.Token` returned by `set_outer_deadline`.
+    """
+    if token is not None:
+        _OUTER_DEADLINE.reset(token)
+
+
+def _remaining_budget() -> float | None:
+    dl = _OUTER_DEADLINE.get()
+    if dl is None:
+        return None
+    return max(0.0, dl - time.monotonic())
+
+
+def _is_budget_spent() -> bool:
+    rem = _remaining_budget()
+    return rem is not None and rem <= 0.0
+
+
+def resolve_effective_attempt_timeout():
+    """Resolve the timeout to use for a single provider attempt.
+
+    Combines the configured per-attempt timeout with any remaining outer budget:
+    - If `DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS` is `0` or `None`, returns `0`
+      callers should skip `asyncio.wait_for` in this case and rely on the outer cap.
+    - If positive and an outer deadline is present, returns
+      `min(per_attempt, remaining_budget)`.
+    - If positive and no outer deadline is present, returns `per_attempt`.
+
+    Returns:
+        float: Seconds to use for the inner per-attempt timeout. `0` means
+        disable inner timeout and rely on the outer budget instead.
+    """
+    per_attempt = float(
+        get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
+    )
+    # 0 or None disable inner wait_for. That means rely on outer task cap for timeouts instead.
+    if per_attempt <= 0:
+        return 0
+    # If we do have a positive per-attempt, use up to remaining outer budget.
+    rem = _remaining_budget()
+    if rem is not None:
+        return max(0.0, min(per_attempt, rem))
+    return per_attempt
+
 
 # --------------------------
 # Policy description
@@ -399,9 +476,10 @@ def make_after_log(slug: str):
         if not _logger.isEnabledFor(after_level):
             return
 
+        show_trace = bool(get_settings().DEEPEVAL_LOG_STACK_TRACES)
         exc_info = (
             (type(exc), exc, getattr(exc, "__traceback__", None))
-            if after_level >= logging.ERROR
+            if show_trace
             else None
         )
 
@@ -416,7 +494,7 @@ def make_after_log(slug: str):
     return _after
 
 
-def _make_timeout_error(timeout_seconds: float) -> TimeoutError:
+def _make_timeout_error(timeout_seconds: float) -> asyncio.TimeoutError:
     settings = get_settings()
     if logger.isEnabledFor(logging.DEBUG):
         logger.debug(
@@ -427,12 +505,12 @@ def _make_timeout_error(timeout_seconds: float) -> TimeoutError:
         )
     msg = (
         f"call timed out after {timeout_seconds:g}s (per attempt). "
-        "Increase DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS (0 disables) or reduce work per attempt."
+        "Increase DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE (None disables) or reduce work per attempt."
     )
-    return TimeoutError(msg)
+    return asyncio.TimeoutError(msg)
 
 
-def _run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
+def run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
     """
     Run a synchronous callable with a soft timeout enforced by a helper thread,
     with a global cap on concurrent timeout-workers.
@@ -499,9 +577,11 @@ def _run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
     done = threading.Event()
     result = {"value": None, "exc": None}
 
+    context = copy_context()
+
     def target():
         try:
-            result["value"] = func(*args, **kwargs)
+            result["value"] = context.run(func, *args, **kwargs)
         except BaseException as e:
             result["exc"] = e
         finally:
@@ -562,37 +642,40 @@ def create_retry_decorator(provider: Provider):
 
             @functools.wraps(func)
             async def attempt(*args, **kwargs):
-                timeout_seconds = (
-                    get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
-                )
+                if _is_budget_spent():
+                    raise _make_timeout_error(0)
+
+                per_attempt_timeout = resolve_effective_attempt_timeout()
+
                 coro = func(*args, **kwargs)
-                if timeout_seconds > 0:
+                if per_attempt_timeout > 0:
                     try:
-                        return await asyncio.wait_for(coro, timeout_seconds)
-                    except asyncio.TimeoutError as e:
+                        return await asyncio.wait_for(coro, per_attempt_timeout)
+                    except (asyncio.TimeoutError, TimeoutError) as e:
                         if (
                             logger.isEnabledFor(logging.DEBUG)
                             and get_settings().DEEPEVAL_VERBOSE_MODE is True
                         ):
                             logger.debug(
                                 "async timeout after %.3fs (active_threads=%d, tasks=%d)",
-                                timeout_seconds,
+                                per_attempt_timeout,
                                 threading.active_count(),
                                 len(asyncio.all_tasks()),
                             )
-                        raise _make_timeout_error(timeout_seconds) from e
+                        raise _make_timeout_error(per_attempt_timeout) from e
                 return await coro
 
             return base_retry(attempt)
 
         @functools.wraps(func)
         def attempt(*args, **kwargs):
-            timeout_seconds = (
-                get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
-            )
-            if timeout_seconds > 0:
-                return _run_sync_with_timeout(
-                    func, timeout_seconds, *args, **kwargs
+            if _is_budget_spent():
+                raise _make_timeout_error(0)
+
+            per_attempt_timeout = resolve_effective_attempt_timeout()
+            if per_attempt_timeout > 0:
+                return run_sync_with_timeout(
+                    func, per_attempt_timeout, *args, **kwargs
                 )
             return func(*args, **kwargs)
 

deepeval/openai/__init__.py

@@ -15,5 +15,7 @@ except ImportError:
 
 if OpenAI or AsyncOpenAI:
     from deepeval.openai.patch import patch_openai_classes
+    from deepeval.telemetry import capture_tracing_integration
 
-    patch_openai_classes()
+    with capture_tracing_integration("openai"):
+        patch_openai_classes()

deepeval/openai/extractors.py

@@ -4,13 +4,13 @@ from typing import Any, Union, Dict
 from openai.types.responses import Response
 
 from deepeval.test_case.llm_test_case import ToolCall
+
+from deepeval.model_integrations.types import InputParameters, OutputParameters
 from deepeval.openai.utils import (
     render_response_input,
     stringify_multimodal_content,
     render_messages,
 )
-from deepeval.openai.types import InputParameters, OutputParameters
-from deepeval.tracing.types import Message
 
 
 # guarding against errors to be compatible with legacy APIs

deepeval/openai/utils.py

@@ -1,6 +1,6 @@
 import json
 import uuid
-from typing import Any, Dict, List, Optional, Iterable
+from typing import Any, Dict, List, Iterable
 
 from openai.types.chat.chat_completion_message_param import (
     ChatCompletionMessageParam,
@@ -8,32 +8,8 @@ from openai.types.chat.chat_completion_message_param import (
 
 from deepeval.tracing.types import ToolSpan, TraceSpanStatus
 from deepeval.tracing.context import current_span_context
-from deepeval.utils import shorten, len_long
-from deepeval.openai.types import OutputParameters
-
-
-_URL_MAX = 200
-_JSON_MAX = max(
-    len_long(), 400
-)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
-
-
-def _compact_dump(value: Any) -> str:
-    try:
-        dumped = json.dumps(
-            value, ensure_ascii=False, default=str, separators=(",", ":")
-        )
-    except Exception:
-        dumped = repr(value)
-    return shorten(dumped, max_len=_JSON_MAX)
-
-
-def _fmt_url(url: Optional[str]) -> str:
-    if not url:
-        return ""
-    if url.startswith("data:"):
-        return "[data-uri]"
-    return shorten(url, max_len=_URL_MAX)
+from deepeval.model_integrations.types import OutputParameters
+from deepeval.model_integrations.utils import compact_dump, fmt_url
 
 
 def create_child_tool_spans(output_parameters: OutputParameters):
@@ -111,7 +87,7 @@ def stringify_multimodal_content(content: Any) -> str:
                 url = image_url
             else:
                 url = (image_url or {}).get("url") or content.get("url")
-            return f"[image:{_fmt_url(url)}]"
+            return f"[image:{fmt_url(url)}]"
 
         # Responses API variants
         if t == "input_text":
@@ -122,14 +98,14 @@ def stringify_multimodal_content(content: Any) -> str:
                 url = image_url
             else:
                 url = (image_url or {}).get("url") or content.get("url")
-            return f"[image:{_fmt_url(url)}]"
+            return f"[image:{fmt_url(url)}]"
 
         # readability for other input_* types we don't currently handle
         if t and t.startswith("input_"):
             return f"[{t}]"
 
     # unknown dicts and types returned as shortened JSON
-    return _compact_dump(content)
+    return compact_dump(content)
 
 
 def render_messages(
@@ -228,7 +204,7 @@ def _render_content(content: Dict[str, Any], indent: int = 0) -> str:
             lines.append(f"{prefix}{key}:")
             lines.append(_render_content(value, indent + 1))
         elif isinstance(value, list):
-            lines.append(f"{prefix}{key}: {_compact_dump(value)}")
+            lines.append(f"{prefix}{key}: {compact_dump(value)}")
         else:
             lines.append(f"{prefix}{key}: {value}")
 

deepeval/prompt/api.py

@@ -1,8 +1,10 @@
-from pydantic import BaseModel, Field, AliasChoices
+from pydantic import BaseModel, Field, AliasChoices, ConfigDict
 from enum import Enum
 from typing import List, Optional
 from pydantic import TypeAdapter
 
+from deepeval.utils import make_model_config
+
 ###################################
 # Model Settings
 ###################################
@@ -92,6 +94,8 @@ class SchemaDataType(Enum):
 
 
 class OutputSchemaField(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     id: str
     type: SchemaDataType
     name: str
@@ -102,9 +106,6 @@ class OutputSchemaField(BaseModel):
         validation_alias=AliasChoices("parent_id", "parentId"),
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class OutputSchema(BaseModel):
     fields: Optional[List[OutputSchemaField]] = None
@@ -187,6 +188,10 @@ class PromptHttpResponse(BaseModel):
 
 
 class PromptPushRequest(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
+    model_config = ConfigDict(use_enum_values=True)
+
     alias: str
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
@@ -203,11 +208,10 @@ class PromptPushRequest(BaseModel):
         default=None, serialization_alias="outputType"
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class PromptUpdateRequest(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
     interpolation_type: PromptInterpolationType = Field(
@@ -223,9 +227,6 @@ class PromptUpdateRequest(BaseModel):
         default=None, serialization_alias="outputType"
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class PromptApi(BaseModel):
     id: str

deepeval/prompt/prompt.py

@@ -5,11 +5,13 @@ from rich.console import Console
 import time
 import json
 import os
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel, ValidationError, ConfigDict
 import asyncio
 import portalocker
 import threading
 
+from deepeval.utils import make_model_config
+
 from deepeval.prompt.api import (
     PromptHttpResponse,
     PromptMessage,
@@ -77,6 +79,8 @@ class CustomEncoder(json.JSONEncoder):
 
 
 class CachedPrompt(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     alias: str
     version: str
     label: Optional[str] = None
@@ -89,9 +93,6 @@ class CachedPrompt(BaseModel):
     output_type: Optional[OutputType]
     output_schema: Optional[OutputSchema]
 
-    class Config:
-        use_enum_values = True
-
 
 class Prompt:
 

deepeval/simulator/conversation_simulator.py

@@ -35,7 +35,6 @@ class ConversationSimulator:
         self,
         model_callback: Callable[[str], str],
         simulator_model: Optional[Union[str, DeepEvalBaseLLM]] = None,
-        opening_message: Optional[str] = None,
         max_concurrent: int = 5,
         async_mode: bool = True,
         language: str = "English",
@@ -45,7 +44,6 @@ class ConversationSimulator:
         self.is_callback_async = inspect.iscoroutinefunction(
             self.model_callback
         )
-        self.opening_message = opening_message
         self.semaphore = asyncio.Semaphore(max_concurrent)
         self.async_mode = async_mode
         self.language = language
@@ -68,6 +66,9 @@ class ConversationSimulator:
         self,
         conversational_goldens: List[ConversationalGolden],
         max_user_simulations: int = 10,
+        on_simulation_complete: Optional[
+            Callable[[ConversationalTestCase, int], None]
+        ] = None,
     ) -> List[ConversationalTestCase]:
         self.simulation_cost = 0 if self.using_native_model else None
 
@@ -87,6 +88,7 @@ class ConversationSimulator:
                     self._a_simulate(
                         conversational_goldens=conversational_goldens,
                         max_user_simulations=max_user_simulations,
+                        on_simulation_complete=on_simulation_complete,
                         progress=progress,
                         pbar_id=pbar_id,
                     )
@@ -103,6 +105,7 @@ class ConversationSimulator:
                             index=conversation_index,
                             progress=progress,
                             pbar_id=pbar_id,
+                            on_simulation_complete=on_simulation_complete,
                         )
                     )
                     conversational_test_cases.append(conversational_test_case)
@@ -115,6 +118,9 @@ class ConversationSimulator:
         self,
         conversational_goldens: List[ConversationalGolden],
         max_user_simulations: int,
+        on_simulation_complete: Optional[
+            Callable[[ConversationalTestCase, int], None]
+        ] = None,
         progress: Optional[Progress] = None,
         pbar_id: Optional[int] = None,
     ) -> List[ConversationalTestCase]:
@@ -131,6 +137,7 @@ class ConversationSimulator:
                     index=conversation_index,
                     progress=progress,
                     pbar_id=pbar_id,
+                    on_simulation_complete=on_simulation_complete,
                 )
 
         tasks = [
@@ -150,6 +157,9 @@ class ConversationSimulator:
         index: int,
         progress: Optional[Progress] = None,
         pbar_id: Optional[int] = None,
+        on_simulation_complete: Optional[
+            Callable[[ConversationalTestCase, int], None]
+        ] = None,
     ) -> ConversationalTestCase:
         simulation_counter = 0
         if max_user_simulations <= 0:
@@ -166,8 +176,6 @@ class ConversationSimulator:
         user_input = None
         thread_id = str(uuid.uuid4())
         turns: List[Turn] = []
-        if self.opening_message and golden.turns is None:
-            turns.append(Turn(role="assistant", content=self.opening_message))
 
         if golden.turns is not None:
             turns.extend(golden.turns)
@@ -187,11 +195,7 @@ class ConversationSimulator:
             if simulation_counter >= max_user_simulations:
                 update_pbar(progress, pbar_max_user_simluations_id)
                 break
-            if len(turns) == 0 or (
-                len(turns) == 1
-                and self.opening_message
-                and golden.turns is None
-            ):
+            if len(turns) == 0:
                 # Generate first user input
                 user_input = self.generate_first_user_input(golden)
                 turns.append(Turn(role="user", content=user_input))
@@ -225,7 +229,7 @@ class ConversationSimulator:
             turns.append(turn)
 
         update_pbar(progress, pbar_id)
-        return ConversationalTestCase(
+        conversational_test_case = ConversationalTestCase(
             turns=turns,
             scenario=golden.scenario,
             expected_outcome=golden.expected_outcome,
@@ -241,6 +245,9 @@ class ConversationSimulator:
             _dataset_alias=golden._dataset_alias,
             _dataset_id=golden._dataset_id,
         )
+        if on_simulation_complete:
+            on_simulation_complete(conversational_test_case, index)
+        return conversational_test_case
 
     async def _a_simulate_single_conversation(
         self,
@@ -249,6 +256,9 @@ class ConversationSimulator:
         index: Optional[int] = None,
         progress: Optional[Progress] = None,
         pbar_id: Optional[int] = None,
+        on_simulation_complete: Optional[
+            Callable[[ConversationalTestCase, int], None]
+        ] = None,
     ) -> ConversationalTestCase:
         simulation_counter = 0
         if max_user_simulations <= 0:
@@ -265,8 +275,6 @@ class ConversationSimulator:
         user_input = None
         thread_id = str(uuid.uuid4())
         turns: List[Turn] = []
-        if self.opening_message and golden.turns is None:
-            turns.append(Turn(role="assistant", content=self.opening_message))
 
         if golden.turns is not None:
             turns.extend(golden.turns)
@@ -286,11 +294,7 @@ class ConversationSimulator:
             if simulation_counter >= max_user_simulations:
                 update_pbar(progress, pbar_max_user_simluations_id)
                 break
-            if len(turns) == 0 or (
-                len(turns) == 1
-                and self.opening_message
-                and golden.turns is None
-            ):
+            if len(turns) == 0:
                 # Generate first user input
                 user_input = await self.a_generate_first_user_input(golden)
                 turns.append(Turn(role="user", content=user_input))
@@ -324,7 +328,7 @@ class ConversationSimulator:
             turns.append(turn)
 
         update_pbar(progress, pbar_id)
-        return ConversationalTestCase(
+        conversational_test_case = ConversationalTestCase(
             turns=turns,
             scenario=golden.scenario,
             expected_outcome=golden.expected_outcome,
@@ -340,6 +344,9 @@ class ConversationSimulator:
             _dataset_alias=golden._dataset_alias,
             _dataset_id=golden._dataset_id,
         )
+        if on_simulation_complete:
+            on_simulation_complete(conversational_test_case, index)
+        return conversational_test_case
 
     ############################################
     ### Generate User Inputs ###################

deepeval/synthesizer/chunking/context_generator.py

@@ -249,8 +249,16 @@ class ContextGenerator:
 
                 except Exception as exc:
                     # record and continue with other docs
+                    show_trace = bool(get_settings().DEEPEVAL_LOG_STACK_TRACES)
+                    exc_info = (
+                        (type(exc), exc, getattr(exc, "__traceback__", None))
+                        if show_trace
+                        else None
+                    )
                     logger.exception(
-                        "Document pipeline failed for %s", path, exc_info=exc
+                        "Document pipeline failed for %s",
+                        path,
+                        exc_info=exc_info,
                     )
                 finally:
                     # drop the collection asap to avoid too many open collections