deepeval 3.6.7__py3-none-any.whl → 3.6.9__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.6.7"
+__version__: str = "3.6.9"

deepeval/config/settings.py

@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional, NamedTuple
 from deepeval.config.utils import (
     parse_bool,
     coerce_to_list,
+    constrain_between,
     dedupe_preserve_order,
 )
 from deepeval.constants import SUPPORTED_PROVIDER_SLUGS, slugify
@@ -336,6 +337,7 @@ class Settings(BaseSettings):
     IGNORE_DEEPEVAL_ERRORS: Optional[bool] = None
     SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None
     DEEPEVAL_VERBOSE_MODE: Optional[bool] = None
+    DEEPEVAL_LOG_STACK_TRACES: Optional[bool] = None
     ENABLE_DEEPEVAL_CACHE: Optional[bool] = None
 
     CONFIDENT_TRACE_FLUSH: Optional[bool] = None
@@ -355,11 +357,19 @@ class Settings(BaseSettings):
     #
     MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05
     MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0
-    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS: per-attempt timeout for provider calls enforced by our retry decorator.
-    # This timeout interacts with retry policy and the task level budget (DEEPEVAL_PER_TASK_TIMEOUT_SECONDS) below.
-    # If you leave this at 0/None, the computed outer budget defaults to 180s.
-    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS: Optional[confloat(ge=0)] = (
-        None  # per-attempt timeout. Set 0/None to disable
+    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE
+    # Per-attempt timeout (seconds) for provider calls used by the retry policy.
+    # This is an OVERRIDE setting. The effective value you should rely on at runtime is
+    # the computed property: DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS.
+    #
+    # If this is None or 0 the DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS is computed from either:
+    #   - DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: slice the outer budget
+    #     across attempts after subtracting expected backoff and a small safety buffer
+    #   - the default outer budget (180s) if no outer override is set.
+    #
+    # Tip: Set this OR the outer override, but generally not both
+    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(gt=0)] = (
+        None
     )
 
     #
@@ -373,76 +383,115 @@ class Settings(BaseSettings):
     #
     DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = 128
     DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = 5.0
-    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS is the outer time budget for one metric/task.
-    # It is computed from per-attempt timeout + retries/backoff unless you explicitly override it.
-    # - OVERRIDE = None or 0 -> auto compute as:
-    #     attempts * per_attempt_timeout + sum(backoff_sleeps) + ~jitter/2 per sleep + 1s safety
-    #   (If per_attempt_timeout is 0/None, the auto outer budget defaults to 180s.)
-    # - OVERRIDE > 0         -> use that exact value. A warning is logged if it is likely too small
-    #   to permit the configured attempts/backoff.
+    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE
+    # Outer time budget (seconds) for a single metric/test-case, including retries and backoff.
+    # This is an OVERRIDE setting. If None or 0 the DEEPEVAL_PER_TASK_TIMEOUT_SECONDS field is computed:
+    #     attempts * per_attempt_timeout + expected_backoff + 1s safety
+    # (When neither override is set 180s is used.)
+    #
+    # If > 0, we use the value exactly and log a warning if it is likely too small
+    # to accommodate the configured attempts/backoff.
     #
-    # Tip:
-    #   Most users only need to set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS and DEEPEVAL_RETRY_MAX_ATTEMPTS.
-    #   Leave the outer budget on auto unless you have very strict SLAs.
-    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[conint(ge=0)] = None
+    # usage:
+    #   - set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE along with DEEPEVAL_RETRY_MAX_ATTEMPTS, or
+    #   - set DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE alone.
+    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = None
 
     # Buffer time for gathering results from all tasks, added to the longest task duration
     # Increase if many tasks are running concurrently
-    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = 60
+    # DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = (
+    #     30  # 15s seemed like not enough. we may make this computed later.
+    # )
+    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE: Optional[confloat(ge=0)] = (
+        None
+    )
 
     ###################
     # Computed Fields #
     ###################
 
-    def _calc_auto_outer_timeout(self) -> int:
+    def _calc_auto_outer_timeout(self) -> float:
         """Compute outer budget from per-attempt timeout + retries/backoff.
         Never reference the computed property itself here.
         """
         attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1
-        timeout_seconds = float(self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+        timeout_seconds = float(
+            self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE or 0
+        )
         if timeout_seconds <= 0:
             # No per-attempt timeout set -> default outer budget
             return 180
 
-        sleeps = max(0, attempts - 1)
-        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)
-        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)
-        base = float(self.DEEPEVAL_RETRY_EXP_BASE)
-        jitter = float(self.DEEPEVAL_RETRY_JITTER)
-
-        backoff = 0.0
-        for _ in range(sleeps):
-            backoff += min(cap, cur)
-            cur *= base
-        backoff += sleeps * (jitter / 2.0)  # expected jitter
-
+        backoff = self._expected_backoff(attempts)
         safety_overhead = 1.0
-        return int(
+        return float(
             math.ceil(attempts * timeout_seconds + backoff + safety_overhead)
         )
 
     @computed_field
     @property
-    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> int:
+    def DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS(self) -> float:
+        over = self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE
+        if over is not None and float(over) > 0:
+            return float(over)
+
+        attempts = int(self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1)
+        outer_over = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE
+
+        # If the user set an outer override, slice it up
+        if outer_over and float(outer_over) > 0 and attempts > 0:
+            backoff = self._expected_backoff(attempts)
+            safety = 1.0
+            usable = max(0.0, float(outer_over) - backoff - safety)
+            return 0.0 if usable <= 0 else (usable / attempts)
+
+        # NEW: when neither override is set, derive from the default outer (180s)
+        default_outer = 180.0
+        backoff = self._expected_backoff(attempts)
+        safety = 1.0
+        usable = max(0.0, default_outer - backoff - safety)
+        # Keep per-attempt sensible (cap to at least 1s)
+        return 0.0 if usable <= 0 else max(1.0, usable / attempts)
+
+    @computed_field
+    @property
+    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> float:
         """If OVERRIDE is set (nonzero), return it; else return the derived budget."""
         outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE
         if outer not in (None, 0):
             # Warn if user-provided outer is likely to truncate retries
             if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:
                 min_needed = self._calc_auto_outer_timeout()
-                if int(outer) < min_needed:
+                if float(outer) < min_needed:
                     if self.DEEPEVAL_VERBOSE_MODE:
                         logger.warning(
                             "Metric timeout (outer=%ss) is less than attempts × per-attempt "
                             "timeout + backoff (≈%ss). Retries may be cut short.",
-                            int(outer),
+                            float(outer),
                             min_needed,
                         )
-            return int(outer)
+            return float(outer)
 
         # Auto mode
         return self._calc_auto_outer_timeout()
 
+    @computed_field
+    @property
+    def DEEPEVAL_TASK_GATHER_BUFFER_SECONDS(self) -> float:
+        """
+        Buffer time we add to the longest task’s duration to allow gather/drain
+        to complete. If an override is provided, use it; otherwise derive a
+        sensible default from the task-level budget:
+            buffer = constrain_between(0.15 * DEEPEVAL_PER_TASK_TIMEOUT_SECONDS, 10, 60)
+        """
+        over = self.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS_OVERRIDE
+        if over is not None and float(over) >= 0:
+            return float(over)
+
+        outer = float(self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS or 0.0)
+        base = 0.15 * outer
+        return constrain_between(base, 10.0, 60.0)
+
     ##############
     # Validators #
     ##############
@@ -810,6 +859,25 @@ class Settings(BaseSettings):
             ctx.switch_model_provider(target)
         return ctx.result
 
+    def _expected_backoff(self, attempts: int) -> float:
+        """Sum of expected sleeps for (attempts-1) retries, including jitter expectation."""
+        sleeps = max(0, attempts - 1)
+        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)
+        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)
+        base = float(self.DEEPEVAL_RETRY_EXP_BASE)
+        jitter = float(self.DEEPEVAL_RETRY_JITTER)
+
+        backoff = 0.0
+        for _ in range(sleeps):
+            backoff += min(cap, cur)
+            cur *= base
+        backoff += sleeps * (jitter / 2.0)  # expected jitter
+        return backoff
+
+    def _constrain_between(self, value: float, lo: float, hi: float) -> float:
+        """Return value constrained to the inclusive range [lo, hi]."""
+        return min(max(value, lo), hi)
+
 
 _settings_singleton: Optional[Settings] = None
 

deepeval/config/utils.py

@@ -137,3 +137,8 @@ def dedupe_preserve_order(items: Iterable[str]) -> List[str]:
             seen.add(x)
             out.append(x)
     return out
+
+
+def constrain_between(value: float, lo: float, hi: float) -> float:
+    """Return value constrained to the inclusive range [lo, hi]."""
+    return min(max(value, lo), hi)

deepeval/dataset/dataset.py

@@ -951,6 +951,8 @@ class EvaluationDataset:
                     context=golden.context,
                     name=golden.name,
                     comments=golden.comments,
+                    additional_metadata=golden.additional_metadata,
+                    custom_column_key_values=golden.custom_column_key_values,
                 )
                 for golden in self.goldens
             ]
@@ -965,6 +967,10 @@ class EvaluationDataset:
                     name=golden.name,
                     comments=golden.comments,
                     source_file=golden.source_file,
+                    tools_called=golden.tools_called,
+                    expected_tools=golden.expected_tools,
+                    additional_metadata=golden.additional_metadata,
+                    custom_column_key_values=golden.custom_column_key_values,
                 )
                 for golden in self.goldens
             ]
@@ -995,36 +1001,68 @@ class EvaluationDataset:
         if file_type == "json":
             with open(full_file_path, "w", encoding="utf-8") as file:
                 if self._multi_turn:
-                    json_data = [
-                        {
-                            "scenario": golden.scenario,
-                            "turns": (
-                                format_turns(golden.turns)
-                                if golden.turns
-                                else None
-                            ),
-                            "expected_outcome": golden.expected_outcome,
-                            "user_description": golden.user_description,
-                            "context": golden.context,
-                            "name": golden.name,
-                            "comments": golden.comments,
-                        }
-                        for golden in goldens
-                    ]
+                    json_data = []
+                    for golden in goldens:
+                        # Serialize turns as structured list of dicts
+                        turns_list = (
+                            json.loads(format_turns(golden.turns))
+                            if golden.turns
+                            else None
+                        )
+                        json_data.append(
+                            {
+                                "scenario": golden.scenario,
+                                "turns": turns_list,
+                                "expected_outcome": golden.expected_outcome,
+                                "user_description": golden.user_description,
+                                "context": golden.context,
+                                "name": golden.name,
+                                "comments": golden.comments,
+                                "additional_metadata": golden.additional_metadata,
+                                "custom_column_key_values": golden.custom_column_key_values,
+                            }
+                        )
                 else:
-                    json_data = [
-                        {
-                            "input": golden.input,
-                            "actual_output": golden.actual_output,
-                            "expected_output": golden.expected_output,
-                            "retrieval_context": golden.retrieval_context,
-                            "context": golden.context,
-                            "name": golden.name,
-                            "comments": golden.comments,
-                            "source_file": golden.source_file,
-                        }
-                        for golden in goldens
-                    ]
+                    json_data = []
+                    for golden in goldens:
+                        # Convert ToolCall lists to list[dict]
+                        def _dump_tools(tools):
+                            if not tools:
+                                return None
+                            dumped = []
+                            for t in tools:
+                                if hasattr(t, "model_dump"):
+                                    dumped.append(
+                                        t.model_dump(
+                                            by_alias=True, exclude_none=True
+                                        )
+                                    )
+                                elif hasattr(t, "dict"):
+                                    dumped.append(t.dict(exclude_none=True))
+                                else:
+                                    dumped.append(t)
+                            return dumped if len(dumped) > 0 else None
+
+                        json_data.append(
+                            {
+                                "input": golden.input,
+                                "actual_output": golden.actual_output,
+                                "expected_output": golden.expected_output,
+                                "retrieval_context": golden.retrieval_context,
+                                "context": golden.context,
+                                "name": golden.name,
+                                "comments": golden.comments,
+                                "source_file": golden.source_file,
+                                "tools_called": _dump_tools(
+                                    golden.tools_called
+                                ),
+                                "expected_tools": _dump_tools(
+                                    golden.expected_tools
+                                ),
+                                "additional_metadata": golden.additional_metadata,
+                                "custom_column_key_values": golden.custom_column_key_values,
+                            }
+                        )
                 json.dump(json_data, file, indent=4, ensure_ascii=False)
         elif file_type == "csv":
             with open(
@@ -1041,6 +1079,8 @@ class EvaluationDataset:
                             "context",
                             "name",
                             "comments",
+                            "additional_metadata",
+                            "custom_column_key_values",
                         ]
                     )
                     for golden in goldens:
@@ -1054,6 +1094,21 @@ class EvaluationDataset:
                             if golden.turns is not None
                             else None
                         )
+                        additional_metadata = (
+                            json.dumps(
+                                golden.additional_metadata, ensure_ascii=False
+                            )
+                            if golden.additional_metadata is not None
+                            else None
+                        )
+                        custom_cols = (
+                            json.dumps(
+                                golden.custom_column_key_values,
+                                ensure_ascii=False,
+                            )
+                            if golden.custom_column_key_values
+                            else None
+                        )
                         writer.writerow(
                             [
                                 golden.scenario,
@@ -1063,6 +1118,8 @@ class EvaluationDataset:
                                 context,
                                 golden.name,
                                 golden.comments,
+                                additional_metadata,
+                                custom_cols,
                             ]
                         )
                 else:
@@ -1076,6 +1133,10 @@ class EvaluationDataset:
                             "name",
                             "comments",
                             "source_file",
+                            "tools_called",
+                            "expected_tools",
+                            "additional_metadata",
+                            "custom_column_key_values",
                         ]
                     )
                     for golden in goldens:
@@ -1089,6 +1150,42 @@ class EvaluationDataset:
                             if golden.context is not None
                             else None
                         )
+
+                        # Dump tools as JSON strings for CSV
+                        def _dump_tools_csv(tools):
+                            if not tools:
+                                return None
+                            dumped = []
+                            for t in tools:
+                                if hasattr(t, "model_dump"):
+                                    dumped.append(
+                                        t.model_dump(
+                                            by_alias=True, exclude_none=True
+                                        )
+                                    )
+                                elif hasattr(t, "dict"):
+                                    dumped.append(t.dict(exclude_none=True))
+                                else:
+                                    dumped.append(t)
+                            return json.dumps(dumped, ensure_ascii=False)
+
+                        tools_called = _dump_tools_csv(golden.tools_called)
+                        expected_tools = _dump_tools_csv(golden.expected_tools)
+                        additional_metadata = (
+                            json.dumps(
+                                golden.additional_metadata, ensure_ascii=False
+                            )
+                            if golden.additional_metadata is not None
+                            else None
+                        )
+                        custom_cols = (
+                            json.dumps(
+                                golden.custom_column_key_values,
+                                ensure_ascii=False,
+                            )
+                            if golden.custom_column_key_values
+                            else None
+                        )
                         writer.writerow(
                             [
                                 golden.input,
@@ -1099,6 +1196,10 @@ class EvaluationDataset:
                                 golden.name,
                                 golden.comments,
                                 golden.source_file,
+                                tools_called,
+                                expected_tools,
+                                additional_metadata,
+                                custom_cols,
                             ]
                         )
         elif file_type == "jsonl":
@@ -1106,7 +1207,9 @@ class EvaluationDataset:
                 for golden in goldens:
                     if self._multi_turn:
                         turns = (
-                            format_turns(golden.turns) if golden.turns else None
+                            json.loads(format_turns(golden.turns))
+                            if golden.turns
+                            else None
                         )
                         record = {
                             "scenario": golden.scenario,
@@ -1114,6 +1217,10 @@ class EvaluationDataset:
                             "expected_outcome": golden.expected_outcome,
                             "user_description": golden.user_description,
                             "context": golden.context,
+                            "name": golden.name,
+                            "comments": golden.comments,
+                            "additional_metadata": golden.additional_metadata,
+                            "custom_column_key_values": golden.custom_column_key_values,
                         }
                     else:
                         retrieval_context = (
@@ -1126,12 +1233,37 @@ class EvaluationDataset:
                             if golden.context is not None
                             else None
                         )
+
+                        # Convert ToolCall lists to list[dict]
+                        def _dump_tools(tools):
+                            if not tools:
+                                return None
+                            dumped = []
+                            for t in tools:
+                                if hasattr(t, "model_dump"):
+                                    dumped.append(
+                                        t.model_dump(
+                                            by_alias=True, exclude_none=True
+                                        )
+                                    )
+                                elif hasattr(t, "dict"):
+                                    dumped.append(t.dict(exclude_none=True))
+                                else:
+                                    dumped.append(t)
+                            return dumped if len(dumped) > 0 else None
+
                         record = {
                             "input": golden.input,
                             "actual_output": golden.actual_output,
                             "expected_output": golden.expected_output,
                             "retrieval_context": retrieval_context,
                             "context": context,
+                            "tools_called": _dump_tools(golden.tools_called),
+                            "expected_tools": _dump_tools(
+                                golden.expected_tools
+                            ),
+                            "additional_metadata": golden.additional_metadata,
+                            "custom_column_key_values": golden.custom_column_key_values,
                         }
 
                     file.write(json.dumps(record, ensure_ascii=False) + "\n")

deepeval/dataset/utils.py

@@ -111,12 +111,36 @@ def trimAndLoadJson(input_string: str) -> Any:
 def format_turns(turns: List[Turn]) -> str:
     res = []
     for turn in turns:
+        # Safely convert nested Pydantic models (ToolCall/MCP calls) to dicts
+        def _dump_list(models):
+            if not models:
+                return None
+            dumped = []
+            for m in models:
+                if hasattr(m, "model_dump"):
+                    dumped.append(
+                        m.model_dump(by_alias=True, exclude_none=True)
+                    )
+                elif hasattr(m, "dict"):
+                    dumped.append(m.dict(exclude_none=True))
+                else:
+                    dumped.append(m)
+            return dumped if len(dumped) > 0 else None
+
         cur_turn = {
             "role": turn.role,
             "content": turn.content,
+            "user_id": turn.user_id if turn.user_id is not None else None,
             "retrieval_context": (
                 turn.retrieval_context if turn.retrieval_context else None
             ),
+            "tools_called": _dump_list(turn.tools_called),
+            "mcp_tools_called": _dump_list(turn.mcp_tools_called),
+            "mcp_resources_called": _dump_list(turn.mcp_resources_called),
+            "mcp_prompts_called": _dump_list(turn.mcp_prompts_called),
+            "additional_metadata": (
+                turn.additional_metadata if turn.additional_metadata else None
+            ),
         }
         res.append(cur_turn)
     try:
@@ -125,11 +149,17 @@ def format_turns(turns: List[Turn]) -> str:
         raise ValueError(f"Error serializing turns: {e}")
 
 
-def parse_turns(turns_str: str) -> List[Turn]:
-    try:
-        parsed = json.loads(turns_str)
-    except json.JSONDecodeError as e:
-        raise ValueError(f"Invalid JSON: {e}")
+def parse_turns(turns_str: Any) -> List[Turn]:
+    # Accept either a JSON string or a Python list
+    if isinstance(turns_str, str):
+        try:
+            parsed = json.loads(turns_str)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON: {e}")
+    elif isinstance(turns_str, list):
+        parsed = turns_str
+    else:
+        raise TypeError("Expected a JSON string or a list of turns.")
 
     if not isinstance(parsed, list):
         raise TypeError("Expected a list of turns.")
@@ -145,15 +175,13 @@ def parse_turns(turns_str: str) -> List[Turn]:
         if "content" not in turn or not isinstance(turn["content"], str):
             raise ValueError(f"Turn at index {i} is missing a valid 'content'.")
 
-        retrieval_context = turn.get("retrieval_context")
+        try:
+            # Pydantic v2
+            res.append(Turn.model_validate(turn))
+        except AttributeError:
+            # Pydantic v1 fallback
+            res.append(Turn.parse_obj(turn))
 
-        res.append(
-            Turn(
-                role=turn["role"],
-                content=turn["content"],
-                retrieval_context=retrieval_context,
-            )
-        )
     return res
 
 

deepeval/errors.py

@@ -1,6 +1,24 @@
-class MissingTestCaseParamsError(Exception):
+class DeepEvalError(Exception):
+    """Base class for framework-originated errors.
+    If raised and not handled, it will abort the current operation.
+    We may also stringify instances of this class and attach them to traces or spans to surface
+    non-fatal diagnostics while allowing the run to continue.
+    """
+
+
+class UserAppError(Exception):
+    """Represents exceptions thrown by user LLM apps/tools.
+    We record these on traces or spans and keep the overall evaluation run alive.
+    """
+
+
+class MissingTestCaseParamsError(DeepEvalError):
+    """Required test case fields are missing."""
+
     pass
 
 
-class MismatchedTestCaseInputsError(Exception):
+class MismatchedTestCaseInputsError(DeepEvalError):
+    """Inputs provided to a metric or test case are inconsistent or invalid."""
+
     pass