deepeval 3.6.5__py3-none-any.whl → 3.6.6__py3-none-any.whl

deepeval/__init__.py

@@ -1,24 +1,56 @@
+from __future__ import annotations
+
+import logging
 import os
-import warnings
 import re
+import warnings
 
-# load environment variables before other imports
+# IMPORTANT: load environment variables before other imports
 from deepeval.config.settings import autoload_dotenv, get_settings
 
+logging.getLogger("deepeval").addHandler(logging.NullHandler())
 autoload_dotenv()
 
-from ._version import __version__
-from deepeval.evaluate import evaluate, assert_test
-from deepeval.evaluate.compare import compare
-from deepeval.test_run import on_test_run_end, log_hyperparameters
-from deepeval.utils import login
-from deepeval.telemetry import *
+
+def _expose_public_api() -> None:
+    # All other imports must happen after env is loaded
+    # Do not do this at module level or ruff will complain with E402
+    global __version__, evaluate, assert_test, compare
+    global on_test_run_end, log_hyperparameters, login, telemetry
+
+    from ._version import __version__ as _version
+    from deepeval.evaluate import (
+        evaluate as _evaluate,
+        assert_test as _assert_test,
+    )
+    from deepeval.evaluate.compare import compare as _compare
+    from deepeval.test_run import (
+        on_test_run_end as _on_end,
+        log_hyperparameters as _log_hparams,
+    )
+    from deepeval.utils import login as _login
+    import deepeval.telemetry as _telemetry
+
+    __version__ = _version
+    evaluate = _evaluate
+    assert_test = _assert_test
+    compare = _compare
+    on_test_run_end = _on_end
+    log_hyperparameters = _log_hparams
+    login = _login
+    telemetry = _telemetry
+
+
+_expose_public_api()
 
 
 settings = get_settings()
+
 if not settings.DEEPEVAL_GRPC_LOGGING:
-    os.environ.setdefault("GRPC_VERBOSITY", "ERROR")
-    os.environ.setdefault("GRPC_TRACE", "")
+    if os.getenv("GRPC_VERBOSITY") is None:
+        os.environ["GRPC_VERBOSITY"] = settings.GRPC_VERBOSITY or "ERROR"
+    if os.getenv("GRPC_TRACE") is None:
+        os.environ["GRPC_TRACE"] = settings.GRPC_TRACE or ""
 
 
 __all__ = [

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.6.5"
+__version__: str = "3.6.6"

deepeval/config/logging.py

@@ -0,0 +1,33 @@
+"""
+Minimal logging configuration helpers for DeepEval.
+
+This module centralizes how the library-level logger ("deepeval") is configured. We
+intentionally keep configuration lightweight so application code retains control
+over handlers and formatters.
+"""
+
+import logging
+from deepeval.config.settings import get_settings
+
+
+def apply_deepeval_log_level() -> None:
+    """
+    Apply DeepEval's current log level to the package logger.
+
+    This function reads `LOG_LEVEL` from `deepeval.config.settings.get_settings()`
+    and sets the level of the `"deepeval"` logger accordingly. If `LOG_LEVEL` is
+    unset (None), INFO is used as a default. The logger's `propagate` flag is set
+    to True so records bubble up to the application's handlers. DeepEval does not
+    install its own handlers here (a NullHandler is attached in `__init__.py`).
+
+    The function is idempotent and safe to call multiple times. It is invoked
+    automatically when settings are first constructed and whenever `LOG_LEVEL`
+    is changed via `settings.edit`.
+    """
+    settings = get_settings()
+    log_level = settings.LOG_LEVEL
+    logging.getLogger("deepeval").setLevel(
+        log_level if log_level is not None else logging.INFO
+    )
+    # ensure we bubble up to app handlers
+    logging.getLogger("deepeval").propagate = True

deepeval/config/settings.py

@@ -10,12 +10,20 @@ Central config for DeepEval.
 """
 
 import logging
+import math
 import os
 import re
 
 from dotenv import dotenv_values
 from pathlib import Path
-from pydantic import AnyUrl, SecretStr, field_validator, confloat
+from pydantic import (
+    AnyUrl,
+    computed_field,
+    confloat,
+    conint,
+    field_validator,
+    SecretStr,
+)
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing import Any, Dict, List, Optional, NamedTuple
 
@@ -155,7 +163,7 @@ class Settings(BaseSettings):
     #
 
     APP_ENV: str = "dev"
-    LOG_LEVEL: str = "info"
+    LOG_LEVEL: Optional[int] = None
     PYTHONPATH: str = "."
     CONFIDENT_REGION: Optional[str] = None
     CONFIDENT_OPEN_BROWSER: Optional[bool] = True
@@ -287,9 +295,33 @@ class Settings(BaseSettings):
     #
     # Retry Policy
     #
-    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = None
-    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = None  # default -> INFO
+    # Controls how Tenacity retries provider calls when the SDK isn't doing its own retries.
+    # Key concepts:
+    # - attempts count includes the first call. e.g. 1 = no retries, 2 = one retry.
+    # - backoff sleeps follow exponential growth with a cap, plus jitter. Expected jitter
+    #   contribution is ~ JITTER/2 per sleep.
+    # - logging levels are looked up dynamically each attempt, so if you change LOG_LEVEL at runtime,
+    #   the retry loggers will honor it without restart.
+    DEEPEVAL_SDK_RETRY_PROVIDERS: Optional[List[str]] = (
+        None  # ["*"] to delegate all retries to SDKs
+    )
+    DEEPEVAL_RETRY_BEFORE_LOG_LEVEL: Optional[int] = (
+        None  # default is LOG_LEVEL if set, else INFO
+    )
     DEEPEVAL_RETRY_AFTER_LOG_LEVEL: Optional[int] = None  # default -> ERROR
+    DEEPEVAL_RETRY_MAX_ATTEMPTS: conint(ge=1) = (
+        2  # attempts = first try + retries
+    )
+    DEEPEVAL_RETRY_INITIAL_SECONDS: confloat(ge=0) = (
+        1.0  # first sleep before retry, if any
+    )
+    DEEPEVAL_RETRY_EXP_BASE: confloat(ge=1) = (
+        2.0  # exponential growth factor for sleeps
+    )
+    DEEPEVAL_RETRY_JITTER: confloat(ge=0) = 2.0  # uniform jitter
+    DEEPEVAL_RETRY_CAP_SECONDS: confloat(ge=0) = (
+        5.0  # cap for each backoff sleep
+    )
 
     #
     # Telemetry and Debug
@@ -316,19 +348,87 @@ class Settings(BaseSettings):
     #
     MEDIA_IMAGE_CONNECT_TIMEOUT_SECONDS: float = 3.05
     MEDIA_IMAGE_READ_TIMEOUT_SECONDS: float = 10.0
+    # DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS: per-attempt timeout for provider calls enforced by our retry decorator.
+    # This timeout interacts with retry policy and the task level budget (DEEPEVAL_PER_TASK_TIMEOUT_SECONDS) below.
+    # If you leave this at 0/None, the computed outer budget defaults to 180s.
+    DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS: Optional[confloat(ge=0)] = (
+        None  # per-attempt timeout. Set 0/None to disable
+    )
 
     #
     # Async Task Configuration
     #
-
-    # Maximum time allowed for a single task to complete
-    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS: int = (
-        300  # Set to float('inf') to disable timeout
-    )
+    DEEPEVAL_TIMEOUT_THREAD_LIMIT: conint(ge=1) = 128
+    DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS: confloat(ge=0) = 5.0
+    # DEEPEVAL_PER_TASK_TIMEOUT_SECONDS is the outer time budget for one metric/task.
+    # It is computed from per-attempt timeout + retries/backoff unless you explicitly override it.
+    # - OVERRIDE = None or 0 -> auto compute as:
+    #     attempts * per_attempt_timeout + sum(backoff_sleeps) + ~jitter/2 per sleep + 1s safety
+    #   (If per_attempt_timeout is 0/None, the auto outer budget defaults to 180s.)
+    # - OVERRIDE > 0         -> use that exact value. A warning is logged if it is likely too small
+    #   to permit the configured attempts/backoff.
+    #
+    # Tip:
+    #   Most users only need to set DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS and DEEPEVAL_RETRY_MAX_ATTEMPTS.
+    #   Leave the outer budget on auto unless you have very strict SLAs.
+    DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE: Optional[conint(ge=0)] = None
 
     # Buffer time for gathering results from all tasks, added to the longest task duration
     # Increase if many tasks are running concurrently
-    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: int = 60
+    DEEPEVAL_TASK_GATHER_BUFFER_SECONDS: confloat(ge=0) = 60
+
+    ###################
+    # Computed Fields #
+    ###################
+
+    def _calc_auto_outer_timeout(self) -> int:
+        """Compute outer budget from per-attempt timeout + retries/backoff.
+        Never reference the computed property itself here.
+        """
+        attempts = self.DEEPEVAL_RETRY_MAX_ATTEMPTS or 1
+        timeout_seconds = float(self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
+        if timeout_seconds <= 0:
+            # No per-attempt timeout set -> default outer budget
+            return 180
+
+        sleeps = max(0, attempts - 1)
+        cur = float(self.DEEPEVAL_RETRY_INITIAL_SECONDS)
+        cap = float(self.DEEPEVAL_RETRY_CAP_SECONDS)
+        base = float(self.DEEPEVAL_RETRY_EXP_BASE)
+        jitter = float(self.DEEPEVAL_RETRY_JITTER)
+
+        backoff = 0.0
+        for _ in range(sleeps):
+            backoff += min(cap, cur)
+            cur *= base
+        backoff += sleeps * (jitter / 2.0)  # expected jitter
+
+        safety_overhead = 1.0
+        return int(
+            math.ceil(attempts * timeout_seconds + backoff + safety_overhead)
+        )
+
+    @computed_field
+    @property
+    def DEEPEVAL_PER_TASK_TIMEOUT_SECONDS(self) -> int:
+        """If OVERRIDE is set (nonzero), return it; else return the derived budget."""
+        outer = self.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS_OVERRIDE
+        if outer not in (None, 0):
+            # Warn if user-provided outer is likely to truncate retries
+            if (self.DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0) > 0:
+                min_needed = self._calc_auto_outer_timeout()
+                if int(outer) < min_needed:
+                    if self.DEEPEVAL_VERBOSE_MODE:
+                        logger.warning(
+                            "Metric timeout (outer=%ss) is less than attempts × per-attempt "
+                            "timeout + backoff (≈%ss). Retries may be cut short.",
+                            int(outer),
+                            min_needed,
+                        )
+            return int(outer)
+
+        # Auto mode
+        return self._calc_auto_outer_timeout()
 
     ##############
     # Validators #
@@ -474,7 +574,9 @@ class Settings(BaseSettings):
             if s in SUPPORTED_PROVIDER_SLUGS:
                 normalized.append(s)
             else:
-                if cls.DEEPEVAL_VERBOSE_MODE:
+                if parse_bool(
+                    os.getenv("DEEPEVAL_VERBOSE_MODE"), default=False
+                ):
                     logger.warning("Unknown provider slug %r dropped", item)
 
         if star:
@@ -487,6 +589,7 @@ class Settings(BaseSettings):
     @field_validator(
         "DEEPEVAL_RETRY_BEFORE_LOG_LEVEL",
         "DEEPEVAL_RETRY_AFTER_LOG_LEVEL",
+        "LOG_LEVEL",
         mode="before",
     )
     @classmethod
@@ -524,6 +627,10 @@ class Settings(BaseSettings):
     # Persistence support #
     #######################
     class _SettingsEditCtx:
+        COMPUTED_FIELDS: frozenset[str] = frozenset(
+            {"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS"}
+        )
+
         def __init__(
             self,
             settings: "Settings",
@@ -559,8 +666,11 @@ class Settings(BaseSettings):
             # lazy import legacy JSON store deps
             from deepeval.key_handler import KEY_FILE_HANDLER
 
+            model_fields = type(self._s).model_fields
+            # Exclude computed fields from persistence
+
             # compute diff of changed fields
-            after = {k: getattr(self._s, k) for k in type(self._s).model_fields}
+            after = {k: getattr(self._s, k) for k in model_fields}
 
             before_norm = {
                 k: _normalize_for_env(v) for k, v in self._before.items()
@@ -570,12 +680,21 @@ class Settings(BaseSettings):
             changed_keys = {
                 k for k in after_norm if after_norm[k] != before_norm.get(k)
             }
+            changed_keys -= self.COMPUTED_FIELDS
+
             if not changed_keys:
                 self.result = PersistResult(False, None, {})
                 return False
 
             updates = {k: after[k] for k in changed_keys}
 
+            if "LOG_LEVEL" in updates:
+                from deepeval.config.logging import (
+                    apply_deepeval_log_level,
+                )
+
+                apply_deepeval_log_level()
+
             #
             # .deepeval JSON support
             #
@@ -681,4 +800,27 @@ def get_settings() -> Settings:
     global _settings_singleton
     if _settings_singleton is None:
         _settings_singleton = Settings()
+        from deepeval.config.logging import apply_deepeval_log_level
+
+        apply_deepeval_log_level()
     return _settings_singleton
+
+
+def reset_settings(*, reload_dotenv: bool = False) -> Settings:
+    """
+    Drop the cached Settings singleton and rebuild it from the current process
+    environment.
+
+    Args:
+        reload_dotenv: When True, call `autoload_dotenv()` before re-instantiating,
+                       which merges .env values into os.environ (never overwriting
+                       existing process env vars).
+
+    Returns:
+        The fresh Settings instance.
+    """
+    global _settings_singleton
+    if reload_dotenv:
+        autoload_dotenv()
+    _settings_singleton = None
+    return get_settings()

deepeval/evaluate/execute.py

@@ -91,7 +91,6 @@ from deepeval.config.settings import get_settings
 
 
 logger = logging.getLogger(__name__)
-settings = get_settings()
 
 
 async def _snapshot_tasks():
@@ -100,6 +99,18 @@ async def _snapshot_tasks():
     return {t for t in asyncio.all_tasks() if t is not cur}
 
 
+def _per_task_timeout() -> float:
+    return get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+
+
+def _gather_timeout() -> float:
+    s = get_settings()
+    return (
+        s.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+        + s.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS
+    )
+
+
 ###########################################
 ### E2E Evals #############################
 ###########################################
@@ -838,7 +849,7 @@ def execute_agentic_test_cases(
                         loop.run_until_complete(
                             asyncio.wait_for(
                                 coro,
-                                timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                                timeout=_per_task_timeout(),
                             )
                         )
                     else:
@@ -1196,7 +1207,7 @@ async def _a_execute_agentic_test_case(
             if asyncio.iscoroutinefunction(observed_callback):
                 await asyncio.wait_for(
                     observed_callback(golden.input),
-                    timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                    timeout=_per_task_timeout(),
                 )
             else:
                 observed_callback(golden.input)
@@ -1753,11 +1764,6 @@ def a_execute_agentic_test_cases_from_loop(
     _is_assert_test: bool = False,
 ) -> Iterator[TestResult]:
 
-    GATHER_TIMEOUT_SECONDS = (
-        settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
-        + settings.DEEPEVAL_TASK_GATHER_BUFFER_SECONDS
-    )
-
     semaphore = asyncio.Semaphore(async_config.max_concurrent)
     original_create_task = asyncio.create_task
 
@@ -1772,7 +1778,7 @@ def a_execute_agentic_test_cases_from_loop(
     async def execute_callback_with_semaphore(coroutine: Awaitable):
         async with semaphore:
             return await asyncio.wait_for(
-                coroutine, timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
+                coroutine, timeout=_per_task_timeout()
             )
 
     def evaluate_test_cases(
@@ -1814,7 +1820,7 @@ def a_execute_agentic_test_cases_from_loop(
             }
 
             def on_task_done(t: asyncio.Task):
-                if settings.DEEPEVAL_DEBUG_ASYNC:
+                if get_settings().DEEPEVAL_DEBUG_ASYNC:
                     # Using info level here to make it easy to spot these logs.
                     # We are gated by DEEPEVAL_DEBUG_ASYNC
                     meta = task_meta.get(t, {})
@@ -1888,7 +1894,7 @@ def a_execute_agentic_test_cases_from_loop(
                 loop.run_until_complete(
                     asyncio.wait_for(
                         asyncio.gather(*created_tasks, return_exceptions=True),
-                        timeout=GATHER_TIMEOUT_SECONDS,
+                        timeout=_gather_timeout(),
                     )
                 )
             except asyncio.TimeoutError:
@@ -1903,16 +1909,13 @@ def a_execute_agentic_test_cases_from_loop(
                     elapsed_time = time.perf_counter() - start_time
 
                     # Determine if it was a per task or gather timeout based on task's elapsed time
-                    if (
-                        elapsed_time
-                        >= settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS
-                    ):
+                    if elapsed_time >= _per_task_timeout():
                         timeout_type = "per-task"
                     else:
                         timeout_type = "gather"
 
                     logger.warning(
-                        f"[deepeval] gather TIMEOUT after {GATHER_TIMEOUT_SECONDS}s; "
+                        f"[deepeval] gather TIMEOUT after {_gather_timeout()}s; "
                         f"pending={len(pending)} tasks. Timeout type: {timeout_type}. "
                         f"To give tasks more time, consider increasing "
                         f"DEEPEVAL_PER_TASK_TIMEOUT_SECONDS for longer task completion time or "
@@ -1926,7 +1929,7 @@ def a_execute_agentic_test_cases_from_loop(
                         elapsed_time,
                         meta,
                     )
-                    if loop.get_debug() and settings.DEEPEVAL_DEBUG_ASYNC:
+                    if loop.get_debug() and get_settings().DEEPEVAL_DEBUG_ASYNC:
                         frames = t.get_stack(limit=6)
                         if frames:
                             logger.info("    stack:")
@@ -1965,7 +1968,7 @@ def a_execute_agentic_test_cases_from_loop(
                 if not leftovers:
                     return
 
-                if settings.DEEPEVAL_DEBUG_ASYNC:
+                if get_settings().DEEPEVAL_DEBUG_ASYNC:
                     logger.warning(
                         "[deepeval] %d stray task(s) not tracked; cancelling...",
                         len(leftovers),
@@ -1985,7 +1988,7 @@ def a_execute_agentic_test_cases_from_loop(
                     )
                 except RuntimeError:
                     # If the loop is closing here, just continue
-                    if settings.DEEPEVAL_DEBUG_ASYNC:
+                    if get_settings().DEEPEVAL_DEBUG_ASYNC:
                         logger.warning(
                             "[deepeval] failed to drain stray tasks because loop is closing"
                         )

deepeval/metrics/g_eval/g_eval.py

@@ -1,5 +1,7 @@
 """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
+import asyncio
+
 from typing import Optional, List, Tuple, Union, Type
 from deepeval.metrics import BaseMetric
 from deepeval.test_case import (
@@ -16,7 +18,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.g_eval.schema import *
+from deepeval.metrics.g_eval import schema as gschema
 from deepeval.metrics.g_eval.utils import (
     Rubric,
     construct_g_eval_params_string,
@@ -29,6 +31,7 @@ from deepeval.metrics.g_eval.utils import (
     number_evaluation_steps,
     get_score_range,
 )
+from deepeval.config.settings import get_settings
 
 
 class GEval(BaseMetric):
@@ -81,12 +84,16 @@ class GEval(BaseMetric):
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
+                coro = self.a_measure(
+                    test_case,
+                    _show_indicator=False,
+                    _in_component=_in_component,
+                    _additional_context=_additional_context,
+                )
                 loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _additional_context=_additional_context,
+                    asyncio.wait_for(
+                        coro,
+                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
                     )
                 )
             else:
@@ -177,7 +184,9 @@ class GEval(BaseMetric):
             return data["steps"]
         else:
             try:
-                res: Steps = await self.model.a_generate(prompt, schema=Steps)
+                res: gschema.Steps = await self.model.a_generate(
+                    prompt, schema=gschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = await self.model.a_generate(prompt)
@@ -201,7 +210,9 @@ class GEval(BaseMetric):
             return data["steps"]
         else:
             try:
-                res: Steps = self.model.generate(prompt, schema=Steps)
+                res: gschema.Steps = self.model.generate(
+                    prompt, schema=gschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = self.model.generate(prompt)
@@ -264,7 +275,7 @@ class GEval(BaseMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except (
             AttributeError
@@ -276,8 +287,8 @@ class GEval(BaseMetric):
                 return data["score"], data["reason"]
             else:
                 try:
-                    res: ReasonScore = await self.model.a_generate(
-                        prompt, schema=ReasonScore
+                    res: gschema.ReasonScore = await self.model.a_generate(
+                        prompt, schema=gschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -338,7 +349,7 @@ class GEval(BaseMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
@@ -349,8 +360,8 @@ class GEval(BaseMetric):
                 return data["score"], data["reason"]
             else:
                 try:
-                    res: ReasonScore = self.model.generate(
-                        prompt, schema=ReasonScore
+                    res: gschema.ReasonScore = self.model.generate(
+                        prompt, schema=gschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -364,7 +375,7 @@ class GEval(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/prompt_alignment/prompt_alignment.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from typing import Optional, List, Union
 
 from deepeval.utils import get_or_create_event_loop, prettify_list
@@ -15,7 +17,8 @@ from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.prompt_alignment.template import PromptAlignmentTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.prompt_alignment.schema import *
+from deepeval.metrics.prompt_alignment import schema as paschema
+from deepeval.config.settings import get_settings
 
 
 class PromptAlignmentMetric(BaseMetric):
@@ -62,15 +65,19 @@ class PromptAlignmentMetric(BaseMetric):
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
+                coro = self.a_measure(
+                    test_case,
+                    _show_indicator=False,
+                    _in_component=_in_component,
+                )
                 loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
+                    asyncio.wait_for(
+                        coro,
+                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
                     )
                 )
             else:
-                self.verdicts: Verdicts = self._generate_verdicts(
+                self.verdicts: paschema.Verdicts = self._generate_verdicts(
                     test_case.input, test_case.actual_output
                 )
                 self.score = self._calculate_score()
@@ -105,7 +112,7 @@ class PromptAlignmentMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: Verdicts = await self._a_generate_verdicts(
+            self.verdicts: paschema.Verdicts = await self._a_generate_verdicts(
                 test_case.input, test_case.actual_output
             )
             self.score = self._calculate_score()
@@ -141,14 +148,17 @@ class PromptAlignmentMetric(BaseMetric):
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(
-                prompt, schema=PromptAlignmentScoreReason
+                prompt, schema=paschema.PromptAlignmentScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: PromptAlignmentScoreReason = await self.model.a_generate(
-                    prompt=prompt, schema=PromptAlignmentScoreReason
+                res: paschema.PromptAlignmentScoreReason = (
+                    await self.model.a_generate(
+                        prompt=prompt,
+                        schema=paschema.PromptAlignmentScoreReason,
+                    )
                 )
                 return res.reason
             except TypeError:
@@ -173,14 +183,14 @@ class PromptAlignmentMetric(BaseMetric):
         )
         if self.using_native_model:
             res, cost = self.model.generate(
-                prompt, schema=PromptAlignmentScoreReason
+                prompt, schema=paschema.PromptAlignmentScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: PromptAlignmentScoreReason = self.model.generate(
-                    prompt=prompt, schema=PromptAlignmentScoreReason
+                res: paschema.PromptAlignmentScoreReason = self.model.generate(
+                    prompt=prompt, schema=paschema.PromptAlignmentScoreReason
                 )
                 return res.reason
             except TypeError:
@@ -190,48 +200,56 @@ class PromptAlignmentMetric(BaseMetric):
 
     async def _a_generate_verdicts(
         self, input: str, actual_output: str
-    ) -> Verdicts:
+    ) -> paschema.Verdicts:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            res, cost = await self.model.a_generate(
+                prompt, schema=paschema.Verdicts
+            )
             self.evaluation_cost += cost
             return [item for item in res.verdicts]
         else:
             try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
+                res: paschema.Verdicts = await self.model.a_generate(
+                    prompt, schema=paschema.Verdicts
                 )
                 return [item for item in res.verdicts]
             except TypeError:
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return [
-                    PromptAlignmentVerdict(**item) for item in data["verdicts"]
+                    paschema.PromptAlignmentVerdict(**item)
+                    for item in data["verdicts"]
                 ]
 
-    def _generate_verdicts(self, input: str, actual_output: str) -> Verdicts:
+    def _generate_verdicts(
+        self, input: str, actual_output: str
+    ) -> paschema.Verdicts:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
+            res, cost = self.model.generate(prompt, schema=paschema.Verdicts)
             self.evaluation_cost += cost
             return [item for item in res.verdicts]
         else:
             try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                res: paschema.Verdicts = self.model.generate(
+                    prompt, schema=paschema.Verdicts
+                )
                 return [item for item in res.verdicts]
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return [
-                    PromptAlignmentVerdict(**item) for item in data["verdicts"]
+                    paschema.PromptAlignmentVerdict(**item)
+                    for item in data["verdicts"]
                 ]
 
     def _calculate_score(self):
@@ -253,7 +271,7 @@ class PromptAlignmentMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/models/retry_policy.py

@@ -33,9 +33,13 @@ Retry logging (settings; read at call time):
 
 from __future__ import annotations
 
+import asyncio
+import inspect
+import itertools
+import functools
+import threading
 import logging
 
-from deepeval.utils import read_env_int, read_env_float
 from dataclasses import dataclass, field
 from typing import Callable, Iterable, Mapping, Optional, Sequence, Tuple, Union
 from collections.abc import Mapping as ABCMapping
@@ -58,6 +62,9 @@ from deepeval.config.settings import get_settings
 
 logger = logging.getLogger(__name__)
 Provider = Union[str, PS]
+_MAX_TIMEOUT_THREADS = get_settings().DEEPEVAL_TIMEOUT_THREAD_LIMIT
+_TIMEOUT_SEMA = threading.BoundedSemaphore(_MAX_TIMEOUT_THREADS)
+_WORKER_ID = itertools.count(1)
 
 # --------------------------
 # Policy description
@@ -184,6 +191,12 @@ def extract_error_code(
 # Predicate factory
 # --------------------------
 
+_BUILTIN_TIMEOUT_EXCS = (
+    (TimeoutError,)
+    if asyncio.TimeoutError is TimeoutError
+    else (TimeoutError, asyncio.TimeoutError)
+)
+
 
 def make_is_transient(
     policy: ErrorPolicy,
@@ -213,6 +226,9 @@ def make_is_transient(
     )
 
     def _pred(e: Exception) -> bool:
+        if isinstance(e, _BUILTIN_TIMEOUT_EXCS):
+            return True
+
         if isinstance(e, policy.auth_excs):
             return False
 
@@ -245,18 +261,23 @@ def make_is_transient(
 
 class StopFromEnv(stop_base):
     def __call__(self, retry_state):
-        attempts = read_env_int("DEEPEVAL_RETRY_MAX_ATTEMPTS", 2, min_value=1)
+        settings = get_settings()
+        attempts = (
+            settings.DEEPEVAL_RETRY_MAX_ATTEMPTS
+        )  # TODO: add constraints in settings
         return stop_after_attempt(attempts)(retry_state)
 
 
 class WaitFromEnv(wait_base):
     def __call__(self, retry_state):
-        initial = read_env_float(
-            "DEEPEVAL_RETRY_INITIAL_SECONDS", 1.0, min_value=0.0
-        )
-        exp_base = read_env_float("DEEPEVAL_RETRY_EXP_BASE", 2.0, min_value=1.0)
-        jitter = read_env_float("DEEPEVAL_RETRY_JITTER", 2.0, min_value=0.0)
-        cap = read_env_float("DEEPEVAL_RETRY_CAP_SECONDS", 5.0, min_value=0.0)
+        settings = get_settings()
+        initial = settings.DEEPEVAL_RETRY_INITIAL_SECONDS
+        exp_base = settings.DEEPEVAL_RETRY_EXP_BASE
+        jitter = settings.DEEPEVAL_RETRY_JITTER
+        cap = settings.DEEPEVAL_RETRY_CAP_SECONDS
+
+        if cap == 0:  # <- 0 means no backoff sleeps or jitter
+            return 0
         return wait_exponential_jitter(
             initial=initial, exp_base=exp_base, jitter=jitter, max=cap
         )(retry_state)
@@ -324,10 +345,11 @@ def dynamic_retry(provider: Provider):
 
 def _retry_log_levels():
     s = get_settings()
+    base_level = s.LOG_LEVEL if s.LOG_LEVEL is not None else logging.INFO
     before_level = s.DEEPEVAL_RETRY_BEFORE_LOG_LEVEL
     after_level = s.DEEPEVAL_RETRY_AFTER_LOG_LEVEL
     return (
-        before_level if before_level is not None else logging.INFO,
+        before_level if before_level is not None else base_level,
         after_level if after_level is not None else logging.ERROR,
     )
 
@@ -394,21 +416,190 @@ def make_after_log(slug: str):
     return _after
 
 
+def _make_timeout_error(timeout_seconds: float) -> TimeoutError:
+    settings = get_settings()
+    if logger.isEnabledFor(logging.DEBUG):
+        logger.debug(
+            "retry config: per_attempt=%s s, max_attempts=%s, per_task_budget=%s s",
+            timeout_seconds,
+            settings.DEEPEVAL_RETRY_MAX_ATTEMPTS,
+            settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+        )
+    msg = (
+        f"call timed out after {timeout_seconds:g}s (per attempt). "
+        "Increase DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS (0 disables) or reduce work per attempt."
+    )
+    return TimeoutError(msg)
+
+
+def _run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
+    """
+    Run a synchronous callable with a soft timeout enforced by a helper thread,
+    with a global cap on concurrent timeout-workers.
+
+    How it works
+    ------------
+    - A module-level BoundedSemaphore (size = settings.DEEPEVAL_TIMEOUT_THREAD_LIMIT)
+      gates creation of timeout worker threads. If no permit is available, this call
+      blocks until a slot frees up. If settings.DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS
+      > 0 and acquisition takes longer than that, a warning is logged before continuing
+      to wait.
+    - Once a permit is acquired, a daemon thread executes `func(*args, **kwargs)`.
+    - We wait up to `timeout_seconds` for completion. If the timeout elapses, we raise
+      `TimeoutError`. The worker thread is not killed, it continues and releases the semaphore when it eventually finishes.
+    - If the worker finishes in time, we return its result or re-raise its exception
+      (with original traceback).
+
+    Cancellation semantics
+    ----------------------
+    This is a soft timeout: Python threads cannot be forcibly terminated. When timeouts
+    are rare this is fine. If timeouts are common, consider moving to:
+      - a shared ThreadPoolExecutor (caps threads and amortizes creation), or
+      - worker process (supports killing in-flight processes)
+
+    Concurrency control & logging
+    -----------------------------
+    - Concurrency is bounded by `DEEPEVAL_TIMEOUT_THREAD_LIMIT`.
+    - If acquisition exceeds `DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS`, we log a
+      warning and then block until a slot is available.
+    - On timeout, if DEBUG is enabled and `DEEPEVAL_VERBOSE_MODE` is True, we log a short
+      thread sample to help diagnose pressure.
+
+    Args:
+        func: Synchronous callable to execute.
+        timeout_seconds: Float seconds for the soft timeout (0/None disables).
+        *args, **kwargs: Passed through to `func`.
+
+    Returns:
+        Whatever `func` returns.
+
+    Raises:
+        TimeoutError: If `timeout_seconds` elapse before completion.
+        BaseException: If `func` raises, the same exception is re-raised with its
+                       original traceback.
+    """
+    if not timeout_seconds or timeout_seconds <= 0:
+        return func(*args, **kwargs)
+
+    # try to respect the global cap on concurrent timeout workers
+    warn_after = float(
+        get_settings().DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS or 0.0
+    )
+    if warn_after > 0:
+        acquired = _TIMEOUT_SEMA.acquire(timeout=warn_after)
+        if not acquired:
+            logger.warning(
+                "timeout thread limit reached (%d); waiting for a slot...",
+                _MAX_TIMEOUT_THREADS,
+            )
+            _TIMEOUT_SEMA.acquire()
+    else:
+        _TIMEOUT_SEMA.acquire()
+
+    done = threading.Event()
+    result = {"value": None, "exc": None}
+
+    def target():
+        try:
+            result["value"] = func(*args, **kwargs)
+        except BaseException as e:
+            result["exc"] = e
+        finally:
+            done.set()
+            _TIMEOUT_SEMA.release()
+
+    t = threading.Thread(
+        target=target,
+        daemon=True,
+        name=f"deepeval-timeout-worker-{next(_WORKER_ID)}",
+    )
+
+    try:
+        t.start()
+    except BaseException:
+        _TIMEOUT_SEMA.release()
+        raise
+
+    finished = done.wait(timeout_seconds)
+    if not finished:
+        if (
+            logger.isEnabledFor(logging.DEBUG)
+            and get_settings().DEEPEVAL_VERBOSE_MODE
+        ):
+            names = [th.name for th in threading.enumerate()[:10]]
+            logger.debug(
+                "timeout after %.3fs (active_threads=%d, sample=%s)",
+                timeout_seconds,
+                threading.active_count(),
+                names,
+            )
+        raise _make_timeout_error(timeout_seconds)
+
+    # Completed within time: return or raise
+    if result["exc"] is not None:
+        exc = result["exc"]
+        raise exc.with_traceback(getattr(exc, "__traceback__", None))
+    return result["value"]
+
+
 def create_retry_decorator(provider: Provider):
     """
     Build a Tenacity @retry decorator wired to our dynamic retry policy
     for the given provider slug.
     """
     slug = slugify(provider)
-
-    return retry(
+    base_retry = retry(
         wait=dynamic_wait(),
         stop=dynamic_stop(),
         retry=dynamic_retry(slug),
         before_sleep=make_before_sleep_log(slug),
         after=make_after_log(slug),
+        reraise=False,
     )
 
+    def _decorator(func):
+        if inspect.iscoroutinefunction(func):
+
+            @functools.wraps(func)
+            async def attempt(*args, **kwargs):
+                timeout_seconds = (
+                    get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
+                )
+                coro = func(*args, **kwargs)
+                if timeout_seconds > 0:
+                    try:
+                        return await asyncio.wait_for(coro, timeout_seconds)
+                    except asyncio.TimeoutError as e:
+                        if (
+                            logger.isEnabledFor(logging.DEBUG)
+                            and get_settings().DEEPEVAL_VERBOSE_MODE is True
+                        ):
+                            logger.debug(
+                                "async timeout after %.3fs (active_threads=%d, tasks=%d)",
+                                timeout_seconds,
+                                threading.active_count(),
+                                len(asyncio.all_tasks()),
+                            )
+                        raise _make_timeout_error(timeout_seconds) from e
+                return await coro
+
+            return base_retry(attempt)
+
+        @functools.wraps(func)
+        def attempt(*args, **kwargs):
+            timeout_seconds = (
+                get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
+            )
+            if timeout_seconds > 0:
+                return _run_sync_with_timeout(
+                    func, timeout_seconds, *args, **kwargs
+                )
+            return func(*args, **kwargs)
+
+        return base_retry(attempt)
+
+    return _decorator
+
 
 def _httpx_net_excs() -> tuple[type, ...]:
     try:

deepeval/tracing/otel/exporter.py

@@ -90,12 +90,6 @@ class ConfidentSpanExporter(SpanExporter):
         api_key: Optional[str] = None,  # dynamic api key,
         _test_run_id: Optional[str] = None,
     ) -> SpanExportResult:
-        # build forest of spans
-        # for span in spans:
-        #     print("--------------------------------")
-        #     print(span.to_json())
-        #     print("--------------------------------")
-        # return SpanExportResult.SUCCESS
 
         ################ Build Forest of Spans ################
         forest = self._build_span_forest(spans)

deepeval/tracing/otel/utils.py

@@ -109,8 +109,24 @@ def check_llm_input_from_gen_ai_attributes(
     input = None
     output = None
     try:
-        input = json.loads(span.attributes.get("gen_ai.input.messages"))
-        input = _flatten_input(input)
+        # check for system instructions
+        system_instructions = []
+        system_instructions_raw = span.attributes.get(
+            "gen_ai.system_instructions"
+        )
+        if system_instructions_raw and isinstance(system_instructions_raw, str):
+            system_instructions_json = json.loads(system_instructions_raw)
+            system_instructions = _flatten_system_instructions(
+                system_instructions_json
+            )
+
+        input_messages = []
+        input_messages_raw = span.attributes.get("gen_ai.input.messages")
+        if input_messages_raw and isinstance(input_messages_raw, str):
+            input_messages_json = json.loads(input_messages_raw)
+            input_messages = _flatten_input(input_messages_json)
+
+        input = system_instructions + input_messages
 
     except Exception:
         pass
@@ -137,6 +153,20 @@ def check_llm_input_from_gen_ai_attributes(
     return input, output
 
 
+def _flatten_system_instructions(system_instructions: list) -> list:
+    if isinstance(system_instructions, list):
+        for system_instruction in system_instructions:
+            if isinstance(system_instruction, dict):
+                role = system_instruction.get("role")
+                if not role:
+                    system_instruction["role"] = "System Instruction"
+        return _flatten_input(system_instructions)
+    elif isinstance(system_instructions, str):
+        return [{"role": "System Instruction", "content": system_instructions}]
+
+    return []
+
+
 def _flatten_input(input: list) -> list:
     if input and isinstance(input, list):
         try:
@@ -411,10 +441,23 @@ def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     return None
 
 
+def _extract_non_thinking_part_of_last_message(message: dict) -> dict:
+
+    if isinstance(message, dict) and message.get("role") == "assistant":
+        parts = message.get("parts")
+        if parts:
+            # Iterate from the last part
+            for part in reversed(parts):
+                if isinstance(part, dict) and part.get("type") == "text":
+                    # Return a modified message with only the text content
+                    return {"role": "assistant", "content": part.get("content")}
+    return None
+
+
 def check_pydantic_ai_agent_input_output(
     span: ReadableSpan,
 ) -> Tuple[Optional[Any], Optional[Any]]:
-    input_val: Optional[Any] = None
+    input_val: list = []
     output_val: Optional[Any] = None
 
     # Get normalized messages once
@@ -445,14 +488,21 @@ def check_pydantic_ai_agent_input_output(
         if span.attributes.get("confident.span.type") == "agent":
             output_val = span.attributes.get("final_result")
             if not output_val and normalized:
-                # Extract the last message if no final_result is available
-                output_val = normalized[-1]
+                output_val = _extract_non_thinking_part_of_last_message(
+                    normalized[-1]
+                )
     except Exception:
         pass
 
+    system_instructions = []
+    system_instruction_raw = span.attributes.get("gen_ai.system_instructions")
+    if system_instruction_raw and isinstance(system_instruction_raw, str):
+        system_instructions = _flatten_system_instructions(
+            json.loads(system_instruction_raw)
+        )
+
     input_val = _flatten_input(input_val)
-    output_val = _flatten_input(output_val)
-    return input_val, output_val
+    return system_instructions + input_val, output_val
 
 
 def check_tool_output(span: ReadableSpan):

deepeval/tracing/trace_test_manager.py

@@ -0,0 +1,19 @@
+from typing import Optional, Dict, Any
+import asyncio
+from time import monotonic
+
+
+class TraceTestingManager:
+    test_name: Optional[str] = None
+    test_dict: Optional[Dict[str, Any]] = None
+
+    async def wait_for_test_dict(
+        self, timeout: float = 10.0, poll_interval: float = 0.05
+    ) -> Dict[str, Any]:
+        deadline = monotonic() + timeout
+        while self.test_dict is None and monotonic() < deadline:
+            await asyncio.sleep(poll_interval)
+        return self.test_dict or {}
+
+
+trace_testing_manager = TraceTestingManager()

deepeval/tracing/tracing.py

@@ -53,7 +53,7 @@ from deepeval.utils import dataclass_to_dict
 from deepeval.tracing.context import current_span_context, current_trace_context
 from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.tracing.api import PromptApi
-from tests.test_integrations.manager import trace_testing_manager
+from deepeval.tracing.trace_test_manager import trace_testing_manager
 
 EVAL_DUMMY_SPAN_NAME = "evals_iterator"
 

{deepeval-3.6.5.dist-info → deepeval-3.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.5
+Version: 3.6.6
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

{deepeval-3.6.5.dist-info → deepeval-3.6.6.dist-info}/RECORD

@@ -1,5 +1,5 @@
-deepeval/__init__.py,sha256=6fsb813LD_jNhqR-xZnSdE5E-KsBbC3tc4oIg5ZMgTw,2115
-deepeval/_version.py,sha256=7XydZTr-OhyEmxjczbOo90U1nYQK6hBYF4GXri8UIcY,27
+deepeval/__init__.py,sha256=IqShG98ALpA1gm_qL2Jq56AJoafAHpcUTSvpgH4HpZM,3062
+deepeval/_version.py,sha256=yb70ATorTjss8Uu310wa6TkPe0yTadiC7Lxb0-KZxMA,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -141,7 +141,8 @@ deepeval/confident/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVG
 deepeval/confident/api.py,sha256=2ZhrQOtfxcnQSyY6OxrjY17y1yn-NB7pfIiJa20B1Pk,8519
 deepeval/confident/types.py,sha256=-slFhDof_1maMgpLxqDRZv6kz6ZVY2hP_0uj_aveJKU,533
 deepeval/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/config/settings.py,sha256=EwCcYQYQDuayRVmRjYPtxUm9EHKLRu1eLaGfzNdaDEI,21827
+deepeval/config/logging.py,sha256=ivqmhOSB-oHOOU3MvnhImrZwkkxzxKJgoKxesnWfHjg,1314
+deepeval/config/settings.py,sha256=Ifw7HDSTaYCFk3zGHRf15uMCDZuy3NXAuNtWm4jcIUA,27575
 deepeval/config/settings_manager.py,sha256=enahSZN8krRu7-L94OBCt99fwUIqQtMRL97PlzsuKEY,4021
 deepeval/config/utils.py,sha256=gSOVv18Tx1R72GucbdQesbZLFL-Y9EzbS4p7qd2w_xE,3799
 deepeval/constants.py,sha256=Qe-es-WDPJndgBspEQXxddDCVanrAu03YWCpXsUkdo0,1368
@@ -159,7 +160,7 @@ deepeval/evaluate/api.py,sha256=rkblH0ZFAAdyuF0Ymh7JE1pIJPR9yFuPrn9SQaCEQp4,435
 deepeval/evaluate/compare.py,sha256=tdSJY4E7YJ_zO3dzvpwngZHLiUI2YQcTWJOLI83htsQ,9855
 deepeval/evaluate/configs.py,sha256=QfWjaWNxLsgEe8-5j4PIs5WcSyEckiWt0qdpXSpl57M,928
 deepeval/evaluate/evaluate.py,sha256=HoEERRLj8SVCcU1r70VQdSL4LQcSc9p20OhcD1nhEuQ,10594
-deepeval/evaluate/execute.py,sha256=M0o4dpUSkvXnzEK6QIgy-2pa0HQx6w6ZRbXoI03tJeI,88931
+deepeval/evaluate/execute.py,sha256=vkiWaQGBAFbLIJ1tTYpGpu1YDpDSpH6o-oPftqPlNpM,88875
 deepeval/evaluate/types.py,sha256=zsL_lNbFMG20czzRQeWNDbLSzL8Uy7IIgvILe-X0kN0,918
 deepeval/evaluate/utils.py,sha256=oBJFcUDYmmsRvXW7rXkQy3gI1Tuu5bixgvHx0yvnw1c,23563
 deepeval/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -240,7 +241,7 @@ deepeval/metrics/faithfulness/faithfulness.py,sha256=KXI5VPcOsPZ2Pk1-69tR-kq4x-F
 deepeval/metrics/faithfulness/schema.py,sha256=2dU9dwwmqpGJcWvY2webERWIfH_tn02xgLghHkAY_eM,437
 deepeval/metrics/faithfulness/template.py,sha256=RuZ0LFm4BjZ8lhVrKPgU3ecHszwkF0fe5-BxAkaP5AA,5839
 deepeval/metrics/g_eval/__init__.py,sha256=HAhsQFVq9LIpZXPN00Jc_WrMXrh47NIT86VnUpWM4_4,102
-deepeval/metrics/g_eval/g_eval.py,sha256=CaW7VHPW-SyXt18IE1rSatgagY238s3It-j6SLRI4H4,14395
+deepeval/metrics/g_eval/g_eval.py,sha256=zd4_M7UaT_l1GxHrA_g9nzCl5LXH-NYzKYLEWKLhpaU,14875
 deepeval/metrics/g_eval/schema.py,sha256=V629txuDrr_2IEKEsgJVYYZb_pkdfcltQV9ZjvxK5co,287
 deepeval/metrics/g_eval/template.py,sha256=mHj4-mr_HQwbCjpHg7lM_6UesoSatL3g8UGGQAOdT0U,4509
 deepeval/metrics/g_eval/utils.py,sha256=uUT86jRXVYvLDzcnZvvfWssDyGoBHb66nWcJSg4i1u4,8784
@@ -327,7 +328,7 @@ deepeval/metrics/pii_leakage/pii_leakage.py,sha256=EIQMS_hOiYhEW5x4nYJwS6AhWl9jh
 deepeval/metrics/pii_leakage/schema.py,sha256=Jk9jdf4HAa76J237mnosWOCV71pBBNdLfaVhf-4dKEg,313
 deepeval/metrics/pii_leakage/template.py,sha256=DEW21CyR2lEI1y2C_fXgZnGJlYw0fvnB-LF-HEKZnqo,2418
 deepeval/metrics/prompt_alignment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/metrics/prompt_alignment/prompt_alignment.py,sha256=vQQa-nNBYXsHcR6kyPW0Efcqu3jRoahh2ZM0Ft9Cq8Y,9481
+deepeval/metrics/prompt_alignment/prompt_alignment.py,sha256=JW6AjOUvJssTTwOzggT5QPp4zAb5Z4e2H-xQS83Pv3M,10083
 deepeval/metrics/prompt_alignment/schema.py,sha256=ann3tH5XfZCLhePE2UXTSK_gkF-_udP0RP_IVHW4mF0,315
 deepeval/metrics/prompt_alignment/template.py,sha256=6-A1rWOZWOEauSsQAXC-gVF2oXkYfgV0XqiriGJJfY0,3182
 deepeval/metrics/ragas.py,sha256=I4EdwbsRGHlSEraKFBrDGUBrzRUBqlWqJAGg0efrZ0w,17637
@@ -386,7 +387,7 @@ deepeval/models/mlllms/__init__.py,sha256=19nN6kUB5XI0nUWUQX0aD9GBUMM8WWGvsDgKju
 deepeval/models/mlllms/gemini_model.py,sha256=7tHIWD4w_fBz3L7jkKWygn1QpBPk9nl2Kw-yb0Jc3PI,10167
 deepeval/models/mlllms/ollama_model.py,sha256=_YtYtw8oIMVVI-CFsDicsdeEJUPhw_9ArPxB_1olsJA,4798
 deepeval/models/mlllms/openai_model.py,sha256=KgvYgQwWZ1A_Gcl6-4-W7IMqbUF9K8sNY37j5Ag7kQQ,9014
-deepeval/models/retry_policy.py,sha256=U7DjJJcCPei14Ws_7U6-JK8ZZYCGVx9YgMPySYr0HtM,24545
+deepeval/models/retry_policy.py,sha256=efMJwjQasAE_3fstfBBmxLOzxUxws26zHP5yvEBcPfQ,31458
 deepeval/models/summac_model.py,sha256=wKeH7pWQRXrTlzlIw_r1YCb8b7jUhWq6jUz9FiNUCSg,1992
 deepeval/models/unbias_model.py,sha256=umOMhQLTmnD7uOuhiQufEl4Wlti4q2s3EtKOpds7zhs,597
 deepeval/models/utils.py,sha256=-3XDgg1U7PZ0jpLFiYXxqdBhp7idvlo7RPZv5SoD8lc,1130
@@ -453,18 +454,19 @@ deepeval/tracing/offline_evals/span.py,sha256=pXqTVXs-WnjRVpCYYEbNe0zSM6Wz9GsKHs
 deepeval/tracing/offline_evals/thread.py,sha256=bcSGFcZJKnszArOLIlWvnCyt0zSmsd7Xsw5rl4RTVFg,1981
 deepeval/tracing/offline_evals/trace.py,sha256=vTflaTKysKRiYvKA-Nx6PUJ3J6NrRLXiIdWieVcm90E,1868
 deepeval/tracing/otel/__init__.py,sha256=HQsaF5yLPwyW5qg8AOV81_nG_7pFHnatOTHi9Wx3HEk,88
-deepeval/tracing/otel/exporter.py,sha256=wPO1ITKpjueLOSNLO6nD2QL9LAd8Xcu6en8hRkB61Wo,28891
+deepeval/tracing/otel/exporter.py,sha256=eykoPzrRn3ljVO_JKzUeXThZ5iApyImGCDgnimIoUXs,28640
 deepeval/tracing/otel/test_exporter.py,sha256=bezihPGWJpwUEF3ZghxqhhorocVFTO2b43jRM-JHYMU,1076
-deepeval/tracing/otel/utils.py,sha256=yAXyPvTjax2HdLcvbVv9pyOVW4S7elIp3RLGuBTr_8o,15113
+deepeval/tracing/otel/utils.py,sha256=Zw2_PcDm3Dtds1xUZWiVwVIKd4N98bVC8OZ_pkXv7qY,17190
 deepeval/tracing/patchers.py,sha256=DAPNkhrDtoeyJIVeQDUMhTz-xGcXu00eqjQZmov8FiU,3096
 deepeval/tracing/perf_epoch_bridge.py,sha256=iyAPddB6Op7NpMtPHJ29lDm53Btz9yLaN6xSCfTRQm4,1825
 deepeval/tracing/trace_context.py,sha256=jmOH3oBKz1FeNz_J61CUfkuQ3SpyFc6n7mo_HVX6JfU,352
-deepeval/tracing/tracing.py,sha256=EhpZnKjYNCr_K5dTr9gqtK5uzKzhvE-lrk_t8OF5EOE,42903
+deepeval/tracing/trace_test_manager.py,sha256=wt4y7EWTRc4Bw938-UFFtXHkdFFOrnx6JaIk7J5Iulw,555
+deepeval/tracing/tracing.py,sha256=-9GE0fjtv5xKfGZHT9LLEt-38NbqkgXRp1uZ0U1W158,42907
 deepeval/tracing/types.py,sha256=l_utWKerNlE5H3mOKpeUJLsvpP3cMyjH7HRANNgTmSQ,5306
 deepeval/tracing/utils.py,sha256=6SXJ7JJu-6OUziFZ_1IJppuVv7Rlq4cw3c3B7DL_eRQ,5295
 deepeval/utils.py,sha256=J1JNzjAlmn-UsFTK8-c5bhUuk5crwFGVCrRvle-nNmA,21533
-deepeval-3.6.5.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.6.5.dist-info/METADATA,sha256=jVsdK4BG21hV9kceAgzk5Ug34I0d1T7s-R5BIKGiHiQ,18754
-deepeval-3.6.5.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.6.5.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
-deepeval-3.6.5.dist-info/RECORD,,
+deepeval-3.6.6.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.6.6.dist-info/METADATA,sha256=n1o8egypf7Pr3YqgzhZ5ZnO7uSaZrWy3i5x4lyKoHmA,18754
+deepeval-3.6.6.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.6.6.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
+deepeval-3.6.6.dist-info/RECORD,,