deepeval 3.6.4__py3-none-any.whl → 3.6.6__py3-none-any.whl

deepeval/metrics/g_eval/g_eval.py

@@ -1,5 +1,7 @@
 """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
+import asyncio
+
 from typing import Optional, List, Tuple, Union, Type
 from deepeval.metrics import BaseMetric
 from deepeval.test_case import (
@@ -16,7 +18,7 @@ from deepeval.metrics.utils import (
 )
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.g_eval.schema import *
+from deepeval.metrics.g_eval import schema as gschema
 from deepeval.metrics.g_eval.utils import (
     Rubric,
     construct_g_eval_params_string,
@@ -29,6 +31,7 @@ from deepeval.metrics.g_eval.utils import (
     number_evaluation_steps,
     get_score_range,
 )
+from deepeval.config.settings import get_settings
 
 
 class GEval(BaseMetric):
@@ -81,12 +84,16 @@ class GEval(BaseMetric):
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
+                coro = self.a_measure(
+                    test_case,
+                    _show_indicator=False,
+                    _in_component=_in_component,
+                    _additional_context=_additional_context,
+                )
                 loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
-                        _additional_context=_additional_context,
+                    asyncio.wait_for(
+                        coro,
+                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
                     )
                 )
             else:
@@ -177,7 +184,9 @@ class GEval(BaseMetric):
             return data["steps"]
         else:
             try:
-                res: Steps = await self.model.a_generate(prompt, schema=Steps)
+                res: gschema.Steps = await self.model.a_generate(
+                    prompt, schema=gschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = await self.model.a_generate(prompt)
@@ -201,7 +210,9 @@ class GEval(BaseMetric):
             return data["steps"]
         else:
             try:
-                res: Steps = self.model.generate(prompt, schema=Steps)
+                res: gschema.Steps = self.model.generate(
+                    prompt, schema=gschema.Steps
+                )
                 return res.steps
             except TypeError:
                 res = self.model.generate(prompt)
@@ -264,7 +275,7 @@ class GEval(BaseMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except (
             AttributeError
@@ -276,8 +287,8 @@ class GEval(BaseMetric):
                 return data["score"], data["reason"]
             else:
                 try:
-                    res: ReasonScore = await self.model.a_generate(
-                        prompt, schema=ReasonScore
+                    res: gschema.ReasonScore = await self.model.a_generate(
+                        prompt, schema=gschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -338,7 +349,7 @@ class GEval(BaseMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except (KeyError, AttributeError, TypeError, ValueError):
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
@@ -349,8 +360,8 @@ class GEval(BaseMetric):
                 return data["score"], data["reason"]
             else:
                 try:
-                    res: ReasonScore = self.model.generate(
-                        prompt, schema=ReasonScore
+                    res: gschema.ReasonScore = self.model.generate(
+                        prompt, schema=gschema.ReasonScore
                     )
                     return res.score, res.reason
                 except TypeError:
@@ -364,7 +375,7 @@ class GEval(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/metrics/prompt_alignment/prompt_alignment.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from typing import Optional, List, Union
 
 from deepeval.utils import get_or_create_event_loop, prettify_list
@@ -15,7 +17,8 @@ from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.prompt_alignment.template import PromptAlignmentTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.prompt_alignment.schema import *
+from deepeval.metrics.prompt_alignment import schema as paschema
+from deepeval.config.settings import get_settings
 
 
 class PromptAlignmentMetric(BaseMetric):
@@ -62,15 +65,19 @@ class PromptAlignmentMetric(BaseMetric):
         ):
             if self.async_mode:
                 loop = get_or_create_event_loop()
+                coro = self.a_measure(
+                    test_case,
+                    _show_indicator=False,
+                    _in_component=_in_component,
+                )
                 loop.run_until_complete(
-                    self.a_measure(
-                        test_case,
-                        _show_indicator=False,
-                        _in_component=_in_component,
+                    asyncio.wait_for(
+                        coro,
+                        timeout=get_settings().DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
                     )
                 )
             else:
-                self.verdicts: Verdicts = self._generate_verdicts(
+                self.verdicts: paschema.Verdicts = self._generate_verdicts(
                     test_case.input, test_case.actual_output
                 )
                 self.score = self._calculate_score()
@@ -105,7 +112,7 @@ class PromptAlignmentMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: Verdicts = await self._a_generate_verdicts(
+            self.verdicts: paschema.Verdicts = await self._a_generate_verdicts(
                 test_case.input, test_case.actual_output
             )
             self.score = self._calculate_score()
@@ -141,14 +148,17 @@ class PromptAlignmentMetric(BaseMetric):
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(
-                prompt, schema=PromptAlignmentScoreReason
+                prompt, schema=paschema.PromptAlignmentScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: PromptAlignmentScoreReason = await self.model.a_generate(
-                    prompt=prompt, schema=PromptAlignmentScoreReason
+                res: paschema.PromptAlignmentScoreReason = (
+                    await self.model.a_generate(
+                        prompt=prompt,
+                        schema=paschema.PromptAlignmentScoreReason,
+                    )
                 )
                 return res.reason
             except TypeError:
@@ -173,14 +183,14 @@ class PromptAlignmentMetric(BaseMetric):
         )
         if self.using_native_model:
             res, cost = self.model.generate(
-                prompt, schema=PromptAlignmentScoreReason
+                prompt, schema=paschema.PromptAlignmentScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: PromptAlignmentScoreReason = self.model.generate(
-                    prompt=prompt, schema=PromptAlignmentScoreReason
+                res: paschema.PromptAlignmentScoreReason = self.model.generate(
+                    prompt=prompt, schema=paschema.PromptAlignmentScoreReason
                 )
                 return res.reason
             except TypeError:
@@ -190,48 +200,56 @@ class PromptAlignmentMetric(BaseMetric):
 
     async def _a_generate_verdicts(
         self, input: str, actual_output: str
-    ) -> Verdicts:
+    ) -> paschema.Verdicts:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            res, cost = await self.model.a_generate(
+                prompt, schema=paschema.Verdicts
+            )
             self.evaluation_cost += cost
             return [item for item in res.verdicts]
         else:
             try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
+                res: paschema.Verdicts = await self.model.a_generate(
+                    prompt, schema=paschema.Verdicts
                 )
                 return [item for item in res.verdicts]
             except TypeError:
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return [
-                    PromptAlignmentVerdict(**item) for item in data["verdicts"]
+                    paschema.PromptAlignmentVerdict(**item)
+                    for item in data["verdicts"]
                 ]
 
-    def _generate_verdicts(self, input: str, actual_output: str) -> Verdicts:
+    def _generate_verdicts(
+        self, input: str, actual_output: str
+    ) -> paschema.Verdicts:
         prompt = PromptAlignmentTemplate.generate_verdicts(
             prompt_instructions=self.prompt_instructions,
             input=input,
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
+            res, cost = self.model.generate(prompt, schema=paschema.Verdicts)
             self.evaluation_cost += cost
             return [item for item in res.verdicts]
         else:
             try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                res: paschema.Verdicts = self.model.generate(
+                    prompt, schema=paschema.Verdicts
+                )
                 return [item for item in res.verdicts]
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 return [
-                    PromptAlignmentVerdict(**item) for item in data["verdicts"]
+                    paschema.PromptAlignmentVerdict(**item)
+                    for item in data["verdicts"]
                 ]
 
     def _calculate_score(self):
@@ -253,7 +271,7 @@ class PromptAlignmentMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success
 

deepeval/models/retry_policy.py

@@ -33,9 +33,13 @@ Retry logging (settings; read at call time):
 
 from __future__ import annotations
 
+import asyncio
+import inspect
+import itertools
+import functools
+import threading
 import logging
 
-from deepeval.utils import read_env_int, read_env_float
 from dataclasses import dataclass, field
 from typing import Callable, Iterable, Mapping, Optional, Sequence, Tuple, Union
 from collections.abc import Mapping as ABCMapping
@@ -58,6 +62,9 @@ from deepeval.config.settings import get_settings
 
 logger = logging.getLogger(__name__)
 Provider = Union[str, PS]
+_MAX_TIMEOUT_THREADS = get_settings().DEEPEVAL_TIMEOUT_THREAD_LIMIT
+_TIMEOUT_SEMA = threading.BoundedSemaphore(_MAX_TIMEOUT_THREADS)
+_WORKER_ID = itertools.count(1)
 
 # --------------------------
 # Policy description
@@ -184,6 +191,12 @@ def extract_error_code(
 # Predicate factory
 # --------------------------
 
+_BUILTIN_TIMEOUT_EXCS = (
+    (TimeoutError,)
+    if asyncio.TimeoutError is TimeoutError
+    else (TimeoutError, asyncio.TimeoutError)
+)
+
 
 def make_is_transient(
     policy: ErrorPolicy,
@@ -213,6 +226,9 @@ def make_is_transient(
     )
 
     def _pred(e: Exception) -> bool:
+        if isinstance(e, _BUILTIN_TIMEOUT_EXCS):
+            return True
+
         if isinstance(e, policy.auth_excs):
             return False
 
@@ -245,18 +261,23 @@ def make_is_transient(
 
 class StopFromEnv(stop_base):
     def __call__(self, retry_state):
-        attempts = read_env_int("DEEPEVAL_RETRY_MAX_ATTEMPTS", 2, min_value=1)
+        settings = get_settings()
+        attempts = (
+            settings.DEEPEVAL_RETRY_MAX_ATTEMPTS
+        )  # TODO: add constraints in settings
         return stop_after_attempt(attempts)(retry_state)
 
 
 class WaitFromEnv(wait_base):
     def __call__(self, retry_state):
-        initial = read_env_float(
-            "DEEPEVAL_RETRY_INITIAL_SECONDS", 1.0, min_value=0.0
-        )
-        exp_base = read_env_float("DEEPEVAL_RETRY_EXP_BASE", 2.0, min_value=1.0)
-        jitter = read_env_float("DEEPEVAL_RETRY_JITTER", 2.0, min_value=0.0)
-        cap = read_env_float("DEEPEVAL_RETRY_CAP_SECONDS", 5.0, min_value=0.0)
+        settings = get_settings()
+        initial = settings.DEEPEVAL_RETRY_INITIAL_SECONDS
+        exp_base = settings.DEEPEVAL_RETRY_EXP_BASE
+        jitter = settings.DEEPEVAL_RETRY_JITTER
+        cap = settings.DEEPEVAL_RETRY_CAP_SECONDS
+
+        if cap == 0:  # <- 0 means no backoff sleeps or jitter
+            return 0
         return wait_exponential_jitter(
             initial=initial, exp_base=exp_base, jitter=jitter, max=cap
         )(retry_state)
@@ -324,10 +345,11 @@ def dynamic_retry(provider: Provider):
 
 def _retry_log_levels():
     s = get_settings()
+    base_level = s.LOG_LEVEL if s.LOG_LEVEL is not None else logging.INFO
     before_level = s.DEEPEVAL_RETRY_BEFORE_LOG_LEVEL
     after_level = s.DEEPEVAL_RETRY_AFTER_LOG_LEVEL
     return (
-        before_level if before_level is not None else logging.INFO,
+        before_level if before_level is not None else base_level,
         after_level if after_level is not None else logging.ERROR,
     )
 
@@ -394,21 +416,190 @@ def make_after_log(slug: str):
     return _after
 
 
+def _make_timeout_error(timeout_seconds: float) -> TimeoutError:
+    settings = get_settings()
+    if logger.isEnabledFor(logging.DEBUG):
+        logger.debug(
+            "retry config: per_attempt=%s s, max_attempts=%s, per_task_budget=%s s",
+            timeout_seconds,
+            settings.DEEPEVAL_RETRY_MAX_ATTEMPTS,
+            settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+        )
+    msg = (
+        f"call timed out after {timeout_seconds:g}s (per attempt). "
+        "Increase DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS (0 disables) or reduce work per attempt."
+    )
+    return TimeoutError(msg)
+
+
+def _run_sync_with_timeout(func, timeout_seconds, *args, **kwargs):
+    """
+    Run a synchronous callable with a soft timeout enforced by a helper thread,
+    with a global cap on concurrent timeout-workers.
+
+    How it works
+    ------------
+    - A module-level BoundedSemaphore (size = settings.DEEPEVAL_TIMEOUT_THREAD_LIMIT)
+      gates creation of timeout worker threads. If no permit is available, this call
+      blocks until a slot frees up. If settings.DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS
+      > 0 and acquisition takes longer than that, a warning is logged before continuing
+      to wait.
+    - Once a permit is acquired, a daemon thread executes `func(*args, **kwargs)`.
+    - We wait up to `timeout_seconds` for completion. If the timeout elapses, we raise
+      `TimeoutError`. The worker thread is not killed, it continues and releases the semaphore when it eventually finishes.
+    - If the worker finishes in time, we return its result or re-raise its exception
+      (with original traceback).
+
+    Cancellation semantics
+    ----------------------
+    This is a soft timeout: Python threads cannot be forcibly terminated. When timeouts
+    are rare this is fine. If timeouts are common, consider moving to:
+      - a shared ThreadPoolExecutor (caps threads and amortizes creation), or
+      - worker process (supports killing in-flight processes)
+
+    Concurrency control & logging
+    -----------------------------
+    - Concurrency is bounded by `DEEPEVAL_TIMEOUT_THREAD_LIMIT`.
+    - If acquisition exceeds `DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS`, we log a
+      warning and then block until a slot is available.
+    - On timeout, if DEBUG is enabled and `DEEPEVAL_VERBOSE_MODE` is True, we log a short
+      thread sample to help diagnose pressure.
+
+    Args:
+        func: Synchronous callable to execute.
+        timeout_seconds: Float seconds for the soft timeout (0/None disables).
+        *args, **kwargs: Passed through to `func`.
+
+    Returns:
+        Whatever `func` returns.
+
+    Raises:
+        TimeoutError: If `timeout_seconds` elapse before completion.
+        BaseException: If `func` raises, the same exception is re-raised with its
+                       original traceback.
+    """
+    if not timeout_seconds or timeout_seconds <= 0:
+        return func(*args, **kwargs)
+
+    # try to respect the global cap on concurrent timeout workers
+    warn_after = float(
+        get_settings().DEEPEVAL_TIMEOUT_SEMAPHORE_WARN_AFTER_SECONDS or 0.0
+    )
+    if warn_after > 0:
+        acquired = _TIMEOUT_SEMA.acquire(timeout=warn_after)
+        if not acquired:
+            logger.warning(
+                "timeout thread limit reached (%d); waiting for a slot...",
+                _MAX_TIMEOUT_THREADS,
+            )
+            _TIMEOUT_SEMA.acquire()
+    else:
+        _TIMEOUT_SEMA.acquire()
+
+    done = threading.Event()
+    result = {"value": None, "exc": None}
+
+    def target():
+        try:
+            result["value"] = func(*args, **kwargs)
+        except BaseException as e:
+            result["exc"] = e
+        finally:
+            done.set()
+            _TIMEOUT_SEMA.release()
+
+    t = threading.Thread(
+        target=target,
+        daemon=True,
+        name=f"deepeval-timeout-worker-{next(_WORKER_ID)}",
+    )
+
+    try:
+        t.start()
+    except BaseException:
+        _TIMEOUT_SEMA.release()
+        raise
+
+    finished = done.wait(timeout_seconds)
+    if not finished:
+        if (
+            logger.isEnabledFor(logging.DEBUG)
+            and get_settings().DEEPEVAL_VERBOSE_MODE
+        ):
+            names = [th.name for th in threading.enumerate()[:10]]
+            logger.debug(
+                "timeout after %.3fs (active_threads=%d, sample=%s)",
+                timeout_seconds,
+                threading.active_count(),
+                names,
+            )
+        raise _make_timeout_error(timeout_seconds)
+
+    # Completed within time: return or raise
+    if result["exc"] is not None:
+        exc = result["exc"]
+        raise exc.with_traceback(getattr(exc, "__traceback__", None))
+    return result["value"]
+
+
 def create_retry_decorator(provider: Provider):
     """
     Build a Tenacity @retry decorator wired to our dynamic retry policy
     for the given provider slug.
     """
     slug = slugify(provider)
-
-    return retry(
+    base_retry = retry(
         wait=dynamic_wait(),
         stop=dynamic_stop(),
         retry=dynamic_retry(slug),
         before_sleep=make_before_sleep_log(slug),
         after=make_after_log(slug),
+        reraise=False,
     )
 
+    def _decorator(func):
+        if inspect.iscoroutinefunction(func):
+
+            @functools.wraps(func)
+            async def attempt(*args, **kwargs):
+                timeout_seconds = (
+                    get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
+                )
+                coro = func(*args, **kwargs)
+                if timeout_seconds > 0:
+                    try:
+                        return await asyncio.wait_for(coro, timeout_seconds)
+                    except asyncio.TimeoutError as e:
+                        if (
+                            logger.isEnabledFor(logging.DEBUG)
+                            and get_settings().DEEPEVAL_VERBOSE_MODE is True
+                        ):
+                            logger.debug(
+                                "async timeout after %.3fs (active_threads=%d, tasks=%d)",
+                                timeout_seconds,
+                                threading.active_count(),
+                                len(asyncio.all_tasks()),
+                            )
+                        raise _make_timeout_error(timeout_seconds) from e
+                return await coro
+
+            return base_retry(attempt)
+
+        @functools.wraps(func)
+        def attempt(*args, **kwargs):
+            timeout_seconds = (
+                get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0
+            )
+            if timeout_seconds > 0:
+                return _run_sync_with_timeout(
+                    func, timeout_seconds, *args, **kwargs
+                )
+            return func(*args, **kwargs)
+
+        return base_retry(attempt)
+
+    return _decorator
+
 
 def _httpx_net_excs() -> tuple[type, ...]:
     try:

deepeval/test_run/__init__.py

@@ -11,7 +11,7 @@ from .test_run import (
 )
 
 from .hooks import on_test_run_end, invoke_test_run_end_hook
-from .api import MetricData
+from .api import MetricData, TurnApi
 from .hyperparameters import log_hyperparameters
 
 
@@ -28,5 +28,6 @@ __all__ = [
     "on_test_run_end",
     "invoke_test_run_end_hook",
     "MetricData",
+    "TurnApi",
     "log_hyperparameters",
 ]

deepeval/test_run/api.py

@@ -99,6 +99,7 @@ class TurnApi(BaseModel):
     role: str
     content: str
     order: int
+    user_id: Optional[str] = Field(None, alias="userId")
     retrieval_context: Optional[list] = Field(None, alias="retrievalContext")
     tools_called: Optional[List[ToolCall]] = Field(None, alias="toolsCalled")
     additional_metadata: Optional[Dict] = Field(