dreadnode 2.0.9__tar.gz → 2.0.11__tar.gz

{dreadnode-2.0.9 → dreadnode-2.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dreadnode
-Version: 2.0.9
+Version: 2.0.11
 Summary: Dreadnode SDK
 Project-URL: Homepage, https://dreadnode.io
 Project-URL: Documentation, https://docs.dreadnode.io
@@ -19,7 +19,7 @@ Requires-Dist: gepa>=0.1.1
 Requires-Dist: httpx<1.0.0,>=0.28.0
 Requires-Dist: jsonpath-ng>=1.7.0
 Requires-Dist: jsonref>=1.1.0
-Requires-Dist: litellm<=1.82.3,>=1.80.11
+Requires-Dist: litellm<=1.83.0,>=1.83.0
 Requires-Dist: logfire<=3.20.0,>=3.5.3
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: mcp<2.0.0,>=1.25.0

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/agents/agent.py

@@ -1,7 +1,10 @@
 import asyncio
+import contextlib
 import inspect
+import random
+import time
 import typing as t
-from contextlib import AsyncExitStack, asynccontextmanager, nullcontext, suppress
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
 from copy import deepcopy
 from pathlib import Path
 from textwrap import dedent
@@ -30,6 +33,7 @@ from dreadnode.agents.events import (
     GenerationContent,
     GenerationEnd,
     GenerationError,
+    GenerationRetry,
     GenerationStart,
     GenerationStep,
     ReactStep,
@@ -80,6 +84,45 @@ def _raise_exception(error: BaseException) -> t.NoReturn:
     raise error
 
 
+_TRANSIENT_LITELLM_EXCEPTION_NAMES: tuple[str, ...] = (
+    "RateLimitError",
+    "Timeout",
+    "APIConnectionError",
+    "APIConnectionTimeoutError",
+    "ServiceUnavailableError",
+    "InternalServerError",
+    "BadGatewayError",
+    "APIError",
+)
+
+
+def _is_transient_api_error(error: BaseException) -> bool:
+    """Classify an error as a transient LLM API failure worth retrying.
+
+    Matches an explicit allow-list of ``litellm.exceptions`` classes that
+    represent recoverable conditions: rate limits, timeouts, connection
+    failures, and 5xx responses. Notably **does not** match
+    ``BadRequestError``, ``AuthenticationError``,
+    ``ContextWindowExceededError`` (handled by overflow recovery), or other
+    deterministic client errors.
+
+    The allow-list is walked dynamically because ``litellm.exceptions`` does
+    not expose a single common ancestor for its transient exceptions.
+    Returns ``False`` if ``litellm`` is not importable.
+    """
+    with contextlib.suppress(ImportError):
+        import litellm.exceptions as _litellm_exc
+
+        classes = tuple(
+            cls
+            for name in _TRANSIENT_LITELLM_EXCEPTION_NAMES
+            if (cls := getattr(_litellm_exc, name, None)) is not None
+        )
+        if classes and isinstance(error, classes):
+            return True
+    return False
+
+
 class Agent(Executor[AgentEvent, Trajectory]):
     """
     Agent abstraction for applying tools, event logic, and message state to LLM generation.
@@ -128,6 +171,14 @@ class Agent(Executor[AgentEvent, Trajectory]):
     """Extra parameters merged into GenerateParams for every generation (e.g. thinking config)."""
     working_dir: Path | None = Config(default=None)
     """Working directory used for tool output offloading and other IO."""
+    backoff_max_tries: int = Config(default=8, ge=0)
+    """Maximum retries on transient LLM API errors per step. ``0`` disables retry."""
+    backoff_max_time: float = Config(default=300.0, ge=0)
+    """Maximum total seconds to spend retrying transient LLM API errors per step."""
+    backoff_base_factor: float = Config(default=1.0, ge=0)
+    """Base factor for exponential backoff: wait = base_factor * 2 ** (attempt - 1)."""
+    backoff_jitter: bool = Config(default=True)
+    """Whether to add up to ``backoff_base_factor`` seconds of random jitter to each wait."""
 
     # Private state
     _generator: Generator | None = PrivateAttr(None, init=False)
@@ -381,6 +432,49 @@ class Agent(Executor[AgentEvent, Trajectory]):
 
         raise winning_reaction
 
+    def _try_backoff(
+        self,
+        error: BaseException,
+        *,
+        tries: int,
+        start_time: float,
+    ) -> float | None:
+        """Compute a retry sleep for a transient LLM API error.
+
+        Returns the sleep duration in seconds if the caller should retry, or
+        ``None`` if the error is not transient, tries are exhausted, or the
+        per-step time budget has been spent. Caller owns sleeping and emitting
+        the ``GenerationRetry`` event. Backoff is agent-owned at the error
+        site, mirroring ``_try_overflow_recovery`` — no step is consumed and
+        no hook indirection is involved.
+        """
+        if self.backoff_max_tries <= 0:
+            return None
+
+        if not _is_transient_api_error(error):
+            return None
+
+        if tries >= self.backoff_max_tries:
+            logger.warning("Backoff aborted: max tries ({}) exceeded.", self.backoff_max_tries)
+            return None
+
+        remaining = self.backoff_max_time - (time.monotonic() - start_time)
+        if remaining <= 0:
+            logger.warning("Backoff aborted: max time ({:.2f}s) exceeded.", self.backoff_max_time)
+            return None
+
+        seconds = self.backoff_base_factor * (2**tries)
+        if self.backoff_jitter:
+            seconds += random.uniform(0, self.backoff_base_factor)
+        if seconds > remaining:
+            logger.warning(
+                "Backoff aborted: next sleep ({:.2f}s) would exceed remaining budget ({:.2f}s).",
+                seconds,
+                remaining,
+            )
+            return None
+        return seconds
+
     async def _try_overflow_recovery(
         self,
         error: BaseException,
@@ -586,7 +680,7 @@ class Agent(Executor[AgentEvent, Trajectory]):
                 agent_name=self.name,
                 inputs={"goal": self._current_input},
                 params={
-                    "session_id": str(self.trajectory.session_id),
+                    "session_id": str(traj.session_id),
                     "max_steps": self.max_steps,
                     **({"model": self.model_name} if self.model_name else {}),
                     **({"tools": [tool.name for tool in self.all_tools]} if self.all_tools else {}),
@@ -628,6 +722,43 @@ class Agent(Executor[AgentEvent, Trajectory]):
 
                     step_chat = await self._generate(messages)
 
+                    # In-place transient-error backoff — rate limits and
+                    # other litellm.APIError failures get retried at the
+                    # error site with exponential backoff. No step budget
+                    # consumed; clients observe GenerationRetry events
+                    # rather than a spurious terminal GenerationError.
+                    backoff_tries = 0
+                    backoff_started = time.monotonic()
+                    while step_chat.failed and step_chat.error:
+                        wait = self._try_backoff(
+                            step_chat.error,
+                            tries=backoff_tries,
+                            start_time=backoff_started,
+                        )
+                        if wait is None:
+                            break
+                        backoff_tries += 1
+                        logger.warning(
+                            "Backing off {:.2f}s (try {}/{})",
+                            wait,
+                            backoff_tries,
+                            self.backoff_max_tries,
+                        )
+                        retry_event = GenerationRetry(
+                            agent_id=self.agent_id,
+                            agent_name=self.name,
+                            step=step_count,
+                            attempt=backoff_tries,
+                            max_attempts=self.backoff_max_tries,
+                            wait_seconds=wait,
+                            error_type=type(step_chat.error).__name__,
+                            error_message=str(step_chat.error),
+                        )
+                        async for event in self._dispatch(retry_event):
+                            yield event
+                        await asyncio.sleep(wait)
+                        step_chat = await self._generate(messages)
+
                     # In-place overflow recovery — at most once per step,
                     # no step budget consumed, preserves step_count for
                     # user_message inclusion in trajectory.
@@ -683,7 +814,7 @@ class Agent(Executor[AgentEvent, Trajectory]):
                         )
 
                     # Check stop conditions INSIDE span
-                    if any(cond(self.trajectory.steps) for cond in self.stop_conditions):
+                    if any(cond(traj.steps) for cond in self.stop_conditions):
                         logger.info("A stop condition was met. Ending run.")
                         gen_end = GenerationEnd(
                             agent_id=self.agent_id,
@@ -872,7 +1003,7 @@ class Agent(Executor[AgentEvent, Trajectory]):
                         )
 
                     # Check stop conditions again INSIDE span
-                    if any(cond(self.trajectory.steps) for cond in self.stop_conditions):
+                    if any(cond(traj.steps) for cond in self.stop_conditions):
                         break
 
             except Continue as e:
@@ -899,7 +1030,7 @@ class Agent(Executor[AgentEvent, Trajectory]):
             stop_reason = "error"
         elif max_steps_reached:
             stop_reason = "max_steps_reached"
-        elif self.trajectory.steps and isinstance(self.trajectory.steps[-1], AgentStalled):
+        elif traj.steps and isinstance(traj.steps[-1], AgentStalled):
             stop_reason = "stalled"
 
         async for event in self._dispatch(
@@ -961,17 +1092,12 @@ class Agent(Executor[AgentEvent, Trajectory]):
                 if hasattr(tool_container, "__aenter__") and hasattr(tool_container, "__aexit__"):
                     await stack.enter_async_context(tool_container)
 
-            # Parent span only on new conversation (not when borrowing external trajectory)
             display_name = self.name or str(self.agent_id)[:8]
-            ctx = (
-                dreadnode.task_span(
-                    f"agent:{display_name}",
-                    type="agent",
-                    label=display_name,
-                    tags=["agent"],
-                )
-                if reset and trajectory is None
-                else nullcontext()
+            ctx = dreadnode.task_span(
+                f"agent:{display_name}",
+                type="agent",
+                label=display_name,
+                tags=["agent"],
             )
 
             with ctx as parent_span:

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/agents/events.py

@@ -311,6 +311,32 @@ class CompactionEvent(AgentEvent):
         }
 
 
+class GenerationRetry(AgentEvent):
+    """Lifecycle event: the agent is about to sleep and retry a failed generation.
+
+    Emitted by the agent loop when a transient LLM API error (rate limit, etc.)
+    is recovered in place via ``Agent._try_backoff``. This is a lifecycle signal
+    only — it does not consume a step or land in the trajectory.
+    """
+
+    step: int
+    attempt: int
+    max_attempts: int
+    wait_seconds: float
+    error_type: str
+    error_message: str
+
+    def _get_data(self) -> dict[str, t.Any]:
+        return {
+            "step": self.step,
+            "attempt": self.attempt,
+            "max_attempts": self.max_attempts,
+            "wait_seconds": self.wait_seconds,
+            "error_type": self.error_type,
+            "error_message": self.error_message,
+        }
+
+
 class AgentStalled(AgentEvent):
     """Event: The agent is stalled and there are no tool calls, or stop condition).
 
@@ -1017,6 +1043,7 @@ EVENT_TYPES: dict[str, type[AgentEvent]] = {
     "GenerationStep": GenerationStep,
     "GenerationContent": GenerationContent,
     "GenerationError": GenerationError,
+    "GenerationRetry": GenerationRetry,
     "ReactStep": ReactStep,
     "UserInputRequired": UserInputRequired,
     "Heartbeat": Heartbeat,

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/agents/hooks.py

@@ -1,19 +1,15 @@
-"""Core agent hooks: backoff, metrics, and summarization.
+"""Optional agent hooks: tool metrics and conversation summarization.
 
-These hooks are included by default in every agent created by the server.
+These hooks are opt-in — users register them explicitly on an ``Agent`` via
+the ``hooks=`` constructor argument. Transient-error backoff is handled inline
+by the agent loop (see ``Agent._try_backoff``) and is not a hook.
 """
 
-import asyncio
 import contextlib
-import random
-import time
 import typing as t
 from dataclasses import dataclass
 
-from loguru import logger
-
 from dreadnode.agents.events import (
-    AgentError,
     AgentEvent,
     AgentStep,
     ToolEnd,
@@ -28,73 +24,6 @@ if t.TYPE_CHECKING:
     from datetime import datetime
 
 
-# =============================================================================
-# Backoff
-# =============================================================================
-
-
-def _make_backoff_hook(
-    exception_types: tuple[type[Exception], ...],
-    *,
-    max_tries: int = 8,
-    max_time: float = 300.0,
-    base_factor: float = 1.0,
-    jitter: bool = True,
-):
-    """Create a backoff hook for specific exception types."""
-    tries = 0
-    start_time: float | None = None
-    last_step_seen = -1
-
-    @hook(AgentEvent)
-    async def backoff_hook(event: AgentEvent) -> Reaction | None:
-        nonlocal tries, start_time, last_step_seen
-
-        if isinstance(event, AgentStep):
-            if event.step > last_step_seen:
-                tries = 0
-                start_time = None
-                last_step_seen = event.step
-            return None
-
-        if not isinstance(event, AgentError) or not isinstance(event.error, exception_types):
-            return None
-
-        if start_time is None:
-            start_time = time.monotonic()
-
-        if tries >= max_tries:
-            logger.warning(f"Backoff aborted: max tries ({max_tries}) exceeded.")
-            return None
-
-        if (time.monotonic() - start_time) >= max_time:
-            logger.warning(f"Backoff aborted: max time ({max_time:.2f}s) exceeded.")
-            return None
-
-        tries += 1
-        seconds = base_factor * (2 ** (tries - 1))
-        if jitter:
-            seconds += random.uniform(0, base_factor)
-
-        logger.warning(f"Backing off {seconds:.2f}s (try {tries}/{max_tries})")
-        await asyncio.sleep(seconds)
-        return Retry()
-
-    return backoff_hook
-
-
-# Pre-instantiated for common LLM rate limit errors
-backoff_on_ratelimit: Hook | None = None
-try:
-    import litellm.exceptions
-
-    backoff_on_ratelimit = _make_backoff_hook(
-        (litellm.exceptions.RateLimitError, litellm.exceptions.APIError),
-    )
-except ImportError:
-    pass
-
-
 # =============================================================================
 # Tool Metrics
 # =============================================================================

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/agents/tools.py

@@ -170,7 +170,8 @@ async def offload_tool_output(
     working_dir: Path,
 ) -> tuple[str, Path]:
     """Write tool output to disk and return middle-out summary plus file path."""
-    output_dir = working_dir / ".dreadnode" / "tool-output"
+    output_root = working_dir if working_dir.name == ".dreadnode" else working_dir / ".dreadnode"
+    output_dir = output_root / "tool-output"
     await asyncio.to_thread(output_dir.mkdir, parents=True, exist_ok=True)
 
     safe_id = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_call_id)
@@ -178,7 +179,8 @@ async def offload_tool_output(
     async with aiofiles.open(file_path, "w") as f:
         await f.write(content)
 
-    relative_path = file_path.relative_to(working_dir)
+    relative_base = working_dir.parent if working_dir.name == ".dreadnode" else working_dir
+    relative_path = file_path.relative_to(relative_base)
 
     half = OFFLOAD_THRESHOLD // 2
     start = content[:half]

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/beast.py

@@ -18,6 +18,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import GenerateParams, Generator, get_generator
@@ -232,6 +233,7 @@ def beast_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -247,6 +249,7 @@ def beast_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     attack = Study[str](

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/crescendo.py

@@ -11,6 +11,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import Generator, get_generator
@@ -221,6 +222,7 @@ Return ONLY the question text."""
         llm_judge(
             evaluator_model,
             eval_rubric,
+            input=TrialCandidate(),
             min_score=0,
             max_score=1,
         )
@@ -235,6 +237,7 @@ Return ONLY the question text."""
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=objective_judge,
+            include_input=True,
         )
 
     # Resolve model names for span attributes

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/deep_inception.py

@@ -18,6 +18,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import Generator
@@ -135,6 +136,7 @@ def deep_inception_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -150,6 +152,7 @@ def deep_inception_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     attack = Study[str](

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/drattack.py

@@ -19,6 +19,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import GenerateParams, Generator, get_generator
@@ -137,6 +138,7 @@ def drattack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -152,6 +154,7 @@ def drattack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     attack = Study[str](

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/goat.py

@@ -8,6 +8,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import Generator
@@ -114,6 +115,7 @@ def goat_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -141,6 +143,7 @@ def goat_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     # Resolve model names for span attributes

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/gptfuzzer.py

@@ -17,6 +17,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import Generator
@@ -150,6 +151,7 @@ def gptfuzzer_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -165,6 +167,7 @@ def gptfuzzer_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     attack = Study[str](

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/pair.py

@@ -15,6 +15,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import Generator
@@ -142,6 +143,7 @@ def pair_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -157,6 +159,7 @@ def pair_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     # Resolve model names for span attributes

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/rainbow.py

@@ -18,6 +18,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import Generator
@@ -170,6 +171,7 @@ def rainbow_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -185,6 +187,7 @@ def rainbow_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     attack = Study[str](

{dreadnode-2.0.9 → dreadnode-2.0.11}/dreadnode/airt/renellm.py

@@ -18,6 +18,7 @@ from dreadnode.airt.compliance import (
     tag_attack,
 )
 from dreadnode.airt.prompt import OBJECTIVE_SCORE_KEY, _evaluate_prompt
+from dreadnode.core.meta import TrialCandidate
 from dreadnode.core.scorer import Scorer
 from dreadnode.core.transforms import Transform, TransformsLike
 from dreadnode.generators.generator import GenerateParams, Generator, get_generator
@@ -165,6 +166,7 @@ def renellm_attack(
         llm_judge(
             evaluator_model,
             rubric,
+            input=TrialCandidate(),
             min_score=1,
             max_score=10,
         )
@@ -180,6 +182,7 @@ def renellm_attack(
             target=target,
             fitted_transforms=fitted_transforms,
             scorer=prompt_judge,
+            include_input=True,
         )
 
     attack = Study[str](