dreadnode 2.0.10__tar.gz → 2.0.12__tar.gz

{dreadnode-2.0.10 → dreadnode-2.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dreadnode
-Version: 2.0.10
+Version: 2.0.12
 Summary: Dreadnode SDK
 Project-URL: Homepage, https://dreadnode.io
 Project-URL: Documentation, https://docs.dreadnode.io
@@ -10,6 +10,7 @@ Requires-Python: <3.14,>=3.11
 Requires-Dist: aiofiles<25.0.0,>=24.1.0
 Requires-Dist: art<7.0.0,>=6.5
 Requires-Dist: coolname<3.0.0,>=2.2.0
+Requires-Dist: croniter<7.0.0,>=6.0.0
 Requires-Dist: cyclopts>=4.2.0
 Requires-Dist: datasets>=4.5.0
 Requires-Dist: fastapi>=0.115.0

{dreadnode-2.0.10 → dreadnode-2.0.12}/dreadnode/agents/agent.py

@@ -1,5 +1,8 @@
 import asyncio
+import contextlib
 import inspect
+import random
+import time
 import typing as t
 from contextlib import AsyncExitStack, asynccontextmanager, suppress
 from copy import deepcopy
@@ -30,6 +33,7 @@ from dreadnode.agents.events import (
     GenerationContent,
     GenerationEnd,
     GenerationError,
+    GenerationRetry,
     GenerationStart,
     GenerationStep,
     ReactStep,
@@ -80,6 +84,45 @@ def _raise_exception(error: BaseException) -> t.NoReturn:
     raise error
 
 
+_TRANSIENT_LITELLM_EXCEPTION_NAMES: tuple[str, ...] = (
+    "RateLimitError",
+    "Timeout",
+    "APIConnectionError",
+    "APIConnectionTimeoutError",
+    "ServiceUnavailableError",
+    "InternalServerError",
+    "BadGatewayError",
+    "APIError",
+)
+
+
+def _is_transient_api_error(error: BaseException) -> bool:
+    """Classify an error as a transient LLM API failure worth retrying.
+
+    Matches an explicit allow-list of ``litellm.exceptions`` classes that
+    represent recoverable conditions: rate limits, timeouts, connection
+    failures, and 5xx responses. Notably **does not** match
+    ``BadRequestError``, ``AuthenticationError``,
+    ``ContextWindowExceededError`` (handled by overflow recovery), or other
+    deterministic client errors.
+
+    The allow-list is walked dynamically because ``litellm.exceptions`` does
+    not expose a single common ancestor for its transient exceptions.
+    Returns ``False`` if ``litellm`` is not importable.
+    """
+    with contextlib.suppress(ImportError):
+        import litellm.exceptions as _litellm_exc
+
+        classes = tuple(
+            cls
+            for name in _TRANSIENT_LITELLM_EXCEPTION_NAMES
+            if (cls := getattr(_litellm_exc, name, None)) is not None
+        )
+        if classes and isinstance(error, classes):
+            return True
+    return False
+
+
 class Agent(Executor[AgentEvent, Trajectory]):
     """
     Agent abstraction for applying tools, event logic, and message state to LLM generation.
@@ -128,6 +171,14 @@ class Agent(Executor[AgentEvent, Trajectory]):
     """Extra parameters merged into GenerateParams for every generation (e.g. thinking config)."""
     working_dir: Path | None = Config(default=None)
     """Working directory used for tool output offloading and other IO."""
+    backoff_max_tries: int = Config(default=8, ge=0)
+    """Maximum retries on transient LLM API errors per step. ``0`` disables retry."""
+    backoff_max_time: float = Config(default=300.0, ge=0)
+    """Maximum total seconds to spend retrying transient LLM API errors per step."""
+    backoff_base_factor: float = Config(default=1.0, ge=0)
+    """Base factor for exponential backoff: wait = base_factor * 2 ** (attempt - 1)."""
+    backoff_jitter: bool = Config(default=True)
+    """Whether to add up to ``backoff_base_factor`` seconds of random jitter to each wait."""
 
     # Private state
     _generator: Generator | None = PrivateAttr(None, init=False)
@@ -381,6 +432,49 @@ class Agent(Executor[AgentEvent, Trajectory]):
 
         raise winning_reaction
 
+    def _try_backoff(
+        self,
+        error: BaseException,
+        *,
+        tries: int,
+        start_time: float,
+    ) -> float | None:
+        """Compute a retry sleep for a transient LLM API error.
+
+        Returns the sleep duration in seconds if the caller should retry, or
+        ``None`` if the error is not transient, tries are exhausted, or the
+        per-step time budget has been spent. Caller owns sleeping and emitting
+        the ``GenerationRetry`` event. Backoff is agent-owned at the error
+        site, mirroring ``_try_overflow_recovery`` — no step is consumed and
+        no hook indirection is involved.
+        """
+        if self.backoff_max_tries <= 0:
+            return None
+
+        if not _is_transient_api_error(error):
+            return None
+
+        if tries >= self.backoff_max_tries:
+            logger.warning("Backoff aborted: max tries ({}) exceeded.", self.backoff_max_tries)
+            return None
+
+        remaining = self.backoff_max_time - (time.monotonic() - start_time)
+        if remaining <= 0:
+            logger.warning("Backoff aborted: max time ({:.2f}s) exceeded.", self.backoff_max_time)
+            return None
+
+        seconds = self.backoff_base_factor * (2**tries)
+        if self.backoff_jitter:
+            seconds += random.uniform(0, self.backoff_base_factor)
+        if seconds > remaining:
+            logger.warning(
+                "Backoff aborted: next sleep ({:.2f}s) would exceed remaining budget ({:.2f}s).",
+                seconds,
+                remaining,
+            )
+            return None
+        return seconds
+
     async def _try_overflow_recovery(
         self,
         error: BaseException,
@@ -393,6 +487,7 @@ class Agent(Executor[AgentEvent, Trajectory]):
         than via hook indirection.
         """
         from dreadnode.agents.hooks import (
+            _get_model_context_budget,
             _is_context_length_error,
             find_summarization_boundary,
             summarize_conversation,
@@ -407,19 +502,38 @@ class Agent(Executor[AgentEvent, Trajectory]):
         work = list(messages)
         system_message: Message | None = work.pop(0) if work and work[0].role == "system" else None
 
-        boundary = find_summarization_boundary(work, min_messages_to_keep=10)
+        summarizer = self._generator
+        if summarizer is None:
+            return None
+
+        # Cap summarizer input at ~60% of the model's token budget (~3 chars/token,
+        # conservative to prefer over-truncating on code/JSON-heavy content).
+        # This keeps the recovery call itself from overflowing the same provider
+        # context that triggered recovery.
+        budget_tokens = _get_model_context_budget(summarizer)
+        max_summarize_chars = int(budget_tokens * 0.6 * 3)
+
+        boundary = find_summarization_boundary(
+            work,
+            min_messages_to_keep=10,
+            max_summarize_chars=max_summarize_chars,
+        )
         if boundary == 0:
+            logger.info(
+                "Overflow recovery: no safe summarization window fits summarizer budget "
+                "(budget_tokens={}, max_chars={})",
+                budget_tokens,
+                max_summarize_chars,
+            )
             return None
 
         to_summarize = work[:boundary]
         to_keep = work[boundary:]
 
-        summarizer = self._generator
-        if summarizer is None:
-            return None
-
         logger.info(
-            f"Overflow recovery: summarizing {len(to_summarize)} messages, keeping {len(to_keep)}"
+            "Overflow recovery: summarizing {} messages, keeping {}",
+            len(to_summarize),
+            len(to_keep),
         )
 
         try:
@@ -628,6 +742,43 @@ class Agent(Executor[AgentEvent, Trajectory]):
 
                     step_chat = await self._generate(messages)
 
+                    # In-place transient-error backoff — rate limits and
+                    # other litellm.APIError failures get retried at the
+                    # error site with exponential backoff. No step budget
+                    # consumed; clients observe GenerationRetry events
+                    # rather than a spurious terminal GenerationError.
+                    backoff_tries = 0
+                    backoff_started = time.monotonic()
+                    while step_chat.failed and step_chat.error:
+                        wait = self._try_backoff(
+                            step_chat.error,
+                            tries=backoff_tries,
+                            start_time=backoff_started,
+                        )
+                        if wait is None:
+                            break
+                        backoff_tries += 1
+                        logger.warning(
+                            "Backing off {:.2f}s (try {}/{})",
+                            wait,
+                            backoff_tries,
+                            self.backoff_max_tries,
+                        )
+                        retry_event = GenerationRetry(
+                            agent_id=self.agent_id,
+                            agent_name=self.name,
+                            step=step_count,
+                            attempt=backoff_tries,
+                            max_attempts=self.backoff_max_tries,
+                            wait_seconds=wait,
+                            error_type=type(step_chat.error).__name__,
+                            error_message=str(step_chat.error),
+                        )
+                        async for event in self._dispatch(retry_event):
+                            yield event
+                        await asyncio.sleep(wait)
+                        step_chat = await self._generate(messages)
+
                     # In-place overflow recovery — at most once per step,
                     # no step budget consumed, preserves step_count for
                     # user_message inclusion in trajectory.
@@ -650,6 +801,12 @@ class Agent(Executor[AgentEvent, Trajectory]):
                             step_chat = await self._generate(messages)
 
                     if step_chat.failed and step_chat.error:
+                        from dreadnode.agents.hooks import _describe_generation_error
+
+                        logger.error(
+                            "Generation step failed: {}",
+                            _describe_generation_error(step_chat.error),
+                        )
                         error_event = GenerationError(
                             agent_id=self.agent_id,
                             agent_name=self.name,

{dreadnode-2.0.10 → dreadnode-2.0.12}/dreadnode/agents/events.py

@@ -311,6 +311,32 @@ class CompactionEvent(AgentEvent):
         }
 
 
+class GenerationRetry(AgentEvent):
+    """Lifecycle event: the agent is about to sleep and retry a failed generation.
+
+    Emitted by the agent loop when a transient LLM API error (rate limit, etc.)
+    is recovered in place via ``Agent._try_backoff``. This is a lifecycle signal
+    only — it does not consume a step or land in the trajectory.
+    """
+
+    step: int
+    attempt: int
+    max_attempts: int
+    wait_seconds: float
+    error_type: str
+    error_message: str
+
+    def _get_data(self) -> dict[str, t.Any]:
+        return {
+            "step": self.step,
+            "attempt": self.attempt,
+            "max_attempts": self.max_attempts,
+            "wait_seconds": self.wait_seconds,
+            "error_type": self.error_type,
+            "error_message": self.error_message,
+        }
+
+
 class AgentStalled(AgentEvent):
     """Event: The agent is stalled and there are no tool calls, or stop condition).
 
@@ -1017,6 +1043,7 @@ EVENT_TYPES: dict[str, type[AgentEvent]] = {
     "GenerationStep": GenerationStep,
     "GenerationContent": GenerationContent,
     "GenerationError": GenerationError,
+    "GenerationRetry": GenerationRetry,
     "ReactStep": ReactStep,
     "UserInputRequired": UserInputRequired,
     "Heartbeat": Heartbeat,

{dreadnode-2.0.10 → dreadnode-2.0.12}/dreadnode/agents/hooks.py

@@ -1,19 +1,15 @@
-"""Core agent hooks: backoff, metrics, and summarization.
+"""Optional agent hooks: tool metrics and conversation summarization.
 
-These hooks are included by default in every agent created by the server.
+These hooks are opt-in — users register them explicitly on an ``Agent`` via
+the ``hooks=`` constructor argument. Transient-error backoff is handled inline
+by the agent loop (see ``Agent._try_backoff``) and is not a hook.
 """
 
-import asyncio
 import contextlib
-import random
-import time
 import typing as t
 from dataclasses import dataclass
 
-from loguru import logger
-
 from dreadnode.agents.events import (
-    AgentError,
     AgentEvent,
     AgentStep,
     ToolEnd,
@@ -28,73 +24,6 @@ if t.TYPE_CHECKING:
     from datetime import datetime
 
 
-# =============================================================================
-# Backoff
-# =============================================================================
-
-
-def _make_backoff_hook(
-    exception_types: tuple[type[Exception], ...],
-    *,
-    max_tries: int = 8,
-    max_time: float = 300.0,
-    base_factor: float = 1.0,
-    jitter: bool = True,
-):
-    """Create a backoff hook for specific exception types."""
-    tries = 0
-    start_time: float | None = None
-    last_step_seen = -1
-
-    @hook(AgentEvent)
-    async def backoff_hook(event: AgentEvent) -> Reaction | None:
-        nonlocal tries, start_time, last_step_seen
-
-        if isinstance(event, AgentStep):
-            if event.step > last_step_seen:
-                tries = 0
-                start_time = None
-                last_step_seen = event.step
-            return None
-
-        if not isinstance(event, AgentError) or not isinstance(event.error, exception_types):
-            return None
-
-        if start_time is None:
-            start_time = time.monotonic()
-
-        if tries >= max_tries:
-            logger.warning(f"Backoff aborted: max tries ({max_tries}) exceeded.")
-            return None
-
-        if (time.monotonic() - start_time) >= max_time:
-            logger.warning(f"Backoff aborted: max time ({max_time:.2f}s) exceeded.")
-            return None
-
-        tries += 1
-        seconds = base_factor * (2 ** (tries - 1))
-        if jitter:
-            seconds += random.uniform(0, base_factor)
-
-        logger.warning(f"Backing off {seconds:.2f}s (try {tries}/{max_tries})")
-        await asyncio.sleep(seconds)
-        return Retry()
-
-    return backoff_hook
-
-
-# Pre-instantiated for common LLM rate limit errors
-backoff_on_ratelimit: Hook | None = None
-try:
-    import litellm.exceptions
-
-    backoff_on_ratelimit = _make_backoff_hook(
-        (litellm.exceptions.RateLimitError, litellm.exceptions.APIError),
-    )
-except ImportError:
-    pass
-
-
 # =============================================================================
 # Tool Metrics
 # =============================================================================
@@ -151,6 +80,9 @@ CONTEXT_LENGTH_ERROR_PATTERNS = [
     "token limit",
     "maximum context length",
     "is too long",
+    "chunk too big",
+    "prompt is too long",
+    "too many tokens",
 ]
 
 
@@ -264,32 +196,94 @@ def _is_context_length_error(error: BaseException) -> bool:
     return any(pattern in error_str for pattern in CONTEXT_LENGTH_ERROR_PATTERNS)
 
 
+def _describe_generation_error(error: BaseException) -> dict[str, t.Any]:
+    """Best-effort structured dump of a generation error for diagnostic logs.
+
+    Returns a dict of litellm-known attributes when present. Never raises:
+    attribute access that fails is silently omitted, so callers can log the
+    result safely even for hostile or malformed exception objects.
+
+    Intended for one-per-failure diagnostic logging at the generation error
+    site. Presence flags are included for fields whose *content* is not
+    itself safe to log (raw bodies may be arbitrary length); they tell a
+    future debugger whether structured data exists to look at without
+    dumping it into every log line.
+    """
+    fields: dict[str, t.Any] = {"type": type(error).__name__}
+
+    with contextlib.suppress(Exception):
+        msg = str(error)
+        if msg:
+            fields["message"] = msg if len(msg) <= 500 else msg[:500] + "...<truncated>"
+
+    for attr in ("status_code", "llm_provider", "model", "request_id"):
+        with contextlib.suppress(Exception):
+            val = getattr(error, attr, None)
+            if val is not None:
+                fields[attr] = val
+
+    with contextlib.suppress(Exception):
+        fields["has_body"] = bool(getattr(error, "body", None))
+
+    with contextlib.suppress(Exception):
+        resp = getattr(error, "response", None)
+        if resp is not None:
+            text = getattr(resp, "text", "") or ""
+            fields["has_response_text"] = bool(text)
+
+    return fields
+
+
 def find_summarization_boundary(
     messages: list[Message],
     min_messages_to_keep: int = 10,
+    max_summarize_chars: int | None = None,
 ) -> int:
     """Find a clean message boundary for summarization.
 
-    Walks messages from the start and finds the latest safe split point that
+    Walks messages from the start and enumerates every safe split point that
     leaves at least ``min_messages_to_keep`` messages in the "keep" portion.
     A safe boundary is after a simple assistant message (no tool calls) —
     this is the natural end of a complete conversational turn.
 
+    When ``max_summarize_chars`` is provided, returns the largest safe split
+    whose cumulative ``len(str(message))`` stays within the cap. This keeps
+    the summarizer call from overflowing the same provider context that
+    triggered recovery. ``str(message)`` is exactly what the summarizer
+    receives (see ``Agent._try_overflow_recovery``) so the cap and the actual
+    serialized input measure the same string — including elision of image
+    URLs (``ContentImageUrl.__str__``) and tool-call arguments
+    (``ToolCall.__str__``).
+
     Returns:
         Index splitting ``messages[:boundary]`` (to summarize) from
         ``messages[boundary:]`` (to keep).  Returns ``0`` when no valid
         boundary exists.
     """
-    best_boundary = 0
+    # Enumerate every safe boundary with its cumulative serialized char count.
+    # (0, 0) is always a valid "no compaction" candidate.
+    candidates: list[tuple[int, int]] = [(0, 0)]
+    running_chars = 0
     for i, message in enumerate(messages):
         if len(messages) - i <= min_messages_to_keep:
             break
+        running_chars += len(str(message))
         is_simple_assistant = message.role == "assistant" and not getattr(
             message, "tool_calls", None
         )
         if is_simple_assistant:
-            best_boundary = i + 1
-    return best_boundary
+            candidates.append((i + 1, running_chars))
+
+    if max_summarize_chars is None:
+        return candidates[-1][0]
+
+    # Cumulative sizes are monotonic in boundary order, so the largest
+    # boundary whose size fits the cap is the best match. Walk candidates
+    # from latest to earliest and return the first that fits.
+    for boundary, size in reversed(candidates):
+        if size <= max_summarize_chars:
+            return boundary
+    return 0
 
 
 def _get_model_context_budget(model_or_generator: "str | Generator | None") -> int:

{dreadnode-2.0.10 → dreadnode-2.0.12}/dreadnode/app/api/client.py

@@ -36,6 +36,7 @@ from dreadnode.app.api.models import (
     UserSecretsList,
     Workspace,
 )
+from dreadnode.core.exceptions import InsufficientCreditsError
 from dreadnode.version import VERSION
 
 
@@ -109,6 +110,11 @@ class ApiClient:
         else:
             return f"{response.status_code}: {detail}"
 
+    @staticmethod
+    def _is_credit_related_429(detail: str) -> bool:
+        lowered = detail.lower()
+        return "credit" in lowered
+
     def _request(
         self,
         method: str,
@@ -141,6 +147,11 @@ class ApiClient:
         response = self._request(method, path, params, json_data, data, headers)
         if response.status_code == 401:
             raise AuthenticationError(self._get_error_message(response))
+        if response.status_code == 429:
+            error_message = self._get_error_message(response)
+            detail = error_message.split(": ", 1)[1] if ": " in error_message else error_message
+            if self._is_credit_related_429(detail):
+                raise InsufficientCreditsError(detail)
         try:
             response.raise_for_status()
         except httpx.HTTPStatusError as e:
@@ -1086,6 +1097,64 @@ class ApiClient:
         )
         return t.cast("dict[str, t.Any]", response.json())
 
+    def create_runtime(
+        self,
+        org: str,
+        workspace: str,
+        project: str | None = None,
+        *,
+        key: str | None = None,
+        name: str | None = None,
+        description: str | None = None,
+        config: dict[str, t.Any] | None = None,
+    ) -> dict[str, t.Any]:
+        """POST /org/{org}/ws/{workspace}/runtimes - Ensure a runtime exists."""
+        payload: dict[str, t.Any] = {}
+        if project is not None:
+            payload["project"] = project
+        if key is not None:
+            payload["key"] = key
+        if name is not None:
+            payload["name"] = name
+        if description is not None:
+            payload["description"] = description
+        if config is not None:
+            payload["config"] = config
+        response = self.request(
+            "POST",
+            f"/org/{org}/ws/{workspace}/runtimes",
+            json_data=payload,
+        )
+        return t.cast("dict[str, t.Any]", response.json())
+
+    def get_runtime_config(
+        self,
+        org: str,
+        workspace: str,
+        runtime_id: str,
+    ) -> dict[str, t.Any]:
+        """GET /org/{org}/ws/{workspace}/runtimes/{runtime_id}/config - Get runtime config."""
+        response = self.request(
+            "GET",
+            f"/org/{org}/ws/{workspace}/runtimes/{runtime_id}/config",
+        )
+        return t.cast("dict[str, t.Any]", response.json())
+
+    def update_runtime_config(
+        self,
+        org: str,
+        workspace: str,
+        runtime_id: str,
+        config: dict[str, t.Any],
+    ) -> dict[str, t.Any]:
+        """PUT /org/{org}/ws/{workspace}/runtimes/{runtime_id}/config - Replace runtime config."""
+        response = self.request(
+            "PUT",
+            f"/org/{org}/ws/{workspace}/runtimes/{runtime_id}/config",
+            json_data={"config": config},
+        )
+        return t.cast("dict[str, t.Any]", response.json())
+
     def pause_runtime(self, org: str, workspace: str, runtime_id: str) -> dict[str, t.Any]:
         """POST /org/{org}/ws/{workspace}/runtimes/{runtime_id}/pause - Pause a runtime."""
         response = self.request(
@@ -1722,6 +1791,23 @@ class ApiClient:
         )
         return t.cast("dict[str, t.Any]", response.json())
 
+    def set_runtime_capability_flags(
+        self,
+        org: str,
+        workspace: str,
+        runtime_id: str,
+        binding_id: str,
+        *,
+        flags: dict[str, bool | None],
+    ) -> dict[str, t.Any]:
+        """PATCH /org/{org}/ws/{workspace}/runtimes/{runtime_id}/capabilities/{binding_id}."""
+        response = self.request(
+            "PATCH",
+            f"/org/{org}/ws/{workspace}/runtimes/{runtime_id}/capabilities/{binding_id}",
+            json_data={"flags": flags},
+        )
+        return t.cast("dict[str, t.Any]", response.json())
+
     def update_runtime_capability(
         self,
         org: str,
@@ -1881,6 +1967,7 @@ class ApiClient:
         title: str | None = None,
         message_count: int = 0,
         project_id: str | None = None,
+        runtime_id: str | None = None,
     ) -> dict[str, t.Any]:
         """POST /org/{org}/ws/{workspace}/sessions - Create or save a session."""
         payload: dict[str, t.Any] = {
@@ -1894,6 +1981,8 @@ class ApiClient:
             payload["title"] = title
         if project_id is not None:
             payload["project_id"] = project_id
+        if runtime_id is not None:
+            payload["runtime_id"] = runtime_id
         response = self.request("POST", f"/org/{org}/ws/{workspace}/sessions", json_data=payload)
         return t.cast("dict[str, t.Any]", response.json())
 
@@ -2500,6 +2589,7 @@ class ApiClient:
         *,
         name: str,
         project_id: str,
+        runtime_id: str | None = None,
         description: str | None = None,
         session_id: str | None = None,
         target_model: str | None = None,
@@ -2517,6 +2607,8 @@ class ApiClient:
             payload["description"] = description
         if session_id is not None:
             payload["session_id"] = session_id
+        if runtime_id is not None:
+            payload["runtime_id"] = runtime_id
         if target_model is not None:
             payload["target_model"] = target_model
         if attacker_model is not None: