dreadnode 1.14.0__tar.gz → 1.15.0__tar.gz

{dreadnode-1.14.0 → dreadnode-1.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dreadnode
-Version: 1.14.0
+Version: 1.15.0
 Summary: Dreadnode SDK
 Project-URL: Homepage, https://github.com/dreadnode/sdk
 Project-URL: Repository, https://github.com/dreadnode/sdk
@@ -213,16 +213,17 @@ Requires-Dist: coolname<3.0.0,>=2.2.0
 Requires-Dist: cyclopts<4.0.0,>=3.22.2
 Requires-Dist: fsspec[s3]<=2025.3.0,>=2023.1.0
 Requires-Dist: httpx<1.0.0,>=0.28.0
-Requires-Dist: jsonpath>=0.82.2
 Requires-Dist: logfire<=3.20.0,>=3.5.3
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: numpy<=2.2.6
 Requires-Dist: optuna<5.0.0,>=4.5.0
 Requires-Dist: pandas<3.0.0,>=2.2.3
 Requires-Dist: pydantic<3.0.0,>=2.9.2
+Requires-Dist: python-jsonpath>=2.0.1
 Requires-Dist: python-ulid<4.0.0,>=3.0.0
+Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: rigging<4.0.0,>=3.2.1
-Requires-Dist: universal-pathlib<0.3.0,>=0.2.6
+Requires-Dist: universal-pathlib<0.4.0,>=0.3.3
 Provides-Extra: all
 Requires-Dist: confusables<2.0.0,>=1.2.0; extra == 'all'
 Requires-Dist: datasets<5.0.0,>=4.0.0; extra == 'all'
@@ -230,7 +231,7 @@ Requires-Dist: moviepy<3.0.0,>=2.1.2; extra == 'all'
 Requires-Dist: nltk<4.0.0,>=3.9.1; extra == 'all'
 Requires-Dist: pillow<12.0.0,>=11.2.1; extra == 'all'
 Requires-Dist: presidio-analyzer<3.0.0,>=2.2.359; extra == 'all'
-Requires-Dist: pyarrow<20.0.0,>=19.0.1; extra == 'all'
+Requires-Dist: pyarrow<22.1.0,>=22.0.0; extra == 'all'
 Requires-Dist: rapidfuzz>=3.14.1; extra == 'all'
 Requires-Dist: scikit-learn<2.0.0,>=1.7.1; extra == 'all'
 Requires-Dist: sentence-transformers<6.0.0,>=5.1.0; extra == 'all'
@@ -253,7 +254,7 @@ Requires-Dist: textblob<1.0.0,>=0.19.0; extra == 'scoring'
 Requires-Dist: textstat<1.0.0,>=0.7.10; extra == 'scoring'
 Provides-Extra: training
 Requires-Dist: datasets<5.0.0,>=4.0.0; extra == 'training'
-Requires-Dist: pyarrow<20.0.0,>=19.0.1; extra == 'training'
+Requires-Dist: pyarrow<22.1.0,>=22.0.0; extra == 'training'
 Requires-Dist: transformers<5.0.0,>=4.41.0; extra == 'training'
 Provides-Extra: transforms
 Requires-Dist: art<7.0.0,>=6.5; extra == 'transforms'

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/__init__.py

@@ -37,6 +37,7 @@ from dreadnode.version import VERSION
 
 if t.TYPE_CHECKING:
     from dreadnode import agent, airt, eval, optimization, scorers, transforms  # noqa: A004
+    from dreadnode.agent import Agent, tool, tool_method
     from dreadnode.data_types import Audio, Image, Table, Video
 
 logger.disable("dreadnode")
@@ -75,6 +76,7 @@ __version__ = VERSION
 
 __all__ = [
     "DEFAULT_INSTANCE",
+    "Agent",
     "Audio",
     "Code",
     "Config",
@@ -138,6 +140,8 @@ __all__ = [
     "task",
     "task_and_run",
     "task_span",
+    "tool",
+    "tool_method",
     "transforms",
 ]
 
@@ -147,6 +151,9 @@ __lazy_components__: dict[str, str] = {
     "Image": "dreadnode.data_types",
     "Table": "dreadnode.data_types",
     "Video": "dreadnode.data_types",
+    "Agent": "dreadnode.agent",
+    "tool": "dreadnode.agent",
+    "tool_method": "dreadnode.agent",
 }
 
 

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/__init__.py

@@ -4,6 +4,7 @@ from dreadnode.agent import error, events, hooks, reactions, result, stop, tools
 from dreadnode.agent.agent import Agent
 from dreadnode.agent.result import AgentResult
 from dreadnode.agent.thread import Thread
+from dreadnode.agent.tools import tool, tool_method
 
 Agent.model_rebuild()
 Thread.model_rebuild()
@@ -20,5 +21,7 @@ __all__ = [
     "reactions",
     "result",
     "stop",
+    "tool",
+    "tool_method",
     "tools",
 ]

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/agent.py

@@ -4,8 +4,10 @@ from contextlib import aclosing, asynccontextmanager
 from copy import deepcopy
 
 import rigging as rg
-from pydantic import ConfigDict, Field, PrivateAttr, field_validator
-from rigging.message import inject_system_content  # can't access via rg
+from loguru import logger
+from pydantic import ConfigDict, Field, PrivateAttr, SkipValidation, field_validator
+from rigging.message import inject_system_content
+from ulid import ULID  # can't access via rg
 
 from dreadnode.agent.error import MaxStepsError
 from dreadnode.agent.events import (
@@ -83,7 +85,7 @@ class Agent(Model):
     caching: rg.caching.CacheMode | None = Config(default=None, repr=False)
     """How to handle cache_control entries on inference messages."""
 
-    tools: list[AnyTool | Toolset] = Config(default_factory=list)
+    tools: t.Annotated[list[AnyTool | Toolset], SkipValidation] = Config(default_factory=list)
     """Tools the agent can use."""
     tool_mode: ToolMode = Config(default="auto", repr=False)
     """The tool calling mode to use."""
@@ -236,6 +238,8 @@ class Agent(Model):
         try:
             messages = rg.caching.apply_cache_mode_to_messages(self.caching, [messages])[0]
 
+            logger.trace(f"Generating with model '{generator.model}'. Messages: {messages!r}")
+
             generated = (await generator.generate_messages([messages], [params]))[0]
             if isinstance(generated, BaseException):
                 raise generated  # noqa: TRY301
@@ -251,6 +255,7 @@ class Agent(Model):
             )
 
         except Exception as error:  # noqa: BLE001
+            logger.error(f"Error during generation: {error}", exc_info=True)
             chat = rg.Chat(
                 messages,
                 [],
@@ -275,6 +280,14 @@ class Agent(Model):
     ) -> t.AsyncGenerator[AgentEvent, None]:
         events: list[AgentEvent] = []
         stop_conditions = self.stop_conditions
+        session_id = ULID()
+
+        logger.info(
+            f"Starting Agent '{self.name}' ({session_id}): "
+            f"model='{self.model}', "
+            f"max_steps={self.max_steps}, "
+            f"tools={[tool.name for tool in self.all_tools]}"
+        )
 
         # Event dispatcher
 
@@ -290,6 +303,11 @@ class Agent(Model):
             if not applicable_hooks:
                 return
 
+            logger.debug(
+                f"Agent '{self.name}' ({session_id}) dispatching '{type(event).__name__}': "
+                f"applicable_hooks={[get_callable_name(h, short=True) for h in applicable_hooks]}"
+            )
+
             # Run all applicable hooks and collect their reactions
             hook_reactions: dict[str, Reaction | None] = {}
             for hook in applicable_hooks:
@@ -308,6 +326,10 @@ class Agent(Model):
                 if reaction is None:
                     continue
 
+                logger.debug(
+                    f"Agent '{self.name}' ({session_id}) hook '{hook_name}' returned reaction: {reaction!r}"
+                )
+
                 if not isinstance(reaction, Reaction):
                     warn_at_user_stacklevel(
                         f"Hook '{hook_name}' returned {reaction}, but expected a Reaction.",
@@ -368,6 +390,7 @@ class Agent(Model):
                 "unknown",
             )
             reacted_event = Reacted(
+                session_id=session_id,
                 agent=self,
                 thread=thread,
                 messages=messages,
@@ -383,6 +406,9 @@ class Agent(Model):
                 return
 
             if isinstance(winning_reaction, RetryWithFeedback):
+                logger.debug(
+                    f"Agent '{self.name}' ({session_id}) injecting feedback for retry: '{winning_reaction.feedback}'"
+                )
                 messages.append(rg.Message("user", winning_reaction.feedback))
                 raise Retry(messages=messages) from winning_reaction
 
@@ -395,6 +421,7 @@ class Agent(Model):
         ) -> t.AsyncGenerator[AgentEvent, None]:
             async for event in _dispatch(
                 ToolStart(
+                    session_id=session_id,
                     agent=self,
                     thread=thread,
                     messages=messages,
@@ -404,6 +431,10 @@ class Agent(Model):
             ):
                 yield event
 
+            logger.debug(
+                f"Executing tool '{tool_call.name}' with args: {tool_call.function.arguments}"
+            )
+
             message: rg.Message
             stop = False
             tool = next((t for t in self.all_tools if t.name == tool_call.name), None)
@@ -416,6 +447,7 @@ class Agent(Model):
                 except Exception as e:
                     async for event in _dispatch(
                         AgentError(
+                            session_id=session_id,
                             agent=self,
                             thread=thread,
                             messages=messages,
@@ -426,12 +458,14 @@ class Agent(Model):
                         yield event
                     raise
             else:
+                logger.warning(f"Tool '{tool_call.name}' not found.")
                 message = rg.Message.from_model(
                     rg.model.SystemErrorModel(content=f"Tool '{tool_call.name}' not found.")
                 )
 
             async for event in _dispatch(
                 ToolEnd(
+                    session_id=session_id,
                     agent=self,
                     thread=thread,
                     messages=messages,
@@ -447,6 +481,7 @@ class Agent(Model):
 
         async for event in _dispatch(
             AgentStart(
+                session_id=session_id,
                 agent=self,
                 thread=thread,
                 messages=messages,
@@ -464,6 +499,7 @@ class Agent(Model):
             try:
                 async for event in _dispatch(
                     StepStart(
+                        session_id=session_id,
                         agent=self,
                         thread=thread,
                         messages=messages,
@@ -479,6 +515,7 @@ class Agent(Model):
                 if step_chat.failed and step_chat.error:
                     async for event in _dispatch(
                         AgentError(
+                            session_id=session_id,
                             agent=self,
                             thread=thread,
                             messages=messages,
@@ -493,6 +530,7 @@ class Agent(Model):
 
                 async for event in _dispatch(
                     GenerationEnd(
+                        session_id=session_id,
                         agent=self,
                         thread=thread,
                         messages=messages,
@@ -506,6 +544,7 @@ class Agent(Model):
                 # Check for stop conditions
 
                 if any(cond(events) for cond in stop_conditions):
+                    logger.info("A stop condition was met. Ending run.")
                     break
 
                 # Check if stalled
@@ -514,8 +553,13 @@ class Agent(Model):
                     if not stop_conditions:
                         break
 
+                    logger.warning(
+                        f"Agent '{self.name}' ({session_id}) stalled: No tool calls and no stop conditions met."
+                    )
+
                     async for event in _dispatch(
                         AgentStalled(
+                            session_id=session_id,
                             agent=self,
                             thread=thread,
                             messages=messages,
@@ -578,7 +622,25 @@ class Agent(Model):
             thread.messages = messages
             thread.events.extend(events)
 
+        total_usage = _total_usage_from_events(events)
+        log_message = (
+            f"Agent '{self.name}' finished: "
+            f"reason='{stop_reason}', "
+            f"steps={step - 1}, "
+            f"total_tokens={total_usage.total_tokens}, "
+            f"in_tokens={total_usage.input_tokens}, "
+            f"out_tokens={total_usage.output_tokens}"
+        )
+
+        if stop_reason == "finished":
+            logger.success(log_message)
+        elif stop_reason == "error":
+            logger.error(f"{log_message}, error='{error!r}'")
+        else:
+            logger.warning(log_message)
+
         yield AgentEnd(
+            session_id=session_id,
             agent=self,
             thread=thread,
             messages=messages,
@@ -646,9 +708,10 @@ class Agent(Model):
                 "user_input": user_input,
                 "messages": messages,
                 "instructions": self.instructions,
-                "tools": [t.name for t in self.all_tools],
                 "hooks": [get_callable_name(hook, short=True) for hook in self.hooks],
                 "stop_conditions": [s.name for s in self.stop_conditions],
+                "tools": [t.name for t in self.all_tools],
+                "tool_schemas": [t.api_definition for t in self.all_tools],
             }
         )
         trace_params.update(
@@ -669,13 +732,17 @@ class Agent(Model):
                         yield event
             finally:
                 if last_event is not None:
-                    log_outputs(messages=last_event.messages, token_usage=last_event.total_usage)
+                    # TODO(nick): Don't love having to inject here, but it's the most accurate in
+                    # in terms of ensuring we don't miss the system component of messages
+                    final_messages = inject_system_content(last_event.messages, self.get_prompt())
+                    log_outputs(messages=final_messages, token_usage=last_event.total_usage)
 
                 if isinstance(last_event, AgentEnd):
                     log_outputs(
                         to="both",
                         steps_taken=min(0, last_event.result.steps - 1),
                         reason=last_event.stop_reason,
+                        failed=last_event.result.failed,
                     )
                     if last_event.result.error:
                         log_output("error", last_event.result.error, to="both")

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/events.py

@@ -9,6 +9,7 @@ from rich.panel import Panel
 from rich.rule import Rule
 from rich.table import Table
 from rich.text import Text
+from ulid import ULID
 
 from dreadnode.agent.format import format_message
 from dreadnode.agent.reactions import (
@@ -39,6 +40,8 @@ class AgentEvent:
     )
     """The timestamp of when the event occurred (UTC)."""
 
+    session_id: ULID = field(repr=False)
+    """The unique identifier for the agent run session."""
     agent: "Agent" = field(repr=False)
     """The agent associated with this event."""
     thread: "Thread" = field(repr=False)

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/hooks/__init__.py

@@ -1,3 +1,4 @@
+from dreadnode.agent.hooks.backoff import backoff_on_error, backoff_on_ratelimit
 from dreadnode.agent.hooks.base import (
     Hook,
     retry_with_feedback,
@@ -6,6 +7,8 @@ from dreadnode.agent.hooks.summarize import summarize_when_long
 
 __all__ = [
     "Hook",
+    "backoff_on_error",
+    "backoff_on_ratelimit",
     "retry_with_feedback",
     "summarize_when_long",
 ]

dreadnode-1.15.0/dreadnode/agent/hooks/backoff.py

@@ -0,0 +1,134 @@
+import asyncio
+import random
+import time
+import typing as t
+from dataclasses import dataclass
+
+from loguru import logger
+
+from dreadnode.agent.events import AgentError, AgentEvent, StepStart
+from dreadnode.agent.reactions import Reaction, Retry
+
+if t.TYPE_CHECKING:
+    from ulid import ULID
+
+    from dreadnode.agent.hooks.base import Hook
+
+
+@dataclass
+class BackoffState:
+    tries: int = 0
+    start_time: float | None = None
+    last_step_seen: int = -1
+
+    def reset(self, step: int = -1) -> None:
+        self.tries = 0
+        self.start_time = None
+        self.last_step_seen = step
+
+
+def backoff_on_error(
+    exception_types: type[Exception] | t.Iterable[type[Exception]],
+    *,
+    max_tries: int = 8,
+    max_time: float = 300.0,
+    base_factor: float = 1.0,
+    jitter: bool = True,
+) -> "Hook":
+    """
+    Creates a hook that retries with exponential backoff when specific errors occur.
+
+    It listens for `AgentError` events and, if the error matches, waits for an
+    exponentially increasing duration before issuing a `Retry` reaction.
+
+    Args:
+        exception_types: An exception type or iterable of types to catch.
+        max_tries: The maximum number of retries before giving up.
+        max_time: The maximum total time in seconds to wait before giving up.
+        base_factor: The base duration (in seconds) for the backoff calculation.
+        jitter: If True, adds a random jitter to the wait time to prevent synchronized retries.
+
+    Returns:
+        An agent hook that implements the backoff logic.
+    """
+    exceptions = (
+        tuple(exception_types) if isinstance(exception_types, t.Iterable) else (exception_types,)
+    )
+
+    session_states: dict[ULID, BackoffState] = {}
+
+    async def backoff_hook(event: "AgentEvent") -> "Reaction | None":
+        state = session_states.setdefault(event.session_id, BackoffState())
+
+        if isinstance(event, StepStart):
+            if event.step > state.last_step_seen:
+                state.reset(event.step)
+            return None
+
+        if not isinstance(event, AgentError) or not isinstance(event.error, exceptions):
+            return None
+
+        if state.start_time is None:
+            state.start_time = time.monotonic()
+
+        if state.tries >= max_tries:
+            logger.warning(
+                f"Backoff aborted for session {event.session_id}: maximum tries ({max_tries}) exceeded."
+            )
+            return None
+
+        if (time.monotonic() - state.start_time) >= max_time:
+            logger.warning(
+                f"Backoff aborted for session {event.session_id}: maximum time ({max_time:.2f}s) exceeded."
+            )
+            return None
+
+        state.tries += 1
+
+        seconds = base_factor * (2 ** (state.tries - 1))
+        if jitter:
+            seconds += random.uniform(0, base_factor)  # noqa: S311 # nosec
+
+        logger.warning(
+            f"Backing off for {seconds:.2f}s (try {state.tries}/{max_tries}) on session {event.session_id} due to error: {event.error}"
+        )
+
+        await asyncio.sleep(seconds)
+        return Retry()
+
+    return backoff_hook
+
+
+def backoff_on_ratelimit(
+    *,
+    max_tries: int = 8,
+    max_time: float = 300.0,
+    base_factor: float = 1.0,
+    jitter: bool = True,
+) -> "Hook":
+    """
+    A convenient default backoff hook for common, ephemeral LLM errors.
+
+    This hook retries on `litellm.exceptions.RateLimitError` and `litellm.exceptions.APIError`
+    with an exponential backoff strategy for up to 5 minutes.
+
+    See `backoff_on_error` for more details.
+
+    Args:
+        max_tries: The maximum number of retries before giving up.
+        max_time: The maximum total time in seconds to wait before giving up.
+        base_factor: The base duration (in seconds) for the backoff calculation.
+        jitter: If True, adds a random jitter to the wait time to prevent synchronized retries.
+
+    Returns:
+        An agent hook that implements the backoff logic.
+    """
+    import litellm.exceptions
+
+    return backoff_on_error(
+        (litellm.exceptions.RateLimitError, litellm.exceptions.APIError),
+        max_time=max_time,
+        max_tries=max_tries,
+        base_factor=base_factor,
+        jitter=jitter,
+    )

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/reactions.py

@@ -19,7 +19,7 @@ class Continue(Reaction):
 
 @dataclass
 class Retry(Reaction):
-    messages: list[rg.Message] | None = Field(None, repr=False)
+    messages: list[rg.Message] | None = Field(default=None, repr=False)
 
 
 @dataclass

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/tools/__init__.py

@@ -3,7 +3,12 @@ import typing as t
 
 from dreadnode.agent.tools.base import (
     AnyTool,
+    FunctionCall,
+    FunctionDefinition,
     Tool,
+    ToolCall,
+    ToolDefinition,
+    ToolMode,
     Toolset,
     discover_tools_on_obj,
     tool,
@@ -15,7 +20,12 @@ if t.TYPE_CHECKING:
 
 __all__ = [
     "AnyTool",
+    "FunctionCall",
+    "FunctionDefinition",
     "Tool",
+    "ToolCall",
+    "ToolDefinition",
+    "ToolMode",
     "Toolset",
     "discover_tools_on_obj",
     "fs",

{dreadnode-1.14.0 → dreadnode-1.15.0}/dreadnode/agent/tools/base.py

@@ -8,6 +8,10 @@ from dreadnode.meta import Component, Config, Model
 
 Tool = tools.Tool
 ToolMode = tools.ToolMode
+ToolCall = tools.ToolCall
+FunctionCall = tools.FunctionCall
+ToolDefinition = tools.ToolDefinition
+FunctionDefinition = tools.FunctionDefinition
 
 AnyTool = Tool[t.Any, t.Any]
 

dreadnode-1.15.0/dreadnode/airt/__init__.py

@@ -0,0 +1,29 @@
+from dreadnode.airt import attack, search
+from dreadnode.airt.attack import (
+    Attack,
+    goat_attack,
+    hop_skip_jump_attack,
+    nes_attack,
+    prompt_attack,
+    simba_attack,
+    tap_attack,
+    zoo_attack,
+)
+from dreadnode.airt.target import CustomTarget, LLMTarget, Target
+
+__all__ = [
+    "Attack",
+    "CustomTarget",
+    "LLMTarget",
+    "Target",
+    "attack",
+    "goat_attack",
+    "hop_skip_jump_attack",
+    "nes_attack",
+    "prompt_attack",
+    "search",
+    "simba_attack",
+    "tap_attack",
+    "target",
+    "zoo_attack",
+]

dreadnode-1.15.0/dreadnode/airt/attack/__init__.py

@@ -0,0 +1,19 @@
+from dreadnode.airt.attack.base import Attack
+from dreadnode.airt.attack.goat import goat_attack
+from dreadnode.airt.attack.hop_skip_jump import hop_skip_jump_attack
+from dreadnode.airt.attack.nes import nes_attack
+from dreadnode.airt.attack.prompt import prompt_attack
+from dreadnode.airt.attack.simba import simba_attack
+from dreadnode.airt.attack.tap import tap_attack
+from dreadnode.airt.attack.zoo import zoo_attack
+
+__all__ = [
+    "Attack",
+    "goat_attack",
+    "hop_skip_jump_attack",
+    "nes_attack",
+    "prompt_attack",
+    "simba_attack",
+    "tap_attack",
+    "zoo_attack",
+]