dreadnode 2.0.14__tar.gz → 2.0.16__tar.gz

{dreadnode-2.0.14 → dreadnode-2.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dreadnode
-Version: 2.0.14
+Version: 2.0.16
 Summary: Dreadnode SDK
 Project-URL: Homepage, https://dreadnode.io
 Project-URL: Documentation, https://docs.dreadnode.io
@@ -13,6 +13,7 @@ Requires-Dist: coolname<3.0.0,>=2.2.0
 Requires-Dist: croniter<7.0.0,>=6.0.0
 Requires-Dist: cyclopts>=4.2.0
 Requires-Dist: datasets>=4.5.0
+Requires-Dist: ddgs>=9.0.0
 Requires-Dist: fastapi>=0.115.0
 Requires-Dist: fastmcp>=2.14.0
 Requires-Dist: fsspec[s3]<=2025.10.0,>=2025.10.0
@@ -20,9 +21,10 @@ Requires-Dist: gepa>=0.1.1
 Requires-Dist: httpx<1.0.0,>=0.28.0
 Requires-Dist: jsonpath-ng>=1.7.0
 Requires-Dist: jsonref>=1.1.0
-Requires-Dist: litellm<=1.83.0,>=1.83.0
+Requires-Dist: litellm<=1.83.7,>=1.83.7
 Requires-Dist: logfire<=3.20.0,>=3.5.3
 Requires-Dist: loguru>=0.7.3
+Requires-Dist: markdownify<2.0.0,>=1.1.0
 Requires-Dist: mcp<2.0.0,>=1.25.0
 Requires-Dist: moviepy<3.0.0,>=2.1.2
 Requires-Dist: numpy<=2.3.5,>=2.3.0

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/__init__.py

@@ -154,6 +154,7 @@ __all__ = [
     "push_capability",
     "push_dataset",
     "push_environment",
+    "push_hf_dataset",
     "push_model",
     "push_update",
     "run",
@@ -290,6 +291,7 @@ __instance_methods__: list[str] = [
     "push_capability",
     "push_dataset",
     "push_environment",
+    "push_hf_dataset",
     "push_model",
 ]
 

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/agents/__init__.py

@@ -20,12 +20,14 @@ from dreadnode.core.hook import Hook
 
 __all__ = [
     "Agent",
+    "AgentJudge",
     "Continue",
     "Fail",
     "Finish",
     "FunctionCall",
     "FunctionDefinition",
     "Hook",
+    "JudgeResult",
     "Reaction",
     "Retry",
     "RetryWithFeedback",
@@ -59,6 +61,8 @@ _LAZY_NAMES = frozenset(
         "reactions",
         "stopping",
         "Agent",
+        "AgentJudge",
+        "JudgeResult",
         "Trajectory",
         "Continue",
         "Fail",
@@ -77,6 +81,7 @@ def _lazy_init() -> None:
     _stopping = importlib.import_module("dreadnode.agents.stopping")
     _agent = importlib.import_module("dreadnode.agents.agent")
     _trajectory = importlib.import_module("dreadnode.agents.trajectory")
+    _judge = importlib.import_module("dreadnode.agents.judge")
 
     _Agent = _agent.Agent  # noqa: N806
     _Trajectory = _trajectory.Trajectory  # noqa: N806
@@ -84,11 +89,18 @@ def _lazy_init() -> None:
     _Agent.model_rebuild()
     _Trajectory.model_rebuild()
 
+    # Expose AgentJudge as ``Agent.Judge`` for discoverability. ``Agent`` is
+    # the primary abstraction; ``Agent.Judge(...)`` is the specialized
+    # construction mirror for trajectory-scoring use cases.
+    _Agent.Judge = _judge.AgentJudge
+
     g = globals()
     g["events"] = _events
     g["reactions"] = _reactions
     g["stopping"] = _stopping
     g["Agent"] = _Agent
+    g["AgentJudge"] = _judge.AgentJudge
+    g["JudgeResult"] = _judge.JudgeResult
     g["Trajectory"] = _Trajectory
     g["Continue"] = _reactions.Continue
     g["Fail"] = _reactions.Fail

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/agents/agent.py

@@ -6,7 +6,6 @@ import time
 import typing as t
 from contextlib import AsyncExitStack, asynccontextmanager, suppress
 from copy import deepcopy
-from pathlib import Path
 from textwrap import dedent
 from uuid import UUID, uuid4
 
@@ -169,8 +168,6 @@ class Agent(Executor[AgentEvent, Trajectory]):
     """Timeout in seconds for each LLM generation call. None = no timeout."""
     generate_params_extra: dict[str, t.Any] = Config(default_factory=dict)
     """Extra parameters merged into GenerateParams for every generation (e.g. thinking config)."""
-    working_dir: Path | None = Config(default=None)
-    """Working directory used for tool output offloading and other IO."""
     backoff_max_tries: int = Config(default=8, ge=0)
     """Maximum retries on transient LLM API errors per step. ``0`` disables retry."""
     backoff_max_time: float = Config(default=300.0, ge=0)
@@ -184,6 +181,15 @@ class Agent(Executor[AgentEvent, Trajectory]):
     _generator: Generator | None = PrivateAttr(None, init=False)
     _current_input: str = PrivateAttr("", init=False)
 
+    # Discoverability namespace for the AgentJudge specialized construction.
+    # Assigned at module-load time by ``dreadnode.agents.__init__._lazy_init``
+    # to break the import cycle between Agent and AgentJudge (which in turn
+    # imports the llm_judge scorer stack).
+    if t.TYPE_CHECKING:
+        from dreadnode.agents.judge import AgentJudge
+
+        Judge: t.ClassVar[type[AgentJudge]]
+
     @field_validator("tools", mode="before")
     @classmethod
     def validate_tools(cls, value: t.Any) -> t.Any:
@@ -219,14 +225,9 @@ class Agent(Executor[AgentEvent, Trajectory]):
         return StopCondition.fit_many(value)
 
     @model_validator(mode="after")
-    def bind_tool_working_dir(self) -> "Agent":
+    def ensure_agent_tag(self) -> "Agent":
         if "agent" not in self.tags:
             self.tags.insert(0, "agent")
-
-        working_dir = self.working_dir or Path.cwd()
-        for tool in self.all_tools:
-            if tool.working_dir is None:
-                tool.working_dir = working_dir
         return self
 
     @property
@@ -640,9 +641,6 @@ class Agent(Executor[AgentEvent, Trajectory]):
 
                 return
 
-            if tool.working_dir is None:
-                tool.working_dir = self.working_dir or Path.cwd()
-
             try:
                 message, stop = await tool.handle_tool_call(tool_call)
 

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/agents/events.py

@@ -152,6 +152,15 @@ class AgentStep(AgentEvent):
 
     @property
     def estimated_cost(self) -> float | None:
+        # Prefer the cost litellm attached to the response - it dispatches
+        # per-provider (anthropic_cost_per_token, openai_cost_per_token,
+        # bedrock_*, vertex_*, gemini_*, ...) and honours cache_read /
+        # cache_creation rates, reasoning tokens, tiered pricing, and
+        # region/service-tier multipliers. The naive math below ignores
+        # all of that and is wrong by 30-80% under prompt caching.
+        if self.usage.cost_usd is not None:
+            return self.usage.cost_usd
+
         import litellm
 
         if self.generator is None:
@@ -701,6 +710,7 @@ class GenerationStep(AgentStep):
                 "input_tokens": self.usage.input_tokens,
                 "output_tokens": self.usage.output_tokens,
                 "total_tokens": self.usage.total_tokens,
+                "cost_usd": self.estimated_cost,
             },
             "stop_reason": self.stop_reason,
             "model": self.generator.model if self.generator else None,

dreadnode-2.0.16/dreadnode/agents/judge.py

@@ -0,0 +1,196 @@
+"""Agent.Judge — specialized agent for evaluating trajectories against a rubric.
+
+Thin wrapper over ``dreadnode.scorers.judge.llm_judge`` that adapts the scorer
+to consume a full :class:`Trajectory` (multi-turn message log) rather than a
+single output string. Used by the hosted training ``llm_judge`` verification
+method and by anyone grading rollouts in a notebook.
+
+Exposed as ``Agent.Judge(...)`` (class attribute) and as ``AgentJudge`` directly
+from ``dreadnode.agents``.
+"""
+
+import typing as t
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dreadnode.core.metric import Metric
+
+if t.TYPE_CHECKING:
+    from dreadnode.agents.trajectory import Trajectory
+    from dreadnode.core.scorer import Scorer
+
+
+@dataclass(slots=True)
+class JudgeResult:
+    """Structured result of an :class:`AgentJudge` evaluation.
+
+    Attributes:
+        passed: Whether the trajectory cleared the judge's passing threshold.
+        score: Scalar in ``[0, 1]`` — the rubric-driven judge score.
+        reason: Free-text rationale returned by the judge LLM.
+        metrics: Raw metric list from the underlying scorer (score metric
+            first, followed by the boolean pass metric).
+    """
+
+    passed: bool
+    score: float
+    reason: str
+    metrics: list[Metric] = field(default_factory=list)
+
+
+class AgentJudge:
+    """Specialized agent that evaluates a :class:`Trajectory` against a rubric.
+
+    Example::
+
+        from dreadnode.agents import Agent
+
+        judge = Agent.Judge(
+            model="openai/gpt-4o",
+            rubric="rce",
+            passing_threshold=0.7,
+        )
+        result = await judge.evaluate(rollout.trajectory)
+        if result.passed:
+            print("win!", result.score, result.reason)
+
+    Reuses the production :func:`dreadnode.scorers.judge.llm_judge` scorer
+    (rubric YAML loading, XML response parsing, fallback regex, etc.) — this
+    class only adapts the transcript shape and surfaces a :class:`JudgeResult`.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        rubric: str | Path,
+        passing_threshold: float = 0.5,
+        system_prompt: str | None = None,
+        model_params: dict[str, t.Any] | None = None,
+    ) -> None:
+        if not 0.0 <= passing_threshold <= 1.0:
+            raise ValueError(
+                f"passing_threshold must be between 0.0 and 1.0 (got {passing_threshold!r})"
+            )
+
+        # Lazy import keeps agents/__init__ import graph cheap — scorers.judge
+        # pulls in yaml + the rubrics dir.
+        from dreadnode.scorers.judge import llm_judge
+
+        self._model = model
+        self._rubric = rubric
+        self._passing_threshold = passing_threshold
+        self._system_prompt = system_prompt
+        self._model_params = model_params
+        self._scorer: Scorer[t.Any] = llm_judge(
+            model=model,
+            rubric=rubric,
+            passing=lambda score: score >= passing_threshold,
+            system_prompt=system_prompt,
+            model_params=model_params,
+        )
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    @property
+    def rubric(self) -> str | Path:
+        return self._rubric
+
+    @property
+    def passing_threshold(self) -> float:
+        return self._passing_threshold
+
+    async def evaluate(
+        self,
+        trajectory: "Trajectory",
+        *,
+        context: dict[str, t.Any] | None = None,
+    ) -> JudgeResult:
+        """Score ``trajectory`` against the configured rubric.
+
+        Flattens ``trajectory.messages`` to a transcript formatted as
+        ``[role] content`` lines, optionally prepended with a
+        ``# Task context`` block built from ``context`` (task instruction,
+        env service URLs, flag hints, etc.).
+
+        Args:
+            trajectory: The agent run to evaluate. Uses ``trajectory.messages``
+                which already chains per-step messages in chat order.
+            context: Optional per-call context merged into the judge's input.
+                Good use: pass the rendered task instruction and any known
+                ground-truth hints so the judge can evaluate faithfulness.
+
+        Returns:
+            A :class:`JudgeResult` with ``passed`` / ``score`` / ``reason``.
+
+        Raises:
+            ValueError: if the scorer returned fewer than two metrics (the
+                scorer contract is score metric + pass metric).
+        """
+
+        transcript = self._trajectory_to_transcript(trajectory, context=context)
+        metrics = await self._scorer.normalize_and_score(transcript)
+        if not metrics:
+            raise ValueError(
+                "AgentJudge scorer returned no metrics — check the judge model's output format."
+            )
+
+        score_metric = metrics[0]
+        score = float(score_metric.value)
+        reason = str(score_metric.attributes.get("reason", ""))
+
+        # Second metric is the boolean pass flag (see scorers/judge.py). If
+        # absent, fall back to threshold comparison so callers still get a
+        # sensible ``passed`` value.
+        if len(metrics) >= 2:
+            passed = bool(metrics[1].value)
+        else:
+            passed = score >= self._passing_threshold
+
+        return JudgeResult(
+            passed=passed,
+            score=score,
+            reason=reason,
+            metrics=list(metrics),
+        )
+
+    @staticmethod
+    def _trajectory_to_transcript(
+        trajectory: "Trajectory",
+        *,
+        context: dict[str, t.Any] | None = None,
+    ) -> str:
+        """Render a transcript the judge LLM can consume.
+
+        Format:
+            # Task context
+            - key: value
+            - ...
+
+            # Conversation transcript
+            [system] ...
+            [user] ...
+            [assistant] ...
+            ...
+
+        Context is optional. The conversation section always appears, even
+        for empty trajectories (judge then scores on instruction only).
+        """
+
+        lines: list[str] = []
+        if context:
+            lines.append("# Task context")
+            for key, value in context.items():
+                lines.append(f"- {key}: {value}")
+            lines.append("")
+        lines.append("# Conversation transcript")
+        for msg in trajectory.messages:
+            role = getattr(msg, "role", "unknown")
+            content = getattr(msg, "content", "") or ""
+            lines.append(f"[{role}] {content}")
+        return "\n".join(lines)
+
+
+__all__ = ["AgentJudge", "JudgeResult"]

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/agents/skills.py

@@ -15,6 +15,7 @@ import yaml
 from loguru import logger
 
 from dreadnode.agents.tools import tool
+from dreadnode.capabilities.capability import Capability
 
 # CAP-IDENT-010 (skill surface): unsafe characters are replaced with `_` and
 # the segment is trimmed of leading/trailing `_`. Unlike the tool wire surface,
@@ -522,3 +523,13 @@ def create_skill_tool(skills: list[Skill]) -> t.Any:
         return resolve_skill(name, skills).render_content()
 
     return skill
+
+
+def attach_capability_skills(*, agent: t.Any, capability: Capability) -> None:
+    """Attach capability-local skills to the reconstructed agent, if any."""
+
+    all_skills: list[Skill] = []
+    for skills_path in capability.skills_paths or []:
+        all_skills.extend(discover_skills(skills_path))
+    if all_skills:
+        agent.tools.append(create_skill_tool(all_skills))

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/agents/tools.py

@@ -7,7 +7,6 @@ import re
 import typing as t
 from pathlib import Path
 
-import aiofiles
 import typing_extensions as te
 from pydantic import (
     BaseModel,
@@ -167,20 +166,19 @@ async def offload_tool_output(
     content: str,
     tool_call_id: str,
     tool_name: str,  # noqa: ARG001 - callers pass as keyword arg
-    working_dir: Path,
 ) -> tuple[str, Path]:
-    """Write tool output to disk and return middle-out summary plus file path."""
-    output_root = working_dir if working_dir.name == ".dreadnode" else working_dir / ".dreadnode"
-    output_dir = output_root / "tool-output"
-    await asyncio.to_thread(output_dir.mkdir, parents=True, exist_ok=True)
+    """Write tool output to disk and return middle-out summary plus file path.
 
-    safe_id = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_call_id)
-    file_path = output_dir / f"{safe_id}.txt"
-    async with aiofiles.open(file_path, "w") as f:
-        await f.write(content)
+    Output lands at ``<cache>/tool-output/<YYYYMMDD-HHMMSS>-<tool_call_id>.txt``,
+    where ``<cache>`` is the active Dreadnode instance's cache directory
+    (``~/.dreadnode`` by default; honors ``configure(cache=...)``).
+    """
+    from dreadnode import _get_default_instance
+    from dreadnode.storage.storage import write_timestamped
 
-    relative_base = working_dir.parent if working_dir.name == ".dreadnode" else working_dir
-    relative_path = file_path.relative_to(relative_base)
+    output_dir = _get_default_instance().storage.tool_output_path
+    safe_id = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_call_id)
+    file_path = await asyncio.to_thread(write_timestamped, output_dir, f"{safe_id}.txt", content)
 
     half = OFFLOAD_THRESHOLD // 2
     start = content[:half]
@@ -189,7 +187,7 @@ async def offload_tool_output(
 
     truncated = (
         f"{start}\n\n"
-        f"... [{middle_lines} lines truncated — full output saved to {relative_path}] ...\n\n"
+        f"... [{middle_lines} lines truncated — full output saved to {file_path}] ...\n\n"
         f"{end}"
     )
     return truncated, file_path
@@ -304,8 +302,6 @@ class Tool(BaseModel, t.Generic[P, R]):
     """If set, the maximum number of characters to truncate any tool output to."""
     offload: bool = True
     """Whether large tool outputs should be offloaded to disk."""
-    working_dir: Path | None = None
-    """Working directory used to store offloaded tool outputs."""
 
     _signature: inspect.Signature | None = PrivateAttr(default=None, init=False)
     _type_adapter: TypeAdapter[t.Any] | None = PrivateAttr(default=None, init=False)
@@ -535,16 +531,24 @@ class Tool(BaseModel, t.Generic[P, R]):
             message = message.shorten(self.truncate)
 
         if self.offload and message.content and len(message.content) > OFFLOAD_THRESHOLD:
-            working_dir = self.working_dir or Path.cwd()
+            from dreadnode import _get_default_instance
+
             truncated_content, output_file = await offload_tool_output(
                 message.content,
                 tool_call.id,
                 tool_call.name,
-                working_dir,
             )
             message = message.clone()
             message.content_parts = [ContentText(text=truncated_content)]
-            message.metadata["output_file"] = str(output_file.relative_to(working_dir))
+            # Strip cache root before persisting on the span — absolute paths
+            # leak user home / hostname to the platform without giving the
+            # server anything actionable (the file is local-only).
+            cache_root = _get_default_instance().cache
+            try:
+                metadata_path = output_file.relative_to(cache_root)
+            except ValueError:
+                metadata_path = output_file
+            message.metadata["output_file"] = str(metadata_path)
 
         return message, stop
 
@@ -564,7 +568,6 @@ class Tool(BaseModel, t.Generic[P, R]):
             catch=self.catch,
             truncate=self.truncate,
             offload=self.offload,
-            working_dir=self.working_dir,
         )
 
         new._signature = self._signature

{dreadnode-2.0.14 → dreadnode-2.0.16}/dreadnode/airt/assessment.py

@@ -24,7 +24,6 @@ import contextvars
 import os
 import signal
 import typing as t
-import uuid
 from pathlib import Path
 
 from loguru import logger
@@ -127,7 +126,6 @@ class Assessment:
 
         self._assessment_id: str | None = None
         self._attack_results: list[AttackResult] = []
-        self._attack_run_ids: dict[int, str] = {}  # index in _attack_results -> platform run_id
         self._tracing_enabled: bool = False
         self._auto_registered: bool = False
         self._context_token: contextvars.Token[Assessment | None] | None = None
@@ -232,14 +230,10 @@ class Assessment:
             yield a
 
     async def _finalize(self) -> None:
-        """Upload pending results, flush traces, mark complete."""
+        """Flush pending OTEL spans and mark complete."""
         if not self._attack_results:
             return
 
-        for i, result in enumerate(self._attack_results):
-            if i not in self._attack_run_ids:
-                await self.upload_result(result)
-
         # Flush pending OTEL spans BEFORE marking complete so CH has data
         # when the API materializes findings on status change.
         # dn.shutdown() would kill the trace provider, breaking any
@@ -327,7 +321,6 @@ class Assessment:
             execution_time_s=getattr(result, "execution_time_s", 0.0),
         )
         self._attack_results.append(ar)
-        await self.upload_result(ar)
 
         return result
 
@@ -496,65 +489,6 @@ class Assessment:
                     _current_assessment.reset(self._context_token)
                 self._context_token = None
 
-    # =========================================================================
-    # Result Upload
-    # =========================================================================
-
-    async def upload_result(self, result: AttackResult) -> str | None:
-        """Register and upload an attack result to the platform.
-
-        Args:
-            result: The AttackResult to upload.
-
-        Returns:
-            The platform run ID, or None if offline.
-        """
-        ctx = _get_platform_context()
-        if ctx is None or self._assessment_id is None:
-            return None
-
-        api, profile = ctx
-        org = profile.org_key
-        ws = profile.workspace_key
-
-        try:
-            trace_id = uuid.uuid4().hex
-            run_data = api.register_airt_attack_run(
-                org,
-                ws,
-                self._assessment_id,
-                attack_name=result.attack_name,
-                goal=result.goal,
-                goal_category=result.goal_category.value,
-                transforms_applied=result.transforms_applied,
-                compliance_tags=result.compliance_tags,
-                trace_id=trace_id,
-            )
-            run_id = run_data["id"]
-
-            api.upload_airt_attack_result(
-                org,
-                ws,
-                self._assessment_id,
-                run_id,
-                attack_result_json=result.to_dict(),
-                best_score=result.best_score,
-                total_trials=result.total_trials,
-                finished_trials=len(result.finished_trials),
-                execution_time_s=result.execution_time_s,
-            )
-        except Exception as e:
-            logger.error(f"Failed to upload attack result: {e}")
-            return None
-
-        idx = len(self._attack_results) - 1
-        if result in self._attack_results:
-            idx = self._attack_results.index(result)
-        self._attack_run_ids[idx] = run_id
-
-        logger.info(f"Attack result uploaded: {result.attack_name} -> {run_id}")
-        return run_id
-
     # =========================================================================
     # Lifecycle
     # =========================================================================