dreadnode 2.0.17__tar.gz → 2.0.19__tar.gz

{dreadnode-2.0.17 → dreadnode-2.0.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dreadnode
-Version: 2.0.17
+Version: 2.0.19
 Summary: Dreadnode SDK
 Project-URL: Homepage, https://dreadnode.io
 Project-URL: Documentation, https://docs.dreadnode.io
@@ -21,7 +21,7 @@ Requires-Dist: gepa>=0.1.1
 Requires-Dist: httpx<1.0.0,>=0.28.0
 Requires-Dist: jsonpath-ng>=1.7.0
 Requires-Dist: jsonref>=1.1.0
-Requires-Dist: litellm<=1.83.7,>=1.83.7
+Requires-Dist: litellm<=1.83.14,>=1.83.14
 Requires-Dist: logfire<=3.20.0,>=3.5.3
 Requires-Dist: loguru>=0.7.3
 Requires-Dist: markdownify<2.0.0,>=1.1.0

{dreadnode-2.0.17 → dreadnode-2.0.19}/dreadnode/agents/__init__.py

@@ -28,6 +28,8 @@ __all__ = [
     "FunctionDefinition",
     "Hook",
     "JudgeResult",
+    "ProcessDecision",
+    "ProcessJudge",
     "Reaction",
     "Retry",
     "RetryWithFeedback",
@@ -44,6 +46,7 @@ __all__ = [
     "exceptions",
     "hooks",
     "mcp",
+    "process_judge_hook",
     "reactions",
     "skills",
     "stopping",
@@ -63,6 +66,9 @@ _LAZY_NAMES = frozenset(
         "Agent",
         "AgentJudge",
         "JudgeResult",
+        "ProcessDecision",
+        "ProcessJudge",
+        "process_judge_hook",
         "Trajectory",
         "Continue",
         "Fail",
@@ -94,6 +100,9 @@ def _lazy_init() -> None:
     # construction mirror for trajectory-scoring use cases.
     _Agent.Judge = _judge.AgentJudge
 
+    _process_judge = importlib.import_module("dreadnode.agents.process_judge")
+    _hooks = importlib.import_module("dreadnode.agents.hooks")
+
     g = globals()
     g["events"] = _events
     g["reactions"] = _reactions
@@ -101,6 +110,9 @@ def _lazy_init() -> None:
     g["Agent"] = _Agent
     g["AgentJudge"] = _judge.AgentJudge
     g["JudgeResult"] = _judge.JudgeResult
+    g["ProcessDecision"] = _process_judge.ProcessDecision
+    g["ProcessJudge"] = _process_judge.ProcessJudge
+    g["process_judge_hook"] = _hooks.process_judge_hook
     g["Trajectory"] = _Trajectory
     g["Continue"] = _reactions.Continue
     g["Fail"] = _reactions.Fail

{dreadnode-2.0.17 → dreadnode-2.0.19}/dreadnode/agents/hooks.py

@@ -12,10 +12,11 @@ from dataclasses import dataclass
 from dreadnode.agents.events import (
     AgentEvent,
     AgentStep,
+    GenerationStart,
     ToolEnd,
     ToolStart,
 )
-from dreadnode.agents.reactions import Reaction, Retry
+from dreadnode.agents.reactions import Fail, Finish, Reaction, Retry, RetryWithFeedback
 from dreadnode.core.hook import Hook, hook
 from dreadnode.generators.generator import Generator
 from dreadnode.generators.message import Message, make_compaction_message
@@ -23,6 +24,8 @@ from dreadnode.generators.message import Message, make_compaction_message
 if t.TYPE_CHECKING:
     from datetime import datetime
 
+    from dreadnode.agents.process_judge import ProcessJudge
+
 
 # =============================================================================
 # Tool Metrics
@@ -410,3 +413,251 @@ def _make_summarize_hook(
 
 # Pre-instantiated with defaults -- this is what the wrapper discovers
 summarize_when_long = _make_summarize_hook()
+
+
+# =============================================================================
+# Process Judge (guard policy)
+# =============================================================================
+
+TranscriptStrategy = t.Literal[
+    "rubric_only",
+    "intent_only",
+    "intent_plus_calls",
+    "intent_plus_outputs",
+    "full",
+]
+OnDeny = t.Literal["retry", "finish"]
+OnJudgeError = t.Literal["deny", "allow", "fail"]
+
+
+def _strip_intent(messages: list[Message], strategy: TranscriptStrategy) -> list[Message]:
+    """Slice ``messages`` per ``strategy``.
+
+    - ``rubric_only``: empty list. Judge sees only the proposed call.
+    - ``intent_only``: system + user-authored only. Smallest prompt and
+      narrowest attack surface — tool outputs and tool calls never reach
+      the judge.
+    - ``intent_plus_calls``: above + assistant messages that carry
+      ``tool_calls`` (no tool result messages, no text-only assistant
+      turns). Lets the judge see the tool-call sequence leading up to
+      this call without exposing it to attacker-controllable tool
+      output content.
+    - ``intent_plus_outputs``: ``intent_only`` + tool-result messages.
+      Tool calls themselves are stripped; the judge sees responses
+      without knowing which call produced them. Widens the injection
+      surface — adversarial tool output can target the judge.
+    - ``full``: messages unchanged. Maximum context, maximum surface.
+    """
+    if strategy == "full":
+        return list(messages)
+    if strategy == "rubric_only":
+        return []
+    if strategy == "intent_only":
+        return [m for m in messages if m.role in ("system", "user")]
+    if strategy == "intent_plus_calls":
+        return [
+            m
+            for m in messages
+            if m.role in ("system", "user")
+            or (m.role == "assistant" and getattr(m, "tool_calls", None))
+        ]
+    if strategy == "intent_plus_outputs":
+        return [m for m in messages if m.role in ("system", "user", "tool")]
+    raise ValueError(f"unknown transcript_strategy: {strategy!r}")
+
+
+def _trim_intent_to_budget(
+    intent: list[Message],
+    model: "str | Generator | None",
+) -> tuple[list[Message], int]:
+    """Drop oldest non-protected messages until intent fits the judge's budget.
+
+    Returns ``(trimmed, dropped_count)``. The system message and the first
+    user message (the original task) are always preserved — the rest is
+    eligible for eviction, oldest first.
+
+    ``str(message)`` matches what :func:`ProcessJudge._render_intent` writes
+    into the judge prompt, so the char count and the actual rendered prompt
+    measure the same string. We allocate ~80 % of the judge model's char
+    budget (roughly 4 chars/token of its max input window) so the rubric
+    block and the proposed-call block still have headroom.
+    """
+    budget_tokens = _get_model_context_budget(model)
+    char_budget = int(budget_tokens * 4 * 0.8)
+
+    if not intent:
+        return [], 0
+
+    protected_end = 0
+    if intent[0].role == "system":
+        protected_end = 1
+    for i in range(protected_end, len(intent)):
+        if intent[i].role == "user":
+            protected_end = i + 1
+            break
+
+    current = sum(len(str(m)) for m in intent)
+    if current <= char_budget:
+        return list(intent), 0
+
+    # Walk a cursor forward, subtracting each evicted message's char count
+    # from the running total. Single slice at the end — no repeated full-list
+    # walks, no in-place mutation.
+    drop_from = protected_end
+    dropped = 0
+    while current > char_budget and drop_from < len(intent):
+        current -= len(str(intent[drop_from]))
+        drop_from += 1
+        dropped += 1
+
+    return intent[:protected_end] + intent[drop_from:], dropped
+
+
+def _deny_reaction(reason: str, on_deny: OnDeny) -> Reaction:
+    if on_deny == "retry":
+        return RetryWithFeedback(feedback=reason)
+    return Finish(reason=f"policy denied: {reason}")
+
+
+def _judge_error_reaction(exc: Exception, on_judge_error: OnJudgeError) -> Reaction | None:
+    from loguru import logger
+
+    if on_judge_error == "deny":
+        return Finish(reason=f"process judge unreachable: {exc}")
+    if on_judge_error == "allow":
+        logger.warning("process judge errored, allowing tool call: {}", exc)
+        return None
+    return Fail(error=exc)
+
+
+def process_judge_hook(
+    judge: "ProcessJudge",
+    *,
+    transcript_strategy: TranscriptStrategy = "intent_plus_calls",
+    on_deny: OnDeny = "retry",
+    on_judge_error: OnJudgeError = "deny",
+    always_allow: t.Sequence[str] = (),
+    always_deny: t.Sequence[str] = (),
+    context_provider: t.Callable[[ToolStart], dict[str, t.Any]] | None = None,
+) -> Hook:
+    """Pre-tool-call gating hook backed by a :class:`ProcessJudge`.
+
+    Listens to ``GenerationStart`` to snapshot the message state going
+    into each generation, then judges every ``ToolStart`` against that
+    snapshot. ``always_allow`` / ``always_deny`` short-circuit the judge
+    call. ``always_deny`` wins ties. The captured intent is sliced per
+    ``transcript_strategy`` and then trimmed to fit the judge model's
+    context window (oldest non-protected messages drop first; the system
+    message and the original user task are always preserved).
+
+    Decisions map to reactions:
+
+    - allow → ``None`` (tool runs).
+    - deny + ``on_deny="retry"`` → :class:`RetryWithFeedback`.
+    - deny + ``on_deny="finish"`` → :class:`Finish` with ``"policy denied: …"``.
+    - judge raises + ``on_judge_error="deny"`` → :class:`Finish`.
+    - judge raises + ``on_judge_error="allow"`` → ``None`` plus warn-level log.
+    - judge raises + ``on_judge_error="fail"`` → :class:`Fail`.
+    """
+    import time
+
+    deny_set = set(always_deny)
+    allow_set = set(always_allow) - deny_set
+
+    _last_messages: list[Message] = []
+
+    @hook(AgentEvent)
+    async def process_judge_hook_inner(event: AgentEvent) -> Reaction | None:
+        nonlocal _last_messages
+        from dreadnode import log_metric
+
+        if isinstance(event, GenerationStart):
+            _last_messages = list(event.messages)
+            return None
+
+        if not isinstance(event, ToolStart):
+            return None
+
+        tool_name = event.tool_call.name
+        start = time.monotonic()
+
+        if tool_name in deny_set:
+            log_metric(
+                "process_judge.deny",
+                1,
+                attributes={
+                    "tool_name": tool_name,
+                    "short_circuit": True,
+                    "reason": "always_deny",
+                },
+            )
+            return _deny_reaction(f"tool {tool_name!r} is in always_deny list", on_deny)
+
+        if tool_name in allow_set:
+            log_metric(
+                "process_judge.allow",
+                1,
+                attributes={"tool_name": tool_name, "short_circuit": True},
+            )
+            return None
+
+        intent = _strip_intent(_last_messages, transcript_strategy)
+        intent, dropped = _trim_intent_to_budget(intent, judge.model)
+        if dropped:
+            log_metric(
+                "process_judge.intent_trimmed",
+                1,
+                attributes={
+                    "tool_name": tool_name,
+                    "strategy": transcript_strategy,
+                    "dropped_messages": dropped,
+                },
+            )
+        context = context_provider(event) if context_provider else None
+
+        try:
+            decision = await judge.evaluate(
+                intent=intent,
+                proposed_call=event.tool_call,
+                context=context,
+            )
+        except Exception as exc:
+            latency_ms = int((time.monotonic() - start) * 1000)
+            log_metric(
+                "process_judge.error",
+                1,
+                attributes={
+                    "tool_name": tool_name,
+                    "error_type": type(exc).__name__,
+                    "error_message": str(exc),
+                    "latency_ms": latency_ms,
+                },
+            )
+            return _judge_error_reaction(exc, on_judge_error)
+
+        latency_ms = int((time.monotonic() - start) * 1000)
+        if decision.allow:
+            log_metric(
+                "process_judge.allow",
+                1,
+                attributes={
+                    "tool_name": tool_name,
+                    "short_circuit": False,
+                    "latency_ms": latency_ms,
+                },
+            )
+            return None
+
+        log_metric(
+            "process_judge.deny",
+            1,
+            attributes={
+                "tool_name": tool_name,
+                "short_circuit": False,
+                "reason": decision.reason,
+                "latency_ms": latency_ms,
+            },
+        )
+        return _deny_reaction(decision.reason, on_deny)
+
+    return process_judge_hook_inner

dreadnode-2.0.19/dreadnode/agents/process_judge.py

@@ -0,0 +1,212 @@
+"""ProcessJudge — LLM-backed gate over (intent, proposed tool call) pairs.
+
+The primitive layer of the guard policy. ``ProcessJudge`` knows
+nothing about ``Trajectory``, ``Hook``, or ``SessionPolicy`` — it takes
+pre-shaped inputs (a list of :class:`Message` plus a :class:`ToolCall`)
+and returns a :class:`ProcessDecision`. Hook code is responsible for the
+trajectory-to-intent transformation.
+
+Composition: reuses the lower-level :func:`dreadnode.scorers.judge.judge`
+function so XML response parsing and the regex fallback stay in one
+place. The default safety-floor rubric ships at
+``dreadnode/policies/rubrics/process/default.yaml`` and is layered with
+any user-supplied rubric under a ``## Project-specific scope`` header
+unless ``replace_default_rubric=True``.
+"""
+
+import typing as t
+from dataclasses import dataclass
+from pathlib import Path
+
+from dreadnode.generators.generator import GenerateParams, Generator, get_generator
+
+if t.TYPE_CHECKING:
+    from dreadnode.agents.tools import ToolCall
+    from dreadnode.generators.message import Message
+
+
+_DEFAULT_RUBRIC_PATH: Path = (
+    Path(__file__).parent.parent / "policies" / "rubrics" / "process" / "default.yaml"
+)
+
+_USER_RUBRIC_HEADER = "## Project-specific scope"
+
+
+@dataclass(slots=True)
+class ProcessDecision:
+    """Outcome of one :class:`ProcessJudge` invocation.
+
+    Two-valued by design — the agent will run an allowed call or won't run
+    a denied one. A "warn" mode is deliberately absent; the gating layer
+    cannot usefully half-allow a tool call.
+    """
+
+    allow: bool
+    reason: str
+
+
+class ProcessJudge:
+    """LLM-backed gate over (intent, proposed tool call) pairs.
+
+    Example::
+
+        from dreadnode.agents.process_judge import ProcessJudge
+
+        judge = ProcessJudge(
+            model="anthropic/claude-haiku-4-5",
+            rubric="In-scope domains: api.example.com",
+        )
+        decision = await judge.evaluate(
+            intent=session_messages,
+            proposed_call=tool_call,
+        )
+        if not decision.allow:
+            print("denied:", decision.reason)
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str | Generator,
+        rubric: str | Path | None = None,
+        replace_default_rubric: bool = False,
+        model_params: dict[str, t.Any] | None = None,
+        system_prompt: str | None = None,
+    ) -> None:
+        if rubric is None and replace_default_rubric:
+            raise ValueError("replace_default_rubric=True requires a user-supplied rubric")
+
+        from dreadnode.scorers.judge import _load_rubric_from_yaml
+
+        default_text, default_system_prompt, _ = _load_rubric_from_yaml(
+            _DEFAULT_RUBRIC_PATH, system_prompt=None, name="process_default"
+        )
+
+        if rubric is None:
+            final_rubric = default_text
+        else:
+            user_text = self._resolve_user_rubric(rubric)
+            final_rubric = (
+                user_text
+                if replace_default_rubric
+                else f"{default_text}\n\n{_USER_RUBRIC_HEADER}\n{user_text}"
+            )
+
+        self._model = model
+        self._rubric = final_rubric
+        self._system_prompt = system_prompt or default_system_prompt
+        self._model_params = model_params
+
+    @property
+    def model(self) -> str | Generator:
+        return self._model
+
+    @property
+    def rubric(self) -> str:
+        return self._rubric
+
+    @property
+    def system_prompt(self) -> str | None:
+        return self._system_prompt
+
+    async def evaluate(
+        self,
+        *,
+        intent: "list[Message]",
+        proposed_call: "ToolCall",
+        context: dict[str, t.Any] | None = None,
+    ) -> ProcessDecision:
+        """Evaluate a proposed tool call against the rubric.
+
+        Args:
+            intent: Trajectory-stripped messages representing the user's
+                intent (typically system + user-authored, optionally tool
+                outputs depending on the hook's strategy).
+            proposed_call: The tool call about to execute. Wire name and
+                arguments are passed verbatim — argument redaction is a
+                separate concern.
+            context: Optional per-call context (task instruction, scope
+                metadata) merged into the rendered intent.
+
+        Returns:
+            A :class:`ProcessDecision` with ``allow`` and a short reason.
+        """
+        from dreadnode.scorers.judge import JudgeInput, judge
+
+        intent_text = self._render_intent(intent, context=context)
+        proposed_call_text = self._render_proposed_call(proposed_call)
+
+        generator = self._resolve_generator()
+        input_data = JudgeInput(
+            input=intent_text,
+            output=proposed_call_text,
+            rubric=self._rubric,
+        )
+
+        judgement = await judge(generator, input_data, system_prompt=self._system_prompt)
+        return ProcessDecision(allow=judgement.passing, reason=judgement.reason)
+
+    def _resolve_generator(self) -> Generator:
+        if isinstance(self._model, Generator):
+            return self._model
+        if not isinstance(self._model, str):
+            raise TypeError("model must be a string identifier or a Generator instance.")
+        params: GenerateParams | None
+        if self._model_params is None:
+            params = None
+        elif isinstance(self._model_params, GenerateParams):
+            params = self._model_params
+        else:
+            params = GenerateParams.model_validate(self._model_params)
+        return get_generator(self._model, params=params)
+
+    @staticmethod
+    def _resolve_user_rubric(rubric: str | Path) -> str:
+        """Resolve a user rubric to its body text.
+
+        Path or YAML-shaped string → load the YAML and read its ``rubric``
+        field. Anything else → treat as a plain string and return as-is.
+        """
+        from dreadnode.scorers.judge import _is_yaml_rubric, _load_rubric_from_yaml
+
+        if isinstance(rubric, Path) or _is_yaml_rubric(rubric):
+            text, _, _ = _load_rubric_from_yaml(rubric, system_prompt=None, name="user")
+            return text
+        return rubric
+
+    @staticmethod
+    def _render_intent(
+        intent: "list[Message]",
+        *,
+        context: dict[str, t.Any] | None = None,
+    ) -> str:
+        """Render intent messages as a transcript the judge can consume.
+
+        Delegates per-message formatting to ``Message.__str__`` so assistant
+        ``tool_calls`` and tool-result ``tool_call_id`` headers actually
+        appear in the rendered output — without this, ``intent_plus_calls``
+        and ``full`` would silently drop the tool-call sequence the judge
+        is supposed to be reasoning about.
+        """
+        lines: list[str] = []
+        if context:
+            lines.append("# Task context")
+            for key, value in context.items():
+                lines.append(f"- {key}: {value}")
+            lines.append("")
+        if intent:
+            lines.append("# Intent transcript")
+            for msg in intent:
+                lines.append(str(msg))
+        return "\n".join(lines)
+
+    @staticmethod
+    def _render_proposed_call(proposed_call: "ToolCall") -> str:
+        """Render the proposed tool call in a structured XML block."""
+        return (
+            f"<tool>{proposed_call.name}</tool>\n"
+            f"<arguments>{proposed_call.function.arguments}</arguments>"
+        )
+
+
+__all__ = ["ProcessDecision", "ProcessJudge"]

{dreadnode-2.0.17 → dreadnode-2.0.19}/dreadnode/app/api/client.py

@@ -56,6 +56,26 @@ class AuthenticationError(RuntimeError):
     """Raised when the platform returns HTTP 401."""
 
 
+class NotFoundError(RuntimeError):
+    """Raised when the platform returns HTTP 404 for a resource lookup.
+
+    Subclass of :class:`RuntimeError` so existing ``except RuntimeError``
+    handlers keep working; callers that want 404-specific recovery (e.g.
+    idempotent deletes that tolerate already-gone resources) can catch
+    this directly instead of pattern-matching on error message strings.
+    """
+
+
+class ConflictError(RuntimeError):
+    """Raised when the platform returns HTTP 409 for a state conflict.
+
+    Subclass of :class:`RuntimeError` so existing ``except RuntimeError``
+    handlers keep working; callers that want conflict-specific UX (e.g.
+    refusing to delete a session linked to an evaluation) can catch this
+    directly instead of pattern-matching on error message strings.
+    """
+
+
 class PlatformBackendUnavailableError(RuntimeError):
     """Raised when the platform endpoint is transiently unavailable.
 
@@ -176,6 +196,10 @@ class ApiClient:
         response = self._request(method, path, params, json_data, data, headers)
         if response.status_code == 401:
             raise AuthenticationError(self._get_error_message(response))
+        if response.status_code == 404:
+            raise NotFoundError(self._get_error_message(response))
+        if response.status_code == 409:
+            raise ConflictError(self._get_error_message(response))
         if response.status_code == 429:
             error_message = self._get_error_message(response)
             detail = error_message.split(": ", 1)[1] if ": " in error_message else error_message
@@ -2243,27 +2267,18 @@ class ApiClient:
         org: str,
         workspace: str,
         session_id: str,
-    ) -> bool:
+    ) -> None:
         """DELETE /org/{org}/ws/{workspace}/sessions/{id}.
 
-        Returns ``True`` when the platform deleted the row, ``False`` when
-        the session was already absent (404). Other failures raise — a
-        local-only session that was never registered with the platform
-        still 404s here, so the caller can safely treat that as success.
+        Raises :class:`NotFoundError` if the session does not exist. Callers
+        that want to tolerate already-gone sessions (e.g. delete propagation
+        for sessions that were never registered with the platform) should
+        catch :class:`NotFoundError` explicitly.
         """
-        response = self._request(
+        self.request(
             "DELETE",
             f"/org/{org}/ws/{workspace}/sessions/{session_id}",
         )
-        if response.status_code == 404:
-            return False
-        if response.status_code == 401:
-            raise AuthenticationError(self._get_error_message(response))
-        try:
-            response.raise_for_status()
-        except httpx.HTTPStatusError as e:
-            raise RuntimeError(self._get_error_message(response)) from e
-        return True
 
     def unarchive_session(
         self,
@@ -2396,6 +2411,60 @@ class ApiClient:
         )
         return t.cast("dict[str, t.Any]", response.json())
 
+    def get_session_trajectory(
+        self,
+        org: str,
+        workspace: str,
+        session_id: str,
+        *,
+        format: t.Literal["atif", "openai", "native"] = "atif",
+        include_compacted: bool = False,
+    ) -> t.Any:
+        """GET /org/{org}/ws/{workspace}/sessions/{id}/trajectory.
+
+        Returns the session's trajectory in the requested format. ``atif``
+        emits an ATIF v1.7 dict; ``openai`` emits a Chat Completions
+        ``messages`` list; ``native`` emits a Dreadnode bundle of session
+        metadata, transcript messages, and ClickHouse events.
+        ``include_compacted=True`` requests full history.
+        """
+        params: dict[str, t.Any] = {
+            "format": format,
+            "include_compacted": include_compacted,
+        }
+        response = self.request(
+            "GET",
+            f"/org/{org}/ws/{workspace}/sessions/{session_id}/trajectory",
+            params=params,
+        )
+        return response.json()
+
+    def get_evaluation_item_trajectory(
+        self,
+        org: str,
+        workspace: str,
+        evaluation_id: str,
+        item_id: str,
+        *,
+        format: t.Literal["atif", "openai"] = "atif",
+        include_compacted: bool = False,
+    ) -> t.Any:
+        """GET /org/{org}/ws/{workspace}/evaluation/{id}/items/{item_id}/trajectory.
+
+        Convenience alias that resolves the linked session and delegates
+        (SES-TRAJ-031). 404 if the eval item has no linked session.
+        """
+        params: dict[str, t.Any] = {
+            "format": format,
+            "include_compacted": include_compacted,
+        }
+        response = self.request(
+            "GET",
+            f"/org/{org}/ws/{workspace}/evaluation/{evaluation_id}/items/{item_id}/trajectory",
+            params=params,
+        )
+        return response.json()
+
     # ------------------------------------------------------------------
     # Session transcripts (message-level persistence)
     # ------------------------------------------------------------------

{dreadnode-2.0.17 → dreadnode-2.0.19}/dreadnode/app/api/models.py

@@ -155,6 +155,8 @@ class User(BaseModel):
     """User's full display name."""
     onboarding_complete: bool = False
     """Whether the user has completed onboarding."""
+    email_opt_in: bool = False
+    """Whether the user has opted in to product/marketing email."""
     is_platform_admin: bool = False
     """Whether the user has platform admin privileges."""
     features: list[str] | None = Field(default_factory=list)