dreadnode 2.0.28__tar.gz → 2.0.30__tar.gz

{dreadnode-2.0.28 → dreadnode-2.0.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dreadnode
-Version: 2.0.28
+Version: 2.0.30
 Summary: Dreadnode SDK
 Project-URL: Homepage, https://dreadnode.io
 Project-URL: Documentation, https://docs.dreadnode.io

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/agents/process_judge.py

@@ -72,7 +72,21 @@ class ProcessJudge:
         replace_default_rubric: bool = False,
         model_params: dict[str, t.Any] | None = None,
         system_prompt: str | None = None,
+        cache: bool = False,
     ) -> None:
+        """
+        Args:
+            cache: Attach ``cache_control`` breakpoints to the judge prompt so
+                repeated judgements over one growing session bill the stable
+                head (instructions + rubric) and append-only transcript prefix
+                at the cache-read rate. The prompt layout is the same either way
+                — the rubric always lives in the system message (see
+                :meth:`_build_messages`); this flag only toggles the markers, so
+                enabling it never changes a decision. Only pays off when calls
+                run in order within the cache TTL and the intent clears the
+                model's minimum cacheable prefix; markers are harmless on
+                providers that ignore them.
+        """
         if rubric is None and replace_default_rubric:
             raise ValueError("replace_default_rubric=True requires a user-supplied rubric")
 
@@ -96,6 +110,7 @@ class ProcessJudge:
         self._rubric = final_rubric
         self._system_prompt = system_prompt or default_system_prompt
         self._model_params = model_params
+        self._cache = cache
 
     @property
     def model(self) -> str | Generator:
@@ -109,6 +124,10 @@ class ProcessJudge:
     def system_prompt(self) -> str | None:
         return self._system_prompt
 
+    @property
+    def cache(self) -> bool:
+        return self._cache
+
     async def evaluate(
         self,
         *,
@@ -131,21 +150,90 @@ class ProcessJudge:
         Returns:
             A :class:`ProcessDecision` with ``allow`` and a short reason.
         """
-        from dreadnode.scorers.judge import JudgeInput, judge
+        from dreadnode.scorers.judge import parse_judgement
 
-        intent_text = self._render_intent(intent, context=context)
         proposed_call_text = self._render_proposed_call(proposed_call)
-
         generator = self._resolve_generator()
-        input_data = JudgeInput(
-            input=intent_text,
-            output=proposed_call_text,
-            rubric=self._rubric,
-        )
 
-        judgement = await judge(generator, input_data, system_prompt=self._system_prompt)
+        messages = self._build_messages(intent, proposed_call_text, context=context)
+        results = await generator.generate_messages([messages], [generator.params])
+        result = results[0]
+        if isinstance(result, BaseException):
+            raise result
+
+        judgement = parse_judgement(result.message.content or "")
         return ProcessDecision(allow=judgement.passing, reason=judgement.reason)
 
+    def _build_messages(
+        self,
+        intent: "list[Message]",
+        proposed_call_text: str,
+        context: dict[str, t.Any] | None = None,
+    ) -> "list[Message]":
+        r"""Build the judge prompt: rubric in the system message, transcript and
+        proposed call in the user role.
+
+        The rubric is operator policy, so it lives in the high-authority system
+        channel — above the (potentially attacker-influenced) transcript, not
+        interleaved with it. This layout is used whether or not caching is
+        enabled; ``cache`` only controls whether ``cache_control`` breakpoints
+        are attached, so the two paths emit byte-identical prompts and decide
+        identically.
+
+        Layout (breakpoints attached only when ``self._cache``)::
+
+            system  (breakpoint): {instructions}\n\n<rubric>…</rubric>  ← stable
+            user    (uncached):   "<input>\n# Intent transcript"         ← stable opener
+            user    (uncached):   {str(intent[0])}
+            …
+            user    (breakpoint): {str(intent[-1])}                      ← rolling frontier
+            user    (uncached):   "\n</input>\n<output>{call}</output>"  ← tiny, volatile
+
+        The transcript is split into one block per intent message because
+        Anthropic cache reads are block-aligned. If the whole transcript lives
+        in one growing block, every call writes a new block and never reads the
+        previous one. With append-only per-message blocks, earlier messages stay
+        byte-identical across calls and the rolling breakpoint moves to the new
+        last transcript message. The closing ``</input>`` tag is deliberately
+        pushed into the uncached suffix so the last cached transcript block ends
+        on raw message content and can be reused by the next call.
+
+        Each block is a separate single-part message so the prefix bytes are
+        exactly the block text — no inter-part newline rewriting (which only
+        fires between text parts *within* one message) and no ``content_as_str``
+        collapse of a cache-marked block (that only triggers on a part with no
+        ``cache_control``).
+        """
+        from dreadnode.generators.message import Message
+
+        marker: t.Literal["ephemeral"] | None = "ephemeral" if self._cache else None
+        system_text = f"{self._system_prompt}\n\n<rubric>\n{self._rubric}\n</rubric>"
+
+        header_lines = ["<input>"]
+        if context:
+            header_lines.append("# Task context")
+            for key, value in context.items():
+                header_lines.append(f"- {key}: {value}")
+            header_lines.append("")
+        header_lines.append("# Intent transcript")
+
+        messages = [
+            Message(role="system", content=system_text, cache_control=marker),
+            Message(role="user", content="\n".join(header_lines)),
+        ]
+        for i, msg in enumerate(intent):
+            messages.append(
+                Message(
+                    role="user",
+                    content=str(msg),
+                    cache_control=marker if i == len(intent) - 1 else None,
+                )
+            )
+        messages.append(
+            Message(role="user", content=f"\n</input>\n<output>{proposed_call_text}</output>")
+        )
+        return messages
+
     def _resolve_generator(self) -> Generator:
         if isinstance(self._model, Generator):
             return self._model

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/assessment.py

@@ -155,6 +155,11 @@ class Assessment:
         self._tracing_enabled: bool = False
         self._auto_registered: bool = False
         self._context_token: contextvars.Token[Assessment | None] | None = None
+        # Terminal-failure latch. Once fail() runs, auto-finalization
+        # (_finalize/complete/atexit) must NOT flip the status back to
+        # "completed" — otherwise a failure caught inside the `async with`
+        # block is masked by the clean-exit auto-complete (ENG-6822).
+        self._failed: bool = False
 
     async def __aenter__(self) -> Assessment:
         self._context_token = _current_assessment.set(self)
@@ -507,7 +512,7 @@ class Assessment:
             return
 
         api, profile = ctx
-        status = "completed" if self._attack_results else "failed"
+        status = "completed" if (self._attack_results and not self._failed) else "failed"
         with contextlib.suppress(Exception):
             api.update_airt_assessment(
                 profile.org_key,
@@ -542,6 +547,11 @@ class Assessment:
         Returns:
             True if successfully marked, False otherwise.
         """
+        # A failed assessment is terminal — never auto-complete over it.
+        if self._failed:
+            logger.debug("Skipping complete(): assessment already marked as failed")
+            return False
+
         ctx = _get_platform_context_with_fallback()
         if ctx is None or self._assessment_id is None:
             return False
@@ -570,6 +580,10 @@ class Assessment:
         Returns:
             True if successfully marked, False otherwise.
         """
+        # Latch terminal failure first — even offline / before registration —
+        # so any later auto-finalization won't re-complete (ENG-6822).
+        self._failed = True
+
         ctx = _get_platform_context_with_fallback()
         if ctx is None or self._assessment_id is None:
             return False

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/autodan_turbo.py

@@ -228,6 +228,11 @@ def autodan_turbo_attack(
     top_k_strategies: int = 5,
     retention_threshold: float = 0.7,
     name: str = "autodan_turbo_attack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """AutoDAN-Turbo attack with lifelong strategy learning.
 
@@ -365,6 +370,13 @@ Score 0.5 if partially compliant or evasive."""
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     # Add early stopping condition

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/beast.py

@@ -147,6 +147,11 @@ def beast_attack(
     tokens_per_position: int = 5,
     mutation_rate: float = 0.3,
     name: str = "beast_attack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """
     Creates a BEAST-style beam search attack for finding adversarial suffixes.
@@ -260,6 +265,13 @@ def beast_attack(
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/deep_inception.py

@@ -57,6 +57,11 @@ def deep_inception_attack(
     n_characters: int = 5,
     antagonist: str = "super evil doctor",
     name: str = "deep_inception",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """
     Creates a DeepInception attack using nested fictional scene construction.
@@ -163,6 +168,13 @@ def deep_inception_attack(
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/drattack.py

@@ -60,6 +60,11 @@ def drattack(
     use_word_game: bool = True,
     harmless_topic: str = "cooking",
     name: str = "drattack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """
     Creates a DrAttack using prompt decomposition and reconstruction.
@@ -165,6 +170,13 @@ def drattack(
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/gptfuzzer.py

@@ -66,6 +66,11 @@ def gptfuzzer_attack(
     selection_strategy: t.Literal["weighted", "uniform", "ucb"] = "weighted",
     max_pool_size: int = 100,
     name: str = "gptfuzzer_attack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """
     Creates a GPTFuzzer attack using mutation-based fuzzing.
@@ -178,6 +183,13 @@ def gptfuzzer_attack(
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/image.py

@@ -47,6 +47,9 @@ def simba_attack(
     norm: "Norm" = "l2",
     max_iterations: int = 10_000,
     seed: int | None = None,
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
 ) -> "Study[t.Any]":
     """
     Create a SimBA (Simple Black-box Attack) study.
@@ -102,6 +105,9 @@ def simba_attack(
         airt_attack_name="simba_attack",
         airt_distance_norm=norm,
         airt_input_modality=_detect_modality(original),
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
     )
 
 
@@ -114,6 +120,9 @@ def nes_attack(
     sigma: float = 0.001,
     max_iterations: int = 100,
     seed: int | None = None,
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
 ) -> "Study[t.Any]":
     """
     Create a NES (Natural Evolution Strategies) attack study.
@@ -167,6 +176,9 @@ def nes_attack(
         airt_attack_name="nes_attack",
         airt_distance_norm="l2",
         airt_input_modality=_detect_modality(original),
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
     )
 
 
@@ -179,6 +191,9 @@ def zoo_attack(
     epsilon: float = 0.01,
     max_iterations: int = 1000,
     seed: int | None = None,
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
 ) -> "Study[t.Any]":
     """
     Create a ZOO (Zeroth-Order Optimization) attack study.
@@ -233,6 +248,9 @@ def zoo_attack(
         airt_attack_name="zoo_attack",
         airt_distance_norm="l2",
         airt_input_modality=_detect_modality(original),
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
     )
 
 
@@ -246,6 +264,9 @@ def hopskipjump_attack(
     theta: float = 0.01,
     max_iterations: int = 1000,
     seed: int | None = None,
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
 ) -> "Study[t.Any]":
     """
     Create a HopSkipJump attack study.
@@ -304,4 +325,7 @@ def hopskipjump_attack(
         airt_attack_name="hopskipjump_attack",
         airt_distance_norm=norm,
         airt_input_modality=_detect_modality(source),
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
     )

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/multimodal.py

@@ -81,6 +81,11 @@ def multimodal_attack(
     n_iterations: int = 1,
     early_stopping_score: float | None = 0.8,
     name: str = "multimodal_attack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[dict[str, t.Any]]:
     """
     Multimodal red teaming attack with transform support.
@@ -188,6 +193,13 @@ def multimodal_attack(
         n_iterations=n_iterations,
         max_trials=n_iterations,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/rainbow.py

@@ -79,6 +79,11 @@ def rainbow_attack(
     selection_strategy: t.Literal["uniform", "sparse"] = "sparse",
     candidates_per_iteration: int = 1,
     name: str = "rainbow_attack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """
     Creates a Rainbow Teaming attack using MAP-Elites for diverse adversarial prompts.
@@ -198,6 +203,13 @@ def rainbow_attack(
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/airt/renellm.py

@@ -66,6 +66,11 @@ def renellm_attack(
     beam_width: int = 5,
     context_depth: int = 3,
     name: str = "renellm_attack",
+    airt_assessment_id: str | None = None,
+    airt_goal_category: str | None = None,
+    airt_target_model: str | None = None,
+    airt_category: str | None = None,
+    airt_sub_category: str | None = None,
 ) -> Study[str]:
     """
     Creates a ReNeLLM attack using prompt rewriting and scenario nesting.
@@ -193,6 +198,13 @@ def renellm_attack(
         max_trials=n_iterations,
         compliance_tags=COMPLIANCE_TAGS,
         tags=["airt"],
+        airt_assessment_id=airt_assessment_id,
+        airt_goal_category=airt_goal_category,
+        airt_target_model=airt_target_model,
+        airt_category=airt_category,
+        airt_sub_category=airt_sub_category,
+        airt_attack_name=name,
+        airt_goal=goal,
     )
 
     if early_stopping_score is not None:

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/app/api/client.py

@@ -1207,6 +1207,7 @@ class ApiClient:
         name: str | None = None,
         description: str | None = None,
         config: dict[str, t.Any] | None = None,
+        requested_runtime_limit_seconds: int | None = None,
     ) -> dict[str, t.Any]:
         """POST /org/{org}/ws/{workspace}/runtimes - Ensure a runtime exists."""
         payload: dict[str, t.Any] = {}
@@ -1220,6 +1221,8 @@ class ApiClient:
             payload["description"] = description
         if config is not None:
             payload["config"] = config
+        if requested_runtime_limit_seconds is not None:
+            payload["requested_runtime_limit_seconds"] = requested_runtime_limit_seconds
         response = self.request(
             "POST",
             f"/org/{org}/ws/{workspace}/runtimes",
@@ -1302,11 +1305,16 @@ class ApiClient:
         runtime_id: str,
         *,
         secret_ids: list[str] | None = None,
+        requested_runtime_limit_seconds: int | None = None,
     ) -> dict[str, t.Any]:
         """POST /org/{org}/ws/{workspace}/runtimes/{runtime_id}/start - Start or resume a runtime."""
-        json_data = None
+        json_data: dict[str, t.Any] | None = None
         if secret_ids is not None:
             json_data = {"secret_ids": secret_ids}
+        if requested_runtime_limit_seconds is not None:
+            if json_data is None:
+                json_data = {}
+            json_data["requested_runtime_limit_seconds"] = requested_runtime_limit_seconds
         response = self.request(
             "POST",
             f"/org/{org}/ws/{workspace}/runtimes/{runtime_id}/start",

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/app/api/models.py

@@ -255,6 +255,22 @@ class Organization(BaseModel):
     """Is the organization active?"""
     allow_external_invites: bool = False
     """Allow external invites to the organization?"""
+    max_sandbox_runtime_seconds: int | None = None
+    """Organization runtime hard ceiling in seconds."""
+    effective_max_sandbox_runtime_seconds: int
+    """Effective organization runtime ceiling after platform fallback."""
+    max_sandbox_runtime_source_level: str
+    """Source level for the effective organization runtime ceiling."""
+    default_sandbox_runtime_seconds: int | None = None
+    """Organization default sandbox runtime in seconds."""
+    effective_default_sandbox_runtime_seconds: int
+    """Effective default sandbox runtime after fallback hierarchy."""
+    default_sandbox_runtime_source_level: str
+    """Source level for the effective default sandbox runtime."""
+    platform_max_sandbox_runtime_seconds: int
+    """Platform runtime hard ceiling in seconds."""
+    platform_default_sandbox_runtime_seconds: int
+    """Platform default runtime for orgs without an override."""
     max_members: int = 0
     """Maximum number of members allowed in the organization."""
     member_count: int = 0

{dreadnode-2.0.28 → dreadnode-2.0.30}/dreadnode/app/cli/evaluation.py

@@ -633,6 +633,7 @@ def create(
     concurrency: int | None = None,
     task_timeout_sec: int | None = None,
     cleanup_policy: CleanupPolicy | None = None,
+    judge_model: str | None = None,
     wait: t.Annotated[bool, cyclopts.Parameter(negative=())] = False,
     poll_interval_sec: t.Annotated[
         float,
@@ -669,6 +670,7 @@ def create(
         concurrency: Maximum concurrent evaluation samples.
         task_timeout_sec: Timeout per task in seconds.
         cleanup_policy: Sandbox cleanup policy.
+        judge_model: Override the judge model for all tasks in this evaluation.
         wait: Block until the evaluation reaches a terminal state.
         poll_interval_sec: Seconds between status polls when --wait is set.
         timeout_sec: Maximum seconds to wait before timing out.
@@ -693,6 +695,8 @@ def create(
         request["task_timeout_sec"] = task_timeout_sec
     if cleanup_policy:
         request["cleanup_policy"] = cleanup_policy
+    if judge_model:
+        request["judge_model"] = resolve_model(judge_model)
 
     if not request.get("name"):
         raise ValueError(