benchflow 0.7.8.dev2342__tar.gz → 0.7.9.dev2354__tar.gz

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/CHANGELOG.md

@@ -2,6 +2,44 @@
 
 ## [Unreleased]
 
+## 0.7.8 — 2026-09-14
+
+### Added
+
+- **Automatic rubric review and scoring.** Tasks with a weighted `rubric.json`
+  run a separate reviewer after verification. Passing requires all required
+  tests and rubric blockers to pass; the final reward is the weighted quality
+  score. Solver evidence and reviewer runs are retained, and `bench eval score`
+  can retry an interrupted review without rerunning the solver. (#1126)
+- **Rubric results in the trajectory viewer.** The Rubric tab displays criterion
+  judgments, explanations, blocker outcomes, and weighted scores. (#1102)
+- **Viewer themes and syntax highlighting.** Trajectory pages support a dark
+  theme and highlighted tool content. (#1098)
+
+### Fixed
+
+- **Automatic rubric review works on Daytona when the task workspace is
+  `/root`.** Daytona's daemon keeps its session tree in `/root/.daytona` (the
+  entrypoint's FIFOs plus every session command's script, log and exit code),
+  and evidence capture aborted on the first FIFO, so the rollout ended in a
+  scoring error with `rewards: null` although its verifier had run. Capture now
+  leaves that tree out as `sandbox_runtime` and records any other socket, FIFO
+  or device node as a `special_file` exclusion instead of aborting. Capture
+  limits, escaping symlinks and concurrent changes still fail closed. (#1128)
+- **Automatic review supports shell-only task images.** Required Python
+  capture tools are provisioned before the solver starts, so a successful
+  shell task does not lose its review to a missing interpreter. (#1127)
+- **Denylist enforcement closes CONNECT and HTTP Host bypasses.** TLS identity
+  and HTTP authority are checked against the allowed destination on Docker and
+  Daytona. (#1122)
+- **IPv6 egress rules are installed when procfs reports zero-size files.**
+  Sandbox firewall setup reads the kernel data instead of relying on the
+  reported file size. (#1124)
+- **Buffered ACP prompts respect their deadlines.** Timeout handling no longer
+  repeats captured provider history. (#1125)
+- **Tool captures retain their content and raw input/output.** ACP trajectories
+  preserve observations needed for later review. (#1100)
+
 ## 0.7.7 — 2026-09-09
 
 ### Added

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.8.dev2342
+Version: 0.7.9.dev2354
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.8.dev2342"
+version = "0.7.9.dev2354"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/src/benchflow/review/evidence.py

@@ -25,7 +25,7 @@ from typing import Literal, Self
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from benchflow.agents.credentials import CREDENTIAL_EVIDENCE_PATHS
-from benchflow.sandbox.protocol import Sandbox
+from benchflow.sandbox.protocol import SANDBOX_RUNTIME_STATE_PATHS, Sandbox
 from benchflow.task.config import ArtifactConfig
 
 logger = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ class EvidenceExclusion(BaseModel):
     model_config = ConfigDict(extra="forbid", frozen=True)
 
     original_path: str
-    reason: Literal["credential", "task_exclude"]
+    reason: Literal["credential", "sandbox_runtime", "special_file", "task_exclude"]
 
 
 class ArtifactEvidence(BaseModel):
@@ -94,8 +94,10 @@ class _CaptureReceipt(BaseModel):
 
 
 # Runs in both Docker and Daytona without requiring BenchFlow in the image.
-# Enumeration is explicit so sockets, devices, FIFOs and quota overflows fail
-# rather than being silently omitted by a tar command.
+# Enumeration is explicit so nothing is silently omitted by a tar command.
+# Sockets, devices and FIFOs hold no bytes to preserve, and provider runtime
+# state is harness-owned, so both are recorded as manifest exclusions; quota
+# overflows, escaping links and concurrent changes still fail the capture.
 _CAPTURE_SCRIPT = r"""
 import fnmatch, hashlib, json, os, pathlib, stat, sys, tarfile, tempfile
 source = pathlib.Path(sys.argv[1]).resolve()
@@ -107,18 +109,26 @@ root = source if source.is_dir() else source.parent
 max_bytes, max_entries = map(int, sys.argv[2:4])
 rules = json.loads(sys.argv[4])
 paths, exclusions, total = [], [], 0
+def record(path, reason):
+    exclusions.append({"original_path": path.as_posix(), "reason": reason})
+    if len(paths) + len(exclusions) > max_entries:
+        raise ValueError("workspace evidence exceeds configured capture limits")
+def component(path, rule):
+    return path.endswith("/" + rule) or "/" + rule + "/" in path
 def excluded(path):
     absolute = path.as_posix()
     relative = path.relative_to(root).as_posix()
-    credential = any(absolute.endswith("/" + rule) or "/" + rule + "/" in absolute for rule in rules["credentials"])
+    credential = any(component(absolute, rule) for rule in rules["credentials"])
     credential |= any(path == pathlib.Path(rule) or path.is_relative_to(rule) for rule in rules["paths"])
     reason = "credential" if credential else None
+    # Relative to the captured root: runtime state below the root is dropped,
+    # but a root that itself lies under such a directory is still captured.
+    if reason is None and any(component("/" + relative, rule) for rule in rules["sandbox_runtime"]):
+        reason = "sandbox_runtime"
     if reason is None and any(fnmatch.fnmatchcase(relative, rule) or fnmatch.fnmatchcase(path.name, rule) for rule in rules["exclude"]):
         reason = "task_exclude"
     if reason:
-        exclusions.append({"original_path": absolute, "reason": reason})
-        if len(paths) + len(exclusions) > max_entries:
-            raise ValueError("workspace evidence exceeds configured capture limits")
+        record(path, reason)
         return True
     return False
 def scan_error(error):
@@ -136,7 +146,8 @@ for directory, dirs, files in walk:
         info = path.lstat()
         if not any(check(info.st_mode) for check in
                    (stat.S_ISREG, stat.S_ISDIR, stat.S_ISLNK)):
-            raise ValueError("unsupported workspace entry: " + str(path))
+            record(path, "special_file")
+            continue
         if path.is_symlink() and not path.resolve().is_relative_to(root):
             raise ValueError("workspace symlink escapes evidence: " + str(path))
         total += info.st_size if stat.S_ISREG(info.st_mode) else 0
@@ -334,7 +345,10 @@ async def capture_workspace(
 
     ``destination`` must not exist. It receives ``workspace/`` and
     ``manifest.json`` together after successful validation. Capture limits are
-    hard errors: this API never labels a truncated tree complete.
+    hard errors: this API never labels a truncated tree complete. Credential
+    files, provider runtime state (``SANDBOX_RUNTIME_STATE_PATHS``) and
+    sockets, FIFOs or device nodes stay out of the bundle, each recorded in
+    ``manifest.exclusions``.
     """
     if destination.exists() or destination.is_symlink():
         raise EvidenceError(f"Evidence destination already exists: {destination}")
@@ -346,6 +360,7 @@ async def capture_workspace(
     rules = json.dumps(
         {
             "credentials": CREDENTIAL_EVIDENCE_PATHS,
+            "sandbox_runtime": SANDBOX_RUNTIME_STATE_PATHS,
             "exclude": list(exclude),
             "paths": list(excluded_paths),
         }

benchflow-0.7.9.dev2354/src/benchflow/review/evidence_runtime.py

@@ -0,0 +1,49 @@
+"""Provision the stdlib runtime used for trusted workspace capture."""
+
+from __future__ import annotations
+
+import shlex
+
+from benchflow.review.evidence import EvidenceError
+from benchflow.sandbox.protocol import Sandbox
+
+_PYTHON_PROBE = "import sys; raise SystemExit(sys.version_info < (3, 10))"
+_SETUP_SCRIPT = f"""
+set -eu
+if python3 -I -c {shlex.quote(_PYTHON_PROBE)} >/dev/null 2>&1; then
+    exit 0
+fi
+if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3
+elif command -v apk >/dev/null 2>&1; then
+    apk add --no-cache python3
+elif command -v dnf >/dev/null 2>&1; then
+    dnf install -y python3
+elif command -v microdnf >/dev/null 2>&1; then
+    microdnf install -y python3
+elif command -v yum >/dev/null 2>&1; then
+    yum install -y python3
+else
+    echo 'Automatic review requires Python 3.10+ in the task image; no supported package manager is available.' >&2
+    exit 1
+fi
+python3 -I -c {shlex.quote(_PYTHON_PROBE)}
+"""
+
+
+async def ensure_evidence_python(env: Sandbox, *, timeout_sec: int = 600) -> None:
+    """Prepare capture before the solver runs, including shell-only images.
+
+    Existing compatible interpreters require no package installation. Failure
+    is a setup error, before a solver can spend its budget producing evidence
+    that the controller cannot preserve.
+    """
+    result = await env.exec(
+        shlex.join(["sh", "-c", _SETUP_SCRIPT]),
+        user="root",
+        timeout_sec=timeout_sec,
+    )
+    if result.return_code != 0:
+        details = (result.stderr or result.stdout or "")[-2000:]
+        raise EvidenceError(f"Could not prepare workspace capture Python: {details}")

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/src/benchflow/rollout/__init__.py

@@ -1189,6 +1189,10 @@ class Rollout:
 
         self._agent_cwd = await _resolve_agent_cwd(self._env, self._task)
 
+        from benchflow.rollout._review import prepare_capture_runtime
+
+        await prepare_capture_runtime(self)
+
         if cfg.primary_agent == "oracle":
             if cfg.sandbox_user:
                 await self._planes.setup_sandbox_user(

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/src/benchflow/rollout/_review.py

@@ -10,6 +10,7 @@ from benchflow._utils.text import describe_exception
 from benchflow.agents.credentials import credential_evidence_overrides
 from benchflow.review.automatic import finish_review, prepare_review
 from benchflow.review.evidence import capture_task_evidence
+from benchflow.review.evidence_runtime import ensure_evidence_python
 from benchflow.review.outcome import scoring_error
 from benchflow.review.persistence import commit_scoring_result, scoring_lock
 
@@ -35,6 +36,14 @@ def prepare_terminal_review(rollout: Rollout) -> None:
         cfg.task_digest = digest
 
 
+async def prepare_capture_runtime(rollout: Rollout) -> None:
+    """Ensure required capture dependencies exist before solver execution."""
+    if rollout._review_plan is not None or rollout._config.purpose == "reviewer":
+        await ensure_evidence_python(
+            rollout._env, timeout_sec=rollout._config.sandbox_setup_timeout
+        )
+
+
 async def capture_terminal_workspace(rollout: Rollout) -> None:
     """Freeze solver or reviewer files before verifier hardening mutates them."""
     if rollout._review_plan is None and rollout._config.purpose != "reviewer":

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/src/benchflow/sandbox/agentcore.py

@@ -50,6 +50,7 @@ import hashlib
 import io
 import json
 import os
+import re
 import shlex
 import tarfile
 import threading
@@ -92,7 +93,11 @@ _MAX_DOWNLOAD_BYTES = 64 * 1024 * 1024
 # whatever the service currently defaults to.
 _PROTOCOL_CONFIGURATION = {"serverProtocol": "HTTP"}
 #: AgentCore offers PUBLIC or VPC only; see the registry's enforces_no_network.
-_NETWORK_CONFIGURATION = {"networkMode": "PUBLIC"}
+_NETWORK_MODES = ("PUBLIC", "VPC")
+#: ``VpcConfig`` caps both lists at 16 and pins the id shapes.
+_MAX_VPC_IDS = 16
+_SUBNET_ID = re.compile(r"^subnet-[0-9a-zA-Z]{8,17}$")
+_SECURITY_GROUP_ID = re.compile(r"^sg-[0-9a-zA-Z]{8,17}$")
 
 # Environment overrides.
 _ENV_REGION = "BENCHFLOW_AGENTCORE_REGION"
@@ -100,6 +105,9 @@ _ENV_ROLE_ARN = "BENCHFLOW_AGENTCORE_ROLE_ARN"
 _ENV_ECR_REPOSITORY = "BENCHFLOW_AGENTCORE_ECR_REPOSITORY"
 _ENV_IDLE_TIMEOUT = "BENCHFLOW_AGENTCORE_IDLE_TIMEOUT_SEC"
 _ENV_MAX_LIFETIME = "BENCHFLOW_AGENTCORE_MAX_LIFETIME_SEC"
+_ENV_NETWORK_MODE = "BENCHFLOW_AGENTCORE_NETWORK_MODE"
+_ENV_SUBNETS = "BENCHFLOW_AGENTCORE_SUBNETS"
+_ENV_SECURITY_GROUPS = "BENCHFLOW_AGENTCORE_SECURITY_GROUPS"
 
 
 class AgentCoreSandbox(BaseSandbox):
@@ -314,7 +322,7 @@ class AgentCoreSandbox(BaseSandbox):
         contract = {
             "imageDigest": image_digest,
             "lifecycleConfiguration": self._lifecycle_configuration(),
-            "networkConfiguration": _NETWORK_CONFIGURATION,
+            "networkConfiguration": self._network_configuration(),
             "protocolConfiguration": _PROTOCOL_CONFIGURATION,
             "roleArn": self._require_role_arn(),
         }
@@ -341,7 +349,7 @@ class AgentCoreSandbox(BaseSandbox):
                     "containerConfiguration": {"containerUri": image_uri}
                 },
                 roleArn=self._require_role_arn(),
-                networkConfiguration=_NETWORK_CONFIGURATION,
+                networkConfiguration=self._network_configuration(),
                 protocolConfiguration=_PROTOCOL_CONFIGURATION,
                 lifecycleConfiguration=self._lifecycle_configuration(),
                 description=f"BenchFlow sandbox for {self.environment_name}",
@@ -381,7 +389,7 @@ class AgentCoreSandbox(BaseSandbox):
                         "containerConfiguration": {"containerUri": image_uri}
                     },
                     roleArn=self._require_role_arn(),
-                    networkConfiguration=_NETWORK_CONFIGURATION,
+                    networkConfiguration=self._network_configuration(),
                     protocolConfiguration=_PROTOCOL_CONFIGURATION,
                     # Omitting this resets idle/max-lifetime to the service
                     # defaults, silently discarding the caller's configured
@@ -396,7 +404,7 @@ class AgentCoreSandbox(BaseSandbox):
                 image_uri,
                 self._lifecycle_configuration(),
                 self._require_role_arn(),
-                _NETWORK_CONFIGURATION,
+                self._network_configuration(),
                 _PROTOCOL_CONFIGURATION,
             )
             self.logger.info("Adopted existing AgentCore runtime %s", arn)
@@ -510,11 +518,6 @@ class AgentCoreSandbox(BaseSandbox):
         # for — wrong permissions, wrong egress, or a shell that never answers.
         for label, expected, found in (
             ("roleArn", role_arn, detail.get("roleArn")),
-            (
-                "networkConfiguration",
-                dict(network),
-                dict(detail.get("networkConfiguration") or {}),
-            ),
             (
                 "protocolConfiguration",
                 dict(protocol),
@@ -523,6 +526,25 @@ class AgentCoreSandbox(BaseSandbox):
         ):
             if found != expected:
                 drifted[label] = (found, expected)
+
+        def _placement(config: dict[str, Any]) -> tuple[Any, ...]:
+            """Only the placement BenchFlow itself pins.
+
+            ``VpcConfig`` also carries service-managed members that cannot be
+            sent on create, and nothing promises the ids come back in the order
+            they were given — comparing the whole document would refuse every
+            adoption of a healthy VPC runtime.
+            """
+            vpc = config.get("networkModeConfig") or {}
+            return (
+                config.get("networkMode"),
+                tuple(sorted(vpc.get("subnets") or ())),
+                tuple(sorted(vpc.get("securityGroups") or ())),
+            )
+
+        found_network = dict(detail.get("networkConfiguration") or {})
+        if _placement(found_network) != _placement(dict(network)):
+            drifted["networkConfiguration"] = (found_network, dict(network))
         if drifted:
             raise SandboxStartupError(
                 f"AgentCore runtime {runtime_id} does not match this run's "
@@ -564,6 +586,72 @@ class AgentCoreSandbox(BaseSandbox):
             "maxLifetime": lifetime,
         }
 
+    def _network_configuration(self) -> dict[str, Any]:
+        """Where the runtime's microVMs are placed on the network.
+
+        ``PUBLIC`` stays the default so existing deployments do not move, but
+        the ``VPC`` opt-in fails loudly rather than quietly: a half-configured
+        runtime that silently stays public is exactly the exposure the operator
+        set out to remove, and it is invisible once the runtime is READY.
+        """
+
+        def _ids(name: str, pattern: re.Pattern[str], shape: str) -> list[str]:
+            raw = os.environ.get(name) or ""
+            # Canonical order and no duplicates: this list feeds the runtime
+            # contract digest, so "a,b" and "b,a" must not register two
+            # runtimes for the same infrastructure.
+            values: list[str] = sorted(
+                {item.strip() for item in raw.split(",") if item.strip()}
+            )
+            for item in values:
+                if not pattern.match(item):
+                    raise ValueError(
+                        f"{name} must be a comma-separated list of {shape} ids, "
+                        f"got {item!r}."
+                    )
+            if len(values) > _MAX_VPC_IDS:
+                raise ValueError(
+                    f"{name} accepts at most {_MAX_VPC_IDS} ids, got {len(values)}."
+                )
+            return values
+
+        subnets = _ids(_ENV_SUBNETS, _SUBNET_ID, "subnet-*")
+        security_groups = _ids(_ENV_SECURITY_GROUPS, _SECURITY_GROUP_ID, "sg-*")
+        mode = (os.environ.get(_ENV_NETWORK_MODE) or "PUBLIC").strip().upper()
+        if mode not in _NETWORK_MODES:
+            raise ValueError(
+                f"{_ENV_NETWORK_MODE} must be one of {', '.join(_NETWORK_MODES)}, "
+                f"got {mode!r}."
+            )
+        if mode == "PUBLIC":
+            if subnets or security_groups:
+                raise ValueError(
+                    f"{_ENV_SUBNETS}/{_ENV_SECURITY_GROUPS} are set but "
+                    f"{_ENV_NETWORK_MODE} is not VPC, so the runtime would still "
+                    f"be created on the public internet. Set "
+                    f"{_ENV_NETWORK_MODE}=VPC to place it in the VPC."
+                )
+            return {"networkMode": "PUBLIC"}
+        missing = [
+            name
+            for name, values in (
+                (_ENV_SUBNETS, subnets),
+                (_ENV_SECURITY_GROUPS, security_groups),
+            )
+            if not values
+        ]
+        if missing:
+            raise ValueError(
+                f"{_ENV_NETWORK_MODE}=VPC requires {' and '.join(missing)}."
+            )
+        return {
+            "networkMode": "VPC",
+            "networkModeConfig": {
+                "subnets": subnets,
+                "securityGroups": security_groups,
+            },
+        }
+
     def configure_agent_timeout(self, timeout_sec: int) -> None:
         """Keep the remote session alive for the effective rollout budget."""
         self._agent_timeout_sec = max(0, int(timeout_sec))

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/src/benchflow/sandbox/protocol.py

@@ -6,6 +6,16 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Protocol, runtime_checkable
 
+# Directories a provider daemon owns inside the sandbox, matched as path
+# components below a captured root. Daytona keeps ~/.daytona/sessions in the
+# home of the account its daemon runs as (root in task images): the
+# entrypoint's stdin/stdout/stderr FIFOs plus every session command's script,
+# output log and exit code. With a /root workspace that tree sits inside the
+# solver's working directory, changes while any command runs (evidence capture
+# included), and is harness state rather than solver output. Docker keeps exec
+# state outside the container filesystem, so it needs no entry.
+SANDBOX_RUNTIME_STATE_PATHS = (".daytona",)
+
 
 @dataclass(frozen=True)
 class ExecResult:

{benchflow-0.7.8.dev2342 → benchflow-0.7.9.dev2354}/tests/test_agentcore_parallel.py

@@ -743,6 +743,161 @@ class TestAdoptionContract:
             self._verify(self._detail(**{field: value}))
 
 
+_VPC_NETWORK = {
+    "networkMode": "VPC",
+    "networkModeConfig": {
+        "subnets": ["subnet-0a1b2c3d4e5f67890"],
+        "securityGroups": ["sg-0a1b2c3d4e5f67890"],
+    },
+}
+
+
+class TestNetworkConfigurationReachesTheRuntime:
+    """The configured network placement has to survive to the control plane.
+
+    ``_network_configuration`` deciding VPC is worth nothing if registration
+    still sends the old constant, and the mistake is invisible: the runtime
+    reaches READY either way, just reachable from the internet.
+    """
+
+    def _vpc_env(self, monkeypatch):
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_ROLE_ARN", "arn:aws:iam::1:role/rt")
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_NETWORK_MODE", "VPC")
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_SUBNETS", "subnet-0a1b2c3d4e5f67890")
+        monkeypatch.setenv(
+            "BENCHFLOW_AGENTCORE_SECURITY_GROUPS", "sg-0a1b2c3d4e5f67890"
+        )
+
+    def _adopt(self, found):
+        control = MagicMock()
+        control.get_agent_runtime.return_value = {
+            "agentRuntimeArtifact": {
+                "containerConfiguration": {"containerUri": "repo@sha256:a"}
+            },
+            "lifecycleConfiguration": {
+                "idleRuntimeSessionTimeout": 600,
+                "maxLifetime": 7200,
+            },
+            "roleArn": "arn:aws:iam::1:role/rt",
+            "networkConfiguration": found,
+            "protocolConfiguration": {"serverProtocol": "HTTP"},
+        }
+        AgentCoreSandbox._verify_adopted_runtime(
+            control,
+            "rt-1",
+            "repo@sha256:a",
+            {"idleRuntimeSessionTimeout": 600, "maxLifetime": 7200},
+            "arn:aws:iam::1:role/rt",
+            _VPC_NETWORK,
+            {"serverProtocol": "HTTP"},
+        )
+
+    def test_a_service_managed_member_does_not_block_adoption(self):
+        """``VpcConfig`` carries members BenchFlow never sends.
+
+        ``requireServiceS3Endpoint`` cannot even be set on create. Adoption is
+        the path every rollout after the first takes, so comparing the whole
+        document would fail a VPC matrix on everything after its first create.
+
+        Guards PR #1082 against the whole-document comparison it replaced.
+        """
+        self._adopt(
+            {
+                "networkMode": "VPC",
+                "networkModeConfig": {
+                    **_VPC_NETWORK["networkModeConfig"],
+                    "requireServiceS3Endpoint": True,
+                },
+            }
+        )
+
+    def test_a_different_subnet_is_still_refused(self):
+        """Tolerating extra members must not tolerate the wrong network."""
+        from benchflow.sandbox.protocol import SandboxStartupError
+
+        with pytest.raises(SandboxStartupError, match="does not match"):
+            self._adopt(
+                {
+                    "networkMode": "VPC",
+                    "networkModeConfig": {
+                        "subnets": ["subnet-0987654321fedcba"],
+                        "securityGroups": ["sg-0a1b2c3d4e5f67890"],
+                    },
+                }
+            )
+
+    @pytest.mark.asyncio
+    async def test_registration_sends_the_configured_vpc(self, tmp_path, monkeypatch):
+        self._vpc_env(monkeypatch)
+        env = _sandbox(_make_task(tmp_path))
+        control = MagicMock()
+        control.create_agent_runtime.return_value = {
+            "agentRuntimeId": "rt-1",
+            "agentRuntimeArn": "arn:rt",
+        }
+        control.get_agent_runtime.return_value = {"status": "READY"}
+
+        with patch.object(env, "_client", return_value=control):
+            await env._create_or_adopt_runtime("bf_x", "repo@sha256:a")
+
+        sent = control.create_agent_runtime.call_args.kwargs
+        assert sent["networkConfiguration"] == _VPC_NETWORK
+
+    @pytest.mark.asyncio
+    async def test_repointing_a_stale_runtime_keeps_the_vpc(
+        self, tmp_path, monkeypatch
+    ):
+        """``update_agent_runtime`` replaces the whole config it is given.
+
+        Omitting the network here is how a VPC runtime silently reverts to
+        PUBLIC on the next image change.
+        """
+        from botocore.exceptions import ClientError
+
+        self._vpc_env(monkeypatch)
+        env = _sandbox(_make_task(tmp_path))
+        control = MagicMock()
+        control.create_agent_runtime.side_effect = ClientError(
+            {"Error": {"Code": "ConflictException", "Message": "exists"}},
+            "CreateAgentRuntime",
+        )
+        control.get_agent_runtime.return_value = {
+            "status": "READY",
+            "agentRuntimeArtifact": {
+                "containerConfiguration": {"containerUri": "repo@sha256:b"}
+            },
+            "lifecycleConfiguration": env._lifecycle_configuration(),
+            "roleArn": "arn:aws:iam::1:role/rt",
+            "networkConfiguration": _VPC_NETWORK,
+            "protocolConfiguration": {"serverProtocol": "HTTP"},
+        }
+
+        with (
+            patch.object(env, "_client", return_value=control),
+            patch.object(
+                provisioning,
+                "find_runtime_by_name",
+                return_value=("arn:rt", "rt-1", "repo@sha256:a"),
+            ),
+        ):
+            await env._create_or_adopt_runtime("bf_x", "repo@sha256:b")
+
+        sent = control.update_agent_runtime.call_args.kwargs
+        assert sent["networkConfiguration"] == _VPC_NETWORK
+
+    def test_a_vpc_runtime_does_not_share_an_identity_with_a_public_one(
+        self, tmp_path, monkeypatch
+    ):
+        """Same name means the VPC run adopts the existing PUBLIC runtime."""
+        monkeypatch.setenv("BENCHFLOW_AGENTCORE_ROLE_ARN", "arn:aws:iam::1:role/rt")
+        env = _sandbox(_make_task(tmp_path))
+        public = env._runtime_contract_digest("sha256:a")
+
+        self._vpc_env(monkeypatch)
+
+        assert env._runtime_contract_digest("sha256:a") != public
+
+
 class TestDeprecatedCleanupAliasIsSafe:
     """`bench environment cleanup` reaches the same destructive code.