benchflow 0.7.8.dev2347__tar.gz → 0.7.9.dev2354__tar.gz

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/CHANGELOG.md

@@ -2,6 +2,44 @@
 
 ## [Unreleased]
 
+## 0.7.8 — 2026-09-14
+
+### Added
+
+- **Automatic rubric review and scoring.** Tasks with a weighted `rubric.json`
+  run a separate reviewer after verification. Passing requires all required
+  tests and rubric blockers to pass; the final reward is the weighted quality
+  score. Solver evidence and reviewer runs are retained, and `bench eval score`
+  can retry an interrupted review without rerunning the solver. (#1126)
+- **Rubric results in the trajectory viewer.** The Rubric tab displays criterion
+  judgments, explanations, blocker outcomes, and weighted scores. (#1102)
+- **Viewer themes and syntax highlighting.** Trajectory pages support a dark
+  theme and highlighted tool content. (#1098)
+
+### Fixed
+
+- **Automatic rubric review works on Daytona when the task workspace is
+  `/root`.** Daytona's daemon keeps its session tree in `/root/.daytona` (the
+  entrypoint's FIFOs plus every session command's script, log and exit code),
+  and evidence capture aborted on the first FIFO, so the rollout ended in a
+  scoring error with `rewards: null` although its verifier had run. Capture now
+  leaves that tree out as `sandbox_runtime` and records any other socket, FIFO
+  or device node as a `special_file` exclusion instead of aborting. Capture
+  limits, escaping symlinks and concurrent changes still fail closed. (#1128)
+- **Automatic review supports shell-only task images.** Required Python
+  capture tools are provisioned before the solver starts, so a successful
+  shell task does not lose its review to a missing interpreter. (#1127)
+- **Denylist enforcement closes CONNECT and HTTP Host bypasses.** TLS identity
+  and HTTP authority are checked against the allowed destination on Docker and
+  Daytona. (#1122)
+- **IPv6 egress rules are installed when procfs reports zero-size files.**
+  Sandbox firewall setup reads the kernel data instead of relying on the
+  reported file size. (#1124)
+- **Buffered ACP prompts respect their deadlines.** Timeout handling no longer
+  repeats captured provider history. (#1125)
+- **Tool captures retain their content and raw input/output.** ACP trajectories
+  preserve observations needed for later review. (#1100)
+
 ## 0.7.7 — 2026-09-09
 
 ### Added

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.8.dev2347
+Version: 0.7.9.dev2354
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.8.dev2347"
+version = "0.7.9.dev2354"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/src/benchflow/review/evidence.py

@@ -25,7 +25,7 @@ from typing import Literal, Self
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from benchflow.agents.credentials import CREDENTIAL_EVIDENCE_PATHS
-from benchflow.sandbox.protocol import Sandbox
+from benchflow.sandbox.protocol import SANDBOX_RUNTIME_STATE_PATHS, Sandbox
 from benchflow.task.config import ArtifactConfig
 
 logger = logging.getLogger(__name__)
@@ -51,7 +51,7 @@ class EvidenceExclusion(BaseModel):
     model_config = ConfigDict(extra="forbid", frozen=True)
 
     original_path: str
-    reason: Literal["credential", "task_exclude"]
+    reason: Literal["credential", "sandbox_runtime", "special_file", "task_exclude"]
 
 
 class ArtifactEvidence(BaseModel):
@@ -94,8 +94,10 @@ class _CaptureReceipt(BaseModel):
 
 
 # Runs in both Docker and Daytona without requiring BenchFlow in the image.
-# Enumeration is explicit so sockets, devices, FIFOs and quota overflows fail
-# rather than being silently omitted by a tar command.
+# Enumeration is explicit so nothing is silently omitted by a tar command.
+# Sockets, devices and FIFOs hold no bytes to preserve, and provider runtime
+# state is harness-owned, so both are recorded as manifest exclusions; quota
+# overflows, escaping links and concurrent changes still fail the capture.
 _CAPTURE_SCRIPT = r"""
 import fnmatch, hashlib, json, os, pathlib, stat, sys, tarfile, tempfile
 source = pathlib.Path(sys.argv[1]).resolve()
@@ -107,18 +109,26 @@ root = source if source.is_dir() else source.parent
 max_bytes, max_entries = map(int, sys.argv[2:4])
 rules = json.loads(sys.argv[4])
 paths, exclusions, total = [], [], 0
+def record(path, reason):
+    exclusions.append({"original_path": path.as_posix(), "reason": reason})
+    if len(paths) + len(exclusions) > max_entries:
+        raise ValueError("workspace evidence exceeds configured capture limits")
+def component(path, rule):
+    return path.endswith("/" + rule) or "/" + rule + "/" in path
 def excluded(path):
     absolute = path.as_posix()
     relative = path.relative_to(root).as_posix()
-    credential = any(absolute.endswith("/" + rule) or "/" + rule + "/" in absolute for rule in rules["credentials"])
+    credential = any(component(absolute, rule) for rule in rules["credentials"])
     credential |= any(path == pathlib.Path(rule) or path.is_relative_to(rule) for rule in rules["paths"])
     reason = "credential" if credential else None
+    # Relative to the captured root: runtime state below the root is dropped,
+    # but a root that itself lies under such a directory is still captured.
+    if reason is None and any(component("/" + relative, rule) for rule in rules["sandbox_runtime"]):
+        reason = "sandbox_runtime"
     if reason is None and any(fnmatch.fnmatchcase(relative, rule) or fnmatch.fnmatchcase(path.name, rule) for rule in rules["exclude"]):
         reason = "task_exclude"
     if reason:
-        exclusions.append({"original_path": absolute, "reason": reason})
-        if len(paths) + len(exclusions) > max_entries:
-            raise ValueError("workspace evidence exceeds configured capture limits")
+        record(path, reason)
         return True
     return False
 def scan_error(error):
@@ -136,7 +146,8 @@ for directory, dirs, files in walk:
         info = path.lstat()
         if not any(check(info.st_mode) for check in
                    (stat.S_ISREG, stat.S_ISDIR, stat.S_ISLNK)):
-            raise ValueError("unsupported workspace entry: " + str(path))
+            record(path, "special_file")
+            continue
         if path.is_symlink() and not path.resolve().is_relative_to(root):
             raise ValueError("workspace symlink escapes evidence: " + str(path))
         total += info.st_size if stat.S_ISREG(info.st_mode) else 0
@@ -334,7 +345,10 @@ async def capture_workspace(
 
     ``destination`` must not exist. It receives ``workspace/`` and
     ``manifest.json`` together after successful validation. Capture limits are
-    hard errors: this API never labels a truncated tree complete.
+    hard errors: this API never labels a truncated tree complete. Credential
+    files, provider runtime state (``SANDBOX_RUNTIME_STATE_PATHS``) and
+    sockets, FIFOs or device nodes stay out of the bundle, each recorded in
+    ``manifest.exclusions``.
     """
     if destination.exists() or destination.is_symlink():
         raise EvidenceError(f"Evidence destination already exists: {destination}")
@@ -346,6 +360,7 @@ async def capture_workspace(
     rules = json.dumps(
         {
             "credentials": CREDENTIAL_EVIDENCE_PATHS,
+            "sandbox_runtime": SANDBOX_RUNTIME_STATE_PATHS,
             "exclude": list(exclude),
             "paths": list(excluded_paths),
         }

benchflow-0.7.9.dev2354/src/benchflow/review/evidence_runtime.py

@@ -0,0 +1,49 @@
+"""Provision the stdlib runtime used for trusted workspace capture."""
+
+from __future__ import annotations
+
+import shlex
+
+from benchflow.review.evidence import EvidenceError
+from benchflow.sandbox.protocol import Sandbox
+
+_PYTHON_PROBE = "import sys; raise SystemExit(sys.version_info < (3, 10))"
+_SETUP_SCRIPT = f"""
+set -eu
+if python3 -I -c {shlex.quote(_PYTHON_PROBE)} >/dev/null 2>&1; then
+    exit 0
+fi
+if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends python3
+elif command -v apk >/dev/null 2>&1; then
+    apk add --no-cache python3
+elif command -v dnf >/dev/null 2>&1; then
+    dnf install -y python3
+elif command -v microdnf >/dev/null 2>&1; then
+    microdnf install -y python3
+elif command -v yum >/dev/null 2>&1; then
+    yum install -y python3
+else
+    echo 'Automatic review requires Python 3.10+ in the task image; no supported package manager is available.' >&2
+    exit 1
+fi
+python3 -I -c {shlex.quote(_PYTHON_PROBE)}
+"""
+
+
+async def ensure_evidence_python(env: Sandbox, *, timeout_sec: int = 600) -> None:
+    """Prepare capture before the solver runs, including shell-only images.
+
+    Existing compatible interpreters require no package installation. Failure
+    is a setup error, before a solver can spend its budget producing evidence
+    that the controller cannot preserve.
+    """
+    result = await env.exec(
+        shlex.join(["sh", "-c", _SETUP_SCRIPT]),
+        user="root",
+        timeout_sec=timeout_sec,
+    )
+    if result.return_code != 0:
+        details = (result.stderr or result.stdout or "")[-2000:]
+        raise EvidenceError(f"Could not prepare workspace capture Python: {details}")

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/src/benchflow/rollout/__init__.py

@@ -1189,6 +1189,10 @@ class Rollout:
 
         self._agent_cwd = await _resolve_agent_cwd(self._env, self._task)
 
+        from benchflow.rollout._review import prepare_capture_runtime
+
+        await prepare_capture_runtime(self)
+
         if cfg.primary_agent == "oracle":
             if cfg.sandbox_user:
                 await self._planes.setup_sandbox_user(

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/src/benchflow/rollout/_review.py

@@ -10,6 +10,7 @@ from benchflow._utils.text import describe_exception
 from benchflow.agents.credentials import credential_evidence_overrides
 from benchflow.review.automatic import finish_review, prepare_review
 from benchflow.review.evidence import capture_task_evidence
+from benchflow.review.evidence_runtime import ensure_evidence_python
 from benchflow.review.outcome import scoring_error
 from benchflow.review.persistence import commit_scoring_result, scoring_lock
 
@@ -35,6 +36,14 @@ def prepare_terminal_review(rollout: Rollout) -> None:
         cfg.task_digest = digest
 
 
+async def prepare_capture_runtime(rollout: Rollout) -> None:
+    """Ensure required capture dependencies exist before solver execution."""
+    if rollout._review_plan is not None or rollout._config.purpose == "reviewer":
+        await ensure_evidence_python(
+            rollout._env, timeout_sec=rollout._config.sandbox_setup_timeout
+        )
+
+
 async def capture_terminal_workspace(rollout: Rollout) -> None:
     """Freeze solver or reviewer files before verifier hardening mutates them."""
     if rollout._review_plan is None and rollout._config.purpose != "reviewer":

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/src/benchflow/sandbox/protocol.py

@@ -6,6 +6,16 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Protocol, runtime_checkable
 
+# Directories a provider daemon owns inside the sandbox, matched as path
+# components below a captured root. Daytona keeps ~/.daytona/sessions in the
+# home of the account its daemon runs as (root in task images): the
+# entrypoint's stdin/stdout/stderr FIFOs plus every session command's script,
+# output log and exit code. With a /root workspace that tree sits inside the
+# solver's working directory, changes while any command runs (evidence capture
+# included), and is harness state rather than solver output. Docker keeps exec
+# state outside the container filesystem, so it needs no entry.
+SANDBOX_RUNTIME_STATE_PATHS = (".daytona",)
+
 
 @dataclass(frozen=True)
 class ExecResult:

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/tests/test_review_automatic.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import os
 from datetime import datetime
 from pathlib import Path
 from types import SimpleNamespace
@@ -14,9 +15,11 @@ import benchflow
 from benchflow._utils.result_paths import iter_task_result_paths
 from benchflow._utils.scoring import classify_score_outcome
 from benchflow.review import automatic, persistence
+from benchflow.review.evidence import EvidenceManifest
 from benchflow.review.options import ReviewerConfig
 from benchflow.rollout import _review
 from benchflow.rollout._results import _build_rollout_result
+from tests.test_review_evidence import LocalTransport
 from tests.test_review_runtime import (
     WEIGHTED_RUBRIC,
     FakeRun,
@@ -308,6 +311,51 @@ async def test_workspace_freezes_before_capture_and_keeps_actual_cwd(
     assert order == ["disconnect", "stop", "capture"]
 
 
+@pytest.mark.asyncio
+async def test_daytona_session_fifos_keep_terminal_evidence_scorable(tmp_path):
+    """Guards terminal capture on Daytona /root workspaces after PR #1126.
+
+    Daytona's entrypoint FIFOs under /root/.daytona made capture fail, which
+    set _export_error; finish_review then ended in a scoring error with
+    rewards null even though the verifier had run.
+    """
+    workspace = tmp_path / "root"
+    entrypoint = workspace / ".daytona/sessions/entrypoint/entrypoint_command"
+    entrypoint.mkdir(parents=True)
+    os.mkfifo(entrypoint / "input.pipe")
+    (workspace / "paper.pdf").write_bytes(b"%PDF-1.7 solver paper")
+    rollout_dir = tmp_path / "rollout"
+    rollout_dir.mkdir()
+
+    async def idle(*_args):
+        return None
+
+    rollout = SimpleNamespace(
+        _review_plan=object(),
+        _config=SimpleNamespace(purpose="task", sandbox_user=None),
+        _env=LocalTransport(),
+        _agent_env={},
+        _planes=SimpleNamespace(quiesce_agent=idle),
+        _task=SimpleNamespace(config=SimpleNamespace(artifacts=[])),
+        _agent_cwd=str(workspace),
+        disconnect=idle,
+        _require_rollout_dir=lambda: rollout_dir,
+    )
+
+    await _review.capture_terminal_workspace(rollout)
+
+    assert getattr(rollout, "_export_error", None) is None
+    manifest = EvidenceManifest.model_validate_json(
+        (rollout_dir / "evidence" / "manifest.json").read_text()
+    )
+    assert [(entry.original_path, entry.reason) for entry in manifest.exclusions] == [
+        (str(workspace / ".daytona"), "sandbox_runtime")
+    ]
+    assert (
+        rollout_dir / "evidence" / "workspace" / "paper.pdf"
+    ).read_bytes() == b"%PDF-1.7 solver paper"
+
+
 def test_reviewer_child_never_recursively_preflights(monkeypatch):
     """Guards child execution recursion after PR #1126."""
     prepare = Mock(side_effect=AssertionError("reviewer cannot review itself"))

{benchflow-0.7.8.dev2347 → benchflow-0.7.9.dev2354}/tests/test_review_evidence.py

@@ -11,6 +11,7 @@ import io
 import os
 import shlex
 import shutil
+import socket
 import sys
 import tarfile
 from pathlib import Path
@@ -157,13 +158,83 @@ async def test_external_symlink_never_imports_outside_files(tmp_path):
 
 
 @pytest.mark.asyncio
-async def test_fifo_is_rejected_not_silently_omitted(tmp_path):
+@pytest.mark.parametrize("kind", ["fifo", "socket"])
+async def test_special_files_are_recorded_not_silently_omitted(
+    tmp_path, monkeypatch, kind
+):
+    """Guards the evidence-capture fix for automatic review from PR #1126.
+
+    A socket or FIFO holds no bytes to preserve. Aborting on one discarded the
+    solver's whole review; it is now an explicit manifest exclusion and the
+    rest of the workspace is still captured.
+    """
     source = tmp_path / "source"
     source.mkdir()
-    os.mkfifo(source / "results-pipe")
+    (source / "answer.txt").write_text("solver output")
+    special = source / "results-pipe"
+    if kind == "fifo":
+        os.mkfifo(special)
+    else:
+        # AF_UNIX addresses are length-limited; bind relative to the workspace.
+        monkeypatch.chdir(source)
+        with socket.socket(socket.AF_UNIX) as server:
+            server.bind(special.name)
+    bundle = tmp_path / "evidence"
 
-    with pytest.raises(EvidenceError, match="unsupported workspace entry"):
-        await capture_workspace(LocalTransport(), str(source), tmp_path / "evidence")
+    manifest = await capture_workspace(LocalTransport(), str(source), bundle)
+
+    assert (bundle / "workspace" / "answer.txt").read_text() == "solver output"
+    assert not (bundle / "workspace" / special.name).exists()
+    assert [(entry.original_path, entry.reason) for entry in manifest.exclusions] == [
+        (str(special), "special_file")
+    ]
+    validate_workspace(bundle / "workspace", manifest)
+
+
+@pytest.mark.asyncio
+async def test_daytona_session_state_never_blocks_or_enters_evidence(tmp_path):
+    """Guards automatic review on Daytona /root workspaces after PR #1126.
+
+    Daytona's daemon keeps ~/.daytona/sessions in /root: the entrypoint's
+    stdin/stdout/stderr FIFOs plus each session command's script, log and exit
+    code (layout observed in a live sandbox). Capture aborted on
+    input.pipe, so every such rollout lost its review and final reward.
+    """
+    source = tmp_path / "root"
+    entrypoint = source / ".daytona/sessions/entrypoint/entrypoint_command"
+    entrypoint.mkdir(parents=True)
+    for pipe in ("input.pipe", "stdout.pipe", "stderr.pipe"):
+        os.mkfifo(entrypoint / pipe)
+    (entrypoint / "cmd.sh").write_text("sleep infinity\n")
+    (entrypoint / "output.log").write_text("")
+    command = source / ".daytona/sessions/benchflow-exec/0b5e4c1d"
+    command.mkdir(parents=True)
+    (command / "cmd.sh").write_text("python3 -c capture\n")
+    (command / "exit_code").write_text("0")
+    (command / "output.log").write_text("harness output")
+    (source / "paper.pdf").write_bytes(b"%PDF-1.7 solver paper")
+    (source / "chains").mkdir()
+    (source / "chains" / "lcdm.csv").write_text("Om,rdh\n0.30,101.0\n")
+    (source / ".daytonarc").write_text("solver file that only resembles provider state")
+    (source / "daytona").mkdir()
+    (source / "daytona" / "notes.md").write_text("solver notes")
+    bundle = tmp_path / "evidence"
+
+    manifest = await capture_workspace(LocalTransport(), str(source), bundle)
+
+    assert [(entry.original_path, entry.reason) for entry in manifest.exclusions] == [
+        (str(source / ".daytona"), "sandbox_runtime")
+    ]
+    assert not (bundle / "workspace" / ".daytona").exists()
+    assert {entry.path for entry in manifest.entries} == {
+        ".daytonarc",
+        "chains",
+        "chains/lcdm.csv",
+        "daytona",
+        "daytona/notes.md",
+        "paper.pdf",
+    }
+    validate_workspace(bundle / "workspace", manifest)
 
 
 @pytest.mark.asyncio
@@ -442,10 +513,12 @@ async def test_external_artifact_failure_does_not_publish_partial_task_evidence(
     source.mkdir()
     (source / "answer").write_text("fine")
     unsupported = tmp_path / "unsupported-artifact"
-    os.mkfifo(unsupported)
+    unsupported.mkdir()
+    (tmp_path / "outside-secret").write_text("do not export")
+    (unsupported / "escape").symlink_to(tmp_path / "outside-secret")
     bundle = tmp_path / "evidence"
 
-    with pytest.raises(EvidenceError, match="unsupported workspace entry"):
+    with pytest.raises(EvidenceError, match="symlink escapes"):
         await capture_task_evidence(
             LocalTransport(), str(source), bundle, artifacts=[str(unsupported)]
         )

benchflow-0.7.9.dev2354/tests/test_review_evidence_runtime.py

@@ -0,0 +1,107 @@
+"""Guards PR #1127's fix for shell-only capture failures introduced by PR #1126."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shlex
+import shutil
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from benchflow.review.evidence import EvidenceError, capture_workspace
+from benchflow.review.evidence_runtime import ensure_evidence_python
+from benchflow.rollout import Rollout, RolloutConfig
+from benchflow.sandbox.protocol import ExecResult
+
+
+class ShellOnlyTransport:
+    """Execute real capture code with Python absent from the initial PATH."""
+
+    def __init__(self, binary_dir: Path):
+        self.binary_dir = binary_dir
+
+    async def exec(self, cmd, *, user="root", timeout_sec=30):
+        assert user == "root"
+        process = await asyncio.create_subprocess_exec(
+            "/bin/sh",
+            "-c",
+            cmd,
+            env={**os.environ, "PATH": str(self.binary_dir)},
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await asyncio.wait_for(process.communicate(), timeout_sec)
+        return ExecResult(process.returncode, stdout.decode(), stderr.decode())
+
+    async def download_file(self, src, dst):
+        shutil.copyfile(src, dst)
+
+
+def shell_image(tmp_path, *, has_python=False, package_manager=False):
+    binary_dir = tmp_path / "bin"
+    binary_dir.mkdir()
+    for name in ("sh", "rm"):
+        (binary_dir / name).symlink_to(shutil.which(name))
+    python = binary_dir / "python3"
+    marker = tmp_path / "package-manager-called"
+    if has_python:
+        python.symlink_to(sys.executable)
+    if package_manager:
+        manager = binary_dir / "apt-get"
+        manager.write_text(
+            "#!/bin/sh\n"
+            f": > {shlex.quote(str(marker))}\n"
+            'if [ "$1" = install ]; then\n'
+            f"  /bin/ln -s {shlex.quote(sys.executable)} {shlex.quote(str(python))}\n"
+            "fi\n"
+        )
+        manager.chmod(0o755)
+    return ShellOnlyTransport(binary_dir), marker
+
+
+@pytest.mark.asyncio
+async def test_shell_only_task_can_capture_after_trusted_setup(tmp_path):
+    """Guards PR #1127: reproduce missing Python, then capture real evidence."""
+    transport, marker = shell_image(tmp_path, package_manager=True)
+    source = tmp_path / "workspace"
+    source.mkdir()
+    (source / "hello.txt").write_text("Hello, world!\n")
+    with pytest.raises(EvidenceError, match=r"python3:.*not found"):
+        await capture_workspace(transport, str(source), tmp_path / "before")
+
+    await ensure_evidence_python(transport)
+    manifest = await capture_workspace(transport, str(source), tmp_path / "after")
+
+    assert marker.exists()
+    assert [entry.path for entry in manifest.entries] == ["hello.txt"]
+    assert (tmp_path / "after/workspace/hello.txt").read_text() == "Hello, world!\n"
+
+
+@pytest.mark.asyncio
+async def test_existing_python_does_not_install_packages(tmp_path):
+    """Guards PR #1127 against changing already compatible task images."""
+    transport, marker = shell_image(tmp_path, has_python=True, package_manager=True)
+    await ensure_evidence_python(transport)
+    assert not marker.exists()
+
+
+@pytest.mark.asyncio
+async def test_missing_capture_runtime_fails_before_agent_setup(tmp_path, monkeypatch):
+    """Guards PR #1127 against spending solver budget before a capture failure."""
+    transport, _ = shell_image(tmp_path)
+    rollout = Rollout(RolloutConfig(task_path=tmp_path, agent="oracle"))
+    rollout._env = transport
+    rollout._task = Mock()
+    rollout._rollout_dir = tmp_path
+    rollout._review_plan = object()
+    rollout._planes.setup_sandbox_user = AsyncMock()
+    monkeypatch.setattr(
+        "benchflow.rollout._resolve_agent_cwd", AsyncMock(return_value="/app")
+    )
+    with pytest.raises(EvidenceError, match="no supported package manager"):
+        await rollout.install_agent()
+    rollout._planes.setup_sandbox_user.assert_not_awaited()