benchflow 0.7.9.dev2404__tar.gz → 0.7.10__tar.gz

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/CHANGELOG.md

@@ -2,6 +2,45 @@
 
 ## [Unreleased]
 
+## 0.7.10 — 2026-10-16
+
+### Added
+
+- **Separate verifier sandboxes run on Docker and Daytona.** A task that sets
+  `[verifier] sandbox_mode = "separate"` (Harbor `environment_mode`) or
+  declares `[verifier.sandbox]` was refused with "separate verifier
+  sandboxes are parsed but not executed". It now runs: after the agent stops,
+  the workspace is frozen into `evidence/`, and a fresh sandbox built from
+  the verifier image (`[verifier.sandbox].docker_image`, `[sandbox]
+  .docker_image`, `tests/Dockerfile`, then the task's own
+  `environment/Dockerfile`) receives only those hash-checked bytes before the
+  normal hardening and `test.sh` run there. Nothing else from the agent's
+  sandbox reaches the verifier. `bench eval run --separate-verifier` (SDK and
+  config `separate_verifier`) forces the mode for every task of a run without
+  editing them, so task digests do not change. A transfer or sandbox failure
+  is a verifier infrastructure error, never a score;
+  `verifier-sandbox/verifier-sandbox.json` records the image source, sandbox
+  id, transfer inventory and timing, and `config.json` records
+  `verifier_sandbox_mode`. Ported from release/0.8's separate verifier,
+  without its declared-artifact and `/logs/artifacts` transfer, which main
+  does not have.
+
+### Fixed
+
+- **Root helpers no longer import modules from the agent's workspace.** The
+  helpers BenchFlow runs as root after the agent stops (disk reclaim, the
+  verifier PATH check, pytest-plugin discovery, the external-symlink purge,
+  the workspace-restore fallback, workspace evidence capture and its
+  reviewer-side admission) ran plain `python3 -c` from the sandbox's default
+  working directory, which is the agent's workspace, so a `json.py` or
+  `glob.py` left there was imported as root before the verifier. They now run
+  `python3 -I`; the `.pth`/`sitecustomize.py` purge keeps its environment
+  but runs from `/`. Pytest-plugin discovery therefore scans only the
+  interpreter's site-packages: declare a plugin installed elsewhere in
+  `verifier.pytest_plugins`.
+
+## 0.7.9 — 2026-10-14
+
 ### Fixed
 
 - **Codex knows `gpt-6.1-sol`: the `codex-acp` pin moves from 1.13.1 to

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.9.dev2404
+Version: 0.7.10
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.9.dev2404"
+version = "0.7.10"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/_utils/scoring.py

@@ -225,6 +225,10 @@ def classify_verifier_error(verifier_error: str | None) -> str | None:
     if not verifier_error:
         return None
     lower = verifier_error.lower()
+    if lower.startswith("separate verifier "):
+        # The verifier sandbox never received the agent's outputs (or never
+        # started): nothing was assessed.
+        return VERIFIER_INFRA
     if "verifier crashed" in verifier_error:
         if contains_verifier_dep_install_marker(lower):
             return VERIFIER_DEP_INSTALL

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/_utils/yaml_loader.py

@@ -125,6 +125,7 @@ def rollout_config_from_dict(
         ),
         context_root=raw.get("context_root"),
         base_image_override=raw.get("base_image_override"),
+        separate_verifier=bool(raw.get("separate_verifier", False)),
         agent=raw.get("agent", "claude-agent-acp"),
         model=raw.get("model"),
         reasoning_effort=raw.get("reasoning_effort"),

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/cli/main.py

@@ -446,6 +446,17 @@ def eval_run(
             ),
         ),
     ] = None,
+    separate_verifier: Annotated[
+        bool,
+        typer.Option(
+            "--separate-verifier",
+            help=(
+                "Run each verifier in a fresh sandbox built from the task image "
+                "that receives only the frozen workspace, as if every task set "
+                "[verifier].sandbox_mode = 'separate'. Docker and Daytona only."
+            ),
+        ),
+    ] = False,
     skills_dir: Annotated[
         Path | None,
         typer.Option("--skills-dir", help="Skills directory to deploy"),
@@ -677,6 +688,7 @@ def eval_run(
         sandbox_setup_timeout=sandbox_setup_timeout,
         context_root=context_root,
         base_image_override=base_image_override,
+        separate_verifier=separate_verifier,
         skills_dir=skills_dir,
         skill_mode=skill_mode,
         skill_creator_dir=skill_creator_dir,
@@ -1019,6 +1031,10 @@ def _run_config_file_eval(plan: "EvalPlan") -> None:
         # run-config file was a no-op.
         if plan.eval_config_override is not None:
             j._config.config_override = plan.eval_config_override
+        # --separate-verifier only ever strengthens isolation, so it wins over
+        # a config file that leaves it off.
+        if plan.request.separate_verifier:
+            j._config.separate_verifier = True
     except subprocess.CalledProcessError as e:
         # A source.repo clone/fetch failure (git exits non-zero) otherwise escapes
         # as a raw traceback — it is not a config-parse error, so give it its own

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/eval_plan.py

@@ -103,6 +103,7 @@ class EvalCreateRequest:
     sandbox_setup_timeout: int = 120
     context_root: Path | None = None
     base_image_override: str | None = None
+    separate_verifier: bool = False
     skills_dir: Path | None = None
     skill_mode: str = SKILL_MODE_NO_SKILL
     skill_creator_dir: Path | None = None
@@ -201,6 +202,7 @@ class EvalPlan:
             sandbox_setup_timeout=req.sandbox_setup_timeout,
             context_root=str(req.context_root) if req.context_root else None,
             base_image_override=req.base_image_override,
+            separate_verifier=req.separate_verifier,
             skills_dir=str(req.skills_dir) if req.skills_dir else None,
             skill_mode=req.skill_mode,
             skill_creator_dir=(

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/eval_sharding.py

@@ -134,6 +134,7 @@ def _config_payload(
         "agent_idle_timeout": config.agent_idle_timeout,
         "context_root": config.context_root,
         "base_image_override": config.base_image_override,
+        "separate_verifier": config.separate_verifier,
         "exclude_tasks": sorted(config.exclude_tasks),
         "include_tasks": sorted(shard.task_names),
         "skill_mode": config.skill_mode,

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/eval_worker.py

@@ -69,6 +69,7 @@ def _evaluation_config(raw: dict[str, Any]) -> EvaluationConfig:
         agent_idle_timeout=raw.get("agent_idle_timeout"),
         context_root=raw.get("context_root"),
         base_image_override=raw.get("base_image_override"),
+        separate_verifier=bool(raw.get("separate_verifier", False)),
         exclude_tasks=set(raw.get("exclude_tasks") or []),
         include_tasks=set(raw.get("include_tasks") or []),
         skill_mode=raw.get("skill_mode") or SKILL_MODE_NO_SKILL,

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/evaluation.py

@@ -501,6 +501,7 @@ class EvaluationConfig:
     agent_idle_timeout: int | None = 600
     context_root: str | None = None
     base_image_override: str | None = None
+    separate_verifier: bool = False
     exclude_tasks: set[str] = field(default_factory=set)
     include_tasks: set[str] = field(default_factory=set)
     skill_mode: str = SKILL_MODE_NO_SKILL
@@ -910,6 +911,7 @@ class Evaluation:
             ),
             context_root=raw.get("context_root"),
             base_image_override=raw.get("base_image_override"),
+            separate_verifier=bool(raw.get("separate_verifier", False)),
             exclude_tasks=exclude,
             include_tasks=include,
             skill_mode=raw.get("skill_mode", SKILL_MODE_NO_SKILL),
@@ -1007,6 +1009,7 @@ class Evaluation:
             ),
             context_root=raw.get("context_root"),
             base_image_override=raw.get("base_image_override"),
+            separate_verifier=bool(raw.get("separate_verifier", False)),
             include_tasks=include,
             exclude_tasks=exclude,
             skill_mode=raw.get("skill_mode", SKILL_MODE_NO_SKILL),
@@ -1284,6 +1287,7 @@ class Evaluation:
             agent_idle_timeout=cfg.agent_idle_timeout,
             context_root=cfg.context_root,
             base_image_override=cfg.base_image_override,
+            separate_verifier=cfg.separate_verifier,
             skill_mode=skill_mode,
             skill_creator_dir=cfg.skill_creator_dir,
             self_gen_no_internet=cfg.self_gen_no_internet,
@@ -1345,6 +1349,7 @@ class Evaluation:
             agent_idle_timeout=cfg.agent_idle_timeout,
             context_root=cfg.context_root,
             base_image_override=cfg.base_image_override,
+            separate_verifier=cfg.separate_verifier,
             skill_mode=cfg.skill_mode,
             skill_creator_dir=cfg.skill_creator_dir,
             self_gen_no_internet=cfg.self_gen_no_internet,

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/review/evidence.py

@@ -368,6 +368,7 @@ async def capture_workspace(
     command = shlex.join(
         [
             "python3",
+            "-I",
             "-c",
             _CAPTURE_SCRIPT,
             workspace,
@@ -475,7 +476,9 @@ async def validate_uploaded_workspace(
     env: Sandbox, workspace: str, manifest_path: str, *, timeout_sec: int = 600
 ) -> None:
     """Reject missing or changed files in the reviewer sandbox after upload."""
-    command = shlex.join(["python3", "-c", _ADMISSION_SCRIPT, workspace, manifest_path])
+    command = shlex.join(
+        ["python3", "-I", "-c", _ADMISSION_SCRIPT, workspace, manifest_path]
+    )
     result = await env.exec(command, user="root", timeout_sec=timeout_sec)
     if result.return_code:
         raise EvidenceError(
@@ -574,7 +577,7 @@ async def install_uploaded_workspace(
     The destination must be fresh; a failed installation is never admissible.
     """
     command = shlex.join(
-        ["python3", "-c", _UNPACK_SCRIPT, archive_path, workspace, manifest_path]
+        ["python3", "-I", "-c", _UNPACK_SCRIPT, archive_path, workspace, manifest_path]
     )
     result = await env.exec(command, user="root", timeout_sec=timeout_sec)
     if result.return_code:
@@ -646,6 +649,7 @@ async def capture_task_evidence(
             command = shlex.join(
                 [
                     "python3",
+                    "-I",
                     "-c",
                     _ARTIFACT_PROBE_SCRIPT,
                     manifest.workspace,

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/rollout/__init__.py

@@ -126,6 +126,7 @@ from benchflow.rollout._review import (
     finish_terminal_review,
     prepare_terminal_review,
 )
+from benchflow.rollout._separate_verifier import run_separate_verifier
 from benchflow.rollout._setup import (
     _agent_launch_with_web_policy as _agent_launch_with_web_policy,
 )
@@ -219,6 +220,10 @@ from benchflow.skill_policy import (
     task_bundled_skills_dir,
 )
 from benchflow.skill_policy import SKILL_MODE_WITH_SKILL as SKILL_MODE_WITH_SKILL
+from benchflow.task.verifier_sandbox import (
+    apply_separate_verifier_flag,
+    separate_verifier_requested,
+)
 from benchflow.trajectories._capture import (
     TrajectoryWriter,
     _capture_session_trajectory,
@@ -967,6 +972,14 @@ class Rollout:
                 self._task.config, cfg.config_override
             )
 
+        if cfg.separate_verifier:
+            # --separate-verifier: this run's verifier gets its own sandbox,
+            # the task's files (and digest) stay as they are. Refused here,
+            # before any sandbox exists, where that sandbox cannot run.
+            self._task.config = apply_separate_verifier_flag(
+                self._task.config, sandbox=cfg.environment, task_dir=cfg.task_path
+            )
+
         prepare_terminal_review(self)
 
         self._disallow_web_tools = (
@@ -1106,6 +1119,11 @@ class Rollout:
             started_at=self._started_at,
             agent_env=self._agent_env,
             base_image_override=cfg.base_image_override,
+            verifier_sandbox_mode=(
+                "separate"
+                if separate_verifier_requested(getattr(self._task, "config", None))
+                else "shared"
+            ),
             usage_tracking=cfg.usage_tracking.with_env_defaults(),
             concurrency=cfg.concurrency,
             agent_idle_timeout=cfg.agent_idle_timeout,
@@ -1899,6 +1917,13 @@ class Rollout:
                     f"Using scraped trajectory ({len(scraped)} events) — UNTRUSTED"
                 )
 
+        if separate_verifier_requested(getattr(self._task, "config", None)):
+            # The verifier never runs in the agent's sandbox: a fresh one gets
+            # only the frozen workspace.
+            self._rewards, self._verifier_error = await run_separate_verifier(self)
+            self._phase = "verified"
+            return self._rewards
+
         await _publish_trajectory_for_verifier(
             self._env, self._trajectory, self._rollout_paths.agent_dir
         )
@@ -1933,6 +1958,14 @@ class Rollout:
         Returns (rewards, verifier_output, verifier_error). The final
         verify() still does full hardening.
         """
+        if separate_verifier_requested(getattr(self._task, "config", None)):
+            # Soft verification would upload tests/ into the agent's sandbox.
+            return (
+                None,
+                None,
+                "soft_verify is unavailable: the task uses a separate verifier "
+                "sandbox, so tests never enter the agent's sandbox",
+            )
         self._rollout_paths.verifier_dir.mkdir(parents=True, exist_ok=True)
         # Clean verifier output dir — chmod 777 so non-root verifier processes can write.
         # Keep /app present for task/verifier paths that still use the legacy

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/rollout/_config.py

@@ -166,6 +166,9 @@ class RolloutConfig:
     generated_skills_root: str = GENERATED_SKILLS_ROOT
     self_gen_no_internet: bool = False
     skip_verify: bool = False
+    # Run the verifier in a fresh sandbox that receives only the frozen
+    # workspace, whatever the task declares (``--separate-verifier``).
+    separate_verifier: bool = False
     reviewer: ReviewerConfig = field(default_factory=ReviewerConfig)
     purpose: Literal["task", "reviewer"] = "task"
     parent_rollout: str | None = None

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/rollout/_results.py

@@ -121,6 +121,7 @@ def _write_config(
     started_at: datetime,
     agent_env: dict[str, str],
     base_image_override: str | None = None,
+    verifier_sandbox_mode: str = "shared",
     reasoning_effort: str | None = None,
     usage_tracking: UsageTrackingConfig | None = None,
     concurrency: int | None = None,
@@ -170,6 +171,7 @@ def _write_config(
         else effective_install_timeout(agent, sandbox_setup_timeout),
         "context_root": str(context_root) if context_root else None,
         "base_image_override": base_image_override,
+        "verifier_sandbox_mode": verifier_sandbox_mode,
         "timeout_sec": timeout,
         "concurrency": concurrency,
         "agent_idle_timeout_sec": agent_idle_timeout,

{benchflow-0.7.9.dev2404 → benchflow-0.7.10}/src/benchflow/rollout/_review.py

@@ -13,6 +13,7 @@ from benchflow.review.evidence import capture_task_evidence
 from benchflow.review.evidence_runtime import ensure_evidence_python
 from benchflow.review.outcome import scoring_error
 from benchflow.review.persistence import commit_scoring_result, scoring_lock
+from benchflow.task.verifier_sandbox import separate_verifier_requested
 
 if TYPE_CHECKING:
     from benchflow.models import RolloutResult
@@ -36,9 +37,22 @@ def prepare_terminal_review(rollout: Rollout) -> None:
         cfg.task_digest = digest
 
 
+def _freeze_requested(rollout: Rollout) -> bool:
+    """Whether the terminal workspace must be frozen into ``evidence/``.
+
+    A rubric review grades it, a reviewer rollout is it, and a separate
+    verifier sandbox receives nothing else from the agent's sandbox.
+    """
+    return (
+        rollout._review_plan is not None
+        or rollout._config.purpose == "reviewer"
+        or separate_verifier_requested(getattr(rollout._task, "config", None))
+    )
+
+
 async def prepare_capture_runtime(rollout: Rollout) -> None:
     """Ensure required capture dependencies exist before solver execution."""
-    if rollout._review_plan is not None or rollout._config.purpose == "reviewer":
+    if _freeze_requested(rollout):
         await ensure_evidence_python(
             rollout._env, timeout_sec=rollout._config.sandbox_setup_timeout
         )
@@ -46,7 +60,7 @@ async def prepare_capture_runtime(rollout: Rollout) -> None:
 
 async def capture_terminal_workspace(rollout: Rollout) -> None:
     """Freeze solver or reviewer files before verifier hardening mutates them."""
-    if rollout._review_plan is None and rollout._config.purpose != "reviewer":
+    if not _freeze_requested(rollout):
         return
     try:
         await rollout.disconnect()