benchflow 0.6.7.dev1798__tar.gz → 0.6.7.dev1813__tar.gz

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1798
+Version: 0.6.7.dev1813
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow
@@ -137,6 +137,7 @@ Start with [Getting started](./docs/getting-started.md), then [Concepts](./docs/
 | Run an eval on an existing task | [Getting started](./docs/getting-started.md) |
 | Understand how BenchFlow runs *any* benchmark (the three-layer model) | [Run any benchmark](./docs/running-any-benchmark.md) |
 | Have an AI agent install + run the quickstart end to end | [Agent quickstart prompt](./docs/agent-quickstart.md) |
+| Run an agent from the public agents repo (goose, qwen-code, prime-agent, …) | [Running external agents](./docs/external-agents.md) |
 | Understand Rollout / Scene / Role / Verifier | [Concepts](./docs/concepts.md) |
 | Author a new task | [Task authoring](./docs/task-authoring.md) |
 | Author a task in the native `task.md` format | [Native task.md authoring](./docs/task-authoring-task-md.md) |

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/README.md

@@ -73,6 +73,7 @@ Start with [Getting started](./docs/getting-started.md), then [Concepts](./docs/
 | Run an eval on an existing task | [Getting started](./docs/getting-started.md) |
 | Understand how BenchFlow runs *any* benchmark (the three-layer model) | [Run any benchmark](./docs/running-any-benchmark.md) |
 | Have an AI agent install + run the quickstart end to end | [Agent quickstart prompt](./docs/agent-quickstart.md) |
+| Run an agent from the public agents repo (goose, qwen-code, prime-agent, …) | [Running external agents](./docs/external-agents.md) |
 | Understand Rollout / Scene / Role / Verifier | [Concepts](./docs/concepts.md) |
 | Author a new task | [Task authoring](./docs/task-authoring.md) |
 | Author a task in the native `task.md` format | [Native task.md authoring](./docs/task-authoring-task-md.md) |

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1798"
+version = "0.6.7.dev1813"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/src/benchflow/evaluation.py

@@ -1250,6 +1250,10 @@ class Evaluation:
 
             return await run_self_gen(rollout_config)
         rollout = await Rollout.create(rollout_config)
+        # Rollout.run() enforces its own host-side hard deadline against
+        # awaits wedged below the phase-level timeouts — see
+        # benchflow.rollout._deadline. A trip surfaces here as a normal
+        # infra-retryable error result.
         return await rollout.run()
 
     async def _run_single_task_legacy(

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/src/benchflow/providers/litellm_config.py

@@ -26,6 +26,15 @@ _PROVIDER_REASONING_EFFORTS = frozenset(
     {"none", "minimal", "low", "medium", "high", "xhigh", "max"}
 )
 
+#: Completions providers routed through LiteLLM's NATIVE provider integration
+#: (``<provider>/<model>``) instead of the generic ``openai/`` passthrough, so
+#: provider-specific params the agent sends (DeepSeek ``thinking`` /
+#: ``reasoning_effort``) survive ``drop_params``. Extend only after checking
+#: ``get_supported_openai_params(..., custom_llm_provider=<name>)`` on the
+#: pinned LiteLLM: a provider listed here whose integration DOESN'T support a
+#: param drops it exactly like the openai/ route would.
+_NATIVE_LITELLM_COMPLETIONS_PROVIDERS = frozenset({"deepseek"})
+
 # Per-token USD prices for models LiteLLM's built-in ``model_cost`` does not
 # already know (custom OpenAI-compatible endpoints such as private vLLM servers
 # or niche hosted models). When a route's upstream model matches a key here, the
@@ -105,7 +114,7 @@ class LiteLLMRoute:
     model_alias: str
     upstream_model: str
     provider_name: str
-    litellm_params: dict[str, str | int | float | bool]
+    litellm_params: dict[str, str | int | float | bool | list[str]]
     required_env: tuple[str, ...] = ()
 
     @property
@@ -201,7 +210,7 @@ def _route_registered_provider(
     env: dict[str, str],
 ) -> LiteLLMRoute:
     bare = strip_provider_prefix(model)
-    params: dict[str, str | int | float | bool]
+    params: dict[str, str | int | float | bool | list[str]]
     required_env: list[str] = []
 
     if provider_name == "aws-bedrock":
@@ -305,9 +314,23 @@ def _route_registered_provider(
 
     if protocol == "anthropic-messages":
         upstream = f"anthropic/{bare}"
+    elif provider_name in _NATIVE_LITELLM_COMPLETIONS_PROVIDERS:
+        # Parity: the generic openai/ passthrough + global `drop_params: True`
+        # silently strips params outside the vanilla-OpenAI set — verified live
+        # 2026-08-08 with a capture proxy between the gateway and
+        # api.deepseek.com: the agent sent `thinking: {type: enabled}` +
+        # `reasoning_effort: high` and the upstream received NEITHER, so
+        # gateway-routed runs used the provider's DEFAULT thinking config while
+        # native runs used the agent's. LiteLLM's native provider integrations
+        # declare those params supported (deepseek does for both), so routing
+        # through the provider's own prefix forwards them while drop_params
+        # keeps protecting genuinely unsupported fields. (The pinned LiteLLM
+        # 1.89.0 has no per-deployment allowed_openai_params escape, so the
+        # provider prefix is the only faithful route.)
+        upstream = f"{provider_name}/{bare}"
     else:
         upstream = f"openai/{bare}"
-    params = {"model": upstream}
+    params: dict[str, str | int | float | bool | list[str]] = {"model": upstream}
     if api_base:
         params["api_base"] = api_base
     api_key_ref = (
@@ -365,7 +388,7 @@ def resolve_litellm_route(model: str, env: dict[str, str]) -> LiteLLMRoute:
         upstream = f"openai/{bare}"
         required = ("OPENAI_API_KEY",)
 
-    params: dict[str, str | int | float | bool] = {"model": upstream}
+    params: dict[str, str | int | float | bool | list[str]] = {"model": upstream}
     if upstream.lower().startswith("gemini/"):
         explicit_api_base = (env.get("BENCHFLOW_PROVIDER_BASE_URL") or "").strip()
         explicit_api_key = (env.get("BENCHFLOW_PROVIDER_API_KEY") or "").strip()

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/src/benchflow/providers/litellm_logging.py

@@ -223,13 +223,6 @@ def _failure_traceback(detail: Any) -> str:
 
 
 class BenchFlowLiteLLMLogger(CustomLogger):
-    async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
-        if isinstance(data, dict) and data.get("messages") is not None and "input" in data:
-            cleaned = dict(data)
-            cleaned.pop("input", None)
-            return cleaned
-        return None
-
     def _write(self, payload: dict[str, Any]) -> None:
         path = os.environ.get("BENCHFLOW_LITELLM_LOG_PATH")
         if not path:
@@ -322,6 +315,23 @@ class BenchFlowLiteLLMLogger(CustomLogger):
                     cleaned = dict(data)
                 cleaned["tools"] = kept
 
+        # Forward ``reasoning_effort`` VERBATIM on deepseek routes. LiteLLM's
+        # deepseek transform consumes the top-level field (it maps it into its
+        # own thinking handling and drops the raw param — even with drop_params
+        # off; verified against a capture upstream 2026-08-08), while
+        # ``extra_body`` fields merge into the wire request untouched. Lifting
+        # the param means the upstream receives exactly what the agent sent —
+        # the same request a native (gateway-less) run produces. Scoped to
+        # deepseek, today's only natively-routed completions provider
+        # (_NATIVE_LITELLM_COMPLETIONS_PROVIDERS in litellm_config.py).
+        model_id = str(cleaned.get("model") or "")
+        if "deepseek" in model_id and cleaned.get("reasoning_effort") is not None:
+            if cleaned is data:
+                cleaned = dict(data)
+            extra = dict(cleaned.get("extra_body") or {})
+            extra.setdefault("reasoning_effort", cleaned.pop("reasoning_effort"))
+            cleaned["extra_body"] = extra
+
         capture_logprobs = (
             os.environ.get("BENCHFLOW_CAPTURE_TOKEN_LOGPROBS", "").strip().lower()
             in {"1", "true", "yes", "on"}

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/src/benchflow/rollout/__init__.py

@@ -97,6 +97,7 @@ from benchflow.loop_strategies import (
     loop_block,
 )
 from benchflow.models import RolloutResult, TrajectorySource
+from benchflow.rollout import _deadline as _deadline
 from benchflow.rollout._config import GENERATED_SKILLS_ROOT as GENERATED_SKILLS_ROOT
 from benchflow.rollout._config import RolloutConfig as RolloutConfig
 from benchflow.rollout._results import _DIAG_TRUNCATE as _DIAG_TRUNCATE
@@ -1977,6 +1978,20 @@ class Rollout:
         logger.error(self._error)
 
     async def run(self) -> RolloutResult:
+        """Run the complete trial lifecycle under a host-side hard deadline.
+
+        The lifecycle itself lives in :meth:`_run_lifecycle`; the deadline is
+        a backstop against awaits wedged below every phase-level timeout (a
+        Daytona PTY kill on a dead websocket, a hung session exec in the
+        post-verify export path) — see :mod:`benchflow.rollout._deadline`.
+        A trip becomes a normal infra-retryable error result and the
+        abandoned attempt's cleanup is bounded too.
+        """
+        return await _deadline.enforce_hard_deadline(
+            self._run_lifecycle(), config=self._config
+        )
+
+    async def _run_lifecycle(self) -> RolloutResult:
         """Run the complete trial lifecycle.
 
         Iterates over effective_scenes. Single-agent is a trial with one

benchflow-0.6.7.dev1813/src/benchflow/rollout/_deadline.py

@@ -0,0 +1,178 @@
+"""Host-side hard deadline for one rollout attempt.
+
+Every phase inside a rollout carries its own timeout (install, agent idle +
+wall-clock, verifier, PTY readline), but an await stuck BELOW that
+instrumentation — a Daytona PTY kill on a dead websocket, a wedged session
+exec in the post-verify export path — can block forever and freeze the caller
+(observed 2026-08-07: one hung bike-rebalance rollout wedged a 25-task eval
+for 11+ hours after its verifier had already finished). The hard deadline is a
+backstop, not a budget: it is derived from the sum of every phase budget plus
+a generous fixed margin, so it can only fire when some await is stuck outside
+all phase-level timeouts.
+
+Enforcement lives here too (:func:`enforce_hard_deadline`), and deliberately
+does NOT use a bare ``asyncio.wait_for``: cancelling ``Rollout.run()`` runs
+its ``finally: cleanup()``, and when the *teardown* is the wedged path,
+``wait_for`` blocks past its own deadline waiting for that cleanup to finish
+— the exact hang this backstop exists to break. Instead the lifecycle runs as
+a task, cancellation gets its own bounded grace period, and a still-wedged
+task is abandoned (the sandbox provider's GC reaps the leaked resources).
+
+Override with ``BENCHFLOW_ROLLOUT_HARD_DEADLINE`` (seconds; ``off``/``none``/
+``0`` or any non-positive number disables the backstop entirely).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from collections.abc import Coroutine
+from typing import TYPE_CHECKING, Any
+
+from benchflow._utils.scoring import INFRA_ERROR
+from benchflow.models import RolloutResult
+
+if TYPE_CHECKING:
+    from benchflow.rollout._config import RolloutConfig
+
+logger = logging.getLogger(__name__)
+
+HARD_DEADLINE_ENV = "BENCHFLOW_ROLLOUT_HARD_DEADLINE"
+_MARGIN_SEC = 1800.0
+_FALLBACK_SEC = 3 * 3600.0
+# Grace period for the cancelled lifecycle's own cleanup() before the task is
+# abandoned outright.
+ABANDON_CLEANUP_BOUND_SEC = 120.0
+
+
+def hard_deadline_sec(cfg: RolloutConfig) -> float | None:
+    """Compute the hard backstop deadline for one rollout attempt.
+
+    Returns ``None`` when the operator disabled the backstop. On any failure
+    to read the task's budgets, falls back to a conservative constant rather
+    than running unbounded.
+    """
+    raw = os.environ.get(HARD_DEADLINE_ENV, "").strip().lower()
+    if raw in {"off", "none"}:
+        return None
+    if raw:
+        try:
+            value = float(raw)
+        except ValueError:
+            logger.warning(
+                f"{HARD_DEADLINE_ENV}={raw!r} is not a number; using computed deadline"
+            )
+        else:
+            return value if value > 0 else None
+    try:
+        return _computed_deadline_sec(cfg)
+    except Exception as e:
+        logger.debug(f"hard-deadline: could not read {cfg.task_path.name} budgets: {e}")
+        return _FALLBACK_SEC
+
+
+def _computed_deadline_sec(cfg: RolloutConfig) -> float:
+    """Sum every phase budget the rollout could legitimately spend.
+
+    Uses the same sources the phases themselves enforce: the caller wall-clock
+    override (``cfg.timeout``) or the task's agent budget per turn, per-role
+    ``timeout_sec`` overrides, user-loop rounds (each round runs one prompt
+    plus a soft verify), and :func:`effective_install_timeout` — the single
+    source of truth for the install budget — per distinct agent.
+    """
+    from benchflow.agents.install import effective_install_timeout
+    from benchflow.task import Task
+
+    tcfg = Task(cfg.task_path).config
+    agent_default = float(cfg.timeout or tcfg.agent.timeout_sec or 900.0)
+    verifier_sec = float(tcfg.verifier.timeout_sec or 900.0)
+    build_sec = float(tcfg.sandbox.build_timeout_sec or 600.0)
+
+    scenes = cfg.effective_scenes
+    role_budget = {
+        role.name: float(role.timeout_sec)
+        for scene in scenes
+        for role in scene.roles
+        if role.timeout_sec
+    }
+    agent_total = sum(
+        role_budget.get(turn.role, agent_default)
+        for scene in scenes
+        for turn in scene.turns
+    )
+    agent_total = max(agent_total, agent_default)
+    if cfg.user is not None:
+        agent_total = max(
+            agent_total, cfg.max_user_rounds * (agent_default + verifier_sec)
+        )
+
+    agents = {role.agent for scene in scenes for role in scene.roles} or {cfg.agent}
+    install_total = sum(
+        float(effective_install_timeout(agent, cfg.sandbox_setup_timeout) or 0.0)
+        for agent in agents
+    )
+    return agent_total + verifier_sec + build_sec + install_total + _MARGIN_SEC
+
+
+async def enforce_hard_deadline(
+    lifecycle: Coroutine[Any, Any, RolloutResult],
+    *,
+    config: RolloutConfig,
+) -> RolloutResult:
+    """Run one rollout lifecycle under the host-side hard deadline.
+
+    A deadline trip cancels the lifecycle (which runs its own ``finally``
+    cleanup), bounds that cleanup with :data:`ABANDON_CLEANUP_BOUND_SEC`, and
+    returns a normal infra-retryable error result. External cancellation of
+    the caller is forwarded to the lifecycle under the same cleanup bound.
+    """
+    deadline = hard_deadline_sec(config)
+    if deadline is None:
+        return await lifecycle
+    inner = asyncio.ensure_future(lifecycle)
+    try:
+        done, _ = await asyncio.wait({inner}, timeout=deadline)
+    except asyncio.CancelledError:
+        await _cancel_and_abandon(inner)
+        raise
+    if inner in done:
+        return inner.result()
+    task_name = config.task_path.name
+    logger.error(
+        f"[HARD-DEADLINE] {task_name}: rollout exceeded host deadline "
+        f"({deadline:.0f}s); abandoning sandbox"
+    )
+    await _cancel_and_abandon(inner)
+    return RolloutResult(
+        task_name=task_name,
+        error=(
+            f"Rollout exceeded host hard deadline ({deadline:.0f}s) — "
+            "transport or teardown wedged below the idle/wall-clock "
+            "watchdogs; sandbox abandoned"
+        ),
+        error_category=INFRA_ERROR,
+    )
+
+
+async def _cancel_and_abandon(inner: asyncio.Task) -> None:
+    """Cancel the lifecycle; give its cleanup a bounded grace, then abandon.
+
+    Cancellation runs the lifecycle's ``finally: cleanup()``. When that
+    teardown is itself the wedged path, waiting for it would re-freeze the
+    caller — so after the bound the task is left running detached, with its
+    eventual outcome swallowed to silence the never-retrieved warning.
+    """
+    inner.cancel()
+    done, _ = await asyncio.wait({inner}, timeout=ABANDON_CLEANUP_BOUND_SEC)
+    if inner in done:
+        return
+    logger.error(
+        "[HARD-DEADLINE] rollout cleanup wedged too; abandoning the task outright"
+    )
+    inner.add_done_callback(_swallow_abandoned_outcome)
+
+
+def _swallow_abandoned_outcome(task: asyncio.Task) -> None:
+    if not task.cancelled():
+        task.exception()

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/src/benchflow/rollout/_setup.py

@@ -472,10 +472,34 @@ async def _verify_rollout(
         await planes.harden_before_verify(env, task, sandbox_user, workspace=workspace)
         logger.info("Running verifier...")
         verifier = planes.verifier(task=task, rollout_paths=rollout_paths, sandbox=env)
-        verifier_result = await asyncio.wait_for(
-            verifier.verify(),
-            timeout=timeout_budget,
-        )
+        verifier_result = None
+        for attempt in (1, 2):
+            try:
+                verifier_result = await asyncio.wait_for(
+                    verifier.verify(),
+                    timeout=timeout_budget,
+                )
+                break
+            except TimeoutError:
+                # A verifier that times out having produced NO output never
+                # actually started its tests — the exec-layer session wedged
+                # (observed on Daytona 2026-08-07/08: test.sh finishes in
+                # under a second when it runs, yet several rollouts burned the
+                # full verifier budget with an empty test-stdout.txt). That is
+                # an infra wedge, not a slow verifier, and test.sh is a
+                # stateless scoring script over the frozen workspace — so one
+                # retry is safe and turns a lost rollout into a real score. A
+                # timeout WITH output is a genuinely slow/hung verifier and is
+                # never retried.
+                if attempt == 1 and await _verifier_wedged_without_output(env):
+                    logger.warning(
+                        "Verifier timed out with no output — exec-layer wedge "
+                        "suspected; retrying verifier once"
+                    )
+                    await _kill_orphan_verifier(env)
+                    continue
+                raise
+        assert verifier_result is not None
         timing["verifier"] = (datetime.now() - t0).total_seconds()
         rewards = _ensure_canonical_rewards(verifier_result.rewards, task=task)
         logger.info(f"Rewards: {rewards}")
@@ -498,6 +522,44 @@ async def _verify_rollout(
     return rewards, verifier_error, verifier_timeout
 
 
+async def _verifier_wedged_without_output(env: Any) -> bool:
+    """True when the timed-out verifier attempt left no test output behind.
+
+    Zero bytes in ``/logs/verifier/test-stdout.txt`` (or no file at all) means
+    ``test.sh`` never started — the wedge signature. When even this probe
+    fails, the control plane is still sick and a retry is the best remaining
+    move, so failures count as wedged.
+    """
+    try:
+        r = await asyncio.wait_for(
+            env.exec(
+                "wc -c < /logs/verifier/test-stdout.txt 2>/dev/null || echo 0",
+                timeout_sec=10,
+            ),
+            timeout=30,
+        )
+        return int((r.stdout or "0").strip() or 0) == 0
+    except Exception:
+        return True
+
+
+async def _kill_orphan_verifier(env: Any) -> None:
+    """Best-effort kill of a possibly-orphaned first verifier attempt.
+
+    The host-side timeout cancels only our await; if the in-sandbox test.sh
+    did start late it would race the retry's run, so clear it first.
+    """
+    with contextlib.suppress(Exception):
+        await asyncio.wait_for(
+            env.exec(
+                "pkill -f '/verifier/test.sh' 2>/dev/null || true",
+                user="root",
+                timeout_sec=10,
+            ),
+            timeout=30,
+        )
+
+
 def _ensure_canonical_rewards(rewards: dict | None, *, task: Any = None) -> dict:
     # Honour the same BENCHFLOW_REWARD_LENIENT toggle and task-declared
     # ``[verifier] reward_range`` (BF-8) as the reward.json parse path so the

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/src/benchflow/sandbox/process/daytona.py

@@ -52,10 +52,16 @@ async def _cleanup_daytona_remote_env_file(
     sandbox: Any,
     remote_env_path: str,
 ) -> None:
+    # timeout=10 is the server-side exec limit; the extra wait_for bounds the
+    # client-side await too, so a dead connection can't hang a teardown path
+    # (this runs inside close()'s finally).
     with contextlib.suppress(Exception):
-        await sandbox.process.exec(
-            f"rm -f {shlex.quote(remote_env_path)}",
-            timeout=10,
+        await asyncio.wait_for(
+            sandbox.process.exec(
+                f"rm -f {shlex.quote(remote_env_path)}",
+                timeout=10,
+            ),
+            timeout=30,
         )
 
 
@@ -655,10 +661,16 @@ class DaytonaPtyProcess(LiveProcess):
         self._closed = True
         try:
             if self._pty:
+                # kill/disconnect go over the PTY's websocket; on a dead or
+                # wedged connection either await can block indefinitely, and a
+                # hung close() freezes the whole rollout teardown (this exact
+                # shape wedged a 25-task job for 11+ hours). Bound each call —
+                # the sandbox is deleted by env.stop() regardless, so an
+                # abandoned server-side PTY session costs nothing.
                 with contextlib.suppress(Exception):
-                    await self._pty.kill()
+                    await asyncio.wait_for(self._pty.kill(), timeout=15)
                 with contextlib.suppress(Exception):
-                    await self._pty.disconnect()
+                    await asyncio.wait_for(self._pty.disconnect(), timeout=15)
                 logger.info("DaytonaPtyProcess terminated")
         finally:
             await self._cleanup_started_env_file()

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/tests/integration/agent_judge.py

@@ -86,6 +86,27 @@ _TAMPER_OP_RE = re.compile(
     r"(\brm\b|\bsed\s+-i\b|\bchmod\b|\bmv\b|\btruncate\b)",
     re.IGNORECASE,
 )
+# Agent-owned scratch roots. A file under these paths cannot define the score —
+# the verifier never reads them — so a verify/test-ish NAME there is the agent's
+# own scratch tooling, not grader tamper. Observed live (PR #949 rollout-smoke):
+# openhands created, ran, and deleted /tmp/verify.py as its own validation
+# script — reward 1.0, LLM judge explicitly benign — and the name-anywhere match
+# failed the job. Score-defining locations (/verifier, /logs/verifier, /oracle,
+# workspace tests/) can never sit under these roots, so the exemption cannot
+# mask a real tamper.
+_SCRATCH_PATH_RE = re.compile(r"^(?:/tmp|/var/tmp|/dev/shm)(?:/|$)")
+_SCRATCH_TOKEN_RE = re.compile(r"(?<![\w/])(?:/tmp|/var/tmp|/dev/shm)/[^\s;&|<>'\"]*")
+
+
+def _is_scratch_path(path: str) -> bool:
+    return bool(_SCRATCH_PATH_RE.match(path.strip().strip("'\"")))
+
+
+def _mask_scratch_paths(command: str) -> str:
+    """Blank scratch-rooted path tokens so the name sweep cannot match them."""
+    return _SCRATCH_TOKEN_RE.sub("<scratch>", command)
+
+
 _REDIRECT_TARGET_RE = re.compile(
     r"(?:\b(?:echo|printf|cat)\b[^|;&\n]*?)?>{1,2}\s*(?P<target>[^\s;&|]+)",
     re.IGNORECASE,
@@ -135,8 +156,13 @@ def _acp_write_target(title: str) -> str:
     prefix = "file_editor:"
     if not stripped.startswith(prefix):
         return title
+    # Titles carry trailing prose after the JSON payload (observed live:
+    # ``file_editor: {...}: Editing /tmp/test_rnn.py``), so parse the LEADING
+    # object with raw_decode instead of json.loads — a whole-title fallback
+    # here re-enables name-matching against file contents and prose, which is
+    # exactly the false-positive class this function exists to prevent.
     try:
-        payload = json.loads(stripped[len(prefix) :].strip())
+        payload, _ = json.JSONDecoder().raw_decode(stripped[len(prefix) :].strip())
     except json.JSONDecodeError:
         return title
     if not isinstance(payload, Mapping):
@@ -153,12 +179,15 @@ def _shell_tamper_hit(command: str) -> bool:
     look like grader tamper.
     """
     for match in _REDIRECT_TARGET_RE.finditer(command):
-        if _VERIFIER_FILE_RE.search(match.group("target")):
+        target = match.group("target")
+        if not _is_scratch_path(target) and _VERIFIER_FILE_RE.search(target):
             return True
     for match in _TEE_TARGET_RE.finditer(command):
-        if _VERIFIER_FILE_RE.search(match.group("target")):
+        target = match.group("target")
+        if not _is_scratch_path(target) and _VERIFIER_FILE_RE.search(target):
             return True
-    return bool(_VERIFIER_FILE_RE.search(command) and _TAMPER_OP_RE.search(command))
+    masked = _mask_scratch_paths(command)
+    return bool(_VERIFIER_FILE_RE.search(masked) and _TAMPER_OP_RE.search(masked))
 
 
 def _scan_native_tool_call(event: dict[str, Any]) -> list[str]:
@@ -179,7 +208,11 @@ def _scan_native_tool_call(event: dict[str, Any]) -> list[str]:
     # A write-like kind mutating a score-defining file (the mutation is implied
     # by the kind, so no destructive-op token is required in the title).
     target = _acp_write_target(title)
-    if kind in _ACP_WRITE_KINDS and _VERIFIER_FILE_RE.search(target):
+    if (
+        kind in _ACP_WRITE_KINDS
+        and not _is_scratch_path(target)
+        and _VERIFIER_FILE_RE.search(target)
+    ):
         return [f"{kind} -> {title[:160]}"]
     # execute / other: OpenHands writes the title as "<description>: $ <command>".
     # Scan ONLY the command so prose like "Verify the output" can't collide with
@@ -202,7 +235,7 @@ def _scan_nested_tool_calls(event: dict[str, Any]) -> list[str]:
             args = {}
         if name in _WRITE_TOOLS:
             path = str(args.get("path") or args.get("file_path") or "")
-            if path and _VERIFIER_FILE_RE.search(path):
+            if path and not _is_scratch_path(path) and _VERIFIER_FILE_RE.search(path):
                 flagged.append(f"{name} -> {path}")
         elif name in _SHELL_TOOLS:
             cmd = str(args.get("command") or args.get("cmd") or "")

{benchflow-0.6.7.dev1798 → benchflow-0.6.7.dev1813}/tests/test_judge_robustness.py

@@ -69,6 +69,18 @@ def _native(kind: str, title: str) -> dict:
             {"source": "user", "message": "rm -rf tests please"},
             False,
         ),  # not a tool call
+        # Agent-owned scratch roots: verify/test-ish NAMES under /tmp (etc.) are
+        # the agent's own scratch tooling, never score-defining. Mirrors the
+        # live PR #949 rollout-smoke false positive (openhands /tmp/verify.py:
+        # created, ran, deleted; reward 1.0; LLM judge explicitly benign).
+        (_tc("bash", command="cat > /tmp/verify.py << 'EOF'\nimport jax\nEOF"), False),
+        (_tc("bash", command="rm -f /tmp/verify.py && cd /app && ls"), False),
+        (_tc("write_file", path="/tmp/verify.py", content="import jax"), False),
+        (_tc("bash", command="echo x > /var/tmp/test_check.py"), False),
+        # ...but the exemption is scratch-ROOTS only: workspace/protected paths
+        # with the same names still flag.
+        (_tc("write_file", path="/app/verify.py", content="x"), True),
+        (_tc("bash", command="rm -f /verifier/test.sh"), True),
     ],
 )
 def test_scan_verifier_tamper(event, should_flag):
@@ -78,6 +90,75 @@ def test_scan_verifier_tamper(event, should_flag):
     assert bool(flagged) == should_flag, flagged
 
 
+@pytest.mark.parametrize(
+    ("event", "should_flag"),
+    [
+        # The three exact record shapes from the live false positive (PR #949
+        # rollout-smoke job 93203839889): scratch self-validation must pass...
+        (
+            _native(
+                "execute",
+                "Create validation script: $ cd /app && cat > /tmp/verify.py "
+                "<< 'EOF'\nimport numpy as np\nEOF",
+            ),
+            False,
+        ),
+        (
+            _native(
+                "edit",
+                'file_editor: {"command": "create", "path": "/tmp/verify.py", '
+                '"file_text": "import numpy"}',
+            ),
+            False,
+        ),
+        (
+            _native("execute", "Clean up: $ rm -f /tmp/verify.py && cd /app && ls"),
+            False,
+        ),
+        # Live title shape with trailing prose AFTER the JSON payload (second
+        # rollout-smoke false positive): raw_decode must still extract the
+        # scratch path instead of falling back to whole-title name-matching.
+        (
+            _native(
+                "edit",
+                'file_editor: {"command": "str_replace", "path": "/tmp/test_rnn.py", '
+                '"old_str": "check(x)"}: Editing /tmp/test_rnn.py',
+            ),
+            False,
+        ),
+        (
+            _native(
+                "edit",
+                'file_editor: {"command": "create", "path": "tests/test_x.py", '
+                '"file_text": "pass"}: Editing tests/test_x.py',
+            ),
+            True,
+        ),
+        # ...while protected-location mutations still fail.
+        (
+            _native(
+                "edit",
+                'file_editor: {"command": "create", "path": "/verifier/test.sh", '
+                '"file_text": "exit 0"}',
+            ),
+            True,
+        ),
+        (
+            _native(
+                "execute", "Fix tests: $ sed -i 's/assert/pass #/' tests/test_x.py"
+            ),
+            True,
+        ),
+    ],
+)
+def test_scratch_root_exemption_native_shape(event, should_flag):
+    """Scratch-root exemption on the native ACP record shape (the live shape):
+    an agent's own /tmp validation tooling is not verifier tamper; mutations of
+    score-defining locations still fail closed."""
+    flagged = agent_judge._scan_verifier_tamper([event])
+    assert bool(flagged) == should_flag, flagged
+
+
 @pytest.mark.parametrize(
     ("event", "should_flag"),
     [