benchflow 0.6.7.dev1827__tar.gz → 0.6.7.dev1831__tar.gz

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1827
+Version: 0.6.7.dev1831
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1827"
+version = "0.6.7.dev1831"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/_utils/live_activity.py

@@ -12,7 +12,34 @@ never to a render error.
 from __future__ import annotations
 
 import threading
-from typing import Any
+from typing import Any, NamedTuple
+
+
+class SessionCounters(NamedTuple):
+    """The live ACP session counters the console heartbeat logs.
+
+    ``total_tokens`` is None until the session has a usage snapshot (i.e.
+    after a completed prompt).
+    """
+
+    tool_calls: int
+    last_tool: str
+    total_tokens: int | None
+
+
+class ActivitySnapshot(NamedTuple):
+    """One dashboard poll of a live rollout.
+
+    ``phase`` is the rollout's lifecycle phase (``Rollout._phase``) and is
+    always present, so the activity cell can label the long non-agent
+    stretches — sandbox create, agent install, verify — instead of blanking
+    in a way that is indistinguishable from a hang. ``counters`` is None
+    until the agent session exists (session-factory agents never grow one).
+    """
+
+    phase: str
+    counters: SessionCounters | None
+
 
 _lock = threading.Lock()
 # Task name -> live Rollout. Task basenames are unique keys by engine-wide
@@ -33,11 +60,13 @@ def unregister(task_name: str) -> None:
         _live.pop(task_name, None)
 
 
-def activity(task_name: str) -> tuple[int, str, int | None] | None:
-    """(tool calls, last tool title, total tokens) for a running task.
+def activity(task_name: str) -> ActivitySnapshot | None:
+    """The :class:`ActivitySnapshot` for a running task, or None when the
+    rollout isn't registered (pre-create, between retries, teardown).
 
-    None until the task's agent session exists — including non-ACP
-    (session-factory) agents, which never grow an ACP client. The dig into
+    ``counters`` inside the snapshot is None until the task's agent session
+    exists — including non-ACP (session-factory) agents, which never grow an
+    ACP client — but ``phase`` is always present. The dig into
     client/session state lives on ``Rollout.activity_snapshot()`` (typed,
     owner-side); the except here only guards renders racing rollout teardown.
     """

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/acp/container_transport.py

@@ -6,12 +6,58 @@ from pathlib import Path
 from typing import Any, TextIO
 
 from benchflow.sandbox.process import LiveProcess
+from benchflow.sandbox.process._base import _ANSI_CSI_RE, _ANSI_OSC_RE
 
 from .transport import Transport, decode_json_rpc_message
 
 logger = logging.getLogger(__name__)
 
 
+def _decode_pty_json_rpc_message(text: str) -> dict[str, Any] | None:
+    """Decode one JSON-RPC message from a line that may carry PTY noise.
+
+    Daytona's PTY transport gives the agent a real terminal, so the shell
+    prompt (with its CSI/OSC escape sequences) can land on the SAME line as
+    the agent's first protocol message — observed on BugSwarm-style images
+    as ``\\x1b[?2004h\\x1b]0;root@…\\x07root@…# \\x1b[?2004l{"jsonrpc":…}``.
+    Whole-line ``json.loads`` fails on that prefix, the response is treated
+    as non-protocol noise, and the handshake "times out" with the completed
+    initialize JSON sitting in the captured agent log — at ANY timeout.
+
+    Recovery is bounded to where the bug lives: :meth:`ContainerTransport.
+    receive` calls this lenient path only until the first successfully
+    decoded protocol message per connection (prompt glue is by construction
+    a pre-first-message PTY phenomenon), and stdio pipes keep the strict
+    whole-line contract in :func:`decode_json_rpc_message`.
+
+    1. fast path — the plain whole-line decode, unchanged;
+    2. retry from the first ``{`` when it is not at position 0 (drops a
+       glued prompt prefix);
+    3. if the line carries ``\\x1b``, strip CSI/OSC sequences (drops escapes
+       trailing or embedded around the JSON) and retry from the first ``{``.
+
+    Every retry still goes through :func:`decode_json_rpc_message`, so the
+    JSON-RPC 2.0 envelope check keeps rejecting agent log lines that merely
+    contain JSON (e.g. ``INFO {"jsonrpc": …}`` with a bad envelope).
+    """
+    message = decode_json_rpc_message(text)
+    if message is not None:
+        return message
+    if "{" not in text:
+        return None
+    brace = text.index("{")
+    if brace:  # brace == 0 would just repeat the whole-line decode
+        message = decode_json_rpc_message(text[brace:])
+        if message is not None:
+            return message
+    if "\x1b" not in text:
+        return None
+    cleaned = _ANSI_CSI_RE.sub("", _ANSI_OSC_RE.sub("", text))
+    if "{" not in cleaned:
+        return None
+    return decode_json_rpc_message(cleaned[cleaned.index("{") :])
+
+
 class ContainerTransport(Transport):
     """ACP transport that speaks to an agent running inside a sandbox.
 
@@ -34,6 +80,12 @@ class ContainerTransport(Transport):
         self._cwd = cwd
         self._agent_log_path = agent_log_path
         self._agent_log_file: TextIO | None = None
+        # First-message latch: the lenient PTY-noise decode applies only
+        # until the first successfully decoded protocol message (prompt glue
+        # is a startup phenomenon). Afterwards the strict whole-line decode
+        # rules, so an agent echoing valid protocol envelopes into its logs
+        # mid-session can never impersonate protocol traffic.
+        self._saw_protocol = False
 
     async def start(self) -> None:
         """Start the agent process inside the sandbox."""
@@ -65,8 +117,12 @@ class ContainerTransport(Transport):
             text = line.decode(errors="replace").strip()
             if not text:
                 continue
-            message = decode_json_rpc_message(text)
+            if self._saw_protocol:
+                message = decode_json_rpc_message(text)
+            else:
+                message = _decode_pty_json_rpc_message(text)
             if message is not None:
+                self._saw_protocol = True
                 return message
             # Capture non-protocol output (agent debug logs, errors, warnings).
             if self._agent_log_path:

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/acp/runtime.py

@@ -6,10 +6,11 @@ Owns the live agent-side of a run:
     - execute_prompts: send each prompt through the session, capture the
       ACP-native trajectory, and report tool-call counts
 
-The one allowed horizontal phase import in this refactor lives here:
-``from benchflow.sandbox.lockdown import build_priv_drop_cmd``. connect_acp wraps
-the agent launch command in the sandbox user's privilege-drop prefix
-before handing it to the transport. It is a single pure-function call
+The allowed horizontal phase imports in this refactor live here:
+``from benchflow.sandbox.lockdown import build_priv_drop_cmd`` (connect_acp
+wraps the agent launch command in the sandbox user's privilege-drop prefix
+before handing it to the transport) and the shared timeout-env parser from
+``benchflow.sandbox.process._base``. Both are single pure-function calls
 with no shared state — not a coupling of concerns.
 
 Does not own:
@@ -46,6 +47,7 @@ from benchflow.sandbox.lockdown import (
     build_priv_drop_cmd,
     enforce_agent_egress_firewall,
 )
+from benchflow.sandbox.process._base import _timeout_sec_from_env
 from benchflow.trajectories._capture import _capture_session_trajectory
 
 # Re-exported for backwards compatibility — tests and downstream code
@@ -65,10 +67,27 @@ logger = logging.getLogger(__name__)
 _ACP_CONNECT_MAX_RETRIES = 3
 _ACP_CONNECT_BASE_DELAY = 2.0
 _PROMPT_CANCEL_DRAIN_TIMEOUT_SEC = 0.25
-_ACP_HANDSHAKE_TIMEOUT_SEC = 60
+_ACP_HANDSHAKE_TIMEOUT_ENV = "BENCHFLOW_ACP_HANDSHAKE_TIMEOUT"
+_ACP_HANDSHAKE_TIMEOUT_DEFAULT_SEC = 60.0
 _OPENHANDS_DISABLE_SUBAGENTS_ENV = "BENCHFLOW_OPENHANDS_DISABLE_SUBAGENTS"
 
 
+def _acp_handshake_timeout_sec() -> float:
+    """Effective timeout for the pre-prompt ACP handshake, in seconds.
+
+    Agent startup cost scales with the task image — heavyweight images
+    (workspace scanning, kernel prep) can push the ``initialize`` response
+    past the 60s default, so operators can override it with
+    ``BENCHFLOW_ACP_HANDSHAKE_TIMEOUT`` (seconds). Parsing (use-time read,
+    positive-float validation, warn-and-fall-back) is shared with the other
+    transport timeout knobs in :mod:`benchflow.sandbox.process._base`.
+    """
+    return _timeout_sec_from_env(
+        _ACP_HANDSHAKE_TIMEOUT_ENV,
+        _ACP_HANDSHAKE_TIMEOUT_DEFAULT_SEC,
+    )
+
+
 async def _prepare_openhands_direct_execution(
     env,
     *,
@@ -129,13 +148,11 @@ async def _prepare_openhands_direct_execution(
 
 
 async def _wait_for_acp_handshake(awaitable, *, phase: str):
+    timeout_sec = _acp_handshake_timeout_sec()
     try:
-        return await asyncio.wait_for(awaitable, timeout=_ACP_HANDSHAKE_TIMEOUT_SEC)
+        return await asyncio.wait_for(awaitable, timeout=timeout_sec)
     except TimeoutError as e:
-        msg = (
-            f"ACP {phase} timed out after {_ACP_HANDSHAKE_TIMEOUT_SEC}s "
-            "before the first prompt"
-        )
+        msg = f"ACP {phase} timed out after {timeout_sec:g}s before the first prompt"
         raise TransportClosedError(
             msg,
             TransportClosedDiagnostic(

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/cli/_live_progress.py

@@ -128,24 +128,49 @@ def _fmt_tokens(n: int) -> str:
     return str(n)
 
 
+# Rollout lifecycle phase (Rollout._phase — the completed phase, except
+# "verifying" which verify() marks at entry) -> what is running now. Shown
+# while the session counters are unavailable — before the agent session
+# exists and after it is torn down — so sandbox create, agent install, and
+# the verifier read as work, not as a hang.
+_PHASE_LABELS = {
+    "created": "creating sandbox…",
+    "setup": "creating sandbox…",
+    "started": "installing agent…",
+    "installed": "running agent…",
+    "connected": "running agent…",
+    "executed": "verifying…",
+    "verifying": "verifying…",
+    "verified": "cleaning up…",
+    "cleaned": "cleaning up…",
+}
+# Unknown phases (e.g. "branched") and pre-register/teardown races: still
+# never blank — a blank cell on a live row reads as a hang.
+_PHASE_FALLBACK = "starting…"
+
+
 def _activity_cell(name: str) -> str:
     """Per-task activity for the "running now" table, e.g.
     ``38 calls · last: file_editor``.
 
     Polls the live rollout's ACP session counters — the same ones the console
     heartbeat logs, which the Live display otherwise mutes — via the
-    live-activity registry. Empty until the agent session exists. Tokens appear
-    only once the session has a usage snapshot (i.e. after a completed prompt).
+    live-activity registry. Until the agent session exists (and after it is
+    closed for the verifier) the cell shows the rollout's lifecycle phase as a
+    label instead, so the row is never blank. Tokens appear only once the
+    session has a usage snapshot (i.e. after a completed prompt).
     """
     snap = live_activity.activity(name)
     if snap is None:
-        return ""
-    calls, last_title, tokens = snap
-    cell = f"{calls} calls"
-    if tokens:
-        cell += f" · {_fmt_tokens(tokens)} tok"
-    if last_title:
-        cell += f" · last: {last_title[:30]}"
+        return _PHASE_FALLBACK
+    counters = snap.counters
+    if counters is None:
+        return _PHASE_LABELS.get(snap.phase, _PHASE_FALLBACK)
+    cell = f"{counters.tool_calls} calls"
+    if counters.total_tokens:
+        cell += f" · {_fmt_tokens(counters.total_tokens)} tok"
+    if counters.last_tool:
+        cell += f" · last: {counters.last_tool[:30]}"
     return cell
 
 

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/cli/_shared.py

@@ -18,10 +18,12 @@ import typer
 from rich.console import Console
 from rich.markup import escape
 
+from benchflow._utils.text import truncate_end
+
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from benchflow.evaluation import EvaluationResult
+    from benchflow.evaluation import EvaluationResult, TaskFailure
 
 console = Console()
 
@@ -100,14 +102,52 @@ def _exit_if_evaluation_had_errors(result: object) -> None:
         raise typer.Exit(1)
 
 
+# Final-block failure lines: keep the block skimmable on big jobs and each
+# line inside a typical terminal width.
+_MAX_FAILURE_LINES = 5
+_FAILURE_LINE_LIMIT = 100
+_FAILURE_REASON_METRICS = 3
+
+
+def _failure_reason(failure: TaskFailure) -> str:
+    """One cheap line explaining why a FAILED (scored, reward != 1) task
+    failed, from evidence already on the result — no file reads.
+
+    Priority: the verifier's own error if set; else the reward plus a compact
+    breakdown of the named metrics in the reward dict (zero/failed metrics
+    first — they explain the miss); else just the reward.
+    """
+    if failure.verifier_error:
+        # Collapse whitespace: verifier errors are routinely multi-line.
+        return " ".join(failure.verifier_error.split())
+    rewards = failure.rewards or {}
+    reward = rewards.get("reward")
+    metrics = [
+        (name, value)
+        for name, value in rewards.items()
+        if name != "reward" and isinstance(value, (bool, int, float))
+    ]
+    if not metrics:
+        return f"reward {reward}"
+    # Zero/failed metrics first (stable within each group), capped so one
+    # metric-happy verifier can't flood the line.
+    metrics.sort(key=lambda kv: kv[1] != 0)
+    shown = ", ".join(
+        f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
+    )
+    return f"reward {reward} — {shown}"
+
+
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
     """Print the Score/errors summary line, colored by outcome, plus artifacts.
 
     A clean pass and a total failure used to look identical (both bold white);
     now the line is green only on a full clean pass, red on a shutout, amber
-    otherwise, and ``errors=N`` is red when non-zero. When ``job_dir`` is given,
-    the result/summary paths are printed so testers know where to look (the
-    guide repeatedly says "read summary.json" but the CLI never said where).
+    otherwise, and ``errors=N`` is red when non-zero. Each FAILED task gets one
+    dim ``✗ task: reason`` line (capped at ``_MAX_FAILURE_LINES``) so the "why"
+    doesn't require opening summary.json. When ``job_dir`` is given, the
+    result/summary paths are printed so testers know where to look (the guide
+    repeatedly says "read summary.json" but the CLI never said where).
     """
     errors = int(getattr(result, "errored", 0) or 0)
     verifier_errors = int(getattr(result, "verifier_errored", 0) or 0)
@@ -132,6 +172,19 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
         f"\n[{style}]{mark} Score: {result.passed}/{result.total} "
         f"({result.score:.1%})[/{style}]{err_part}"
     )
+    # One dim reason line per FAILED task, so "0/1" doesn't force a dig into
+    # summary.json to learn why. getattr(): sharded aggregation and older
+    # SimpleNamespace-style callers don't carry task_failures.
+    failures = getattr(result, "task_failures", None) or []
+    for failure in failures[:_MAX_FAILURE_LINES]:
+        line = truncate_end(
+            f"  ✗ {failure.task_name}: {_failure_reason(failure)}",
+            _FAILURE_LINE_LIMIT,
+        )
+        console.print(f"[dim]{escape(line)}[/dim]")
+    extra = len(failures) - _MAX_FAILURE_LINES
+    if extra > 0:
+        console.print(f"[dim]  … and {extra} more[/dim]")
     if job_dir is not None:
         console.print(f"[dim]Artifacts:[/dim] {escape(str(job_dir))}")
         console.print(f"[dim]Summary:  [/dim] {escape(str(job_dir))}/summary.json")

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/cli/main.py

@@ -389,8 +389,9 @@ def eval_run(
         typer.Option(
             "--quiet",
             help=(
-                "Suppress the per-run console progress heartbeat (the "
-                "'… Nmin, K tool calls' lines during agent execution)."
+                "Suppress live progress output: the Rich dashboard on a TTY "
+                "and the per-run console progress heartbeat (the '… Nmin, K "
+                "tool calls' lines during agent execution)."
             ),
         ),
     ] = False,
@@ -597,6 +598,9 @@ def eval_run(
         # Explicit off beats the session layer's auto-gate; see
         # benchflow.acp.session._console_progress_enabled.
         os.environ["BENCHFLOW_PROGRESS"] = "off"
+        # --quiet silences ALL live progress: the heartbeat lines above and
+        # the TTY Rich dashboard (see cli._live_progress.progress_enabled).
+        os.environ["BENCHFLOW_NO_PROGRESS"] = "1"
 
     request = EvalCreateRequest(
         config_file=config_file,

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/evaluation.py

@@ -60,6 +60,7 @@ from benchflow._utils.scoring import (
     VERIFIER_TIMEOUT,
     api_error_is_transient,
     classify_error,
+    classify_score_outcome,
     classify_verifier_error,
     count_audit_outcomes,
     count_score_outcomes,
@@ -574,6 +575,21 @@ class EvaluationConfig:
             )
 
 
+@dataclass(frozen=True)
+class TaskFailure:
+    """Cheap failure evidence for one FAILED (scored, reward != 1) task.
+
+    Carried on :class:`EvaluationResult` so the CLI's final block can print a
+    one-line reason per failed task from data the engine already holds —
+    without re-reading result.json files. Errored tasks are excluded (they
+    already surface through the error counters and warning replay).
+    """
+
+    task_name: str
+    rewards: dict[str, Any] | None
+    verifier_error: str | None
+
+
 @dataclass
 class EvaluationResult:
     """Aggregated results for a job."""
@@ -588,6 +604,7 @@ class EvaluationResult:
     elapsed_sec: float = 0.0
     memory_score: float | None = None
     memory_scores: dict[str, float] = field(default_factory=dict)
+    task_failures: list[TaskFailure] = field(default_factory=list)
 
     @property
     def score(self) -> float:
@@ -1776,6 +1793,18 @@ class Evaluation:
         score_counts = count_score_outcomes(all_results.values())
         audit_counts = count_audit_outcomes(all_results.values())
         memory, memory_scores = memory_summary(all_results)
+        # Per-task failure evidence for the CLI's final block — FAILED (scored,
+        # reward != 1) tasks only, from data already in memory. Sorted by name
+        # so the printed lines are deterministic across resume/concurrency.
+        task_failures = [
+            TaskFailure(
+                task_name=name,
+                rewards=r.get("rewards"),
+                verifier_error=r.get("verifier_error"),
+            )
+            for name, r in sorted(all_results.items())
+            if classify_score_outcome(r) == "failed"
+        ]
         job_result = EvaluationResult(
             job_name=self._job_name,
             config=cfg,
@@ -1791,6 +1820,7 @@ class Evaluation:
             elapsed_sec=elapsed,
             memory_score=memory["avg_score"],
             memory_scores=memory_scores,
+            task_failures=task_failures,
         )
 
         assert (

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/rollout/__init__.py

@@ -72,6 +72,7 @@ from benchflow._types import Role, Scene, Turn
 # / ``_capture_session_trajectory`` / ``default_rollout_planes`` keep affecting
 # the ``Rollout`` methods that call those names, because those methods stay
 # defined in this module.
+from benchflow._utils.live_activity import ActivitySnapshot, SessionCounters
 from benchflow._utils.scoring import classify_error as classify_error
 from benchflow.acp.types import McpServerSpec
 from benchflow.agents.credentials import upload_credential
@@ -755,22 +756,26 @@ class Rollout:
     def acp_client(self) -> Any:
         return self._acp_client
 
-    def activity_snapshot(self) -> tuple[int, str, int | None] | None:
-        """(tool calls, last tool title, total tokens) for the eval
-        dashboard's activity cell, or None before the agent session exists.
+    def activity_snapshot(self) -> ActivitySnapshot:
+        """The eval dashboard's per-task :class:`ActivitySnapshot`: the
+        current lifecycle phase plus the live :class:`SessionCounters` (tool
+        calls, last tool title, total tokens). ``counters`` is None before
+        the agent session exists — the phase then carries the cell
+        ("creating sandbox…", "verifying…"), so a 90s sandbox create is not
+        indistinguishable from a hang.
 
         Rollout owns the client/session dig so a rename breaks here — in
         typed, tested code — instead of silently blanking the dashboard cell
         (see benchflow._utils.live_activity). Session-factory agents have no
-        ACP client and always return None.
+        ACP client and always report counter-less snapshots.
         """
         session = self._acp_client.session if self._acp_client else None
         if session is None:
-            return None
+            return ActivitySnapshot(self._phase, None)
         calls, last_title = session.progress_snapshot()
         usage = session.latest_usage_totals()
         tokens = usage.get("total_tokens") if usage else None
-        return calls, last_title, tokens
+        return ActivitySnapshot(self._phase, SessionCounters(calls, last_title, tokens))
 
     @property
     def trajectory(self) -> list[dict]:
@@ -1737,6 +1742,12 @@ class Rollout:
         """Run the verifier and return rewards."""
         cfg = self._config
 
+        # Mark the phase at entry (the other transitions mark completion):
+        # the verifier can run for minutes after disconnect() reset the phase
+        # to "installed", and the dashboard's activity cell reads _phase to
+        # label that stretch "verifying…" instead of going blank.
+        self._phase = "verifying"
+
         if not self._trajectory and cfg.primary_agent != "oracle":
             scraped = await _scrape_agent_trajectory(
                 self._env, cfg.primary_agent, cfg.sandbox_user

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/sandbox/process/_base.py

@@ -23,6 +23,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
+import os
 import re
 from abc import ABC, abstractmethod
 
@@ -33,6 +34,41 @@ _DIAG_TRUNCATE = 2000  # max chars for diagnostic stderr in error messages
 _BOOTSTRAP_DONE = "__BENCHFLOW_BOOTSTRAP_DONE__"
 _ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
+# Terminal control sequences a PTY-backed transport can interleave with
+# protocol output: ECMA-48 CSI (parameter bytes 0x30-0x3F, intermediate
+# bytes 0x20-0x2F, one final byte 0x40-0x7E) and OSC (BEL- or ST-terminated,
+# e.g. the ``\x1b]0;<title>\x07`` window-title update shell prompts emit).
+# One canonical pair — the AgentCore stdout chunker and the ACP container
+# transport must not drift apart on what counts as terminal noise.
+_ANSI_CSI_PATTERN = r"\x1b\[[0-?]*[ -/]*[@-~]"
+_ANSI_OSC_PATTERN = r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)"
+_ANSI_CSI_RE = re.compile(_ANSI_CSI_PATTERN)
+_ANSI_OSC_RE = re.compile(_ANSI_OSC_PATTERN)
+_ANSI_CSI_RE_BYTES = re.compile(_ANSI_CSI_PATTERN.encode())
+_ANSI_OSC_RE_BYTES = re.compile(_ANSI_OSC_PATTERN.encode())
+
+
+def _timeout_sec_from_env(env_var: str, default: float) -> float:
+    """Read a positive seconds value from *env_var*, else *default*.
+
+    The one parser for operator-tunable timeout knobs on this plane (PTY
+    readline timeouts, the ACP handshake window). Read at use time so
+    long-lived processes and tests see env changes. Unset or empty means
+    the default; non-numeric, non-positive, or NaN values warn and fall
+    back rather than disabling or breaking the guarded wait.
+    """
+    raw = os.environ.get(env_var)
+    if raw is None or not raw.strip():
+        return default
+    try:
+        value = float(raw)
+    except ValueError:
+        value = float("nan")
+    if not value > 0:  # also rejects NaN
+        logger.warning("Invalid %s=%r; using default %.0fs", env_var, raw, default)
+        return default
+    return value
+
 
 async def drain_oversized_line(reader: asyncio.StreamReader) -> int:
     """Drain an oversized line from *reader* after a buffer overflow.

{benchflow-0.6.7.dev1827 → benchflow-0.6.7.dev1831}/src/benchflow/sandbox/process/agentcore.py

@@ -19,12 +19,17 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
-import re
 import shlex
 import uuid
 from typing import Any
 
-from benchflow.sandbox.process._base import _ENV_KEY_RE, LiveProcess
+from benchflow.sandbox.process._base import (
+    _ANSI_CSI_RE_BYTES,
+    _ANSI_OSC_RE_BYTES,
+    _ENV_KEY_RE,
+    LiveProcess,
+    _timeout_sec_from_env,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -38,26 +43,10 @@ _READER_ENDED = object()
 _START_MARKER_TIMEOUT_SEC = 180
 _READLINE_TIMEOUT_ENV = "BENCHFLOW_AGENTCORE_READLINE_TIMEOUT"
 _READLINE_TIMEOUT_DEFAULT_SEC = 900.0
-_ANSI_CSI_RE = re.compile(rb"\x1b\[[0-?]*[ -/]*[@-~]")
 
 
 def _readline_timeout_sec() -> float:
-    import os
-
-    value = os.environ.get(_READLINE_TIMEOUT_ENV)
-    if value is None:
-        return _READLINE_TIMEOUT_DEFAULT_SEC
-    try:
-        timeout = float(value)
-    except ValueError:
-        logger.warning(
-            "Invalid %s=%r; using default %.0fs",
-            _READLINE_TIMEOUT_ENV,
-            value,
-            _READLINE_TIMEOUT_DEFAULT_SEC,
-        )
-        return _READLINE_TIMEOUT_DEFAULT_SEC
-    return timeout if timeout > 0 else _READLINE_TIMEOUT_DEFAULT_SEC
+    return _timeout_sec_from_env(_READLINE_TIMEOUT_ENV, _READLINE_TIMEOUT_DEFAULT_SEC)
 
 
 class AgentCoreProcess(LiveProcess):
@@ -119,9 +108,12 @@ class AgentCoreProcess(LiveProcess):
                     line, self._partial = self._partial.split(b"\n", 1)
                     # Bash/readline emits bracketed-paste CSI sequences when a
                     # command takes over the PTY, including a standalone
-                    # ``ESC[?2004l`` *after* the startup marker. Those bytes
-                    # are terminal state, never ACP JSON-RPC.
-                    line = _ANSI_CSI_RE.sub(b"", line.replace(b"\r", b""))
+                    # ``ESC[?2004l`` *after* the startup marker, and shell
+                    # prompts emit OSC window-title updates. Those bytes are
+                    # terminal state, never ACP JSON-RPC.
+                    line = line.replace(b"\r", b"")
+                    line = _ANSI_OSC_RE_BYTES.sub(b"", line)
+                    line = _ANSI_CSI_RE_BYTES.sub(b"", line)
                     if line:
                         await self._line_buffer.put(line + b"\n")
         except asyncio.CancelledError:
@@ -308,8 +300,10 @@ class AgentCoreProcess(LiveProcess):
             # before echo suppression takes effect. Substring matching accepts
             # that echoed command and releases startup too early, putting the
             # prompt/command noise into ACP's JSON-RPC stream. The real marker
-            # is its own line, possibly wrapped in bracketed-paste ANSI codes.
-            text = _ANSI_CSI_RE.sub(b"", line).decode(errors="replace").strip()
+            # is its own line, possibly wrapped in bracketed-paste ANSI codes
+            # or an OSC window-title update from the shell prompt.
+            cleaned = _ANSI_CSI_RE_BYTES.sub(b"", _ANSI_OSC_RE_BYTES.sub(b"", line))
+            text = cleaned.decode(errors="replace").strip()
             if text == marker:
                 return