benchflow 0.6.7.dev1829__tar.gz → 0.6.7.dev1833__tar.gz

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.6.7.dev1829
+Version: 0.6.7.dev1833
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.7.dev1829"
+version = "0.6.7.dev1833"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/_utils/evaluation_results.py

@@ -73,6 +73,7 @@ def rollout_result_payload(
     )
     return {
         "task_name": result.task_name,
+        "rollout_name": result.rollout_name,
         "rewards": result.rewards,
         "error": result.error,
         "error_category": result.error_category,

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/acp/container_transport.py

@@ -6,12 +6,58 @@ from pathlib import Path
 from typing import Any, TextIO
 
 from benchflow.sandbox.process import LiveProcess
+from benchflow.sandbox.process._base import _ANSI_CSI_RE, _ANSI_OSC_RE
 
 from .transport import Transport, decode_json_rpc_message
 
 logger = logging.getLogger(__name__)
 
 
+def _decode_pty_json_rpc_message(text: str) -> dict[str, Any] | None:
+    """Decode one JSON-RPC message from a line that may carry PTY noise.
+
+    Daytona's PTY transport gives the agent a real terminal, so the shell
+    prompt (with its CSI/OSC escape sequences) can land on the SAME line as
+    the agent's first protocol message — observed on BugSwarm-style images
+    as ``\\x1b[?2004h\\x1b]0;root@…\\x07root@…# \\x1b[?2004l{"jsonrpc":…}``.
+    Whole-line ``json.loads`` fails on that prefix, the response is treated
+    as non-protocol noise, and the handshake "times out" with the completed
+    initialize JSON sitting in the captured agent log — at ANY timeout.
+
+    Recovery is bounded to where the bug lives: :meth:`ContainerTransport.
+    receive` calls this lenient path only until the first successfully
+    decoded protocol message per connection (prompt glue is by construction
+    a pre-first-message PTY phenomenon), and stdio pipes keep the strict
+    whole-line contract in :func:`decode_json_rpc_message`.
+
+    1. fast path — the plain whole-line decode, unchanged;
+    2. retry from the first ``{`` when it is not at position 0 (drops a
+       glued prompt prefix);
+    3. if the line carries ``\\x1b``, strip CSI/OSC sequences (drops escapes
+       trailing or embedded around the JSON) and retry from the first ``{``.
+
+    Every retry still goes through :func:`decode_json_rpc_message`, so the
+    JSON-RPC 2.0 envelope check keeps rejecting agent log lines that merely
+    contain JSON (e.g. ``INFO {"jsonrpc": …}`` with a bad envelope).
+    """
+    message = decode_json_rpc_message(text)
+    if message is not None:
+        return message
+    if "{" not in text:
+        return None
+    brace = text.index("{")
+    if brace:  # brace == 0 would just repeat the whole-line decode
+        message = decode_json_rpc_message(text[brace:])
+        if message is not None:
+            return message
+    if "\x1b" not in text:
+        return None
+    cleaned = _ANSI_CSI_RE.sub("", _ANSI_OSC_RE.sub("", text))
+    if "{" not in cleaned:
+        return None
+    return decode_json_rpc_message(cleaned[cleaned.index("{") :])
+
+
 class ContainerTransport(Transport):
     """ACP transport that speaks to an agent running inside a sandbox.
 
@@ -34,6 +80,12 @@ class ContainerTransport(Transport):
         self._cwd = cwd
         self._agent_log_path = agent_log_path
         self._agent_log_file: TextIO | None = None
+        # First-message latch: the lenient PTY-noise decode applies only
+        # until the first successfully decoded protocol message (prompt glue
+        # is a startup phenomenon). Afterwards the strict whole-line decode
+        # rules, so an agent echoing valid protocol envelopes into its logs
+        # mid-session can never impersonate protocol traffic.
+        self._saw_protocol = False
 
     async def start(self) -> None:
         """Start the agent process inside the sandbox."""
@@ -65,8 +117,12 @@ class ContainerTransport(Transport):
             text = line.decode(errors="replace").strip()
             if not text:
                 continue
-            message = decode_json_rpc_message(text)
+            if self._saw_protocol:
+                message = decode_json_rpc_message(text)
+            else:
+                message = _decode_pty_json_rpc_message(text)
             if message is not None:
+                self._saw_protocol = True
                 return message
             # Capture non-protocol output (agent debug logs, errors, warnings).
             if self._agent_log_path:

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/acp/runtime.py

@@ -6,10 +6,11 @@ Owns the live agent-side of a run:
     - execute_prompts: send each prompt through the session, capture the
       ACP-native trajectory, and report tool-call counts
 
-The one allowed horizontal phase import in this refactor lives here:
-``from benchflow.sandbox.lockdown import build_priv_drop_cmd``. connect_acp wraps
-the agent launch command in the sandbox user's privilege-drop prefix
-before handing it to the transport. It is a single pure-function call
+The allowed horizontal phase imports in this refactor live here:
+``from benchflow.sandbox.lockdown import build_priv_drop_cmd`` (connect_acp
+wraps the agent launch command in the sandbox user's privilege-drop prefix
+before handing it to the transport) and the shared timeout-env parser from
+``benchflow.sandbox.process._base``. Both are single pure-function calls
 with no shared state — not a coupling of concerns.
 
 Does not own:
@@ -46,6 +47,7 @@ from benchflow.sandbox.lockdown import (
     build_priv_drop_cmd,
     enforce_agent_egress_firewall,
 )
+from benchflow.sandbox.process._base import _timeout_sec_from_env
 from benchflow.trajectories._capture import _capture_session_trajectory
 
 # Re-exported for backwards compatibility — tests and downstream code
@@ -65,10 +67,27 @@ logger = logging.getLogger(__name__)
 _ACP_CONNECT_MAX_RETRIES = 3
 _ACP_CONNECT_BASE_DELAY = 2.0
 _PROMPT_CANCEL_DRAIN_TIMEOUT_SEC = 0.25
-_ACP_HANDSHAKE_TIMEOUT_SEC = 60
+_ACP_HANDSHAKE_TIMEOUT_ENV = "BENCHFLOW_ACP_HANDSHAKE_TIMEOUT"
+_ACP_HANDSHAKE_TIMEOUT_DEFAULT_SEC = 60.0
 _OPENHANDS_DISABLE_SUBAGENTS_ENV = "BENCHFLOW_OPENHANDS_DISABLE_SUBAGENTS"
 
 
+def _acp_handshake_timeout_sec() -> float:
+    """Effective timeout for the pre-prompt ACP handshake, in seconds.
+
+    Agent startup cost scales with the task image — heavyweight images
+    (workspace scanning, kernel prep) can push the ``initialize`` response
+    past the 60s default, so operators can override it with
+    ``BENCHFLOW_ACP_HANDSHAKE_TIMEOUT`` (seconds). Parsing (use-time read,
+    positive-float validation, warn-and-fall-back) is shared with the other
+    transport timeout knobs in :mod:`benchflow.sandbox.process._base`.
+    """
+    return _timeout_sec_from_env(
+        _ACP_HANDSHAKE_TIMEOUT_ENV,
+        _ACP_HANDSHAKE_TIMEOUT_DEFAULT_SEC,
+    )
+
+
 async def _prepare_openhands_direct_execution(
     env,
     *,
@@ -129,13 +148,11 @@ async def _prepare_openhands_direct_execution(
 
 
 async def _wait_for_acp_handshake(awaitable, *, phase: str):
+    timeout_sec = _acp_handshake_timeout_sec()
     try:
-        return await asyncio.wait_for(awaitable, timeout=_ACP_HANDSHAKE_TIMEOUT_SEC)
+        return await asyncio.wait_for(awaitable, timeout=timeout_sec)
     except TimeoutError as e:
-        msg = (
-            f"ACP {phase} timed out after {_ACP_HANDSHAKE_TIMEOUT_SEC}s "
-            "before the first prompt"
-        )
+        msg = f"ACP {phase} timed out after {timeout_sec:g}s before the first prompt"
         raise TransportClosedError(
             msg,
             TransportClosedDiagnostic(

benchflow-0.6.7.dev1833/src/benchflow/cli/_failure_evidence.py

@@ -0,0 +1,122 @@
+"""Verifier-artifact mining for the eval report's failure lines.
+
+The one file-touching corner of the CLI's final report block: when a failed
+task's reason would otherwise be the bare ``reward X`` fallback, mine the
+rollout's verifier artifacts for a one-line explanation. Lives in its own
+module so ``cli/_shared.py`` stays the side-effect-free display helpers it
+advertises.
+
+Contract (the engine stays file-free — these reads are CLI-side, report-time
+only):
+
+- The rollout dir is resolved exactly, from the ``rollout_name`` the engine
+  records on every result — never guessed from globs.
+- Evidence sources are tried in order (CTRF report, then test-stdout tail);
+  the first that yields a line wins.
+- Every read is bounded (``_ARTIFACT_READ_BYTES`` per file) and nothing here
+  raises — any surprise degrades to ``None``, i.e. the bare reward reason.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+# Never read more than this many bytes per file, and evidence is only mined
+# for the <= _MAX_FAILURE_LINES tasks the report block displays.
+_ARTIFACT_READ_BYTES = 64 * 1024
+# Pytest final summary, decoration stripped: "1 failed, 2 passed in 40.86s".
+_PYTEST_SUMMARY_RE = re.compile(r"\d+ failed\b")
+
+
+def _ctrf_failure_line(ctrf_path: Path) -> str | None:
+    """``<test> failed[: <assertion>]`` from the first failed CTRF test.
+
+    The verifier's pytest run writes a CTRF report (``pytest --ctrf``, see the
+    standard path in ``task_authoring/structural_checks.py``). Per test the
+    useful failure evidence is the ``E …`` assertion line inside ``trace``;
+    ``message`` is only a generic phase description, so it is the fallback.
+    """
+    if ctrf_path.stat().st_size > _ARTIFACT_READ_BYTES:
+        # JSON only parses whole — an oversized report is skipped (not
+        # truncated) and the next evidence source gets its turn.
+        return None
+    data = json.loads(ctrf_path.read_text(encoding="utf-8", errors="replace"))
+    tests = (data.get("results") or {}).get("tests") or []
+    for test in tests:
+        if not isinstance(test, dict) or test.get("status") != "failed":
+            continue
+        # Full pytest node ids ("tests/test_x.py::test_build") waste the
+        # 100-char line budget; keep the test function name.
+        name = str(test.get("name") or "test").rsplit("::", 1)[-1]
+        trace = test.get("trace")
+        if isinstance(trace, str):
+            for line in trace.splitlines():
+                stripped = line.strip()
+                if stripped.startswith("E "):
+                    assertion = stripped[2:].strip()
+                    # "<test> failed:" already says it's an assertion — drop
+                    # the prefix to reclaim line budget for the message.
+                    assertion = assertion.removeprefix("AssertionError: ")
+                    return f"{name} failed: {assertion}"
+        message = test.get("message")
+        if isinstance(message, str) and message.strip():
+            return f"{name} failed: {' '.join(message.split())}"
+        return f"{name} failed"
+    return None
+
+
+def _stdout_tail_failure_line(stdout_path: Path) -> str | None:
+    """Last pytest summary ("N failed, M passed …") — else last ``FAILED …``
+    line — from a bounded tail of the verifier's test-stdout capture."""
+    with stdout_path.open("rb") as fh:
+        fh.seek(0, 2)  # SEEK_END
+        fh.seek(max(0, fh.tell() - _ARTIFACT_READ_BYTES))
+        tail = fh.read(_ARTIFACT_READ_BYTES).decode("utf-8", errors="replace")
+    last_failed: str | None = None
+    for line in reversed(tail.splitlines()):
+        bare = line.strip().strip("=").strip()
+        if _PYTEST_SUMMARY_RE.match(bare):
+            return bare
+        if last_failed is None and line.strip().startswith("FAILED "):
+            last_failed = line.strip()
+    return last_failed
+
+
+def artifact_failure_evidence(
+    job_dir: Path, rollout_name: str
+) -> tuple[str, Path] | None:
+    """One-liner mined from the rollout's verifier artifacts, or ``None``.
+
+    Returns the one-liner plus the verifier dir it came from (for the report
+    block's single ``(details: …)`` pointer line). Never raises.
+    """
+    # Local import: RolloutPaths pulls in the task package, which the CLI
+    # shouldn't pay for until a failure actually needs artifact evidence.
+    from benchflow.task.paths import RolloutPaths
+
+    rollout_paths = RolloutPaths(rollout_dir=job_dir / rollout_name)
+    verifier_dir = rollout_paths.verifier_dir
+    attempts: list[tuple[Path, Callable[[Path], str | None]]] = [
+        # ctrf.json has no RolloutPaths property — the verifier recovers it by
+        # literal name (verifier_core.py, `_recover_main_verifier_outputs`).
+        (verifier_dir / "ctrf.json", _ctrf_failure_line),
+        (rollout_paths.test_stdout_path, _stdout_tail_failure_line),
+    ]
+    for path, extract in attempts:
+        try:
+            if not path.is_file():
+                continue
+            detail = extract(path)
+        except Exception:
+            # Unreadable file, bad JSON, permissions … try the next source;
+            # the report must never crash over evidence mining.
+            continue
+        if detail is not None:
+            return detail, verifier_dir
+    return None

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/cli/_shared.py

@@ -3,7 +3,9 @@
 These are the cross-cutting, side-effect-free helpers that several CLI command
 groups (``cli/main.py`` and the ``cli/<group>.py`` modules) need in common: the
 shared Rich :data:`console`, the evaluation-result summary/exit helpers, and the
-agent ``Requires`` rendering used by ``agents``/``agent`` listings.
+agent ``Requires`` rendering used by ``agents``/``agent`` listings. The one
+file-reading step of the eval report (mining verifier artifacts for failure
+evidence) is delegated to :mod:`._failure_evidence` to keep that claim true.
 
 Keeping them here lets each command group import one stable surface instead of
 re-deriving the formatting, and lets ``cli/main.py`` stay a thin app + eval
@@ -19,6 +21,7 @@ from rich.console import Console
 from rich.markup import escape
 
 from benchflow._utils.text import truncate_end
+from benchflow.cli._failure_evidence import artifact_failure_evidence
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -109,17 +112,24 @@ _FAILURE_LINE_LIMIT = 100
 _FAILURE_REASON_METRICS = 3
 
 
-def _failure_reason(failure: TaskFailure) -> str:
+def _failure_reason(
+    failure: TaskFailure, job_dir: Path | None = None
+) -> tuple[str, Path | None]:
     """One cheap line explaining why a FAILED (scored, reward != 1) task
-    failed, from evidence already on the result — no file reads.
+    failed, plus the verifier dir when artifacts supplied the evidence.
 
     Priority: the verifier's own error if set; else the reward plus a compact
     breakdown of the named metrics in the reward dict (zero/failed metrics
-    first — they explain the miss); else just the reward.
+    first — they explain the miss); else the reward plus a one-liner mined
+    from the rollout's verifier artifacts (bounded CLI-side reads resolved via
+    the ``rollout_name`` the engine records — the engine stays file-free);
+    else just the reward. The returned path is non-``None`` only for the
+    artifact tier, so the caller can print one ``(details: …)`` pointer per
+    report block.
     """
     if failure.verifier_error:
         # Collapse whitespace: verifier errors are routinely multi-line.
-        return " ".join(failure.verifier_error.split())
+        return " ".join(failure.verifier_error.split()), None
     rewards = failure.rewards or {}
     reward = rewards.get("reward")
     metrics = [
@@ -127,15 +137,22 @@ def _failure_reason(failure: TaskFailure) -> str:
         for name, value in rewards.items()
         if name != "reward" and isinstance(value, (bool, int, float))
     ]
-    if not metrics:
-        return f"reward {reward}"
-    # Zero/failed metrics first (stable within each group), capped so one
-    # metric-happy verifier can't flood the line.
-    metrics.sort(key=lambda kv: kv[1] != 0)
-    shown = ", ".join(
-        f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
-    )
-    return f"reward {reward} — {shown}"
+    if metrics:
+        # Zero/failed metrics first (stable within each group), capped so one
+        # metric-happy verifier can't flood the line.
+        metrics.sort(key=lambda kv: kv[1] != 0)
+        shown = ", ".join(
+            f"{name} {value}" for name, value in metrics[:_FAILURE_REASON_METRICS]
+        )
+        return f"reward {reward} — {shown}", None
+    # No rollout_name (pre-#957-follow-up result.json, sharded aggregation):
+    # nothing to resolve — the bare reward is as honest as it gets.
+    if job_dir is not None and failure.rollout_name:
+        evidence = artifact_failure_evidence(job_dir, failure.rollout_name)
+        if evidence is not None:
+            detail, verifier_dir = evidence
+            return f"reward {reward} — {detail}", verifier_dir
+    return f"reward {reward}", None
 
 
 def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -> None:
@@ -145,9 +162,13 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
     now the line is green only on a full clean pass, red on a shutout, amber
     otherwise, and ``errors=N`` is red when non-zero. Each FAILED task gets one
     dim ``✗ task: reason`` line (capped at ``_MAX_FAILURE_LINES``) so the "why"
-    doesn't require opening summary.json. When ``job_dir`` is given, the
-    result/summary paths are printed so testers know where to look (the guide
-    repeatedly says "read summary.json" but the CLI never said where).
+    doesn't require opening summary.json; when ``job_dir`` is given, a reason
+    that would otherwise be a bare ``reward X`` is upgraded from the rollout's
+    verifier artifacts (bounded reads, displayed failures only), with a single
+    ``(details: …/verifier)`` pointer line for the block. When ``job_dir`` is
+    given, the result/summary paths are printed so testers know where to look
+    (the guide repeatedly says "read summary.json" but the CLI never said
+    where).
     """
     errors = int(getattr(result, "errored", 0) or 0)
     verifier_errors = int(getattr(result, "verifier_errored", 0) or 0)
@@ -176,15 +197,23 @@ def _report_eval_result(result: EvaluationResult, job_dir: Path | None = None) -
     # summary.json to learn why. getattr(): sharded aggregation and older
     # SimpleNamespace-style callers don't carry task_failures.
     failures = getattr(result, "task_failures", None) or []
+    artifact_pointer: Path | None = None
     for failure in failures[:_MAX_FAILURE_LINES]:
+        reason, verifier_dir = _failure_reason(failure, job_dir)
+        if artifact_pointer is None and verifier_dir is not None:
+            artifact_pointer = verifier_dir
         line = truncate_end(
-            f"  ✗ {failure.task_name}: {_failure_reason(failure)}",
+            f"  ✗ {failure.task_name}: {reason}",
             _FAILURE_LINE_LIMIT,
         )
         console.print(f"[dim]{escape(line)}[/dim]")
     extra = len(failures) - _MAX_FAILURE_LINES
     if extra > 0:
         console.print(f"[dim]  … and {extra} more[/dim]")
+    # One pointer per block (not per line): the first artifact-backed reason's
+    # verifier dir, so "where do I look next" needs no summary.json dig either.
+    if artifact_pointer is not None:
+        console.print(f"[dim]  (details: {escape(str(artifact_pointer))})[/dim]")
     if job_dir is not None:
         console.print(f"[dim]Artifacts:[/dim] {escape(str(job_dir))}")
         console.print(f"[dim]Summary:  [/dim] {escape(str(job_dir))}/summary.json")

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/evaluation.py

@@ -588,6 +588,10 @@ class TaskFailure:
     task_name: str
     rewards: dict[str, Any] | None
     verifier_error: str | None
+    # The task's rollout dir name under the job dir (``<task>__<uuid8>``), so
+    # the CLI can find the rollout's verifier artifacts without guessing.
+    # None on results persisted before the key existed.
+    rollout_name: str | None = None
 
 
 @dataclass
@@ -1801,6 +1805,9 @@ class Evaluation:
                 task_name=name,
                 rewards=r.get("rewards"),
                 verifier_error=r.get("verifier_error"),
+                # `or None`: RolloutResult defaults rollout_name to "" — don't
+                # let that masquerade as a resolvable rollout dir.
+                rollout_name=r.get("rollout_name") or None,
             )
             for name, r in sorted(all_results.items())
             if classify_score_outcome(r) == "failed"

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/sandbox/process/_base.py

@@ -23,6 +23,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
+import os
 import re
 from abc import ABC, abstractmethod
 
@@ -33,6 +34,41 @@ _DIAG_TRUNCATE = 2000  # max chars for diagnostic stderr in error messages
 _BOOTSTRAP_DONE = "__BENCHFLOW_BOOTSTRAP_DONE__"
 _ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
+# Terminal control sequences a PTY-backed transport can interleave with
+# protocol output: ECMA-48 CSI (parameter bytes 0x30-0x3F, intermediate
+# bytes 0x20-0x2F, one final byte 0x40-0x7E) and OSC (BEL- or ST-terminated,
+# e.g. the ``\x1b]0;<title>\x07`` window-title update shell prompts emit).
+# One canonical pair — the AgentCore stdout chunker and the ACP container
+# transport must not drift apart on what counts as terminal noise.
+_ANSI_CSI_PATTERN = r"\x1b\[[0-?]*[ -/]*[@-~]"
+_ANSI_OSC_PATTERN = r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)"
+_ANSI_CSI_RE = re.compile(_ANSI_CSI_PATTERN)
+_ANSI_OSC_RE = re.compile(_ANSI_OSC_PATTERN)
+_ANSI_CSI_RE_BYTES = re.compile(_ANSI_CSI_PATTERN.encode())
+_ANSI_OSC_RE_BYTES = re.compile(_ANSI_OSC_PATTERN.encode())
+
+
+def _timeout_sec_from_env(env_var: str, default: float) -> float:
+    """Read a positive seconds value from *env_var*, else *default*.
+
+    The one parser for operator-tunable timeout knobs on this plane (PTY
+    readline timeouts, the ACP handshake window). Read at use time so
+    long-lived processes and tests see env changes. Unset or empty means
+    the default; non-numeric, non-positive, or NaN values warn and fall
+    back rather than disabling or breaking the guarded wait.
+    """
+    raw = os.environ.get(env_var)
+    if raw is None or not raw.strip():
+        return default
+    try:
+        value = float(raw)
+    except ValueError:
+        value = float("nan")
+    if not value > 0:  # also rejects NaN
+        logger.warning("Invalid %s=%r; using default %.0fs", env_var, raw, default)
+        return default
+    return value
+
 
 async def drain_oversized_line(reader: asyncio.StreamReader) -> int:
     """Drain an oversized line from *reader* after a buffer overflow.

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/sandbox/process/agentcore.py

@@ -19,12 +19,17 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import logging
-import re
 import shlex
 import uuid
 from typing import Any
 
-from benchflow.sandbox.process._base import _ENV_KEY_RE, LiveProcess
+from benchflow.sandbox.process._base import (
+    _ANSI_CSI_RE_BYTES,
+    _ANSI_OSC_RE_BYTES,
+    _ENV_KEY_RE,
+    LiveProcess,
+    _timeout_sec_from_env,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -38,26 +43,10 @@ _READER_ENDED = object()
 _START_MARKER_TIMEOUT_SEC = 180
 _READLINE_TIMEOUT_ENV = "BENCHFLOW_AGENTCORE_READLINE_TIMEOUT"
 _READLINE_TIMEOUT_DEFAULT_SEC = 900.0
-_ANSI_CSI_RE = re.compile(rb"\x1b\[[0-?]*[ -/]*[@-~]")
 
 
 def _readline_timeout_sec() -> float:
-    import os
-
-    value = os.environ.get(_READLINE_TIMEOUT_ENV)
-    if value is None:
-        return _READLINE_TIMEOUT_DEFAULT_SEC
-    try:
-        timeout = float(value)
-    except ValueError:
-        logger.warning(
-            "Invalid %s=%r; using default %.0fs",
-            _READLINE_TIMEOUT_ENV,
-            value,
-            _READLINE_TIMEOUT_DEFAULT_SEC,
-        )
-        return _READLINE_TIMEOUT_DEFAULT_SEC
-    return timeout if timeout > 0 else _READLINE_TIMEOUT_DEFAULT_SEC
+    return _timeout_sec_from_env(_READLINE_TIMEOUT_ENV, _READLINE_TIMEOUT_DEFAULT_SEC)
 
 
 class AgentCoreProcess(LiveProcess):
@@ -119,9 +108,12 @@ class AgentCoreProcess(LiveProcess):
                     line, self._partial = self._partial.split(b"\n", 1)
                     # Bash/readline emits bracketed-paste CSI sequences when a
                     # command takes over the PTY, including a standalone
-                    # ``ESC[?2004l`` *after* the startup marker. Those bytes
-                    # are terminal state, never ACP JSON-RPC.
-                    line = _ANSI_CSI_RE.sub(b"", line.replace(b"\r", b""))
+                    # ``ESC[?2004l`` *after* the startup marker, and shell
+                    # prompts emit OSC window-title updates. Those bytes are
+                    # terminal state, never ACP JSON-RPC.
+                    line = line.replace(b"\r", b"")
+                    line = _ANSI_OSC_RE_BYTES.sub(b"", line)
+                    line = _ANSI_CSI_RE_BYTES.sub(b"", line)
                     if line:
                         await self._line_buffer.put(line + b"\n")
         except asyncio.CancelledError:
@@ -308,8 +300,10 @@ class AgentCoreProcess(LiveProcess):
             # before echo suppression takes effect. Substring matching accepts
             # that echoed command and releases startup too early, putting the
             # prompt/command noise into ACP's JSON-RPC stream. The real marker
-            # is its own line, possibly wrapped in bracketed-paste ANSI codes.
-            text = _ANSI_CSI_RE.sub(b"", line).decode(errors="replace").strip()
+            # is its own line, possibly wrapped in bracketed-paste ANSI codes
+            # or an OSC window-title update from the shell prompt.
+            cleaned = _ANSI_CSI_RE_BYTES.sub(b"", _ANSI_OSC_RE_BYTES.sub(b"", line))
+            text = cleaned.decode(errors="replace").strip()
             if text == marker:
                 return
 

{benchflow-0.6.7.dev1829 → benchflow-0.6.7.dev1833}/src/benchflow/sandbox/process/daytona.py

@@ -18,6 +18,7 @@ from benchflow.sandbox.process._base import (
     _ENV_KEY_RE,
     LiveProcess,
     SubprocessLiveProcess,
+    _timeout_sec_from_env,
 )
 
 logger = logging.getLogger(__name__)
@@ -30,22 +31,10 @@ _DAYTONA_SSH_SERVER_ALIVE_COUNT_MAX = 12
 
 
 def _daytona_pty_readline_timeout_sec() -> float:
-    value = os.environ.get(_DAYTONA_PTY_READLINE_TIMEOUT_ENV)
-    if value is None:
-        return _DAYTONA_PTY_READLINE_TIMEOUT_DEFAULT_SEC
-    try:
-        timeout = float(value)
-    except ValueError:
-        logger.warning(
-            "Invalid %s=%r; using default %.0fs",
-            _DAYTONA_PTY_READLINE_TIMEOUT_ENV,
-            value,
-            _DAYTONA_PTY_READLINE_TIMEOUT_DEFAULT_SEC,
-        )
-        return _DAYTONA_PTY_READLINE_TIMEOUT_DEFAULT_SEC
-    if timeout <= 0:
-        return _DAYTONA_PTY_READLINE_TIMEOUT_DEFAULT_SEC
-    return timeout
+    return _timeout_sec_from_env(
+        _DAYTONA_PTY_READLINE_TIMEOUT_ENV,
+        _DAYTONA_PTY_READLINE_TIMEOUT_DEFAULT_SEC,
+    )
 
 
 async def _cleanup_daytona_remote_env_file(