benchflow 0.6.8.dev1872__tar.gz → 0.6.8.dev1907__tar.gz

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.6.8.dev1872
+Version: 0.6.8.dev1907
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.8.dev1872"
+version = "0.6.8.dev1907"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/src/benchflow/acp/container_transport.py

@@ -7,6 +7,7 @@ from typing import Any, TextIO
 
 from benchflow.sandbox.process import LiveProcess
 from benchflow.sandbox.process._base import _ANSI_CSI_RE, _ANSI_OSC_RE
+from benchflow.trajectories.types import redact_trajectory_text
 
 from .transport import Transport, decode_json_rpc_message
 
@@ -138,3 +139,7 @@ class ContainerTransport(Transport):
             self._agent_log_file.close()
             self._agent_log_file = None
         await self._cp.close()
+        stderr = getattr(self._cp, "stderr_tail", "")
+        if isinstance(stderr, str) and stderr and self._agent_log_path:
+            with self._agent_log_path.open("a") as agent_log:
+                agent_log.write(redact_trajectory_text(stderr))

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/src/benchflow/agents/openclaw_acp_shim.py

@@ -49,6 +49,17 @@ _PARAM_MAP = {
 }
 
 
+def _default_max_tokens(model: str) -> int | None:
+    # OpenClaw derives an invalid ~172k default for GPT-5.4; ACP sends a
+    # BenchFlow-generated LiteLLM alias in normal runs.
+    model = model.removeprefix("openai/")
+    if model == "gpt-5.4" or (
+        model.startswith("benchflow-") and model.endswith("-gpt-5.4")
+    ):
+        return 128000
+    return None
+
+
 # ── ACP stdio I/O ─────────────────────────────────────────────────────────────
 
 
@@ -671,6 +682,7 @@ def main():
 
         elif method == "session/set_model":
             model = params.get("modelId", "")
+            requested_model = model
             # A provider-resolution / config-write failure here must NOT crash the
             # shim: an unhandled exception exits rc=1, which benchflow sees as the
             # ACP transport dying mid-set_model ("Process closed stdout (rc=1)") —
@@ -726,6 +738,25 @@ def main():
                             check=True,
                             timeout=10,
                         )
+                max_tokens = _default_max_tokens(requested_model)
+                configured_max_tokens = os.environ.get("BENCHFLOW_MODEL_MAX_TOKENS")
+                if max_tokens is not None and (
+                    not configured_max_tokens
+                    or not configured_max_tokens.isdigit()
+                    or int(configured_max_tokens) > max_tokens
+                ):
+                    subprocess.run(
+                        [
+                            _OPENCLAW_BIN,
+                            "config",
+                            "set",
+                            _PARAM_MAP["BENCHFLOW_MODEL_MAX_TOKENS"],
+                            str(max_tokens),
+                        ],
+                        capture_output=True,
+                        check=True,
+                        timeout=10,
+                    )
             except Exception as exc:
                 diag = (
                     f"[openclaw-acp-shim] set_model setup failed, continuing: {exc!r}"

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/src/benchflow/sandbox/process/_base.py

@@ -31,6 +31,8 @@ logger = logging.getLogger(__name__)
 
 _BUFFER_LIMIT = 10 * 1024 * 1024  # 10MB readline buffer
 _DIAG_TRUNCATE = 2000  # max chars for diagnostic stderr in error messages
+_STDERR_TAIL_LIMIT = 64 * 1024  # bounded stderr retained for rollout diagnostics
+_STDERR_DRAIN_TIMEOUT_SEC = 2
 _BOOTSTRAP_DONE = "__BENCHFLOW_BOOTSTRAP_DONE__"
 _ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
@@ -136,6 +138,29 @@ class SubprocessLiveProcess(LiveProcess):
 
     _process: asyncio.subprocess.Process | None = None
 
+    def _set_process(self, process: asyncio.subprocess.Process) -> None:
+        """Store a subprocess and drain stderr without blocking its stdout pipe."""
+        self._process = process
+        self._stderr_tail = bytearray()
+        self._stderr_task = (
+            asyncio.create_task(self._drain_stderr(process.stderr))
+            if isinstance(process.stderr, asyncio.StreamReader)
+            else None
+        )
+
+    async def _drain_stderr(self, stderr: asyncio.StreamReader | None) -> None:
+        if stderr is None:
+            return
+        while chunk := await stderr.read(8192):
+            self._stderr_tail.extend(chunk)
+            if len(self._stderr_tail) > _STDERR_TAIL_LIMIT:
+                del self._stderr_tail[:-_STDERR_TAIL_LIMIT]
+
+    @property
+    def stderr_tail(self) -> str:
+        """Bounded stderr captured while the subprocess was alive."""
+        return bytes(getattr(self, "_stderr_tail", b"")).decode(errors="replace")
+
     async def readline(self) -> bytes:
         """Read one line from stdout."""
         if not self._process or not self._process.stdout:
@@ -149,8 +174,16 @@ class SubprocessLiveProcess(LiveProcess):
             # Return empty line — caller will retry readline
             return b""
         if not line:
-            stderr_text = ""
-            if self._process and self._process.stderr:
+            stderr_task = getattr(self, "_stderr_task", None)
+            if stderr_task:
+                with contextlib.suppress(TimeoutError):
+                    await asyncio.wait_for(
+                        asyncio.shield(stderr_task), timeout=_STDERR_DRAIN_TIMEOUT_SEC
+                    )
+                stderr_text = self.stderr_tail.strip()
+            else:
+                stderr_text = ""
+            if not stderr_task and self._process and self._process.stderr:
                 try:
                     stderr_bytes = await asyncio.wait_for(
                         self._process.stderr.read(8192), timeout=2
@@ -179,7 +212,9 @@ class SubprocessLiveProcess(LiveProcess):
             msg = f"Process closed stdout (rc={rc}): {hint}"
             stderr_snippet: str | None = None
             if stderr_text:
-                stderr_snippet = stderr_text[:_DIAG_TRUNCATE]
+                from benchflow.trajectories.types import redact_trajectory_text
+
+                stderr_snippet = redact_trajectory_text(stderr_text)[:_DIAG_TRUNCATE]
                 msg += f"\nstderr: {stderr_snippet}"
             # Raise a structured TransportClosedError at the source so
             # downstream code (rollout._build_rollout_result) doesn't have
@@ -222,6 +257,16 @@ class SubprocessLiveProcess(LiveProcess):
                 except TimeoutError:
                     self._process.kill()
                     await self._process.wait()
+            stderr_task = getattr(self, "_stderr_task", None)
+            if stderr_task:
+                try:
+                    await asyncio.wait_for(
+                        asyncio.shield(stderr_task), timeout=_STDERR_DRAIN_TIMEOUT_SEC
+                    )
+                except TimeoutError:
+                    stderr_task.cancel()
+                    with contextlib.suppress(asyncio.CancelledError):
+                        await stderr_task
             logger.info("Process terminated")
 
     @property

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/src/benchflow/sandbox/process/apple.py

@@ -110,13 +110,14 @@ class AppleContainerProcess(SubprocessLiveProcess):
             args.extend(["--workdir", cwd])
         args.extend([self._container_name, "bash", "-c", command])
         try:
-            self._process = await asyncio.create_subprocess_exec(
+            process = await asyncio.create_subprocess_exec(
                 *args,
                 stdin=asyncio.subprocess.PIPE,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
                 limit=_BUFFER_LIMIT,
             )
+            self._set_process(process)
         except BaseException:
             if env:
                 try:
@@ -129,6 +130,6 @@ class AppleContainerProcess(SubprocessLiveProcess):
             raise
         logger.info(
             "Apple Container process started (pid=%s, container=%s)",
-            self._process.pid,
+            process.pid,
             self._container_name,
         )

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/src/benchflow/sandbox/process/daytona.py

@@ -360,22 +360,23 @@ class DaytonaProcess(SubprocessLiveProcess):
                 "DaytonaProcess: ssh benchflow-daytona %s...",
                 remote_cmd[:100],
             )
-            self._process = await asyncio.create_subprocess_exec(
+            process = await asyncio.create_subprocess_exec(
                 *cmd,
                 stdin=asyncio.subprocess.PIPE,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
                 limit=_BUFFER_LIMIT,
             )
+            self._set_process(process)
         except Exception:
             if remote_env_path:
                 await self._cleanup_remote_env_file(remote_env_path)
             await self.close()
             raise
         self._ssh_config_cleanup_task = asyncio.create_task(
-            self._cleanup_ssh_config_after_exit(self._process, ssh_config_path)
+            self._cleanup_ssh_config_after_exit(process, ssh_config_path)
         )
-        logger.info(f"Daytona process started (pid={self._process.pid})")
+        logger.info(f"Daytona process started (pid={process.pid})")
 
     async def close(self) -> None:
         try:

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/src/benchflow/sandbox/process/docker.py

@@ -161,7 +161,7 @@ class DockerProcess(SubprocessLiveProcess):
 
         logger.debug(f"DockerProcess: {' '.join(cmd[:10])}...")
         try:
-            self._process = await asyncio.create_subprocess_exec(
+            process = await asyncio.create_subprocess_exec(
                 *cmd,
                 stdin=asyncio.subprocess.PIPE,
                 stdout=asyncio.subprocess.PIPE,
@@ -169,6 +169,7 @@ class DockerProcess(SubprocessLiveProcess):
                 env=proc_env,
                 limit=_BUFFER_LIMIT,
             )
+            self._set_process(process)
         except BaseException:
             if env:
                 try:
@@ -180,6 +181,5 @@ class DockerProcess(SubprocessLiveProcess):
                     )
             raise
         logger.info(
-            f"Docker process started (pid={self._process.pid}, "
-            f"project={self._project_name})"
+            f"Docker process started (pid={process.pid}, project={self._project_name})"
         )

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/tests/test_acp.py

@@ -512,6 +512,72 @@ class TestTransportProtocolFiltering:
         assert msg == {"jsonrpc": "2.0", "id": 2, "result": {"ok": True}}
         assert not agent_log.exists()
 
+    @pytest.mark.asyncio
+    async def test_container_transport_keeps_live_subprocess_stderr(
+        self, tmp_path
+    ) -> None:
+        """Guards PR #980 against losing stderr after a valid empty ACP result."""
+        from benchflow.sandbox.process import SubprocessLiveProcess
+
+        class _LocalProcess(SubprocessLiveProcess):
+            async def start(self, command, env=None, cwd=None) -> None:
+                self._set_process(
+                    await asyncio.create_subprocess_exec(
+                        sys.executable,
+                        "-c",
+                        "import sys, time; "
+                        "sys.stderr.write('api_key=AIzaSy12345678901234567890\\n'); "
+                        "sys.stderr.flush(); "
+                        'print(\'{\\"jsonrpc\\": \\"2.0\\", \\"id\\": 2, \\"result\\": {}}\', flush=True); '
+                        "time.sleep(60)",
+                        stdin=asyncio.subprocess.PIPE,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.PIPE,
+                    )
+                )
+
+        agent_log = tmp_path / "agent.log"
+        transport = ContainerTransport(
+            _LocalProcess(), "agent acp", agent_log_path=agent_log
+        )
+        await transport.start()
+        assert await transport.receive() == {
+            "jsonrpc": "2.0",
+            "id": 2,
+            "result": {},
+        }
+        await transport.close()
+        assert "***REDACTED***" in agent_log.read_text()
+        assert "AIzaSy12345678901234567890" not in agent_log.read_text()
+
+    @pytest.mark.asyncio
+    async def test_container_transport_stderr_tail_is_bounded(self, tmp_path) -> None:
+        """Guards PR #980 against unbounded live ACP stderr retention."""
+        from benchflow.sandbox.process import SubprocessLiveProcess
+
+        class _LocalProcess(SubprocessLiveProcess):
+            async def start(self, command, env=None, cwd=None) -> None:
+                self._set_process(
+                    await asyncio.create_subprocess_exec(
+                        sys.executable,
+                        "-c",
+                        "import sys, time; sys.stderr.write('x' * 70000); "
+                        'sys.stderr.flush(); print(\'{\\"jsonrpc\\": \\"2.0\\", \\"id\\": 2, \\"result\\": {}}\', flush=True); time.sleep(60)',
+                        stdin=asyncio.subprocess.PIPE,
+                        stdout=asyncio.subprocess.PIPE,
+                        stderr=asyncio.subprocess.PIPE,
+                    )
+                )
+
+        process = _LocalProcess()
+        transport = ContainerTransport(
+            process, "agent acp", agent_log_path=tmp_path / "agent.log"
+        )
+        await transport.start()
+        await transport.receive()
+        await transport.close()
+        assert len(process.stderr_tail.encode()) == 64 * 1024
+
     @pytest.mark.asyncio
     async def test_container_transport_clears_stale_log_on_retry(
         self, tmp_path
@@ -1542,7 +1608,7 @@ class TestTransportErrorDiagnostics:
                 return b""
 
             async def read(self, _n: int) -> bytes:
-                return b"Connection to sandbox lost"
+                return b"api_key=AIzaSy12345678901234567890"
 
         class _LP(SubprocessLiveProcess):
             async def start(
@@ -1558,7 +1624,8 @@ class TestTransportErrorDiagnostics:
         assert diag.process_exit_code == 255
         assert diag.transport_diagnosis == "process_exited"
         assert diag.stderr_snippet is not None
-        assert "Connection to sandbox lost" in diag.stderr_snippet
+        assert "***REDACTED***" in diag.stderr_snippet
+        assert "AIzaSy12345678901234567890" not in str(exc_info.value)
 
     @pytest.mark.asyncio
     async def test_live_process_raises_typed_transport_error_when_remote_killed(

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/tests/test_bare_model_provider.py

@@ -25,6 +25,7 @@ These tests pin the registry-driven bare-model routing:
 import pytest
 
 from benchflow.agents.openclaw_acp_shim import (
+    _default_max_tokens,
     _infer_provider_prefix,
     _resolve_bare_model_prefix,
     _setup_bare_custom_provider,
@@ -33,6 +34,12 @@ from benchflow.agents.providers import (
     find_provider,
     find_provider_for_bare_model,
 )
+from benchflow.providers.litellm_config import safe_model_alias
+
+
+@pytest.mark.parametrize("model", ["gpt-5.4", "openai/gpt-5.4"])
+def test_gpt54_proxy_alias_has_supported_token_cap(model):
+    assert _default_max_tokens(safe_model_alias(model)) == 128000
 
 
 class TestFindProviderForBareModel:

{benchflow-0.6.8.dev1872 → benchflow-0.6.8.dev1907}/tests/test_process.py

@@ -17,6 +17,37 @@ from benchflow.sandbox.process import (
 )
 
 
+@pytest.mark.asyncio
+async def test_subprocess_close_cancels_stderr_drain_that_never_reaches_eof(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Guards PR #980 against waiting forever for inherited stderr FDs."""
+    from benchflow.sandbox.process import SubprocessLiveProcess
+
+    class _Process:
+        returncode = None
+        stdin = None
+
+        def terminate(self) -> None:
+            self.returncode = 0
+
+        async def wait(self) -> None:
+            return None
+
+    class _LiveProcess(SubprocessLiveProcess):
+        async def start(self, command, env=None, cwd=None) -> None:
+            pass
+
+    process = _LiveProcess()
+    process._process = _Process()  # type: ignore[assignment]
+    process._stderr_task = asyncio.create_task(asyncio.Event().wait())
+    monkeypatch.setattr(
+        "benchflow.sandbox.process._base._STDERR_DRAIN_TIMEOUT_SEC", 0.01
+    )
+    await process.close()
+    assert process._stderr_task.cancelled()
+
+
 class _FakeStdin:
     def __init__(self):
         self.writes = []