benchflow 0.7.6.dev2219__tar.gz → 0.7.6.dev2231__tar.gz

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2219
+Version: 0.7.6.dev2231
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2219"
+version = "0.7.6.dev2231"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/src/benchflow/agents/registry.py

@@ -119,6 +119,7 @@ _BENCHFLOW_BIN_PREFIX = "/opt/benchflow/bin"
 # OpenCode routes through the chat-completions path. Shared with
 # ``benchflow.acp.runtime._format_acp_model`` so set_model targets the same id.
 OPENCODE_PROXY_PROVIDER_ID = "benchflow"
+_CLAUDE_AGENT_ACP_PACKAGE = "@agentclientprotocol/claude-agent-acp@0.73.0"
 _OPENHANDS_CLI_GIT_REV = "2df8a2835d3f1bd2f2eadf5a7a2e1ad0dfb0d271"
 _OPENHANDS_SDK_VERSION = "1.28.1"
 _OPENHANDS_TOOLS_VERSION = "1.28.1"
@@ -517,14 +518,15 @@ AGENTS: dict[str, AgentConfig] = {
         description="Claude Code via ACP (Anthropic's Agent Client Protocol)",
         skill_paths=["$HOME/.claude/skills"],
         home_dirs=[".claude"],
-        # Pinned to 0.40.0: the config-option wiring below (set_config_option +
-        # the "model"/"effort" ids) targets this version's ACP protocol (sdk
-        # 0.24, which dropped session/set_model). The option ids are coupled to
-        # this pin — re-verify them when bumping. runtime.py uses
-        # capability-first dispatch for the rest of the family.
-        install_cmd=_js_agent_install(
-            "claude-agent-acp", "@agentclientprotocol/claude-agent-acp@0.40.0"
-        ),
+        # Pinned to 0.73.0 (bundles @anthropic-ai/claude-agent-sdk 0.3.257):
+        # claude-fable-5-1 rejects Claude Code < 2.1.251 with
+        # `claude_code_version_too_old` (HTTP 400), so the previous 0.40.0 pin
+        # (sdk 0.3.160) cannot run that model at all. The config-option wiring
+        # below (set_config_option + the "model"/"effort" ids) was re-verified
+        # against 0.73.0 with tests/test_acp_pinned_protocol_guard.py; the ids
+        # stay coupled to this pin — re-run that guard when bumping. runtime.py
+        # uses capability-first dispatch for the rest of the family.
+        install_cmd=_js_agent_install("claude-agent-acp", _CLAUDE_AGENT_ACP_PACKAGE),
         launch_cmd=_js_agent_launch("claude-agent-acp"),
         protocol="acp",
         requires_env=["ANTHROPIC_API_KEY"],

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/src/benchflow/sandbox/lockdown.py

@@ -609,7 +609,7 @@ async def _discover_pytest_plugin_flags(env, task: "Task") -> str:
         result = await env.exec(
             f"python3 -c {shlex.quote(_DISCOVER_PYTEST_PLUGINS_SCRIPT)}",
             user="root",
-            timeout_sec=15,
+            timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
         )
         if result.stderr:
             logger.debug(f"Plugin discovery stderr: {result.stderr.strip()}")
@@ -684,7 +684,9 @@ async def _distro_pip_env(env) -> dict[str, str]:
     """
     try:
         result = await env.exec(
-            "cat /etc/os-release 2>/dev/null || true", user="root", timeout_sec=5
+            "cat /etc/os-release 2>/dev/null || true",
+            user="root",
+            timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
         )
     except Exception as e:
         logger.warning("distro detection failed (%s); skipping pip env tweaks", e)
@@ -709,14 +711,16 @@ async def _trusted_verifier_path(
     checks prove they are root-owned directories and not group/world writable.
     Runtime locations and sandbox-user writable locations stay excluded.
     """
-    path_result = await env.exec("printenv PATH", user="root", timeout_sec=10)
+    path_result = await env.exec(
+        "printenv PATH", user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC
+    )
     raw_path = path_result.stdout or ""
     if not raw_path.strip():
         return _SAFE_VERIFIER_PATH
     cmd = _trusted_path_extras_cmd(
         raw_path, _blocked_verifier_path_prefixes(sandbox_user, workspace)
     )
-    result = await env.exec(cmd, user="root", timeout_sec=10)
+    result = await env.exec(cmd, user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC)
     if _exec_return_code(result) != 0:
         logger.debug(
             "Trusted verifier PATH extras unavailable; using safe PATH.%s",
@@ -748,14 +752,16 @@ async def _trusted_verifier_pythonpath(
     is chowned to root before verification.
     """
     pp_result = await env.exec(
-        "printenv PYTHONPATH 2>/dev/null || true", user="root", timeout_sec=10
+        "printenv PYTHONPATH 2>/dev/null || true",
+        user="root",
+        timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
     )
     raw_pp = (pp_result.stdout or "").strip()
     if not raw_pp:
         return ""
     blocked = _blocked_verifier_pythonpath_prefixes(sandbox_user)
     cmd = _trusted_path_extras_cmd(raw_pp, blocked)
-    result = await env.exec(cmd, user="root", timeout_sec=10)
+    result = await env.exec(cmd, user="root", timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC)
     try:
         extras = _json.loads(result.stdout or "[]")
     except _json.JSONDecodeError:
@@ -1052,13 +1058,14 @@ async def _kill_sandbox_user_procs(env, sandbox_user: str) -> None:
     await env.exec(
         f"pkill -u {sandbox_user} 2>/dev/null; "
         f"sleep 1; pkill -9 -u {sandbox_user} 2>/dev/null || true",
-        timeout_sec=10,
+        timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
     )
     # Second pass: catch any processes that slipped through (e.g. cron/at jobs).
     await env.exec(
         f"! pgrep -u {sandbox_user} > /dev/null 2>&1 || "
         f"(sleep 1 && pkill -9 -u {sandbox_user}; sleep 1)",
         user="root",
+        timeout_sec=VERIFIER_SETUP_TIMEOUT_SEC,
     )
 
 

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/tests/test_acp_pinned_protocol_guard.py

@@ -1,5 +1,4 @@
-"""Gated live guard: the pinned claude-agent-acp advertises the config option
-ids the registry wires up.
+"""Gated live guard for the pinned claude-agent-acp config-option contract.
 
 Skipped by default. Run with ``RUN_ACP_DEP_GUARD=1`` (needs ``npm`` + ``node`` +
 network):
@@ -7,13 +6,11 @@ network):
     RUN_ACP_DEP_GUARD=1 uv run --extra dev python -m pytest \
         tests/test_acp_pinned_protocol_guard.py -q
 
-It installs the pinned ``claude-agent-acp@0.40.0``, starts it over ACP stdio,
-runs ``initialize`` + ``session/new``, and asserts the advertised config option
-ids include ``{"model", "effort"}`` — the ids ``benchflow.agents.registry``
-hard-codes for model and reasoning-effort selection. If a future pin keeps
-``session/set_config_option`` but renames an id, this fails (a plain SDK method
-grep would not). ``session/new`` advertises the options without auth, so no
-credentials are needed. Re-run when bumping the ``@agentclientprotocol`` pin.
+It installs the exact package selected by ``benchflow.agents.registry``, starts
+it over ACP stdio, and proves the complete Fable model + effort path works:
+``initialize``, ``session/new``, and both ``session/set_config_option`` calls.
+The adapter exposes and accepts these options without auth, so no credentials
+are needed. Re-run when bumping the ``@agentclientprotocol`` pin.
 """
 
 import asyncio
@@ -25,13 +22,16 @@ from pathlib import Path
 
 import pytest
 
+from benchflow.agents.registry import _CLAUDE_AGENT_ACP_PACKAGE
+
 pytestmark = pytest.mark.skipif(
     os.environ.get("RUN_ACP_DEP_GUARD") != "1",
     reason="gated live ACP guard; set RUN_ACP_DEP_GUARD=1 (needs npm + node + network)",
 )
 
-PINNED_CLAUDE = "@agentclientprotocol/claude-agent-acp@0.40.0"
 EXPECTED_OPTION_IDS = {"model", "effort"}
+FABLE_MODEL = "claude-fable-5-1"
+FABLE_EFFORT = "xhigh"
 
 
 def _tool_or_skip(name: str) -> str:
@@ -41,7 +41,7 @@ def _tool_or_skip(name: str) -> str:
     return path
 
 
-async def _advertised_option_ids(entry: Path) -> set[str]:
+async def _exercise_config_options(entry: Path) -> tuple[set[str], dict[str, str]]:
     from benchflow.acp.client import ACPClient
     from benchflow.acp.transport import StdioTransport
 
@@ -51,23 +51,39 @@ async def _advertised_option_ids(entry: Path) -> set[str]:
         await asyncio.wait_for(client.initialize(), timeout=60)
         await asyncio.wait_for(client.session_new(cwd="/tmp"), timeout=90)
         opts = client.session.config_options or []
-        return {
+        ids = {
             o["id"]
             for o in opts
             if isinstance(o, dict) and isinstance(o.get("id"), str)
         }
+        if ids >= EXPECTED_OPTION_IDS:
+            await asyncio.wait_for(
+                client.set_config_option("model", FABLE_MODEL), timeout=60
+            )
+            await asyncio.wait_for(
+                client.set_config_option("effort", FABLE_EFFORT), timeout=60
+            )
+        current = {
+            o["id"]: o["currentValue"]
+            for o in client.session.config_options or []
+            if isinstance(o, dict)
+            and o.get("id") in EXPECTED_OPTION_IDS
+            and isinstance(o.get("currentValue"), str)
+        }
+        return ids, current
     finally:
         with contextlib.suppress(Exception):
             await client.close()
 
 
-def test_pinned_claude_acp_advertises_model_and_effort_options(tmp_path):
+def test_pinned_claude_acp_supports_fable_model_and_effort(tmp_path):
+    """Guards PR #1086's Fable-compatible adapter and ACP config contract."""
     npm = _tool_or_skip("npm")
     _tool_or_skip("node")
     prefix = tmp_path / "claude"
     prefix.mkdir()
     subprocess.run(
-        [npm, "install", "--prefix", str(prefix), PINNED_CLAUDE],
+        [npm, "install", "--prefix", str(prefix), _CLAUDE_AGENT_ACP_PACKAGE],
         check=True,
         capture_output=True,
         text=True,
@@ -83,11 +99,13 @@ def test_pinned_claude_acp_advertises_model_and_effort_options(tmp_path):
     )
     assert entry.is_file(), f"pinned agent entry not found: {entry}"
 
-    ids = asyncio.run(_advertised_option_ids(entry))
+    ids, current = asyncio.run(_exercise_config_options(entry))
     missing = EXPECTED_OPTION_IDS - ids
     assert not missing, (
-        f"pinned {PINNED_CLAUDE} no longer advertises config option(s) "
+        f"pinned {_CLAUDE_AGENT_ACP_PACKAGE} no longer advertises config option(s) "
         f"{sorted(missing)!r} (advertised: {sorted(ids)!r}); the registry "
         f"model/effort wiring is stale — re-verify acp_model_config_id / "
         f"acp_effort_config_id"
     )
+    assert current.get("model", "").split("[", 1)[0] == FABLE_MODEL, current
+    assert current.get("effort") == FABLE_EFFORT, current

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/tests/test_agent_registry.py

@@ -26,6 +26,7 @@ class TestEnvMappingField:
     """env_mapping exists on AgentConfig and is populated for known agents."""
 
     def test_claude_agent_has_mapping(self):
+        """Guards PR #1086's Fable-compatible adapter pin and ACP config ids."""
         cfg = AGENTS["claude-agent-acp"]
         assert "BENCHFLOW_PROVIDER_BASE_URL" in cfg.env_mapping
         assert cfg.env_mapping["BENCHFLOW_PROVIDER_BASE_URL"] == "ANTHROPIC_BASE_URL"
@@ -33,7 +34,7 @@ class TestEnvMappingField:
         assert cfg.supports_acp_set_model is False
         assert cfg.acp_model_config_id == "model"
         assert cfg.acp_effort_config_id == "effort"
-        assert "@agentclientprotocol/claude-agent-acp@0.40.0" in cfg.install_cmd
+        assert "@agentclientprotocol/claude-agent-acp@0.73.0" in cfg.install_cmd
 
     def test_pi_acp_no_static_mapping(self):
         """pi-acp is multi-protocol — launch wrapper handles env translation."""

{benchflow-0.7.6.dev2219 → benchflow-0.7.6.dev2231}/tests/test_sandbox_hardening.py

@@ -268,6 +268,43 @@ class TestVerifierDirWipe:
                 == VERIFIER_SETUP_TIMEOUT_SEC
             )
 
+    @pytest.mark.asyncio
+    async def test_pr_1062_all_verifier_setup_probes_share_budget(self):
+        """Guards PR #1062 against short or unbounded Daytona setup execs."""
+        from benchflow.sandbox.lockdown import (
+            VERIFIER_SETUP_TIMEOUT_SEC,
+            build_reclaim_caches_cmd,
+            harden_before_verify,
+        )
+
+        def side_effect(command, **kwargs):
+            if command == "printenv PATH":
+                stdout = "/usr/local/bin:/usr/bin"
+            elif command.startswith("printenv PYTHONPATH"):
+                stdout = "/usr/local/lib/python3.12/site-packages"
+            elif command.startswith("python3 -c"):
+                stdout = "[]"
+            elif command.startswith("cat /etc/os-release"):
+                stdout = "ID=ubuntu\n"
+            else:
+                stdout = ""
+            return MagicMock(stdout=stdout, stderr="", exit_code=0)
+
+        env = _make_env(side_effect)
+        await harden_before_verify(
+            env, _make_task(), sandbox_user="agent", workspace="/app"
+        )
+
+        reclaim_command = build_reclaim_caches_cmd("/app")
+        setup_calls = [
+            call for call in env.exec.call_args_list if call.args[0] != reclaim_command
+        ]
+        assert len(setup_calls) >= 12
+        assert all(
+            call.kwargs.get("timeout_sec") == VERIFIER_SETUP_TIMEOUT_SEC
+            for call in setup_calls
+        )
+
     @pytest.mark.asyncio
     async def test_pr_942_workspace_chown_failure_is_fatal(self):
         """Guards PR #942: failed ownership freezing cannot reach verification."""