benchflow 0.7.6.dev2055__tar.gz → 0.7.6.dev2058__tar.gz

{benchflow-0.7.6.dev2055 → benchflow-0.7.6.dev2058}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2055
+Version: 0.7.6.dev2058
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2055 → benchflow-0.7.6.dev2058}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2055"
+version = "0.7.6.dev2058"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2055 → benchflow-0.7.6.dev2058}/src/benchflow/acp/runtime.py

@@ -193,7 +193,11 @@ def _codex_reasoning_effort(model_id: str) -> str:
     return model_id.rsplit("[", 1)[1][:-1]
 
 
-def _codex_session_model_id(model: str, session: object | None) -> str:
+def _codex_session_model_id(
+    model: str,
+    session: object | None,
+    reasoning_effort: str | None = None,
+) -> str:
     """Map a bare Codex model to the exact ACP modelId returned by session/new.
 
     ``@agentclientprotocol/codex-acp`` validates ``session/set_model`` against
@@ -210,6 +214,10 @@ def _codex_session_model_id(model: str, session: object | None) -> str:
     if (
         isinstance(current_model, str)
         and _codex_model_name(current_model) == requested_name
+        and (
+            not reasoning_effort
+            or _codex_reasoning_effort(current_model) == reasoning_effort
+        )
     ):
         return current_model
 
@@ -226,7 +234,13 @@ def _codex_session_model_id(model: str, session: object | None) -> str:
     if not candidates:
         return model
 
-    preferred_efforts = ("medium", "high", "low", "minimal", "none")
+    # A requested reasoning effort rides the codex model id: codex-acp
+    # declares no ACP effort config option, and its session/set_model
+    # validates ``model[effort]`` ids, so picking the matching advertised
+    # variant is the only way the run's effort actually reaches the agent.
+    preferred_efforts: tuple[str, ...] = ("medium", "high", "low", "minimal", "none")
+    if reasoning_effort:
+        preferred_efforts = (reasoning_effort, *preferred_efforts)
     for effort in preferred_efforts:
         for candidate in candidates:
             if _codex_reasoning_effort(candidate) == effort:
@@ -303,11 +317,12 @@ def _select_acp_model_id(
     model: str,
     agent: str,
     session: object | None,
+    reasoning_effort: str | None = None,
 ) -> str:
     """Return the concrete modelId to send through ACP session/set_model."""
     formatted = _format_acp_model(model, agent)
     if agent == "codex-acp":
-        return _codex_session_model_id(formatted, session)
+        return _codex_session_model_id(formatted, session, reasoning_effort)
     return formatted
 
 
@@ -400,12 +415,19 @@ def _resolve_acp_model_option_id(
     This is what lets the ``@agentclientprotocol`` family migrate from
     ``session/set_model`` to a ``"model"`` config option with no registry
     change: a member that advertises a model option is configured through it
-    automatically, while a member that does not (e.g. current ``codex-acp``,
-    which advertises only ``fast-mode``) keeps using ``session/set_model``.
+    automatically, while a member that does not keeps using
+    ``session/set_model``.
+
+    codex-acp is the documented exception: 1.6.0 advertises a "model" config
+    option whose values reject the ``model[effort]`` ids its own
+    ``session/set_model`` requires (-32602 Invalid params, verified live
+    2026-08-19), so codex stays on the set_model path.
     """
     declared = getattr(agent_cfg, "acp_model_config_id", "") or ""
     if declared:
         return declared
+    if getattr(agent_cfg, "name", "") == "codex-acp":
+        return None
     if "model" in _session_config_option_ids(session):
         return "model"
     return None
@@ -476,6 +498,7 @@ async def _configure_acp_session(
     reasoning_effort: str | None,
 ) -> None:
     agent_cfg = AGENTS.get(agent)
+    effort_in_model_id = False
 
     if model and _model_selection_owned_by_env(agent, model, agent_env):
         logger.info(
@@ -483,7 +506,14 @@ async def _configure_acp_session(
         )
     elif model:
         acp_model_input = _resolve_acp_model_input(agent, model, agent_env)
-        acp_model_id = _select_acp_model_id(acp_model_input, agent, session)
+        acp_model_id = _select_acp_model_id(
+            acp_model_input, agent, session, reasoning_effort
+        )
+        effort_in_model_id = bool(
+            reasoning_effort
+            and agent == "codex-acp"
+            and _codex_reasoning_effort(acp_model_id) == reasoning_effort
+        )
         model_option_id = _resolve_acp_model_option_id(agent_cfg, session)
         if model_option_id:
             # Capability-first: the agent advertises a model config option (or
@@ -509,6 +539,14 @@ async def _configure_acp_session(
 
     if not reasoning_effort:
         return
+    if effort_in_model_id:
+        # The effort already rides the selected ``model[effort]`` id (codex),
+        # so there is nothing further to configure.
+        logger.info(
+            f"Reasoning effort {reasoning_effort!r} applied via the model id "
+            f"for {agent}"
+        )
+        return
     if not agent_cfg or not agent_cfg.acp_effort_config_id:
         raise RuntimeError(
             f"reasoning_effort={reasoning_effort!r} was requested for agent "

{benchflow-0.7.6.dev2055 → benchflow-0.7.6.dev2058}/src/benchflow/agents/registry.py

@@ -605,13 +605,16 @@ AGENTS: dict[str, AgentConfig] = {
         # Pinned for reproducibility: an unpinned @agentclientprotocol install
         # floats to latest and can silently break agent activation when the ACP
         # protocol changes (claude-agent-acp above hit exactly this — sdk 0.24
-        # dropped session/set_model). 0.0.45 ships sdk 0.22.x, which still
-        # implements session/set_model. If a future bump advertises a model
-        # config option instead, runtime.py's capability-first dispatch routes
-        # the model through that option — but re-verify model selection when
-        # bumping this pin.
+        # dropped session/set_model). 1.6.0 still implements session/set_model,
+        # but validates its modelId argument as ``model[effort]`` (0.0.45
+        # accepted bare names); _codex_session_model_id maps bare ids onto the
+        # session's advertised ``model[effort]`` variants, so model selection
+        # keeps working across the bump. 1.6.0 also advertises a "model"
+        # config option, but that option rejects ``model[effort]`` ids
+        # (-32602), so runtime.py keeps codex on session/set_model — verified
+        # live 2026-08-19 against gpt-5.6-sol via an Azure provider.
         install_cmd=_js_agent_install(
-            "codex-acp", "@agentclientprotocol/codex-acp@0.0.45"
+            "codex-acp", "@agentclientprotocol/codex-acp@1.6.0"
         ),
         # Self-write ~/.codex/auth.json from OPENAI_API_KEY in the launcher itself,
         # ONLY when the key is set (so subscription/host-auth mode is untouched),

{benchflow-0.7.6.dev2055 → benchflow-0.7.6.dev2058}/tests/test_acp_model_config_dispatch.py

@@ -71,8 +71,8 @@ async def _connect(
 
 @pytest.mark.asyncio
 async def test_codex_with_only_fastmode_option_uses_set_model(tmp_path):
-    """codex-acp@0.0.45 advertises only 'fast-mode' (no 'model'), so dispatch
-    must use session/set_model — capability-first must NOT regress it."""
+    """A codex-acp advertising only 'fast-mode' (no 'model') dispatches via
+    session/set_model — capability-first must NOT regress it."""
     mock_acp = _make_mocks(config_options=[{"id": "fast-mode"}])
     await _connect(mock_acp, agent="codex-acp", model="gpt-5.5", tmp_path=tmp_path)
 
@@ -114,14 +114,43 @@ async def test_codex_litellm_alias_uses_bare_model_for_set_model(tmp_path):
 
 
 @pytest.mark.asyncio
-async def test_codex_with_model_option_uses_config_option(tmp_path):
-    """When a future codex-acp advertises a 'model' config option (as Claude
-    already does), capability-first routes through it — no registry change."""
+async def test_codex_with_model_option_still_uses_set_model(tmp_path):
+    """codex-acp@1.6.0 advertises a 'model' config option whose values reject
+    the ``model[effort]`` ids its own session/set_model requires (-32602
+    Invalid params, verified live 2026-08-19), so codex is the documented
+    exception to capability-first and stays on session/set_model."""
     mock_acp = _make_mocks(config_options=[{"id": "model"}])
     await _connect(mock_acp, agent="codex-acp", model="gpt-5.5", tmp_path=tmp_path)
 
-    mock_acp.set_config_option.assert_awaited_once_with("model", "gpt-5.5")
-    mock_acp.set_model.assert_not_awaited()
+    mock_acp.set_model.assert_awaited_once_with("gpt-5.5")
+    mock_acp.set_config_option.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_codex_reasoning_effort_rides_the_model_id(tmp_path):
+    """codex-acp declares no ACP effort config option; a requested effort must
+    select the matching advertised ``model[effort]`` variant instead of
+    failing closed, and the effort step must treat it as satisfied."""
+    mock_acp = _make_mocks(
+        config_options=[{"id": "fast-mode"}],
+        model_state={
+            "availableModels": [
+                {"modelId": "gpt-5.5[medium]"},
+                {"modelId": "gpt-5.5[xhigh]"},
+            ],
+            "currentModelId": "gpt-5.4-mini[medium]",
+        },
+    )
+    await _connect(
+        mock_acp,
+        agent="codex-acp",
+        model="gpt-5.5",
+        tmp_path=tmp_path,
+        reasoning_effort="xhigh",
+    )
+
+    mock_acp.set_model.assert_awaited_once_with("gpt-5.5[xhigh]")
+    mock_acp.set_config_option.assert_not_awaited()
 
 
 @pytest.mark.asyncio

{benchflow-0.7.6.dev2055 → benchflow-0.7.6.dev2058}/tests/test_agent_registry.py

@@ -52,7 +52,7 @@ class TestEnvMappingField:
         """Same @agentclientprotocol family as claude — pin so a floating latest
         can't silently break activation when upstream drops session/set_model."""
         cfg = AGENTS["codex-acp"]
-        assert "@agentclientprotocol/codex-acp@0.0.45" in cfg.install_cmd
+        assert "@agentclientprotocol/codex-acp@1.6.0" in cfg.install_cmd
 
     def test_gemini_has_mapping(self):
         cfg = AGENTS["gemini"]