benchflow 0.7.6.dev2102__tar.gz → 0.7.6.dev2208__tar.gz

{benchflow-0.7.6.dev2102 → benchflow-0.7.6.dev2208}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.7.6.dev2102
+Version: 0.7.6.dev2208
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.7.6.dev2102 → benchflow-0.7.6.dev2208}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.7.6.dev2102"
+version = "0.7.6.dev2208"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.7.6.dev2102 → benchflow-0.7.6.dev2208}/src/benchflow/agents/openclaw_acp_shim.py

@@ -21,6 +21,7 @@ import base64
 import json
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -49,15 +50,30 @@ _PARAM_MAP = {
 }
 
 
+_TOKEN_CAP_MODEL = (
+    r"(?:(?:(?:openai|us-openai|azure-foundry-openai)/|"
+    r"benchflow-(?:(?:openai|us-openai|azure-foundry-openai)-)?)?gpt-5\.4|"
+    r"(?:(?:anthropic|anthropic-vertex|azure-foundry-anthropic)/|"
+    r"benchflow-(?:(?:anthropic|anthropic-vertex|azure-foundry-anthropic)-)?)?"
+    r"claude-(?:sonnet-4-6|opus-4-[6-8]|(?:sonnet|opus)-5(?:-\d+)?))"
+)
+
+
 def _default_max_tokens(model: str) -> int | None:
-    # OpenClaw derives an invalid ~172k default for GPT-5.4; ACP sends a
-    # BenchFlow-generated LiteLLM alias in normal runs.
-    model = model.removeprefix("openai/")
-    if model == "gpt-5.4" or (
-        model.startswith("benchflow-") and model.endswith("-gpt-5.4")
-    ):
-        return 128000
-    return None
+    # Assume future numeric Sonnet/Opus 5.x releases retain the documented
+    # 128k max output; narrow/update the pattern if that limit changes.
+    return 128000 if re.fullmatch(_TOKEN_CAP_MODEL, model) else None
+
+
+def _max_tokens_value(model: str, configured: str | None) -> str | None:
+    cap = _default_max_tokens(model)
+    if cap is None:
+        return configured
+    try:
+        value = int(configured or "")
+    except ValueError:
+        return str(cap)
+    return configured if 0 < value <= cap else str(cap)
 
 
 # ── ACP stdio I/O ─────────────────────────────────────────────────────────────
@@ -728,35 +744,41 @@ def main():
                         timeout=10,
                     )
 
-                # Apply model generation parameters from env vars
-                for env_key, config_path in _PARAM_MAP.items():
-                    val = os.environ.get(env_key)
-                    if val:
-                        subprocess.run(
-                            [_OPENCLAW_BIN, "config", "set", config_path, val],
-                            capture_output=True,
-                            check=True,
-                            timeout=10,
-                        )
-                max_tokens = _default_max_tokens(requested_model)
-                configured_max_tokens = os.environ.get("BENCHFLOW_MODEL_MAX_TOKENS")
-                if max_tokens is not None and (
-                    not configured_max_tokens
-                    or not configured_max_tokens.isdigit()
-                    or int(configured_max_tokens) > max_tokens
-                ):
+                max_tokens = _max_tokens_value(
+                    requested_model, os.environ.get("BENCHFLOW_MODEL_MAX_TOKENS")
+                )
+                if max_tokens:
                     subprocess.run(
                         [
                             _OPENCLAW_BIN,
                             "config",
                             "set",
                             _PARAM_MAP["BENCHFLOW_MODEL_MAX_TOKENS"],
-                            str(max_tokens),
+                            max_tokens,
                         ],
                         capture_output=True,
                         check=True,
                         timeout=10,
                     )
+                # Apply model generation parameters from env vars
+                for env_key in (
+                    "BENCHFLOW_MODEL_TEMPERATURE",
+                    "BENCHFLOW_MODEL_TOP_P",
+                ):
+                    val = os.environ.get(env_key)
+                    if val:
+                        subprocess.run(
+                            [
+                                _OPENCLAW_BIN,
+                                "config",
+                                "set",
+                                _PARAM_MAP[env_key],
+                                val,
+                            ],
+                            capture_output=True,
+                            check=True,
+                            timeout=10,
+                        )
             except Exception as exc:
                 diag = (
                     f"[openclaw-acp-shim] set_model setup failed, continuing: {exc!r}"

{benchflow-0.7.6.dev2102 → benchflow-0.7.6.dev2208}/tests/test_bare_model_provider.py

@@ -27,6 +27,7 @@ import pytest
 from benchflow.agents.openclaw_acp_shim import (
     _default_max_tokens,
     _infer_provider_prefix,
+    _max_tokens_value,
     _resolve_bare_model_prefix,
     _setup_bare_custom_provider,
 )
@@ -37,9 +38,96 @@ from benchflow.agents.providers import (
 from benchflow.providers.litellm_config import safe_model_alias
 
 
-@pytest.mark.parametrize("model", ["gpt-5.4", "openai/gpt-5.4"])
-def test_gpt54_proxy_alias_has_supported_token_cap(model):
-    assert _default_max_tokens(safe_model_alias(model)) == 128000
+@pytest.mark.parametrize(
+    ("model", "cap"),
+    [
+        ("gpt-5.4", 128000),
+        ("openai/gpt-5.4", 128000),
+        ("us-openai/gpt-5.4", 128000),
+        ("azure-foundry-openai/gpt-5.4", 128000),
+        ("claude-sonnet-4-6", 128000),
+        ("claude-opus-4-6", 128000),
+        ("claude-opus-4-7", 128000),
+        ("claude-opus-4-8", 128000),
+        ("claude-sonnet-5", 128000),
+        ("claude-opus-5-1", 128000),
+        ("anthropic/claude-opus-5", 128000),
+        ("anthropic-vertex/claude-sonnet-5-2", 128000),
+        ("azure-foundry-anthropic/claude-sonnet-4-6", 128000),
+        ("claude-sonnet-4-60", None),
+        ("claude-sonnet-5-beta", None),
+        ("claude-opus-5-1-2", None),
+        ("claude-opus-50", None),
+        ("not-claude-sonnet-5", None),
+        ("evil/claude-opus-5", None),
+        ("evil/gpt-5.4", None),
+    ],
+)
+def test_model_token_cap(model, cap):
+    assert _default_max_tokens(model) == cap
+    assert _default_max_tokens(safe_model_alias(model)) == cap
+
+
+@pytest.mark.parametrize(
+    ("configured", "value"),
+    [
+        (None, "128000"),
+        ("invalid", "128000"),
+        ("-1", "128000"),
+        ("127999", "127999"),
+        ("128000", "128000"),
+        ("128001", "128000"),
+        ("9" * 5000, "128000"),
+    ],
+)
+def test_max_tokens_value(configured, value):
+    assert _max_tokens_value("claude-sonnet-5", configured) == value
+
+
+def test_uncapped_model_preserves_configured_max_tokens():
+    assert _max_tokens_value("other-model", "invalid") == "invalid"
+
+
+def test_set_model_writes_max_tokens_before_optional_params(monkeypatch):
+    import benchflow.agents.openclaw_acp_shim as shim
+
+    monkeypatch.setattr(shim, "setup_openai_auth", lambda: None)
+    monkeypatch.setattr(shim, "setup_gcloud_adc", lambda: None)
+    monkeypatch.setenv("BENCHFLOW_MODEL_MAX_TOKENS", "128001")
+    monkeypatch.setenv("BENCHFLOW_MODEL_TEMPERATURE", "0.5")
+
+    calls = []
+
+    def run(args, **_):
+        calls.append(args)
+        if args[-2] == "agents.defaults.params.temperature":
+            raise RuntimeError("temperature config failed")
+
+    monkeypatch.setattr(shim.subprocess, "run", run)
+    messages = [
+        {
+            "id": 1,
+            "method": "session/set_model",
+            "params": {"modelId": "anthropic/claude-sonnet-5"},
+        }
+    ]
+
+    def recv():
+        if messages:
+            return messages.pop()
+        raise EOFError
+
+    monkeypatch.setattr(shim, "recv", recv)
+    monkeypatch.setattr(shim, "send", lambda _: None)
+
+    shim.main()
+
+    writes = [(call[-2], call[-1]) for call in calls]
+    assert writes == [
+        ("agents.defaults.model", "anthropic/claude-sonnet-5"),
+        ("agents.defaults.params.maxTokens", "128000"),
+        ("agents.defaults.params.temperature", "0.5"),
+    ]
 
 
 class TestFindProviderForBareModel: