benchflow 0.5.3.dev899__tar.gz → 0.5.3.dev906__tar.gz

{benchflow-0.5.3.dev899 → benchflow-0.5.3.dev906}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev899
+Version: 0.5.3.dev906
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev899 → benchflow-0.5.3.dev906}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev899"
+version = "0.5.3.dev906"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev899 → benchflow-0.5.3.dev906}/src/benchflow/agents/registry.py

@@ -106,7 +106,7 @@ def _apt_install(*packages: str) -> str:
 _BENCHFLOW_NODE_PREFIX = "/opt/benchflow/node"
 _BENCHFLOW_JS_AGENT_PREFIX = "/opt/benchflow/js-agents"
 _BENCHFLOW_BIN_PREFIX = "/opt/benchflow/bin"
-_OPENHANDS_CLI_VERSION = "1.16.0"
+_OPENHANDS_CLI_GIT_REV = "3ca17446c5d9c1e35e054803478a3501ec251ecf"
 _OPENHANDS_SDK_VERSION = "1.22.1"
 _OPENHANDS_TOOLS_VERSION = "1.22.1"
 _JS_AGENT_PATH = (
@@ -546,15 +546,15 @@ AGENTS: dict[str, AgentConfig] = {
         install_cmd=(
             "export DEBIAN_FRONTEND=noninteractive && "
             'export PATH="$HOME/.local/bin:$PATH" && '
-            "( command -v curl >/dev/null 2>&1 || "
+            "( command -v curl >/dev/null 2>&1 && command -v git >/dev/null 2>&1 || "
             "  if command -v apt-get >/dev/null 2>&1; then "
-            f"    {_apt_install('curl', 'ca-certificates')}; "
+            f"    {_apt_install('curl', 'ca-certificates', 'git')}; "
             "  elif command -v dnf >/dev/null 2>&1; then "
-            "    dnf -y --allowerasing install curl ca-certificates >/dev/null 2>&1; "
+            "    dnf -y --allowerasing install curl ca-certificates git >/dev/null 2>&1; "
             "  elif command -v apk >/dev/null 2>&1; then "
-            "    apk add --no-cache curl ca-certificates >/dev/null 2>&1; "
+            "    apk add --no-cache curl ca-certificates git >/dev/null 2>&1; "
             "  else "
-            "    echo 'OpenHands install requires curl' >&2; "
+            "    echo 'OpenHands GitHub install requires curl and git' >&2; "
             "    exit 127; "
             "  fi ) && "
             "( UV_OK=0; "
@@ -569,18 +569,18 @@ AGENTS: dict[str, AgentConfig] = {
             "    curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null 2>&1 && "
             '    export PATH="$HOME/.local/bin:$PATH"; '
             "  fi && "
-            # OpenHands CLI 1.16.0 pins openhands-sdk/tools 1.21.0. That one
-            # SDK release makes the synthetic `security_risk` tool field
-            # required whenever LLMSecurityAnalyzer is attached; the ACP path
-            # attaches it even under --always-approve, so Claude Opus can loop
-            # on validation errors until timeout. 1.22.x restores the intended
-            # default-to-UNKNOWN behavior without the API drift seen in 1.26.x.
+            # Pin the OpenHands CLI source so the agent workflow cannot drift
+            # with GitHub main; only override the buggy sdk/tools 1.21.0 pins.
+            # SDK 1.22.x restores default-to-UNKNOWN for the synthetic
+            # `security_risk` tool field without the API drift seen in 1.26.x.
             f"printf 'openhands-sdk=={_OPENHANDS_SDK_VERSION}\\n"
             f"openhands-tools=={_OPENHANDS_TOOLS_VERSION}\\n' "
             "> /tmp/oh-sdk-overrides.txt && "
             "uv tool install --force --refresh "
             "--overrides /tmp/oh-sdk-overrides.txt "
-            f"openhands=={_OPENHANDS_CLI_VERSION} --python 3.12 && "
+            "--from "
+            f"'git+https://github.com/OpenHands/OpenHands-CLI.git@{_OPENHANDS_CLI_GIT_REV}' "
+            "openhands --python 3.12 && "
             "  uv tool list | grep -q '^openhands\\b' ) && "
             # Let sandbox user traverse to uv-managed Python interpreter path.
             "chmod o+x /root /root/.local /root/.local/share "

{benchflow-0.5.3.dev899 → benchflow-0.5.3.dev906}/tests/test_agent_registry.py

@@ -99,18 +99,23 @@ class TestOpenHandsConfig:
         assert "$HOME/.agents/skills" in cfg.skill_paths
         assert "$WORKSPACE/.agents/skills" in cfg.skill_paths
 
-    def test_openhands_install_cmd_pins_stable_pypi_release(self):
+    def test_openhands_install_cmd_pins_cli_git_revision(self):
         cfg = AGENTS["openhands"]
         assert (
-            "apt-get -o Acquire::Retries=3 install -y -qq curl ca-certificates"
+            "apt-get -o Acquire::Retries=3 install -y -qq curl ca-certificates git"
             in cfg.install_cmd
         )
         assert (
             "uv tool install --force --refresh "
             "--overrides /tmp/oh-sdk-overrides.txt "
-            "openhands==1.16.0 --python 3.12" in cfg.install_cmd
+            "--from "
+            "'git+https://github.com/OpenHands/OpenHands-CLI.git@"
+            "3ca17446c5d9c1e35e054803478a3501ec251ecf' "
+            "openhands --python 3.12" in cfg.install_cmd
         )
         assert "OpenHands/OpenHands-CLI.git@main" not in cfg.install_cmd
+        assert "openhands==1.16.0" not in cfg.install_cmd
+        assert "command -v git" in cfg.install_cmd
         assert "install.openhands.dev/install.sh" not in cfg.install_cmd
 
     def test_openhands_install_cmd_overrides_buggy_sdk_pin(self):

{benchflow-0.5.3.dev899 → benchflow-0.5.3.dev906}/tests/test_agent_setup.py

@@ -638,7 +638,9 @@ async def test_install_agent_writes_command_stdout_and_stderr_on_failure(
     assert (
         "uv tool install --force --refresh "
         "--overrides /tmp/oh-sdk-overrides.txt "
-        "openhands==1.16.0 --python 3.12" in log_text
+        "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@"
+        "3ca17446c5d9c1e35e054803478a3501ec251ecf' "
+        "openhands --python 3.12" in log_text
     )
     assert "=== stderr ===" in log_text
     assert "uv: command not found" in log_text

benchflow-0.5.3.dev906/tests/test_skillsbench_publish_scrub.py

@@ -0,0 +1,75 @@
+import importlib.util
+import json
+from pathlib import Path
+
+
+def _load_publish_module():
+    path = (
+        Path(__file__).resolve().parents[1]
+        / "experiments"
+        / "skillsbench-fill"
+        / "publish.py"
+    )
+    spec = importlib.util.spec_from_file_location("skillsbench_fill_publish", path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_safe_bytes_preserves_token_usage_counters(tmp_path):
+    """Guards Hugging Face PR #4 token-usage recovery against scrub redaction."""
+    publish = _load_publish_module()
+    result = {
+        "agent_result": {
+            "n_input_tokens": 123,
+            "n_output_tokens": 45,
+            "n_cache_read_tokens": 678,
+            "n_cache_creation_tokens": 90,
+            "total_tokens": 936,
+            "usage_source": "provider_response",
+        },
+        "final_metrics": {
+            "total_prompt_tokens": 123,
+            "total_completion_tokens": 45,
+            "total_cached_tokens": 678,
+            "total_cost_usd": 1.23,
+        },
+        "HUGGING_FACE_TOKEN": "hf_abcdefghijklmnopqrstuvwxyz",
+    }
+    src = tmp_path / "result.json"
+    src.write_text(json.dumps(result))
+
+    scrubbed = json.loads(publish.safe_bytes(src))
+
+    assert scrubbed["agent_result"]["n_input_tokens"] == 123
+    assert scrubbed["agent_result"]["n_output_tokens"] == 45
+    assert scrubbed["agent_result"]["n_cache_read_tokens"] == 678
+    assert scrubbed["agent_result"]["n_cache_creation_tokens"] == 90
+    assert scrubbed["agent_result"]["total_tokens"] == 936
+    assert scrubbed["final_metrics"]["total_prompt_tokens"] == 123
+    assert scrubbed["final_metrics"]["total_completion_tokens"] == 45
+    assert scrubbed["final_metrics"]["total_cached_tokens"] == 678
+    assert scrubbed["HUGGING_FACE_TOKEN"] == "[REDACTED]"
+
+
+def test_safe_bytes_normalizes_config_without_leaking_secret_tokens(tmp_path):
+    """Guards Hugging Face PR #4 token-usage recovery against credential leakage."""
+    publish = _load_publish_module()
+    config = {
+        "include_task_skills": False,
+        "agent_env": {
+            "AWS_BEARER_TOKEN_BEDROCK": "Bearer abcdefghijklmnop",
+            "OPENAI_API_KEY": "sk-abcdefghijklmnopqrstuvwxyz",
+            "BENCHFLOW_MODEL_MAX_TOKENS": 8192,
+        },
+    }
+    src = tmp_path / "config.json"
+    src.write_text(json.dumps(config))
+
+    scrubbed = json.loads(publish.safe_bytes(src, is_config=True, mode="with"))
+
+    assert scrubbed["include_task_skills"] is True
+    assert scrubbed["agent_env"]["AWS_BEARER_TOKEN_BEDROCK"] == "[REDACTED]"
+    assert scrubbed["agent_env"]["OPENAI_API_KEY"] == "[REDACTED]"
+    assert scrubbed["agent_env"]["BENCHFLOW_MODEL_MAX_TOKENS"] == 8192