benchflow 0.5.3.dev885__tar.gz → 0.5.3.dev899__tar.gz

{benchflow-0.5.3.dev885 → benchflow-0.5.3.dev899}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev885
+Version: 0.5.3.dev899
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev885 → benchflow-0.5.3.dev899}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev885"
+version = "0.5.3.dev899"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev885 → benchflow-0.5.3.dev899}/src/benchflow/agents/registry.py

@@ -106,6 +106,9 @@ def _apt_install(*packages: str) -> str:
 _BENCHFLOW_NODE_PREFIX = "/opt/benchflow/node"
 _BENCHFLOW_JS_AGENT_PREFIX = "/opt/benchflow/js-agents"
 _BENCHFLOW_BIN_PREFIX = "/opt/benchflow/bin"
+_OPENHANDS_CLI_VERSION = "1.16.0"
+_OPENHANDS_SDK_VERSION = "1.22.1"
+_OPENHANDS_TOOLS_VERSION = "1.22.1"
 _JS_AGENT_PATH = (
     f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:"
     f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH"
@@ -543,15 +546,15 @@ AGENTS: dict[str, AgentConfig] = {
         install_cmd=(
             "export DEBIAN_FRONTEND=noninteractive && "
             'export PATH="$HOME/.local/bin:$PATH" && '
-            "( command -v curl >/dev/null 2>&1 && command -v git >/dev/null 2>&1 || "
+            "( command -v curl >/dev/null 2>&1 || "
             "  if command -v apt-get >/dev/null 2>&1; then "
-            f"    {_apt_install('curl', 'ca-certificates', 'git')}; "
+            f"    {_apt_install('curl', 'ca-certificates')}; "
             "  elif command -v dnf >/dev/null 2>&1; then "
-            "    dnf -y --allowerasing install curl ca-certificates git >/dev/null 2>&1; "
+            "    dnf -y --allowerasing install curl ca-certificates >/dev/null 2>&1; "
             "  elif command -v apk >/dev/null 2>&1; then "
-            "    apk add --no-cache curl ca-certificates git >/dev/null 2>&1; "
+            "    apk add --no-cache curl ca-certificates >/dev/null 2>&1; "
             "  else "
-            "    echo 'OpenHands GitHub install requires curl and git' >&2; "
+            "    echo 'OpenHands install requires curl' >&2; "
             "    exit 127; "
             "  fi ) && "
             "( UV_OK=0; "
@@ -566,9 +569,18 @@ AGENTS: dict[str, AgentConfig] = {
             "    curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null 2>&1 && "
             '    export PATH="$HOME/.local/bin:$PATH"; '
             "  fi && "
+            # OpenHands CLI 1.16.0 pins openhands-sdk/tools 1.21.0. That one
+            # SDK release makes the synthetic `security_risk` tool field
+            # required whenever LLMSecurityAnalyzer is attached; the ACP path
+            # attaches it even under --always-approve, so Claude Opus can loop
+            # on validation errors until timeout. 1.22.x restores the intended
+            # default-to-UNKNOWN behavior without the API drift seen in 1.26.x.
+            f"printf 'openhands-sdk=={_OPENHANDS_SDK_VERSION}\\n"
+            f"openhands-tools=={_OPENHANDS_TOOLS_VERSION}\\n' "
+            "> /tmp/oh-sdk-overrides.txt && "
             "uv tool install --force --refresh "
-            "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@main' "
-            "openhands --python 3.12 && "
+            "--overrides /tmp/oh-sdk-overrides.txt "
+            f"openhands=={_OPENHANDS_CLI_VERSION} --python 3.12 && "
             "  uv tool list | grep -q '^openhands\\b' ) && "
             # Let sandbox user traverse to uv-managed Python interpreter path.
             "chmod o+x /root /root/.local /root/.local/share "

{benchflow-0.5.3.dev885 → benchflow-0.5.3.dev899}/tests/test_agent_registry.py

@@ -99,20 +99,29 @@ class TestOpenHandsConfig:
         assert "$HOME/.agents/skills" in cfg.skill_paths
         assert "$WORKSPACE/.agents/skills" in cfg.skill_paths
 
-    def test_openhands_install_cmd_forces_github_main(self):
+    def test_openhands_install_cmd_pins_stable_pypi_release(self):
         cfg = AGENTS["openhands"]
         assert (
-            "apt-get -o Acquire::Retries=3 install -y -qq curl ca-certificates git"
+            "apt-get -o Acquire::Retries=3 install -y -qq curl ca-certificates"
             in cfg.install_cmd
         )
         assert (
             "uv tool install --force --refresh "
-            "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@main' "
-            "openhands --python 3.12" in cfg.install_cmd
+            "--overrides /tmp/oh-sdk-overrides.txt "
+            "openhands==1.16.0 --python 3.12" in cfg.install_cmd
         )
-        assert "command -v git" in cfg.install_cmd
+        assert "OpenHands/OpenHands-CLI.git@main" not in cfg.install_cmd
         assert "install.openhands.dev/install.sh" not in cfg.install_cmd
 
+    def test_openhands_install_cmd_overrides_buggy_sdk_pin(self):
+        """Guards PR #644 against Opus timeouts from OpenHands SDK 1.21.0."""
+        cfg = AGENTS["openhands"]
+
+        assert "openhands-sdk==1.22.1" in cfg.install_cmd
+        assert "openhands-tools==1.22.1" in cfg.install_cmd
+        assert "openhands-sdk>=1.22.0" not in cfg.install_cmd
+        assert "--overrides /tmp/oh-sdk-overrides.txt" in cfg.install_cmd
+
     def test_openhands_install_cmd_does_not_deploy_bedrock_shim(self):
         """Guards the LiteLLM runtime refactor: Bedrock patches live with LiteLLM."""
         cfg = AGENTS["openhands"]

{benchflow-0.5.3.dev885 → benchflow-0.5.3.dev899}/tests/test_agent_setup.py

@@ -637,8 +637,8 @@ async def test_install_agent_writes_command_stdout_and_stderr_on_failure(
     assert log_text.startswith("$ ")
     assert (
         "uv tool install --force --refresh "
-        "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@main' "
-        "openhands --python 3.12" in log_text
+        "--overrides /tmp/oh-sdk-overrides.txt "
+        "openhands==1.16.0 --python 3.12" in log_text
     )
     assert "=== stderr ===" in log_text
     assert "uv: command not found" in log_text

benchflow-0.5.3.dev899/tests/test_hf_scores.py

@@ -0,0 +1,91 @@
+"""Unit tests for the Experiments page's HF PR score aggregation."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+_DASHBOARD = Path(__file__).resolve().parent.parent / "dashboard"
+if str(_DASHBOARD) not in sys.path:
+    sys.path.insert(0, str(_DASHBOARD))
+
+import hf_scores  # noqa: E402
+
+
+def test_build_scoreboard_combines_hf_pr_modes(monkeypatch):
+    """Guards the HF-score dashboard PR against mixing pass-rate and fill progress."""
+
+    def fake_analysis(buckets):
+        hf_scores._add_bucket(
+            buckets,
+            pr=2,
+            harness="openhands",
+            model="gpt-5.5",
+            mode="with-skills",
+            passed=6,
+            failed=4,
+        )
+        hf_scores._add_bucket(
+            buckets,
+            pr=2,
+            harness="openhands",
+            model="gpt-5.5",
+            mode="without-skills",
+            passed=2,
+            failed=8,
+        )
+        return []
+
+    def fake_direct(buckets):
+        hf_scores._add_bucket(
+            buckets,
+            pr=5,
+            harness="openhands",
+            model="gpt-5.5",
+            mode="with-skills",
+            passed=1,
+            failed=0,
+        )
+        return []
+
+    monkeypatch.setattr(hf_scores, "_read_analysis_prs", fake_analysis)
+    monkeypatch.setattr(hf_scores, "_read_direct_prs", fake_direct)
+
+    result = hf_scores.build_scoreboard()
+
+    with_row = result["by_mode"]["with-skills"][0]
+    assert with_row["label"] == "OpenHands GPT-5.5"
+    assert with_row["passed"] == 7
+    assert with_row["total"] == 11
+    assert with_row["prs"] == [2, 5]
+    gain_row = result["by_mode"]["normalized-gain"][0]
+    assert gain_row["without_pass_rate"] == 0.2
+    assert round(gain_row["gain"], 3) == 0.545
+
+
+def test_snapshot_serves_cached_hf_scores_without_inline_refresh(tmp_path, monkeypatch):
+    """Guards the HF-score dashboard PR against blocking live requests on HF."""
+    cache = {
+        "as_of": "2026-06-08T00:00:00+00:00",
+        "source": "HuggingFace PR2/PR3/PR4/PR5",
+        "repo": hf_scores.REPO,
+        "refs": ["refs/pr/2", "refs/pr/3", "refs/pr/4", "refs/pr/5"],
+        "scored_trials": 1,
+        "groups": 1,
+        "by_mode": {"with-skills": [], "without-skills": [], "normalized-gain": []},
+        "warnings": [],
+        "warning_count": 0,
+    }
+    path = tmp_path / "hf_scoreboard_cache.json"
+    path.write_text(json.dumps(cache))
+
+    def fail_refresh():
+        raise AssertionError("snapshot should not refresh inline by default")
+
+    monkeypatch.setattr(hf_scores, "build_scoreboard", fail_refresh)
+
+    result = hf_scores.snapshot(path)
+
+    assert result["cached"] is True
+    assert result["scored_trials"] == 1