benchflow 0.5.3.dev894__tar.gz → 0.5.3.dev902__tar.gz

{benchflow-0.5.3.dev894 → benchflow-0.5.3.dev902}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchflow
-Version: 0.5.3.dev894
+Version: 0.5.3.dev902
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.5.3.dev894 → benchflow-0.5.3.dev902}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.5.3.dev894"
+version = "0.5.3.dev902"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.5.3.dev894 → benchflow-0.5.3.dev902}/src/benchflow/agents/registry.py

@@ -106,6 +106,9 @@ def _apt_install(*packages: str) -> str:
 _BENCHFLOW_NODE_PREFIX = "/opt/benchflow/node"
 _BENCHFLOW_JS_AGENT_PREFIX = "/opt/benchflow/js-agents"
 _BENCHFLOW_BIN_PREFIX = "/opt/benchflow/bin"
+_OPENHANDS_CLI_GIT_REV = "3ca17446c5d9c1e35e054803478a3501ec251ecf"
+_OPENHANDS_SDK_VERSION = "1.22.1"
+_OPENHANDS_TOOLS_VERSION = "1.22.1"
 _JS_AGENT_PATH = (
     f"{_BENCHFLOW_BIN_PREFIX}:{_BENCHFLOW_JS_AGENT_PREFIX}/bin:"
     f"{_BENCHFLOW_NODE_PREFIX}/bin:$PATH"
@@ -566,8 +569,17 @@ AGENTS: dict[str, AgentConfig] = {
             "    curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null 2>&1 && "
             '    export PATH="$HOME/.local/bin:$PATH"; '
             "  fi && "
+            # Pin the OpenHands CLI source so the agent workflow cannot drift
+            # with GitHub main; only override the buggy sdk/tools 1.21.0 pins.
+            # SDK 1.22.x restores default-to-UNKNOWN for the synthetic
+            # `security_risk` tool field without the API drift seen in 1.26.x.
+            f"printf 'openhands-sdk=={_OPENHANDS_SDK_VERSION}\\n"
+            f"openhands-tools=={_OPENHANDS_TOOLS_VERSION}\\n' "
+            "> /tmp/oh-sdk-overrides.txt && "
             "uv tool install --force --refresh "
-            "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@main' "
+            "--overrides /tmp/oh-sdk-overrides.txt "
+            "--from "
+            f"'git+https://github.com/OpenHands/OpenHands-CLI.git@{_OPENHANDS_CLI_GIT_REV}' "
             "openhands --python 3.12 && "
             "  uv tool list | grep -q '^openhands\\b' ) && "
             # Let sandbox user traverse to uv-managed Python interpreter path.

{benchflow-0.5.3.dev894 → benchflow-0.5.3.dev902}/tests/test_agent_registry.py

@@ -99,7 +99,7 @@ class TestOpenHandsConfig:
         assert "$HOME/.agents/skills" in cfg.skill_paths
         assert "$WORKSPACE/.agents/skills" in cfg.skill_paths
 
-    def test_openhands_install_cmd_forces_github_main(self):
+    def test_openhands_install_cmd_pins_cli_git_revision(self):
         cfg = AGENTS["openhands"]
         assert (
             "apt-get -o Acquire::Retries=3 install -y -qq curl ca-certificates git"
@@ -107,12 +107,26 @@ class TestOpenHandsConfig:
         )
         assert (
             "uv tool install --force --refresh "
-            "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@main' "
+            "--overrides /tmp/oh-sdk-overrides.txt "
+            "--from "
+            "'git+https://github.com/OpenHands/OpenHands-CLI.git@"
+            "3ca17446c5d9c1e35e054803478a3501ec251ecf' "
             "openhands --python 3.12" in cfg.install_cmd
         )
+        assert "OpenHands/OpenHands-CLI.git@main" not in cfg.install_cmd
+        assert "openhands==1.16.0" not in cfg.install_cmd
         assert "command -v git" in cfg.install_cmd
         assert "install.openhands.dev/install.sh" not in cfg.install_cmd
 
+    def test_openhands_install_cmd_overrides_buggy_sdk_pin(self):
+        """Guards PR #644 against Opus timeouts from OpenHands SDK 1.21.0."""
+        cfg = AGENTS["openhands"]
+
+        assert "openhands-sdk==1.22.1" in cfg.install_cmd
+        assert "openhands-tools==1.22.1" in cfg.install_cmd
+        assert "openhands-sdk>=1.22.0" not in cfg.install_cmd
+        assert "--overrides /tmp/oh-sdk-overrides.txt" in cfg.install_cmd
+
     def test_openhands_install_cmd_does_not_deploy_bedrock_shim(self):
         """Guards the LiteLLM runtime refactor: Bedrock patches live with LiteLLM."""
         cfg = AGENTS["openhands"]

{benchflow-0.5.3.dev894 → benchflow-0.5.3.dev902}/tests/test_agent_setup.py

@@ -637,7 +637,9 @@ async def test_install_agent_writes_command_stdout_and_stderr_on_failure(
     assert log_text.startswith("$ ")
     assert (
         "uv tool install --force --refresh "
-        "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@main' "
+        "--overrides /tmp/oh-sdk-overrides.txt "
+        "--from 'git+https://github.com/OpenHands/OpenHands-CLI.git@"
+        "3ca17446c5d9c1e35e054803478a3501ec251ecf' "
         "openhands --python 3.12" in log_text
     )
     assert "=== stderr ===" in log_text