hud-python 0.4.47__py3-none-any.whl → 0.4.49__py3-none-any.whl

hud/cli/tests/test_eval.py

@@ -0,0 +1,574 @@
+"""Tests for hud.cli.eval module."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import pytest
+from mcp import types
+
+from hud.cli.eval import (
+    build_agent,
+    run_single_task,
+)
+from hud.types import Task, Trace
+
+
+class TestBuildAgent:
+    """Test the build_agent function."""
+
+    def test_builds_integration_test_agent(self) -> None:
+        """
+        Test building an integration test agent.
+        """
+        with patch("hud.agents.misc.integration_test_agent.IntegrationTestRunner") as mock_runner:
+            mock_instance = Mock()
+            mock_runner.return_value = mock_instance
+
+            # Test with verbose=False
+            result = build_agent("integration_test", verbose=False)
+
+            mock_runner.assert_called_once_with(verbose=False)
+            assert result == mock_instance
+
+    def test_builds_claude_agent(self) -> None:
+        """
+        Test building a Claude agent with default model.
+        """
+        with patch("hud.agents.ClaudeAgent") as mock_runner:
+            mock_instance = Mock()
+            mock_runner.return_value = mock_instance
+
+            # Test with verbose=False
+            result = build_agent("claude", verbose=False)
+
+            mock_runner.assert_called_once_with(model="claude-sonnet-4-20250514", verbose=False)
+            assert result == mock_instance
+
+    def test_builds_claude_agent_with_custom_model_and_allowed_tools(self) -> None:
+        """
+        Test building a Claude agent with custom model name and allowed tools.
+        """
+        with patch("hud.agents.ClaudeAgent") as mock_runner:
+            mock_instance = Mock()
+            mock_runner.return_value = mock_instance
+
+            # Test with verbose=False
+            result = build_agent(
+                "claude",
+                model="claude-sonnet-4-20250514",
+                allowed_tools=["act"],
+                verbose=True,
+            )
+
+            mock_runner.assert_called_once_with(
+                model="claude-sonnet-4-20250514",
+                allowed_tools=["act"],
+                verbose=True,
+            )
+            assert result == mock_instance
+
+
+class TestRunSingleTask:
+    """Test the run_single_task function."""
+
+    @pytest.mark.asyncio
+    async def test_applies_agent_config_from_task(self) -> None:
+        """Test that task.agent_config is applied during agent initialization."""
+        mock_task = Task(
+            prompt="Test",
+            mcp_config={"local": {"url": "http://localhost:8765/mcp"}},
+            agent_config={
+                "system_prompt": "Custom instructions",
+                "allowed_tools": ["tool1", "tool2"],
+                "append_setup_output": False,
+            },
+        )
+        mock_agent = AsyncMock(
+            initialize=AsyncMock(), run=AsyncMock(return_value=Trace(reward=1.0, done=True))
+        )
+
+        with (
+            patch("hud.utils.tasks.load_tasks", return_value=[mock_task]),
+            patch(
+                "hud.agents.misc.integration_test_agent.IntegrationTestRunner",
+                return_value=mock_agent,
+            ),
+            patch("hud.cli.eval.find_environment_dir", return_value=None),
+            patch("hud.cli.eval.hud.trace"),
+        ):
+            await run_single_task("test.json", agent_type="integration_test", max_steps=10)
+
+            # Verify agent.run was called with the task containing agent_config
+            mock_agent.run.assert_called_once()
+            called_task = mock_agent.run.call_args[0][0]
+            assert called_task.agent_config == mock_task.agent_config
+
+    @pytest.mark.asyncio
+    async def test_runs_with_group_size_greater_than_one(self) -> None:
+        """Test that group_size > 1 triggers run_tasks_grouped instead of agent.run."""
+        mock_task = Task(prompt="Test", mcp_config={"local": {"url": "http://localhost:8765/mcp"}})
+
+        with (
+            patch("hud.utils.tasks.load_tasks", return_value=[mock_task]),
+            patch("hud.cli.eval.run_tasks_grouped", new_callable=AsyncMock) as mock_grouped,
+            patch("hud.cli.eval.display_group_statistics"),
+            patch("hud.cli.eval.find_environment_dir", return_value=None),
+            patch("hud.cli.eval.hud.trace"),
+        ):
+            mock_grouped.return_value = [{"task": mock_task, "rewards": [1.0, 0.5]}]
+
+            await run_single_task(
+                "test.json", agent_type="integration_test", group_size=3, max_steps=10
+            )
+
+            # Verify run_tasks_grouped was called with correct group_size
+            mock_grouped.assert_called_once()
+            assert mock_grouped.call_args.kwargs["group_size"] == 3
+            assert mock_grouped.call_args.kwargs["max_steps"] == 10
+
+
+class TestToolFiltering:
+    """Test wildcard tool filtering via agent_config in tasks."""
+
+    @pytest.fixture
+    def mock_mcp_client(self):
+        """Fixture for mock MCP client."""
+        client = MagicMock()
+        client.initialize = AsyncMock()
+        client.mcp_config = {"local": {"url": "http://localhost"}}
+        return client
+
+    @pytest.fixture
+    def mock_model_client(self):
+        """Fixture for mock Anthropic client."""
+        return MagicMock()
+
+    async def _run_agent_with_tools(
+        self,
+        mock_mcp_client: MagicMock,
+        mock_model_client: MagicMock,
+        tools: list[types.Tool],
+        agent_config: dict | None = None,
+    ) -> list[types.Tool]:
+        """Helper to create agent, initialize with tools and config, return filtered tools."""
+        from hud.agents import ClaudeAgent
+
+        mock_mcp_client.list_tools = AsyncMock(return_value=tools)
+
+        task = Task(
+            prompt="Test",
+            mcp_config={"local": {"url": "http://localhost"}},
+            agent_config=agent_config or {},
+        )
+
+        agent = ClaudeAgent(
+            mcp_client=mock_mcp_client,
+            model_client=mock_model_client,
+            model="test",
+            validate_api_key=False,
+        )
+        await agent.initialize(task)
+        return agent.get_available_tools()
+
+    @pytest.mark.asyncio
+    async def test_no_filters_returns_all_tools(self, mock_mcp_client, mock_model_client) -> None:
+        """Test that no filters in agent_config returns all tools."""
+        tools = [
+            types.Tool(name="tool1", description="Tool 1", inputSchema={}),
+            types.Tool(name="tool2", description="Tool 2", inputSchema={}),
+            types.Tool(name="debug_tool", description="Debug", inputSchema={}),
+        ]
+
+        result = await self._run_agent_with_tools(mock_mcp_client, mock_model_client, tools)
+
+        assert len(result) == 3
+
+    @pytest.mark.asyncio
+    async def test_allowed_tools_filters_correctly(
+        self, mock_mcp_client, mock_model_client
+    ) -> None:
+        """Test that allowed_tools in agent_config filters to matching patterns."""
+        tools = [
+            types.Tool(name="screenshot_take", description="Tool 1", inputSchema={}),
+            types.Tool(name="screenshot_full", description="Tool 2", inputSchema={}),
+            types.Tool(name="click", description="Tool 3", inputSchema={}),
+        ]
+        agent_config = {"allowed_tools": ["screenshot_*"]}
+
+        result = await self._run_agent_with_tools(
+            mock_mcp_client, mock_model_client, tools, agent_config
+        )
+
+        assert len(result) == 2
+        assert all("screenshot" in t.name for t in result)
+
+    @pytest.mark.asyncio
+    async def test_disallowed_tools_excludes_correctly(
+        self, mock_mcp_client, mock_model_client
+    ) -> None:
+        """Test that disallowed_tools in agent_config excludes matching patterns."""
+        tools = [
+            types.Tool(name="tool1", description="Tool 1", inputSchema={}),
+            types.Tool(name="debug_tool", description="Tool 2", inputSchema={}),
+            types.Tool(name="internal_secret", description="Tool 3", inputSchema={}),
+        ]
+        agent_config = {"disallowed_tools": ["debug_*", "internal_*"]}
+
+        result = await self._run_agent_with_tools(
+            mock_mcp_client, mock_model_client, tools, agent_config
+        )
+
+        assert len(result) == 1
+        assert result[0].name == "tool1"
+
+    @pytest.mark.asyncio
+    async def test_both_filters_applies_allowed_then_disallowed(
+        self, mock_mcp_client, mock_model_client
+    ) -> None:
+        """Test that both filters in agent_config work together (disallowed takes precedence)."""
+        tools = [
+            types.Tool(name="browser_click", description="Tool 1", inputSchema={}),
+            types.Tool(name="browser_debug", description="Tool 2", inputSchema={}),
+            types.Tool(name="system_click", description="Tool 3", inputSchema={}),
+        ]
+        agent_config = {"allowed_tools": ["browser_*"], "disallowed_tools": ["*_debug"]}
+
+        result = await self._run_agent_with_tools(
+            mock_mcp_client, mock_model_client, tools, agent_config
+        )
+
+        assert len(result) == 1
+        assert result[0].name == "browser_click"
+
+
+class TestRunDatasetToolFiltering:
+    """Test tool filtering via run_dataset with agent_config in both init and task."""
+
+    @pytest.fixture
+    def all_tools(self):
+        """Fixture for a standard set of tools."""
+        return [
+            types.Tool(name="browser_click", description="Click", inputSchema={}),
+            types.Tool(name="browser_type", description="Type", inputSchema={}),
+            types.Tool(name="browser_debug", description="Debug", inputSchema={}),
+            types.Tool(name="system_screenshot", description="Screenshot", inputSchema={}),
+            types.Tool(name="system_execute", description="Execute", inputSchema={}),
+        ]
+
+    @pytest.fixture
+    def captured_agent_fixture(self):
+        """Fixture that returns a dictionary to capture the agent instance."""
+        return {"agent": None}
+
+    @pytest.fixture
+    def mock_run_context(self, captured_agent_fixture):
+        """Fixture for mocking _run_context."""
+
+        async def _mock(self, context, max_steps=10):
+            captured_agent_fixture["agent"] = self
+            return Trace(reward=1.0, done=True, content="Done")
+
+        return _mock
+
+    @pytest.fixture
+    def mock_call_tools(self):
+        """Fixture for mocking call_tools."""
+
+        async def _mock(self, tool_call=None):
+            return []
+
+        return _mock
+
+    @pytest.fixture
+    def mock_client_instance(self, all_tools):
+        """Fixture for mock MCP client instance."""
+        mock_client = MagicMock()
+        mock_client.initialize = AsyncMock()
+        mock_client.list_tools = AsyncMock(return_value=all_tools)
+        mock_client.shutdown = AsyncMock()
+        mock_client.mcp_config = {"local": {"url": "http://localhost:8765/mcp"}}
+        return mock_client
+
+    @pytest.mark.asyncio
+    async def test_agent_config_intersection_union_via_run_dataset(
+        self,
+        all_tools,
+        captured_agent_fixture,
+        mock_run_context,
+        mock_call_tools,
+        mock_client_instance,
+    ) -> None:
+        """Test that allowed_tools intersect and disallowed_tools union when set in both __init__ and task.agent_config."""  # noqa: E501
+        from hud.agents import ClaudeAgent
+        from hud.datasets.runner import run_dataset
+
+        # Create a task with its own agent_config
+        task_dict = {
+            "prompt": "Test task",
+            "mcp_config": {"local": {"url": "http://localhost:8765/mcp"}},
+            "agent_config": {
+                "allowed_tools": [
+                    "browser_*",
+                    "system_screenshot",
+                ],  # Task wants browser_* and system_screenshot
+                "disallowed_tools": [
+                    "*_debug",
+                    "*_execute",
+                ],  # Task disallows *_debug and *_execute
+            },
+        }
+
+        # Agent config passed to __init__ via run_dataset
+        agent_init_config = {
+            "allowed_tools": ["browser_*", "system_*"],  # Agent init wants browser_* and system_*
+            "disallowed_tools": ["browser_debug"],  # Agent init disallows browser_debug
+            "validate_api_key": False,
+        }
+
+        with (
+            patch("hud.job"),
+            patch("hud.trace"),
+            patch.object(ClaudeAgent, "_run_context", mock_run_context),
+            patch.object(ClaudeAgent, "call_tools", mock_call_tools),
+            patch("hud.clients.MCPClient", return_value=mock_client_instance),
+        ):
+            # Run the dataset
+            await run_dataset(
+                name="test_job",
+                dataset=[task_dict],
+                agent_class=ClaudeAgent,
+                agent_config=agent_init_config,
+                max_steps=10,
+            )
+
+            # Verify agent was created and ran
+            captured_agent = captured_agent_fixture["agent"]
+            assert captured_agent is not None
+
+            # Get the filtered tools
+            filtered_tools = captured_agent.get_available_tools()
+            filtered_names = {tool.name for tool in filtered_tools}
+
+            # Expected behavior:
+            # 1. allowed_tools intersection: ["browser_*", "system_*"] ∩ ["browser_*", "system_screenshot"] # noqa: E501
+            #    Exact string intersection: only "browser_*" is in both lists
+            #    So only tools matching browser_* are allowed: browser_click, browser_type, browser_debug # noqa: E501
+            # 2. disallowed_tools union: ["browser_debug"] U ["*_debug", "*_execute"]
+            #    Result: ["browser_debug", "*_debug", "*_execute"] (all patterns included)
+            # 3. Final: {browser_click, browser_type, browser_debug} - {browser_debug}
+            #    Result: browser_click, browser_type
+
+            expected_tools = {"browser_click", "browser_type"}
+            assert filtered_names == expected_tools, (
+                f"Expected {expected_tools}, got {filtered_names}"
+            )
+
+    @pytest.mark.asyncio
+    async def test_no_allowed_tools_keeps_all_tools_except_disallowed(
+        self,
+        all_tools,
+        captured_agent_fixture,
+        mock_run_context,
+        mock_call_tools,
+        mock_client_instance,
+    ) -> None:
+        """Test that when allowed_tools is not set, all tools are available except disallowed ones."""  # noqa: E501
+        from hud.agents import ClaudeAgent
+        from hud.datasets.runner import run_dataset
+
+        # Create a task with its own agent_config (no allowed_tools)
+        task_dict = {
+            "prompt": "Test task",
+            "mcp_config": {"local": {"url": "http://localhost:8765/mcp"}},
+            "agent_config": {
+                # No allowed_tools set - should allow all tools
+                "disallowed_tools": ["*_execute"],  # Task disallows *_execute
+            },
+        }
+
+        # Agent config passed to __init__ via run_dataset (no allowed_tools)
+        agent_init_config = {
+            # No allowed_tools set - should allow all tools
+            "disallowed_tools": ["browser_debug"],  # Agent init disallows browser_debug
+            "validate_api_key": False,
+        }
+
+        with (
+            patch("hud.job"),
+            patch("hud.trace"),
+            patch.object(ClaudeAgent, "_run_context", mock_run_context),
+            patch.object(ClaudeAgent, "call_tools", mock_call_tools),
+            patch("hud.clients.MCPClient", return_value=mock_client_instance),
+        ):
+            # Run the dataset
+            await run_dataset(
+                name="test_job",
+                dataset=[task_dict],
+                agent_class=ClaudeAgent,
+                agent_config=agent_init_config,
+                max_steps=10,
+            )
+
+            # Verify agent was created and ran
+            captured_agent = captured_agent_fixture["agent"]
+            assert captured_agent is not None
+
+            # Get the filtered tools
+            filtered_tools = captured_agent.get_available_tools()
+            filtered_names = {tool.name for tool in filtered_tools}
+
+            # Expected behavior:
+            # 1. allowed_tools: None (no allowed_tools set in either init or task)
+            #    Result: All tools are initially allowed
+            # 2. disallowed_tools union: ["browser_debug"] U ["*_execute"]
+            #    Result: ["browser_debug", "*_execute"] (all patterns included)
+            # 3. Final: {all tools} - {browser_debug, system_execute}
+            #    Result: browser_click, browser_type, system_screenshot
+
+            expected_tools = {"browser_click", "browser_type", "system_screenshot"}
+            assert filtered_names == expected_tools, (
+                f"Expected {expected_tools}, got {filtered_names}"
+            )
+
+
+class TestSystemPromptHandling:
+    """Test system prompt handling through run_dataset flow."""
+
+    @pytest.fixture
+    def mock_mcp_client(self):
+        """Fixture for mock MCP client."""
+        client = MagicMock()
+        client.initialize = AsyncMock()
+        client.list_tools = AsyncMock(return_value=[])
+        client.shutdown = AsyncMock()
+        client.mcp_config = {"local": {"url": "http://localhost:8765/mcp"}}
+        return client
+
+    @pytest.fixture
+    def captured_agent_fixture(self):
+        """Fixture that returns a dictionary to capture the agent instance."""
+        return {"agent": None}
+
+    @pytest.fixture
+    def mock_run_context(self, captured_agent_fixture):
+        """Fixture for mocking _run_context to capture agent."""
+
+        async def _mock(self, context, max_steps=10):
+            captured_agent_fixture["agent"] = self
+            return Trace(reward=1.0, done=True, content="Done")
+
+        return _mock
+
+    @pytest.fixture
+    def mock_call_tools(self):
+        """Fixture for mocking call_tools."""
+
+        async def _mock(self, tool_call=None):
+            return []
+
+        return _mock
+
+    @pytest.mark.asyncio
+    async def test_task_system_prompt_only(
+        self, captured_agent_fixture, mock_run_context, mock_call_tools, mock_mcp_client
+    ) -> None:
+        """Test that task system_prompt is appended when agent has default system prompt."""
+        from hud.agents import ClaudeAgent
+        from hud.agents.base import GLOBAL_SYSTEM_PROMPT
+        from hud.datasets.runner import run_dataset
+
+        task_system_prompt = "Task prompt"
+
+        # Create a task with its own system_prompt in agent_config
+        task_dict = {
+            "prompt": "Test task",
+            "mcp_config": {"local": {"url": "http://localhost:8765/mcp"}},
+            "agent_config": {
+                "system_prompt": task_system_prompt,
+            },
+        }
+
+        # Agent config with no custom system_prompt (will use default)
+        agent_init_config = {
+            "validate_api_key": False,
+        }
+
+        with (
+            patch("hud.job"),
+            patch("hud.trace"),
+            patch.object(ClaudeAgent, "_run_context", mock_run_context),
+            patch.object(ClaudeAgent, "call_tools", mock_call_tools),
+            patch("hud.clients.MCPClient", return_value=mock_mcp_client),
+        ):
+            # Run the dataset
+            await run_dataset(
+                name="test_job",
+                dataset=[task_dict],
+                agent_class=ClaudeAgent,
+                agent_config=agent_init_config,
+                max_steps=10,
+            )
+
+            # Verify agent was created and ran
+            captured_agent = captured_agent_fixture["agent"]
+            assert captured_agent is not None
+
+            # Verify the task system prompt was appended
+            assert captured_agent.system_prompt.endswith(f"\n\n{task_system_prompt}")
+            # Verify it starts with the base global system prompt
+            assert captured_agent.system_prompt.startswith(GLOBAL_SYSTEM_PROMPT)
+
+    @pytest.mark.asyncio
+    async def test_both_agent_and_task_system_prompts(
+        self, captured_agent_fixture, mock_run_context, mock_call_tools, mock_mcp_client
+    ) -> None:
+        """Test that both agent init and task system prompts are present when both are set."""
+        from hud.agents import ClaudeAgent
+        from hud.datasets.runner import run_dataset
+
+        agent_custom_prompt = "Agent init prompt"
+        task_system_prompt = "Task prompt"
+
+        # Create a task with its own system_prompt in agent_config
+        task_dict = {
+            "prompt": "Test task",
+            "mcp_config": {"local": {"url": "http://localhost:8765/mcp"}},
+            "agent_config": {
+                "system_prompt": task_system_prompt,
+            },
+        }
+
+        # Agent config WITH custom system_prompt
+        agent_init_config = {
+            "system_prompt": agent_custom_prompt,
+            "validate_api_key": False,
+        }
+
+        with (
+            patch("hud.job"),
+            patch("hud.trace"),
+            patch.object(ClaudeAgent, "_run_context", mock_run_context),
+            patch.object(ClaudeAgent, "call_tools", mock_call_tools),
+            patch("hud.clients.MCPClient", return_value=mock_mcp_client),
+        ):
+            # Run the dataset
+            await run_dataset(
+                name="test_job",
+                dataset=[task_dict],
+                agent_class=ClaudeAgent,
+                agent_config=agent_init_config,
+                max_steps=10,
+            )
+
+            # Verify agent was created and ran
+            captured_agent = captured_agent_fixture["agent"]
+            assert captured_agent is not None
+
+            # Verify the task system prompt was appended at the end
+            assert captured_agent.system_prompt.endswith(f"\n\n{task_system_prompt}")
+            # Verify it starts with the agent custom prompt
+            assert captured_agent.system_prompt.startswith(agent_custom_prompt)
+            # Verify both prompts are present
+            assert agent_custom_prompt in captured_agent.system_prompt
+            assert task_system_prompt in captured_agent.system_prompt

hud/cli/tests/test_mcp_server.py

@@ -2,101 +2,15 @@
 
 from __future__ import annotations
 
-from pathlib import Path  # noqa: TC003
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
 from hud.cli.dev import (
-    create_proxy_server,
-    get_docker_cmd,
-    get_image_name,
     run_mcp_dev_server,
-    update_pyproject_toml,
 )
 
 
-class TestCreateMCPServer:
-    """Test MCP server creation."""
-
-    def test_create_mcp_server(self) -> None:
-        """Test that MCP server is created with correct configuration."""
-        mcp = create_proxy_server(".", "test-image:latest")
-        assert mcp._mcp_server.name == "HUD Dev Proxy - test-image:latest"
-        # Proxy server doesn't define its own tools, it forwards to Docker containers
-
-
-class TestDockerUtils:
-    """Test Docker utility functions."""
-
-    def test_get_docker_cmd(self) -> None:
-        """Test extracting CMD from Docker image."""
-        with patch("subprocess.run") as mock_run:
-            mock_result = MagicMock()
-            mock_result.returncode = 0
-            mock_result.stdout = '["python", "-m", "server"]'
-            mock_run.return_value = mock_result
-
-            cmd = get_docker_cmd("test-image:latest")
-            assert cmd is None
-
-    def test_get_docker_cmd_failure(self) -> None:
-        """Test handling when Docker inspect fails."""
-        import subprocess
-
-        with patch("subprocess.run") as mock_run:
-            # check=True causes CalledProcessError on non-zero return
-            mock_run.side_effect = subprocess.CalledProcessError(1, "docker inspect")
-
-            cmd = get_docker_cmd("test-image:latest")
-            assert cmd is None
-
-
-class TestImageResolution:
-    """Test image name resolution."""
-
-    def test_get_image_name_override(self) -> None:
-        """Test image name with override."""
-        name, source = get_image_name(".", "custom-image:v1")
-        assert name == "custom-image:v1"
-        assert source == "override"
-
-    def test_get_image_name_from_pyproject(self, tmp_path: Path) -> None:
-        """Test image name from pyproject.toml."""
-        pyproject = tmp_path / "pyproject.toml"
-        pyproject.write_text("""
-[tool.hud]
-image = "my-project:latest"
-""")
-
-        name, source = get_image_name(str(tmp_path))
-        assert name == "my-project:latest"
-        assert source == "cache"
-
-    def test_get_image_name_auto_generate(self, tmp_path: Path) -> None:
-        """Test auto-generated image name."""
-        test_dir = tmp_path / "my_test_project"
-        test_dir.mkdir()
-
-        name, source = get_image_name(str(test_dir))
-        assert name == "my-test-project:dev"
-        assert source == "auto"
-
-    def test_update_pyproject_toml(self, tmp_path: Path) -> None:
-        """Test updating pyproject.toml with image name."""
-        pyproject = tmp_path / "pyproject.toml"
-        pyproject.write_text("""
-[project]
-name = "test"
-""")
-
-        update_pyproject_toml(str(tmp_path), "new-image:v1", silent=True)
-
-        content = pyproject.read_text()
-        assert "[tool.hud]" in content
-        assert 'image = "new-image:v1"' in content
-
-
 class TestRunMCPDevServer:
     """Test the main server runner."""
 
@@ -110,16 +24,13 @@ class TestRunMCPDevServer:
             pytest.raises(click.Abort),
         ):
             run_mcp_dev_server(
-                directory=".",
-                image="missing:latest",
-                build=False,
-                no_cache=False,
-                transport="http",
+                module=".",
+                stdio=False,
                 port=8765,
-                no_reload=False,
                 verbose=False,
                 inspector=False,
-                no_logs=False,
-                docker_args=[],
                 interactive=False,
+                watch=[],
+                docker=False,
+                docker_args=[],
             )