hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/cli/tests/test_dev.py

@@ -0,0 +1,197 @@
+"""Tests for CLI dev module."""
+
+from __future__ import annotations
+
+from unittest import mock
+
+from hud.cli.dev import auto_detect_module, should_use_docker_mode
+
+
+class TestShouldUseDockerMode:
+    """Test Docker mode detection."""
+
+    def test_docker_mode_with_dockerfile(self, tmp_path):
+        """Test detection when Dockerfile exists."""
+        dockerfile = tmp_path / "Dockerfile"
+        dockerfile.write_text("FROM python:3.11")
+
+        assert should_use_docker_mode(tmp_path) is True
+
+    def test_no_docker_mode_without_dockerfile(self, tmp_path):
+        """Test detection when Dockerfile doesn't exist."""
+        assert should_use_docker_mode(tmp_path) is False
+
+    def test_docker_mode_empty_dockerfile(self, tmp_path):
+        """Test detection with empty Dockerfile."""
+        dockerfile = tmp_path / "Dockerfile"
+        dockerfile.write_text("")
+
+        assert should_use_docker_mode(tmp_path) is True
+
+
+class TestAutoDetectModule:
+    """Test MCP module auto-detection."""
+
+    def test_detect_module_from_init_with_mcpserver(self, tmp_path, monkeypatch):
+        """Test detection from __init__.py with MCPServer."""
+        monkeypatch.chdir(tmp_path)
+
+        init_file = tmp_path / "__init__.py"
+        init_file.write_text("""
+from hud.server import MCPServer
+mcp = MCPServer(name='test')
+""")
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name == tmp_path.name
+        assert extra_path is None
+
+    def test_detect_module_from_init_with_fastmcp(self, tmp_path, monkeypatch):
+        """Test detection from __init__.py with FastMCP."""
+        monkeypatch.chdir(tmp_path)
+
+        init_file = tmp_path / "__init__.py"
+        init_file.write_text("""
+from fastmcp import FastMCP
+mcp = FastMCP(name='test')
+""")
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name == tmp_path.name
+        assert extra_path is None
+
+    def test_detect_module_from_main_py(self, tmp_path, monkeypatch):
+        """Test detection from main.py with MCPServer."""
+        monkeypatch.chdir(tmp_path)
+
+        # Need both __init__.py and main.py
+        init_file = tmp_path / "__init__.py"
+        init_file.write_text("")
+
+        main_file = tmp_path / "main.py"
+        main_file.write_text("""
+from hud.server import MCPServer
+mcp = MCPServer(name='test')
+""")
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name == f"{tmp_path.name}.main"
+        assert extra_path == tmp_path.parent
+
+    def test_detect_module_from_init_with_environment(self, tmp_path, monkeypatch):
+        """Test detection from __init__.py with Environment."""
+        monkeypatch.chdir(tmp_path)
+
+        init_file = tmp_path / "__init__.py"
+        init_file.write_text("""
+from hud import Environment
+env = Environment(name='test')
+""")
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name == tmp_path.name
+        assert extra_path is None
+
+    def test_detect_module_from_main_py_with_environment(self, tmp_path, monkeypatch):
+        """Test detection from main.py with Environment."""
+        monkeypatch.chdir(tmp_path)
+
+        # Need both __init__.py and main.py
+        init_file = tmp_path / "__init__.py"
+        init_file.write_text("")
+
+        main_file = tmp_path / "main.py"
+        main_file.write_text("""
+from hud import Environment
+env = Environment(name='test')
+""")
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name == f"{tmp_path.name}.main"
+        assert extra_path == tmp_path.parent
+
+    def test_no_detection_without_mcp_or_env(self, tmp_path, monkeypatch):
+        """Test no detection when neither mcp nor env is defined."""
+        monkeypatch.chdir(tmp_path)
+
+        init_file = tmp_path / "__init__.py"
+        init_file.write_text("# Just a comment")
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name is None
+        assert extra_path is None
+
+    def test_no_detection_empty_dir(self, tmp_path, monkeypatch):
+        """Test no detection in empty directory."""
+        monkeypatch.chdir(tmp_path)
+
+        module_name, extra_path = auto_detect_module()
+
+        assert module_name is None
+        assert extra_path is None
+
+
+class TestShowDevServerInfo:
+    """Test dev server info display."""
+
+    @mock.patch("hud.cli.dev.hud_console")
+    def test_show_dev_server_info_http(self, mock_console):
+        """Test showing server info for HTTP transport."""
+        from hud.cli.dev import show_dev_server_info
+
+        result = show_dev_server_info(
+            server_name="test-server",
+            port=8000,
+            transport="http",
+            inspector=False,
+            interactive=False,
+        )
+
+        # Returns cursor deeplink
+        assert result.startswith("cursor://")
+        assert "test-server" in result
+
+        # Console should have been called
+        assert mock_console.section_title.called
+        assert mock_console.info.called
+
+    @mock.patch("hud.cli.dev.hud_console")
+    def test_show_dev_server_info_stdio(self, mock_console):
+        """Test showing server info for stdio transport."""
+        from hud.cli.dev import show_dev_server_info
+
+        result = show_dev_server_info(
+            server_name="test-server",
+            port=8000,
+            transport="stdio",
+            inspector=False,
+            interactive=False,
+        )
+
+        # Returns cursor deeplink
+        assert result.startswith("cursor://")
+
+    @mock.patch("hud.cli.dev.hud_console")
+    def test_show_dev_server_info_with_telemetry(self, mock_console):
+        """Test showing server info with telemetry URLs."""
+        from hud.cli.dev import show_dev_server_info
+
+        result = show_dev_server_info(
+            server_name="browser-env",
+            port=8000,
+            transport="http",
+            inspector=False,
+            interactive=False,
+            telemetry={
+                "live_url": "https://hud.ai/trace/123",
+                "vnc_url": "http://localhost:5900",
+            },
+        )
+
+        assert result.startswith("cursor://")

hud/cli/tests/test_eval.py

@@ -0,0 +1,251 @@
+"""Tests for hud.cli.eval module and run_dataset function."""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from mcp import types
+
+from hud.environment.router import ToolRouter
+from hud.eval.context import EvalContext
+from hud.types import AgentType, MCPToolResult, Trace
+
+
+class MockEvalContext(EvalContext):
+    """Mock EvalContext for testing."""
+
+    def __init__(
+        self,
+        prompt: str = "Test prompt",
+        tools: list[types.Tool] | None = None,
+    ) -> None:
+        # Core attributes
+        self.prompt = prompt
+        self._tools = tools or []
+        self._submitted: str | None = None
+        self.reward: float | None = None
+        self.results: list[EvalContext] = []
+
+        # Environment attributes
+        self._router = ToolRouter()
+        self._agent_include: list[str] | None = None
+        self._agent_exclude: list[str] | None = None
+
+        # EvalContext attributes
+        self._task = None
+        self.trace_id = "test-trace-id"
+        self.eval_name = "test-eval"
+        self.job_id: str | None = None
+        self.group_id: str | None = None
+        self.index = 0
+        self.variants: dict[str, Any] = {}
+        self.answer: str | None = None
+        self.system_prompt: str | None = None
+        self.error: BaseException | None = None
+        self.metadata: dict[str, Any] = {}
+        self._is_summary = False
+
+    def as_tools(self) -> list[types.Tool]:
+        return self._tools
+
+    @property
+    def has_scenario(self) -> bool:
+        return False
+
+    async def list_tools(self) -> list[types.Tool]:
+        return self._tools
+
+    async def call_tool(self, call: Any, /, **kwargs: Any) -> MCPToolResult:
+        return MCPToolResult(
+            content=[types.TextContent(type="text", text="ok")],
+            isError=False,
+        )
+
+    async def submit(self, answer: str) -> None:
+        self._submitted = answer
+
+
+def _create_mock_agent_cls() -> tuple[MagicMock, MagicMock]:
+    """Create a mock agent class and instance for testing."""
+    mock_agent_instance = MagicMock()
+    mock_agent_instance.run = AsyncMock(return_value=Trace(reward=1.0, done=True))
+    mock_agent_cls = MagicMock()
+    mock_agent_cls.create.return_value = mock_agent_instance
+    return mock_agent_cls, mock_agent_instance
+
+
+class TestRunDataset:
+    """Test the new run_dataset function."""
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_with_task_list(self) -> None:
+        """Test run_dataset with a list of tasks."""
+        from hud.eval.task import Task
+
+        tasks = [
+            Task(env={"name": "test"}, id="task1", scenario="test"),
+            Task(env={"name": "test"}, id="task2", scenario="test"),
+        ]
+        mock_agent_cls, mock_agent_instance = _create_mock_agent_cls()
+
+        # Mock hud.eval to return our mock context
+        mock_ctx = MockEvalContext()
+
+        with (
+            patch("hud.datasets.runner.hud.eval") as mock_eval,
+            patch("hud.agents.claude.ClaudeAgent", mock_agent_cls),
+        ):
+            # Set up the async context manager
+            mock_eval.return_value.__aenter__ = AsyncMock(return_value=mock_ctx)
+            mock_eval.return_value.__aexit__ = AsyncMock(return_value=None)
+
+            from hud.datasets.runner import run_dataset
+
+            await run_dataset(tasks, agent_type="claude", max_steps=5)
+
+            # Verify hud.eval was called with correct params
+            mock_eval.assert_called_once()
+            call_kwargs = mock_eval.call_args[1]
+            assert call_kwargs["group"] == 1
+            assert call_kwargs["max_concurrent"] == 30
+
+            # Agent should have run
+            mock_agent_instance.run.assert_called_once()
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_with_string_source(self) -> None:
+        """Test run_dataset with a string source (loads via load_dataset)."""
+        from hud.eval.task import Task
+
+        mock_tasks = [Task(env={"name": "test"}, id="loaded_task", scenario="loaded")]
+        mock_agent_cls, _ = _create_mock_agent_cls()
+        mock_ctx = MockEvalContext()
+
+        with (
+            patch("hud.datasets.loader.load_tasks", return_value=mock_tasks) as mock_load,
+            patch("hud.datasets.runner.hud.eval") as mock_eval,
+            patch("hud.agents.OpenAIAgent", mock_agent_cls),
+        ):
+            mock_eval.return_value.__aenter__ = AsyncMock(return_value=mock_ctx)
+            mock_eval.return_value.__aexit__ = AsyncMock(return_value=None)
+
+            from hud.datasets.runner import run_dataset
+
+            await run_dataset("my-tasks.json", agent_type="openai")
+
+            # Verify load_dataset was called
+            mock_load.assert_called_once_with("my-tasks.json")
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_empty_tasks_raises(self) -> None:
+        """Test run_dataset raises ValueError for empty tasks."""
+        with patch("hud.datasets.loader.load_dataset", return_value=[]):
+            from hud.datasets.runner import run_dataset
+
+            with pytest.raises(ValueError, match="No tasks to run"):
+                await run_dataset([], agent_type=AgentType.CLAUDE)
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_with_group_size(self) -> None:
+        """Test run_dataset passes group_size to hud.eval."""
+        from hud.eval.task import Task
+
+        tasks = [Task(env={"name": "test"}, id="task1", scenario="test")]
+        mock_agent_cls, _ = _create_mock_agent_cls()
+        mock_ctx = MockEvalContext()
+
+        with (
+            patch("hud.datasets.runner.hud.eval") as mock_eval,
+            patch("hud.agents.claude.ClaudeAgent", mock_agent_cls),
+        ):
+            mock_eval.return_value.__aenter__ = AsyncMock(return_value=mock_ctx)
+            mock_eval.return_value.__aexit__ = AsyncMock(return_value=None)
+
+            from hud.datasets.runner import run_dataset
+
+            await run_dataset(tasks, agent_type="claude", group_size=3)
+
+            call_kwargs = mock_eval.call_args[1]
+            assert call_kwargs["group"] == 3
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_with_max_concurrent(self) -> None:
+        """Test run_dataset passes max_concurrent to hud.eval."""
+        from hud.eval.task import Task
+
+        tasks = [Task(env={"name": "test"}, id="task1", scenario="test")]
+        mock_agent_cls, _ = _create_mock_agent_cls()
+        mock_ctx = MockEvalContext()
+
+        with (
+            patch("hud.datasets.runner.hud.eval") as mock_eval,
+            patch("hud.agents.claude.ClaudeAgent", mock_agent_cls),
+        ):
+            mock_eval.return_value.__aenter__ = AsyncMock(return_value=mock_ctx)
+            mock_eval.return_value.__aexit__ = AsyncMock(return_value=None)
+
+            from hud.datasets.runner import run_dataset
+
+            await run_dataset(tasks, agent_type="claude", max_concurrent=10)
+
+            call_kwargs = mock_eval.call_args[1]
+            assert call_kwargs["max_concurrent"] == 10
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_returns_results(self) -> None:
+        """Test run_dataset returns EvalContext results."""
+        from hud.eval.task import Task
+
+        tasks = [Task(env={"name": "test"}, id="task1", scenario="test")]
+        mock_agent_cls, _ = _create_mock_agent_cls()
+        mock_ctx = MockEvalContext()
+
+        with (
+            patch("hud.datasets.runner.hud.eval") as mock_eval,
+            patch("hud.agents.claude.ClaudeAgent", mock_agent_cls),
+        ):
+            mock_eval.return_value.__aenter__ = AsyncMock(return_value=mock_ctx)
+            mock_eval.return_value.__aexit__ = AsyncMock(return_value=None)
+
+            from hud.datasets.runner import run_dataset
+
+            results = await run_dataset(tasks, agent_type="claude")
+
+            # Should return list with the context
+            assert len(results) == 1
+            assert results[0] is mock_ctx
+
+    @pytest.mark.asyncio
+    async def test_run_dataset_parallel_results(self) -> None:
+        """Test run_dataset returns ctx.results for parallel execution."""
+        from hud.eval.task import Task
+
+        tasks = [Task(env={"name": "test"}, id="task1", scenario="test")]
+        mock_agent_cls, _ = _create_mock_agent_cls()
+
+        # Create mock context with results (parallel execution)
+        mock_result1 = MockEvalContext(prompt="result1")
+        mock_result1.reward = 0.8
+        mock_result2 = MockEvalContext(prompt="result2")
+        mock_result2.reward = 0.9
+
+        mock_ctx = MockEvalContext()
+        mock_ctx.results = [mock_result1, mock_result2]
+
+        with (
+            patch("hud.datasets.runner.hud.eval") as mock_eval,
+            patch("hud.agents.claude.ClaudeAgent", mock_agent_cls),
+        ):
+            mock_eval.return_value.__aenter__ = AsyncMock(return_value=mock_ctx)
+            mock_eval.return_value.__aexit__ = AsyncMock(return_value=None)
+
+            from hud.datasets.runner import run_dataset
+
+            results = await run_dataset(tasks, agent_type="claude")
+
+            # Should return the parallel results
+            assert len(results) == 2
+            assert results[0].reward == 0.8
+            assert results[1].reward == 0.9

hud/cli/tests/test_eval_bedrock.py

@@ -0,0 +1,51 @@
+"""Tests for AWS Bedrock auto-detection in hud.cli.eval."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+import typer
+
+from hud.cli.eval import EvalConfig
+from hud.types import AgentType
+
+
+class TestBedrockAutoDetection:
+    VALID_ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/my-profile"
+
+    def test_get_agent_kwargs_detects_bedrock_arn_from_config_checkpoint_name(self) -> None:
+        """Regression: ARN in [claude].checkpoint_name should trigger Bedrock client."""
+        cfg = EvalConfig(
+            agent_type=AgentType.CLAUDE,
+            model=None,  # no CLI --model
+            agent_config={"claude": {"checkpoint_name": self.VALID_ARN}},
+        )
+
+        with (
+            patch("hud.settings.settings.aws_access_key_id", "AKIATEST"),
+            patch("hud.settings.settings.aws_secret_access_key", "secret"),
+            patch("hud.settings.settings.aws_region", "us-east-1"),
+            patch("anthropic.AsyncAnthropicBedrock", return_value=MagicMock()) as mock_bedrock,
+        ):
+            kwargs = cfg.get_agent_kwargs()
+
+        assert kwargs.get("checkpoint_name") == self.VALID_ARN
+        assert "model_client" in kwargs
+        mock_bedrock.assert_called_once()
+
+    def test_get_agent_kwargs_bedrock_arn_missing_aws_creds_exits(self) -> None:
+        """Should fail fast if ARN is detected but AWS creds are missing."""
+        cfg = EvalConfig(
+            agent_type=AgentType.CLAUDE,
+            model=None,
+            agent_config={"claude": {"checkpoint_name": self.VALID_ARN}},
+        )
+
+        with (
+            patch("hud.settings.settings.aws_access_key_id", None),
+            patch("hud.settings.settings.aws_secret_access_key", None),
+            patch("hud.settings.settings.aws_region", None),
+            pytest.raises(typer.Exit),
+        ):
+            cfg.get_agent_kwargs()

hud/cli/tests/test_init.py

@@ -0,0 +1,124 @@
+"""Tests for CLI init module."""
+
+from __future__ import annotations
+
+from hud.cli.init import _replace_placeholders
+
+
+class TestReplacePlaceholders:
+    """Test placeholder replacement in template files."""
+
+    def test_replace_in_pyproject(self, tmp_path):
+        """Test replacing placeholders in pyproject.toml."""
+        # Create server directory structure
+        server_dir = tmp_path / "server"
+        server_dir.mkdir()
+
+        pyproject = server_dir / "pyproject.toml"
+        pyproject.write_text("""
+[project]
+name = "blank"
+description = "blank environment"
+""")
+
+        modified = _replace_placeholders(tmp_path, "my-cool-env")
+
+        # Normalize paths for cross-platform comparison
+        modified_normalized = [p.replace("\\", "/") for p in modified]
+        assert "server/pyproject.toml" in modified_normalized
+        content = pyproject.read_text()
+        assert "my_cool_env" in content
+        assert "blank" not in content
+
+    def test_replace_in_readme(self, tmp_path):
+        """Test replacing placeholders in README.md."""
+        readme = tmp_path / "README.md"
+        readme.write_text("# blank\n\nThis is the blank environment.")
+
+        modified = _replace_placeholders(tmp_path, "test-env")
+
+        assert "README.md" in modified
+        content = readme.read_text()
+        assert "test_env" in content
+        assert "blank" not in content
+
+    def test_replace_in_tasks_json(self, tmp_path):
+        """Test replacing placeholders in tasks.json."""
+        tasks = tmp_path / "tasks.json"
+        tasks.write_text('{"name": "blank", "tasks": []}')
+
+        modified = _replace_placeholders(tmp_path, "my-tasks")
+
+        assert "tasks.json" in modified
+        content = tasks.read_text()
+        assert "my_tasks" in content
+
+    def test_no_replace_in_non_placeholder_files(self, tmp_path):
+        """Test that non-placeholder files are not modified."""
+        other_file = tmp_path / "other.py"
+        other_file.write_text("# blank comment")
+
+        modified = _replace_placeholders(tmp_path, "test")
+
+        assert "other.py" not in modified
+        content = other_file.read_text()
+        assert "blank" in content  # Should be unchanged
+
+    def test_skip_pycache_directories(self, tmp_path):
+        """Test that __pycache__ directories are skipped."""
+        pycache = tmp_path / "__pycache__"
+        pycache.mkdir()
+
+        cached_file = pycache / "module.pyc"
+        cached_file.write_text("blank")
+
+        modified = _replace_placeholders(tmp_path, "test")
+
+        # __pycache__ files should not be in modified list
+        assert not any("__pycache__" in f for f in modified)
+
+    def test_normalize_special_characters(self, tmp_path):
+        """Test that environment name is normalized for Python identifiers."""
+        server_dir = tmp_path / "server"
+        server_dir.mkdir()
+
+        pyproject = server_dir / "pyproject.toml"
+        pyproject.write_text('name = "blank"')
+
+        _replace_placeholders(tmp_path, "my cool-env.v2!")
+
+        content = pyproject.read_text()
+        # Special characters should be replaced with underscores
+        assert "my_cool_env_v2_" in content
+
+    def test_no_changes_when_no_placeholder(self, tmp_path):
+        """Test that files without placeholder are not modified."""
+        server_dir = tmp_path / "server"
+        server_dir.mkdir()
+
+        pyproject = server_dir / "pyproject.toml"
+        pyproject.write_text('name = "other-name"')
+
+        modified = _replace_placeholders(tmp_path, "test")
+
+        assert "server/pyproject.toml" not in modified
+
+    def test_nested_directory_structure(self, tmp_path):
+        """Test replacement in nested directory structure."""
+        # Create nested structure
+        server_dir = tmp_path / "server"
+        server_dir.mkdir()
+        (server_dir / "pyproject.toml").write_text('name = "blank"')
+
+        env_dir = tmp_path / "environment"
+        env_dir.mkdir()
+        (env_dir / "pyproject.toml").write_text('name = "blank"')
+        (env_dir / "README.md").write_text("# blank environment")
+
+        modified = _replace_placeholders(tmp_path, "nested-test")
+
+        # Normalize paths for cross-platform comparison
+        modified_normalized = [p.replace("\\", "/") for p in modified]
+        assert "server/pyproject.toml" in modified_normalized
+        assert "environment/pyproject.toml" in modified_normalized
+        assert "environment/README.md" in modified_normalized

hud/cli/tests/test_main_module.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 import subprocess
 import sys
 
@@ -20,11 +21,16 @@ class TestMainModule:
     def test_main_module_executes(self):
         """Test that running the module as main executes correctly."""
         # Use subprocess to run the module as __main__ and check it doesn't crash
-        # We expect it to show help/error since we're not providing arguments
+        # Use --version flag for a quick, deterministic test that doesn't require user input
+        env = {**os.environ, "HUD_SKIP_VERSION_CHECK": "1"}
         result = subprocess.run(
-            [sys.executable, "-m", "hud.cli"], capture_output=True, text=True, timeout=10
+            [sys.executable, "-m", "hud.cli", "--version"],
+            capture_output=True,
+            text=True,
+            timeout=30,
+            env=env,
         )
 
-        # Should exit with an error code but not crash
-        # (The actual main function will show help or error for missing args)
-        assert result.returncode != 0  # CLI should exit with error for no args
+        # Should exit successfully with version info
+        assert result.returncode == 0
+        assert "version" in result.stdout.lower() or "hud" in result.stdout.lower()