hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/eval/tests/test_parallel.py

@@ -0,0 +1,168 @@
+"""Tests for hud.eval.parallel module."""
+
+from __future__ import annotations
+
+import ast
+
+import pytest
+
+from hud.eval.parallel import (
+    ASTExtractionError,
+    _extract_body,
+    _find_async_with,
+    _get_end_line,
+    expand_variants,
+    resolve_group_ids,
+)
+
+
+class TestExpandVariants:
+    """Tests for expand_variants helper."""
+
+    def test_none_returns_empty_dict(self) -> None:
+        """None variants returns list with empty dict."""
+        result = expand_variants(None)
+        assert result == [{}]
+
+    def test_empty_dict_returns_empty_dict(self) -> None:
+        """Empty variants returns list with empty dict."""
+        result = expand_variants({})
+        assert result == [{}]
+
+    def test_single_value_stays_single(self) -> None:
+        """Single non-list value stays as single variant."""
+        result = expand_variants({"model": "gpt-4o"})
+        assert result == [{"model": "gpt-4o"}]
+
+    def test_list_expands_to_variants(self) -> None:
+        """List value expands to multiple variants."""
+        result = expand_variants({"model": ["gpt-4o", "claude"]})
+        assert result == [{"model": "gpt-4o"}, {"model": "claude"}]
+
+    def test_multiple_lists_create_combinations(self) -> None:
+        """Multiple lists create all combinations."""
+        result = expand_variants(
+            {
+                "model": ["a", "b"],
+                "temp": [0.0, 1.0],
+            }
+        )
+
+        assert len(result) == 4
+        assert {"model": "a", "temp": 0.0} in result
+        assert {"model": "a", "temp": 1.0} in result
+        assert {"model": "b", "temp": 0.0} in result
+        assert {"model": "b", "temp": 1.0} in result
+
+    def test_mixed_single_and_list(self) -> None:
+        """Mixed single values and lists work correctly."""
+        result = expand_variants(
+            {
+                "model": ["gpt-4o", "claude"],
+                "temp": 0.7,
+            }
+        )
+
+        assert len(result) == 2
+        assert {"model": "gpt-4o", "temp": 0.7} in result
+        assert {"model": "claude", "temp": 0.7} in result
+
+
+class TestResolveGroupIds:
+    """Tests for resolve_group_ids helper."""
+
+    def test_uses_provided_group_ids(self) -> None:
+        """Uses provided group_ids when given."""
+        result = resolve_group_ids(["a", "b", "c"], 3)
+        assert result == ["a", "b", "c"]
+
+    def test_generates_shared_group_id(self) -> None:
+        """Generates shared group_id when not provided."""
+        result = resolve_group_ids(None, 3)
+        assert len(result) == 3
+        # All should be the same
+        assert result[0] == result[1] == result[2]
+        # Should be a valid UUID
+        assert len(result[0]) == 36
+
+    def test_raises_on_length_mismatch(self) -> None:
+        """Raises ValueError when group_ids length doesn't match."""
+        with pytest.raises(ValueError, match="group_ids length"):
+            resolve_group_ids(["a", "b"], 3)
+
+
+class TestASTHelpers:
+    """Tests for AST helper functions."""
+
+    def test_find_async_with_finds_correct_node(self) -> None:
+        """_find_async_with finds the async with containing target line."""
+        source = """
+async def main():
+    x = 1
+    async with something as ctx:
+        do_stuff()
+        more_stuff()
+    y = 2
+"""
+        tree = ast.parse(source)
+
+        # Line 5 is inside the async with
+        node = _find_async_with(tree, 5)
+        assert node is not None
+        assert isinstance(node, ast.AsyncWith)
+
+    def test_find_async_with_returns_none_when_not_found(self) -> None:
+        """_find_async_with returns None when line is outside async with."""
+        source = """
+async def main():
+    x = 1
+    async with something as ctx:
+        do_stuff()
+    y = 2
+"""
+        tree = ast.parse(source)
+
+        # Line 7 is outside the async with
+        node = _find_async_with(tree, 7)
+        assert node is None
+
+    def test_get_end_line(self) -> None:
+        """_get_end_line returns last line of node."""
+        source = """
+async with ctx:
+    line1()
+    line2()
+    line3()
+"""
+        tree = ast.parse(source)
+        async_with = tree.body[0]
+
+        end_line = _get_end_line(async_with)
+        assert end_line >= 4  # At least through line 4
+
+    def test_extract_body(self) -> None:
+        """_extract_body extracts the body source from async with."""
+        source = """async with ctx:
+    do_thing()
+    more_thing()
+"""
+        lines = source.split("\n")
+        lines = [line + "\n" for line in lines]
+
+        tree = ast.parse(source)
+        async_with = tree.body[0]
+        assert isinstance(async_with, ast.AsyncWith)
+
+        body = _extract_body(lines, async_with)
+        assert "do_thing()" in body
+        assert "more_thing()" in body
+
+
+class TestASTExtractionError:
+    """Tests for ASTExtractionError."""
+
+    def test_is_exception(self) -> None:
+        """ASTExtractionError is an exception."""
+        error = ASTExtractionError("test message")
+        assert isinstance(error, Exception)
+        assert str(error) == "test message"

hud/eval/tests/test_task.py

@@ -0,0 +1,291 @@
+"""Tests for hud.eval.task module."""
+
+from __future__ import annotations
+
+import pytest
+
+from hud.eval.task import Task, TaskAgentConfig
+
+
+class TestTaskSerialization:
+    """Tests for Task serialization and roundtrip."""
+
+    def test_v5_task_roundtrip(self) -> None:
+        """v5 Task serializes and deserializes correctly."""
+        task = Task(
+            env={"name": "browser", "include": ["navigate", "click"]},
+            scenario="checkout",
+            id="task-1",
+            args={"user_id": "alice"},
+        )
+
+        # Serialize
+        data = task.model_dump(mode="json")
+
+        # Should have v5 format
+        assert "env" in data
+        assert data["env"]["name"] == "browser"
+        assert data["scenario"] == "checkout"
+        assert data["id"] == "task-1"
+
+        # Recreate from serialized data
+        task2 = Task(**data)
+
+        # Serialize again
+        data2 = task2.model_dump(mode="json")
+
+        # Should be identical
+        assert data == data2
+
+    def test_v4_task_roundtrip(self) -> None:
+        """v4 Task serializes (flattens) and deserializes correctly."""
+        v4_dict = {
+            "prompt": "Go to google.com and search for cats",
+            "mcp_config": {
+                "browser": {"url": "http://localhost:8080"},
+            },
+            "evaluate_tool": {"name": "check_url", "arguments": {"contains": "google"}},
+            "setup_tool": {"name": "navigate", "arguments": {"url": "about:blank"}},
+            "id": "v4-task-1",
+            "agent_config": {"system_prompt": "You are a helpful assistant"},
+            "metadata": {"category": "navigation"},
+        }
+
+        # Create Task from v4 dict
+        task = Task.from_v4(v4_dict)
+
+        # Serialize (should flatten to v4 format)
+        data = task.model_dump(mode="json")
+
+        # Should have v4 format (flat, not nested env)
+        assert "prompt" in data
+        assert "mcp_config" in data
+        assert "evaluate_tool" in data
+        assert data["prompt"] == "Go to google.com and search for cats"
+        assert data["id"] == "v4-task-1"
+
+        # Recreate from serialized data
+        task2 = Task(**data)
+
+        # Serialize again
+        data2 = task2.model_dump(mode="json")
+
+        # Should be identical
+        assert data == data2
+
+    def test_v4_preserves_agent_config(self) -> None:
+        """v4 Task preserves agent_config through roundtrip."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {"system_prompt": "Custom system prompt"},
+        }
+
+        task = Task.from_v4(v4_dict)
+        data = task.model_dump(mode="json")
+
+        # agent_config should preserve system_prompt and restore tool filters
+        agent_config = data.get("agent_config")
+        assert agent_config is not None
+        assert agent_config["system_prompt"] == "Custom system prompt"
+        # allowed_tools defaults to ["*"] when not specified (restored during serialization)
+        assert agent_config["allowed_tools"] == ["*"]
+        # These have default False values from TaskAgentConfig
+        assert agent_config["append_setup_output"] is False
+        assert agent_config["append_setup_tool"] is False
+
+        # Roundtrip
+        task2 = Task(**data)
+        assert task2.agent_config is not None
+        assert isinstance(task2.agent_config, TaskAgentConfig)
+        assert task2.agent_config.system_prompt == "Custom system prompt"
+        # Tool filters should be on Environment after roundtrip
+        assert task2.env is not None
+        assert task2.env._agent_include is None  # ["*"] → None
+
+    def test_v4_preserves_metadata(self) -> None:
+        """v4 Task preserves metadata through roundtrip."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "metadata": {"key1": "value1", "key2": 42},
+        }
+
+        task = Task.from_v4(v4_dict)
+        data = task.model_dump(mode="json")
+
+        assert data.get("metadata") == {"key1": "value1", "key2": 42}
+
+        # Roundtrip
+        task2 = Task(**data)
+        assert task2.metadata == {"key1": "value1", "key2": 42}
+
+
+class TestTaskValidation:
+    """Tests for Task validation."""
+
+    def test_v5_allows_none_env(self) -> None:
+        """v5 Task allows None env (for blank evals)."""
+        task = Task(scenario="test")  # env=None is valid
+        assert task.env is None
+        assert task.scenario == "test"
+
+    def test_v4_requires_evaluate_tool(self) -> None:
+        """v4 Task requires evaluate_tool for validation."""
+        from hud.eval.utils import validate_v4_task
+
+        with pytest.raises(ValueError, match="evaluate_tool"):
+            validate_v4_task(
+                {
+                    "prompt": "test",
+                    "mcp_config": {"server": {}},
+                    # Missing evaluate_tool
+                }
+            )
+
+    def test_agent_config_accepts_dict(self) -> None:
+        """agent_config can be provided as dict and gets converted."""
+        task = Task(
+            env={"name": "browser"},
+            agent_config={"system_prompt": "Hello"},
+        )
+
+        assert isinstance(task.agent_config, TaskAgentConfig)
+        assert task.agent_config.system_prompt == "Hello"
+
+
+class TestV4AgentConfigToolFilters:
+    """Tests for v4 agent_config.allowed_tools and disallowed_tools processing."""
+
+    def test_v4_extracts_allowed_tools(self) -> None:
+        """v4 allowed_tools is extracted and stored on Environment."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {
+                "allowed_tools": ["browser_*", "file_read"],
+            },
+        }
+
+        task = Task.from_v4(v4_dict)
+
+        assert task.env is not None
+        assert task.env._agent_include == ["browser_*", "file_read"]
+
+    def test_v4_extracts_disallowed_tools(self) -> None:
+        """v4 disallowed_tools is extracted and stored on Environment."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {
+                "disallowed_tools": ["*setup*", "*evaluate*", "checkout_branch"],
+            },
+        }
+
+        task = Task.from_v4(v4_dict)
+
+        assert task.env is not None
+        assert task.env._agent_exclude == ["*setup*", "*evaluate*", "checkout_branch"]
+
+    def test_v4_wildcard_star_allowed_converts_to_none(self) -> None:
+        """v4 allowed_tools=['*'] converts to None (meaning include all)."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {
+                "allowed_tools": ["*"],
+            },
+        }
+
+        task = Task.from_v4(v4_dict)
+
+        assert task.env is not None
+        # ["*"] should be converted to None
+        assert task.env._agent_include is None
+
+    def test_v4_both_allowed_and_disallowed(self) -> None:
+        """v4 supports both allowed_tools and disallowed_tools together."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {
+                "allowed_tools": ["*"],
+                "disallowed_tools": ["*setup*", "*evaluate*"],
+            },
+        }
+
+        task = Task.from_v4(v4_dict)
+
+        assert task.env is not None
+        assert task.env._agent_include is None  # ["*"] → None
+        assert task.env._agent_exclude == ["*setup*", "*evaluate*"]
+
+    @pytest.mark.asyncio
+    async def test_v4_tool_filters_applied_in_as_tools(self) -> None:
+        """v4 tool filters are applied when calling env.as_tools()."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {
+                "allowed_tools": ["*"],
+                "disallowed_tools": ["*setup*"],
+            },
+        }
+
+        task = Task.from_v4(v4_dict)
+        env = task.env
+        assert env is not None
+
+        # Add local tools to test filtering
+        @env.tool()
+        def my_setup_tool() -> str:
+            """Should be filtered out."""
+            return "setup"
+
+        @env.tool()
+        def run_query() -> str:
+            """Should be visible."""
+            return "query"
+
+        await env._build_routing()
+
+        tools = env.as_tools()
+        tool_names = [t.name for t in tools]
+
+        assert "my_setup_tool" not in tool_names
+        assert "run_query" in tool_names
+
+    def test_v4_tool_filters_preserved_in_serialization(self) -> None:
+        """v4 tool filters are preserved when serializing for remote execution."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {
+                "allowed_tools": ["*"],
+                "disallowed_tools": ["*setup*", "*evaluate*", "*grade*"],
+            },
+        }
+
+        task = Task.from_v4(v4_dict)
+
+        # Serialize (this is what gets sent to remote execution)
+        data = task.model_dump(mode="json")
+
+        # agent_config must include the tool filters for remote execution
+        assert "agent_config" in data
+        assert data["agent_config"]["allowed_tools"] == ["*"]
+        assert data["agent_config"]["disallowed_tools"] == ["*setup*", "*evaluate*", "*grade*"]
+
+        # Verify roundtrip works (remote worker will deserialize this)
+        task2 = Task(**data)
+        assert task2.env is not None
+        assert task2.env._agent_include is None  # ["*"] → None
+        assert task2.env._agent_exclude == ["*setup*", "*evaluate*", "*grade*"]

hud/eval/types.py

@@ -0,0 +1,65 @@
+"""Types and exceptions for the eval module.
+
+Kept separate to avoid circular imports.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+# =============================================================================
+# Exceptions
+# =============================================================================
+
+
+class ParallelEvalComplete(Exception):
+    """Raised by summary context to skip body re-execution after parallel eval.
+
+    This is caught by the eval() context manager to cleanly exit.
+    The summary context with results is still accessible after the with block.
+    """
+
+
+# =============================================================================
+# Payload Models
+# =============================================================================
+
+
+class EvalPayload(BaseModel):
+    """Base payload for eval enter/exit."""
+
+    prompt: str | None = None
+    code_snippet: str | None = None
+    job_id: str | None = None
+    group_id: str | None = None
+    variants: dict[str, Any] | None = None
+    task_version_id: str | None = None
+    metadata: dict[str, Any] | None = None
+
+
+class EvalExitPayload(EvalPayload):
+    """Exit payload with result fields."""
+
+    reward: float | None = None
+    success: bool = True
+    error_message: str | None = None
+
+
+class JobEnterPayload(BaseModel):
+    """Payload for job/{job_id}/enter - sent once at job start."""
+
+    name: str | None = None
+    variants: dict[str, Any] | None = None  # Full variant config
+    group: int | None = None
+    taskset: str | None = None  # taskset slug to associate job with
+    tasks: list[dict[str, Any]] | None = None  # task definitions to add to taskset
+
+
+__all__ = [
+    "EvalExitPayload",
+    "EvalPayload",
+    "JobEnterPayload",
+    "ParallelEvalComplete",
+]