hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/eval/tests/test_parallel.py

@@ -0,0 +1,168 @@
+"""Tests for hud.eval.parallel module."""
+
+from __future__ import annotations
+
+import ast
+
+import pytest
+
+from hud.eval.parallel import (
+    ASTExtractionError,
+    _extract_body,
+    _find_async_with,
+    _get_end_line,
+    expand_variants,
+    resolve_group_ids,
+)
+
+
+class TestExpandVariants:
+    """Tests for expand_variants helper."""
+
+    def test_none_returns_empty_dict(self) -> None:
+        """None variants returns list with empty dict."""
+        result = expand_variants(None)
+        assert result == [{}]
+
+    def test_empty_dict_returns_empty_dict(self) -> None:
+        """Empty variants returns list with empty dict."""
+        result = expand_variants({})
+        assert result == [{}]
+
+    def test_single_value_stays_single(self) -> None:
+        """Single non-list value stays as single variant."""
+        result = expand_variants({"model": "gpt-4o"})
+        assert result == [{"model": "gpt-4o"}]
+
+    def test_list_expands_to_variants(self) -> None:
+        """List value expands to multiple variants."""
+        result = expand_variants({"model": ["gpt-4o", "claude"]})
+        assert result == [{"model": "gpt-4o"}, {"model": "claude"}]
+
+    def test_multiple_lists_create_combinations(self) -> None:
+        """Multiple lists create all combinations."""
+        result = expand_variants(
+            {
+                "model": ["a", "b"],
+                "temp": [0.0, 1.0],
+            }
+        )
+
+        assert len(result) == 4
+        assert {"model": "a", "temp": 0.0} in result
+        assert {"model": "a", "temp": 1.0} in result
+        assert {"model": "b", "temp": 0.0} in result
+        assert {"model": "b", "temp": 1.0} in result
+
+    def test_mixed_single_and_list(self) -> None:
+        """Mixed single values and lists work correctly."""
+        result = expand_variants(
+            {
+                "model": ["gpt-4o", "claude"],
+                "temp": 0.7,
+            }
+        )
+
+        assert len(result) == 2
+        assert {"model": "gpt-4o", "temp": 0.7} in result
+        assert {"model": "claude", "temp": 0.7} in result
+
+
+class TestResolveGroupIds:
+    """Tests for resolve_group_ids helper."""
+
+    def test_uses_provided_group_ids(self) -> None:
+        """Uses provided group_ids when given."""
+        result = resolve_group_ids(["a", "b", "c"], 3)
+        assert result == ["a", "b", "c"]
+
+    def test_generates_shared_group_id(self) -> None:
+        """Generates shared group_id when not provided."""
+        result = resolve_group_ids(None, 3)
+        assert len(result) == 3
+        # All should be the same
+        assert result[0] == result[1] == result[2]
+        # Should be a valid UUID
+        assert len(result[0]) == 36
+
+    def test_raises_on_length_mismatch(self) -> None:
+        """Raises ValueError when group_ids length doesn't match."""
+        with pytest.raises(ValueError, match="group_ids length"):
+            resolve_group_ids(["a", "b"], 3)
+
+
+class TestASTHelpers:
+    """Tests for AST helper functions."""
+
+    def test_find_async_with_finds_correct_node(self) -> None:
+        """_find_async_with finds the async with containing target line."""
+        source = """
+async def main():
+    x = 1
+    async with something as ctx:
+        do_stuff()
+        more_stuff()
+    y = 2
+"""
+        tree = ast.parse(source)
+
+        # Line 5 is inside the async with
+        node = _find_async_with(tree, 5)
+        assert node is not None
+        assert isinstance(node, ast.AsyncWith)
+
+    def test_find_async_with_returns_none_when_not_found(self) -> None:
+        """_find_async_with returns None when line is outside async with."""
+        source = """
+async def main():
+    x = 1
+    async with something as ctx:
+        do_stuff()
+    y = 2
+"""
+        tree = ast.parse(source)
+
+        # Line 7 is outside the async with
+        node = _find_async_with(tree, 7)
+        assert node is None
+
+    def test_get_end_line(self) -> None:
+        """_get_end_line returns last line of node."""
+        source = """
+async with ctx:
+    line1()
+    line2()
+    line3()
+"""
+        tree = ast.parse(source)
+        async_with = tree.body[0]
+
+        end_line = _get_end_line(async_with)
+        assert end_line >= 4  # At least through line 4
+
+    def test_extract_body(self) -> None:
+        """_extract_body extracts the body source from async with."""
+        source = """async with ctx:
+    do_thing()
+    more_thing()
+"""
+        lines = source.split("\n")
+        lines = [line + "\n" for line in lines]
+
+        tree = ast.parse(source)
+        async_with = tree.body[0]
+        assert isinstance(async_with, ast.AsyncWith)
+
+        body = _extract_body(lines, async_with)
+        assert "do_thing()" in body
+        assert "more_thing()" in body
+
+
+class TestASTExtractionError:
+    """Tests for ASTExtractionError."""
+
+    def test_is_exception(self) -> None:
+        """ASTExtractionError is an exception."""
+        error = ASTExtractionError("test message")
+        assert isinstance(error, Exception)
+        assert str(error) == "test message"

hud/eval/tests/test_task.py

@@ -0,0 +1,145 @@
+"""Tests for hud.eval.task module."""
+
+from __future__ import annotations
+
+import pytest
+
+from hud.eval.task import Task, TaskAgentConfig
+
+
+class TestTaskSerialization:
+    """Tests for Task serialization and roundtrip."""
+
+    def test_v5_task_roundtrip(self) -> None:
+        """v5 Task serializes and deserializes correctly."""
+        task = Task(
+            env={"name": "browser", "include": ["navigate", "click"]},
+            scenario="checkout",
+            id="task-1",
+            args={"user_id": "alice"},
+        )
+
+        # Serialize
+        data = task.model_dump(mode="json")
+
+        # Should have v5 format
+        assert "env" in data
+        assert data["env"]["name"] == "browser"
+        assert data["scenario"] == "checkout"
+        assert data["id"] == "task-1"
+
+        # Recreate from serialized data
+        task2 = Task(**data)
+
+        # Serialize again
+        data2 = task2.model_dump(mode="json")
+
+        # Should be identical
+        assert data == data2
+
+    def test_v4_task_roundtrip(self) -> None:
+        """v4 Task serializes (flattens) and deserializes correctly."""
+        v4_dict = {
+            "prompt": "Go to google.com and search for cats",
+            "mcp_config": {
+                "browser": {"url": "http://localhost:8080"},
+            },
+            "evaluate_tool": {"name": "check_url", "arguments": {"contains": "google"}},
+            "setup_tool": {"name": "navigate", "arguments": {"url": "about:blank"}},
+            "id": "v4-task-1",
+            "agent_config": {"system_prompt": "You are a helpful assistant"},
+            "metadata": {"category": "navigation"},
+        }
+
+        # Create Task from v4 dict
+        task = Task.from_v4(v4_dict)
+
+        # Serialize (should flatten to v4 format)
+        data = task.model_dump(mode="json")
+
+        # Should have v4 format (flat, not nested env)
+        assert "prompt" in data
+        assert "mcp_config" in data
+        assert "evaluate_tool" in data
+        assert data["prompt"] == "Go to google.com and search for cats"
+        assert data["id"] == "v4-task-1"
+
+        # Recreate from serialized data
+        task2 = Task(**data)
+
+        # Serialize again
+        data2 = task2.model_dump(mode="json")
+
+        # Should be identical
+        assert data == data2
+
+    def test_v4_preserves_agent_config(self) -> None:
+        """v4 Task preserves agent_config through roundtrip."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "agent_config": {"system_prompt": "Custom system prompt"},
+        }
+
+        task = Task.from_v4(v4_dict)
+        data = task.model_dump(mode="json")
+
+        assert data.get("agent_config") == {"system_prompt": "Custom system prompt"}
+
+        # Roundtrip
+        task2 = Task(**data)
+        assert task2.agent_config is not None
+        assert isinstance(task2.agent_config, TaskAgentConfig)
+        assert task2.agent_config.system_prompt == "Custom system prompt"
+
+    def test_v4_preserves_metadata(self) -> None:
+        """v4 Task preserves metadata through roundtrip."""
+        v4_dict = {
+            "prompt": "Test prompt",
+            "mcp_config": {"server": {"url": "http://localhost"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+            "metadata": {"key1": "value1", "key2": 42},
+        }
+
+        task = Task.from_v4(v4_dict)
+        data = task.model_dump(mode="json")
+
+        assert data.get("metadata") == {"key1": "value1", "key2": 42}
+
+        # Roundtrip
+        task2 = Task(**data)
+        assert task2.metadata == {"key1": "value1", "key2": 42}
+
+
+class TestTaskValidation:
+    """Tests for Task validation."""
+
+    def test_v5_allows_none_env(self) -> None:
+        """v5 Task allows None env (for blank evals)."""
+        task = Task(scenario="test")  # env=None is valid
+        assert task.env is None
+        assert task.scenario == "test"
+
+    def test_v4_requires_evaluate_tool(self) -> None:
+        """v4 Task requires evaluate_tool for validation."""
+        from hud.eval.utils import validate_v4_task
+
+        with pytest.raises(ValueError, match="evaluate_tool"):
+            validate_v4_task(
+                {
+                    "prompt": "test",
+                    "mcp_config": {"server": {}},
+                    # Missing evaluate_tool
+                }
+            )
+
+    def test_agent_config_accepts_dict(self) -> None:
+        """agent_config can be provided as dict and gets converted."""
+        task = Task(
+            env={"name": "browser"},
+            agent_config={"system_prompt": "Hello"},
+        )
+
+        assert isinstance(task.agent_config, TaskAgentConfig)
+        assert task.agent_config.system_prompt == "Hello"

hud/eval/types.py

@@ -0,0 +1,63 @@
+"""Types and exceptions for the eval module.
+
+Kept separate to avoid circular imports.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+# =============================================================================
+# Exceptions
+# =============================================================================
+
+
+class ParallelEvalComplete(Exception):
+    """Raised by summary context to skip body re-execution after parallel eval.
+
+    This is caught by the eval() context manager to cleanly exit.
+    The summary context with results is still accessible after the with block.
+    """
+
+
+# =============================================================================
+# Payload Models
+# =============================================================================
+
+
+class EvalPayload(BaseModel):
+    """Base payload for eval enter/exit."""
+
+    prompt: str | None = None
+    code_snippet: str | None = None
+    job_id: str | None = None
+    group_id: str | None = None
+    variants: dict[str, Any] | None = None
+    task_version_id: str | None = None
+    metadata: dict[str, Any] | None = None
+
+
+class EvalExitPayload(EvalPayload):
+    """Exit payload with result fields."""
+
+    reward: float | None = None
+    success: bool = True
+    error_message: str | None = None
+
+
+class JobEnterPayload(BaseModel):
+    """Payload for job/{job_id}/enter - sent once at job start."""
+
+    name: str | None = None
+    variants: dict[str, Any] | None = None  # Full variant config
+    group: int | None = None
+
+
+__all__ = [
+    "EvalExitPayload",
+    "EvalPayload",
+    "JobEnterPayload",
+    "ParallelEvalComplete",
+]

hud/eval/utils.py

@@ -0,0 +1,183 @@
+"""Utility functions for the eval module."""
+
+from __future__ import annotations
+
+import logging
+import warnings
+from typing import Any
+
+__all__ = ["build_env_from_v4", "is_v4_format", "validate_v4_task"]
+
+logger = logging.getLogger(__name__)
+
+
+def is_v4_format(data: dict[str, Any]) -> bool:
+    """Detect if dict looks like v4 LegacyTask format.
+
+    Used for branching logic. Checks if data has the core v4 fields
+    (prompt AND mcp_config). Does NOT validate completeness.
+
+    Args:
+        data: Dict to check
+
+    Returns:
+        True if looks like v4 format, False otherwise
+    """
+    if not isinstance(data, dict):
+        return False
+
+    # Core v4 detection: prompt + mcp_config
+    return bool(data.get("prompt")) and bool(data.get("mcp_config"))
+
+
+def validate_v4_task(data: dict[str, Any]) -> None:
+    """Validate v4 task has all required fields.
+
+    A valid v4 task must have all three required fields:
+    - prompt: The task instruction
+    - mcp_config: MCP server configuration
+    - evaluate_tool: How to evaluate success
+
+    Call this after is_v4_format() when you need to ensure completeness.
+
+    Args:
+        data: Dict to validate
+
+    Raises:
+        ValueError: If any required fields are missing
+    """
+    missing = []
+    if not data.get("prompt"):
+        missing.append("prompt")
+    if not data.get("mcp_config"):
+        missing.append("mcp_config")
+    if not data.get("evaluate_tool"):
+        missing.append("evaluate_tool")
+
+    if missing:
+        raise ValueError(f"v4 task missing required fields: {', '.join(missing)}")
+
+
+def build_env_from_v4(source: dict[str, Any] | Any) -> dict[str, Any]:
+    """Build Environment from v4 LegacyTask format.
+
+    Creates an Environment configured with the legacy task's fields.
+    Returns a dict ready to be passed to Task() constructor.
+
+    Args:
+        source: dict or LegacyTask with v4 fields (prompt, mcp_config, etc.)
+
+    Returns:
+        Dict with Task fields: env, id, scenario, args, validation, system_prompt, metadata
+
+    Raises:
+        TypeError: If source is not a dict or LegacyTask
+    """
+    from hud.environment import Environment
+    from hud.types import LegacyTask, MCPToolCall
+
+    # Convert dict to LegacyTask if needed
+    if isinstance(source, dict):
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            legacy = LegacyTask(**source)
+    elif isinstance(source, LegacyTask):
+        legacy = source
+    else:
+        raise TypeError(f"Expected dict or LegacyTask, got {type(source).__name__}")
+
+    # Warn if using local MCP configs (command without url)
+    _warn_local_mcp(legacy.mcp_config)
+
+    # Extract tool filters from agent_config (v4 style)
+    # These are agent-level filters, not connection-level
+    include_tools: list[str] | None = None
+    exclude_tools: list[str] | None = None
+    if legacy.agent_config:
+        include_tools = legacy.agent_config.allowed_tools
+        exclude_tools = legacy.agent_config.disallowed_tools
+
+    # Convert ["*"] wildcard to None (meaning include all)
+    if include_tools == ["*"]:
+        include_tools = None
+
+    # Create Environment - NO connections made here, just config stored
+    env = Environment(legacy.id or "v4-legacy")
+    env.connect_mcp_config(legacy.mcp_config)
+
+    # Store agent-level tool filters on Environment (applied in as_tools())
+    # This allows Environment to call setup/evaluate while hiding them from agent
+    env._agent_include = include_tools
+    env._agent_exclude = exclude_tools
+
+    # Set the prompt
+    env.prompt = legacy.prompt
+
+    # Add setup_tool calls (stored, not executed)
+    if legacy.setup_tool:
+        setup_calls = legacy.setup_tool
+        if not isinstance(setup_calls, list):
+            setup_calls = [setup_calls]
+        for call in setup_calls:
+            env.setup_tool(call.name, **(call.arguments or {}))
+
+    # Add evaluate_tool calls (stored, not executed)
+    if legacy.evaluate_tool:
+        eval_calls = legacy.evaluate_tool
+        if not isinstance(eval_calls, list):
+            eval_calls = [eval_calls]
+        for call in eval_calls:
+            env.evaluate_tool(call.name, **(call.arguments or {}))
+
+    # Build Task fields dict
+    result: dict[str, Any] = {
+        "env": env,
+        "id": legacy.id,
+        "scenario": None,  # v4 uses prompt, not scenarios
+        "args": {},
+    }
+
+    # Map integration_test_tool → validation (same concept: tool calls to verify)
+    if legacy.integration_test_tool:
+        int_test = legacy.integration_test_tool
+        if not isinstance(int_test, list):
+            int_test = [int_test]
+        # Convert to MCPToolCall if needed
+        result["validation"] = [
+            call if isinstance(call, MCPToolCall) else MCPToolCall(**call.model_dump())
+            for call in int_test
+        ]
+
+    # Extract agent_config (just system_prompt for now)
+    if legacy.agent_config and legacy.agent_config.system_prompt:
+        result["agent_config"] = {"system_prompt": legacy.agent_config.system_prompt}
+
+    # Preserve metadata
+    if legacy.metadata:
+        result["metadata"] = legacy.metadata
+
+    return result
+
+
+def _warn_local_mcp(mcp_config: dict[str, Any] | None) -> None:
+    """Warn if mcp_config uses local MCP servers (command without url).
+
+    Local MCP servers can cause port conflicts when running tasks concurrently.
+    """
+    if not mcp_config:
+        return
+
+    has_local = any(
+        isinstance(server_cfg, dict) and "command" in server_cfg and not server_cfg.get("url")
+        for server_cfg in mcp_config.values()
+        if isinstance(server_cfg, dict)
+    )
+
+    if has_local:
+        warnings.warn(
+            "Task uses local MCP configuration (command without url). "
+            "This may cause port conflicts when running tasks concurrently. "
+            "Consider using remote MCP servers for parallel execution.",
+            UserWarning,
+            stacklevel=4,
+        )

hud/patches/__init__.py

@@ -0,0 +1,19 @@
+"""
+HUD runtime patches for third-party libraries.
+
+This module applies monkey-patches to fix issues in dependencies
+without requiring forked packages.
+"""
+
+from hud.patches.mcp_patches import apply_all_patches, suppress_fastmcp_logging
+from hud.patches.warnings import apply_default_warning_filters, suppress_mcp_use_import_warnings
+
+# Apply patches on import
+apply_all_patches()
+
+__all__ = [
+    "apply_all_patches",
+    "apply_default_warning_filters",
+    "suppress_fastmcp_logging",
+    "suppress_mcp_use_import_warnings",
+]