hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/eval/tests/test_context.py

@@ -0,0 +1,178 @@
+"""Tests for hud.eval.context module."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from hud.eval.context import (
+    EvalContext,
+    get_current_trace_headers,
+)
+
+
+class TestEvalContext:
+    """Tests for EvalContext."""
+
+    def test_init_generates_trace_id(self) -> None:
+        """EvalContext generates trace_id if not provided."""
+        ctx = EvalContext(name="test-task", quiet=True)
+
+        assert ctx.trace_id is not None
+        assert len(ctx.trace_id) == 36  # UUID format
+
+    def test_init_uses_provided_trace_id(self) -> None:
+        """EvalContext uses provided trace_id."""
+        ctx = EvalContext(name="test-task", trace_id="custom-id", quiet=True)
+
+        assert ctx.trace_id == "custom-id"
+
+    def test_headers_contains_trace_id(self) -> None:
+        """headers property returns dict with trace ID."""
+        ctx = EvalContext(name="test-task", trace_id="test-123", quiet=True)
+
+        assert ctx.headers == {"Trace-Id": "test-123"}
+
+    def test_success_true_when_no_error(self) -> None:
+        """success property returns True when no error."""
+        ctx = EvalContext(name="test-task", quiet=True)
+
+        assert ctx.success is True
+
+    def test_success_false_when_error(self) -> None:
+        """success property returns False when error is set."""
+        ctx = EvalContext(name="test-task", quiet=True)
+        ctx.error = ValueError("test error")
+
+        assert ctx.success is False
+
+    def test_variants_empty_by_default(self) -> None:
+        """variants is empty dict by default."""
+        ctx = EvalContext(name="test-task", quiet=True)
+
+        assert ctx.variants == {}
+
+    def test_variants_set_from_init(self) -> None:
+        """variants set from parameter."""
+        ctx = EvalContext(
+            name="test-task",
+            variants={"model": "gpt-4o", "temp": 0.7},
+            quiet=True,
+        )
+
+        assert ctx.variants == {"model": "gpt-4o", "temp": 0.7}
+
+    @pytest.mark.asyncio
+    async def test_context_manager_sets_headers(self) -> None:
+        """Context manager sets trace headers in contextvar."""
+        ctx = EvalContext(name="test-task", trace_id="test-123", quiet=True)
+
+        # Mock telemetry calls
+        with (
+            patch.object(ctx, "_eval_enter", new_callable=AsyncMock),
+            patch.object(ctx, "_eval_exit", new_callable=AsyncMock),
+            patch.object(EvalContext, "__aenter__", return_value=ctx),
+            patch.object(EvalContext, "__aexit__", return_value=None),
+        ):
+            assert get_current_trace_headers() is None
+
+            # Manually set token for test
+            from hud.eval.context import _current_trace_headers
+
+            token = _current_trace_headers.set(ctx.headers)
+            try:
+                headers = get_current_trace_headers()
+                assert headers is not None
+                assert headers["Trace-Id"] == "test-123"
+            finally:
+                _current_trace_headers.reset(token)
+
+            assert get_current_trace_headers() is None
+
+    def test_repr(self) -> None:
+        """__repr__ shows useful info."""
+        ctx = EvalContext(
+            name="test-task",
+            trace_id="abc12345-6789-0000-0000-000000000000",
+            quiet=True,
+        )
+        ctx.reward = 0.95
+
+        repr_str = repr(ctx)
+        assert "abc12345" in repr_str
+        assert "test-task" in repr_str
+        assert "0.95" in repr_str
+
+
+class TestEvalContextPrompt:
+    """Tests for EvalContext.prompt feature."""
+
+    def test_prompt_can_be_set(self) -> None:
+        """EvalContext.prompt can be set."""
+        ctx = EvalContext(name="test-task", quiet=True)
+        ctx.prompt = "Test prompt"
+
+        assert ctx.prompt == "Test prompt"
+
+    def test_prompt_included_in_payload(self) -> None:
+        """Prompt is included in eval payload."""
+        ctx = EvalContext(name="test-task", quiet=True)
+        ctx.prompt = "Test prompt"
+
+        payload = ctx._build_base_payload()
+        assert payload.prompt == "Test prompt"
+
+
+class TestEvalContextFromEnvironment:
+    """Tests for EvalContext.from_environment factory."""
+
+    def test_copies_connections(self) -> None:
+        """from_environment copies connections from parent (deep copy)."""
+        from hud.environment import Environment
+
+        parent = Environment("parent-env")
+        # Add a mock connection with copy method
+        mock_conn = MagicMock()
+        mock_conn_copy = MagicMock()
+        mock_conn.copy.return_value = mock_conn_copy
+        parent._connections["test-conn"] = mock_conn
+
+        ctx = EvalContext.from_environment(parent, name="test-task")
+
+        # Verify connection was copied (not same object)
+        assert "test-conn" in ctx._connections
+        mock_conn.copy.assert_called_once()
+        assert ctx._connections["test-conn"] is mock_conn_copy
+
+    def test_copies_prompt(self) -> None:
+        """from_environment copies prompt from parent."""
+        from hud.environment import Environment
+
+        parent = Environment("parent-env")
+        parent.prompt = "Parent prompt"
+
+        ctx = EvalContext.from_environment(parent, name="test-task")
+
+        assert ctx.prompt == "Parent prompt"
+
+    def test_sets_eval_properties(self) -> None:
+        """from_environment sets eval-specific properties."""
+        from hud.environment import Environment
+
+        parent = Environment("parent-env")
+
+        ctx = EvalContext.from_environment(
+            parent,
+            name="test-task",
+            trace_id="custom-trace",
+            variants={"model": "gpt-4o"},
+            group_id="group-123",
+            index=5,
+        )
+
+        assert ctx.eval_name == "test-task"
+        assert ctx.trace_id == "custom-trace"
+        assert ctx.variants == {"model": "gpt-4o"}
+        assert ctx.group_id == "group-123"
+        assert ctx.index == 5

hud/eval/tests/test_eval.py

@@ -0,0 +1,210 @@
+"""Tests for hud.eval.task module (Task class)."""
+
+from __future__ import annotations
+
+import pytest
+
+from hud.eval.task import Task
+
+
+class TestTaskDataclass:
+    """Tests for Task as a Pydantic model."""
+
+    def test_init_defaults(self) -> None:
+        """Task initializes with sensible defaults."""
+        task = Task()
+
+        assert task.env is None
+        assert task.scenario is None
+        assert task.args == {}
+
+    def test_init_with_env_dict(self) -> None:
+        """Task auto-converts env dict to Environment via validator."""
+        from hud.environment import Environment
+
+        task = Task(
+            env={"name": "browser", "include": ["navigate"]},
+            scenario="checkout",
+            args={"user_id": "alice"},
+        )
+
+        # env dict is auto-converted to Environment
+        assert isinstance(task.env, Environment)
+        assert task.scenario == "checkout"
+        assert task.args == {"user_id": "alice"}
+
+    def test_copy_creates_new_instance(self) -> None:
+        """copy() creates a new Task instance."""
+        original = Task(
+            env={"name": "test"},
+            scenario="checkout",
+            args={"user_id": "alice"},
+        )
+        copied = original.copy()
+
+        assert copied is not original
+        assert copied.env is original.env  # Env reference is shared (intentional)
+        assert copied.scenario == original.scenario
+        assert copied.args == original.args
+        assert copied.args is not original.args  # Args are deep copied
+
+
+class TestEnvironmentCall:
+    """Tests for Environment.__call__ returning Task."""
+
+    def test_call_returns_task(self) -> None:
+        """Environment() returns a Task object."""
+        from hud.environment import Environment
+
+        env = Environment("test-env")
+        task = env()
+
+        assert isinstance(task, Task)
+
+    def test_call_with_scenario_sets_scenario(self) -> None:
+        """Environment(scenario) sets scenario name."""
+        from hud.environment import Environment
+
+        env = Environment("test-env")
+        task = env("checkout")
+
+        assert task.scenario == "checkout"
+
+    def test_call_with_args_sets_args(self) -> None:
+        """Environment(scenario, **args) sets args."""
+        from hud.environment import Environment
+
+        env = Environment("test-env")
+        task = env("checkout", user_id="alice", amount=100)
+
+        assert task.args == {"user_id": "alice", "amount": 100}
+
+    def test_call_returns_task_with_env(self) -> None:
+        """Environment() returns Task with env reference."""
+        from hud.environment import Environment
+
+        env = Environment("test-env")
+        task = env()
+
+        # Task has reference to the Environment
+        assert task.env is env
+
+        # With setup_tool (v4 legacy)
+        env2 = Environment("test-env").setup_tool("navigate", url="https://example.com")
+        task2 = env2()
+        assert task2.env is env2
+        assert len(task2.env._setup_calls) == 1
+
+
+class TestTaskFromV4:
+    """Tests for Task.from_v4() migration helper."""
+
+    def test_from_v4_with_legacy_task(self) -> None:
+        """Task.from_v4() accepts LegacyTask object."""
+        import warnings
+
+        # Suppress the deprecation warning from LegacyTask
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            from hud.types import LegacyTask
+
+            legacy = LegacyTask(
+                prompt="Navigate to google.com",
+                mcp_config={"hud": {"url": "https://mcp.hud.ai"}},
+                evaluate_tool={"name": "check", "arguments": {}},
+            )
+
+        task = Task.from_v4(legacy)
+
+        assert isinstance(task, Task)
+        assert task.env is not None
+        assert task.env.prompt == "Navigate to google.com"
+        assert task.scenario is None  # Uses setup/evaluate_tool, not scenarios
+
+    def test_from_v4_with_dict(self) -> None:
+        """Task.from_v4() accepts dict with LegacyTask fields."""
+        task = Task.from_v4(
+            {
+                "prompt": "Navigate to google.com",
+                "mcp_config": {"hud": {"url": "https://mcp.hud.ai"}},
+                "evaluate_tool": {"name": "check", "arguments": {}},
+            }
+        )
+
+        assert isinstance(task, Task)
+        assert task.env is not None
+        assert task.env.prompt == "Navigate to google.com"
+
+    def test_from_v4_with_json_string(self) -> None:
+        """Task.from_v4() accepts JSON string."""
+        import json
+
+        data = {
+            "prompt": "Navigate to google.com",
+            "mcp_config": {"hud": {"url": "https://mcp.hud.ai"}},
+            "evaluate_tool": {"name": "check", "arguments": {}},
+        }
+        task = Task.from_v4(json.dumps(data))
+
+        assert isinstance(task, Task)
+        assert task.env is not None
+        assert task.env.prompt == "Navigate to google.com"
+
+    def test_from_v4_with_setup_tool(self) -> None:
+        """Task.from_v4() preserves setup_tool via env._setup_calls."""
+        task = Task.from_v4(
+            {
+                "prompt": "Check URL",
+                "mcp_config": {"hud": {"url": "https://mcp.hud.ai"}},
+                "setup_tool": {"name": "navigate", "arguments": {"url": "https://google.com"}},
+                "evaluate_tool": {"name": "check", "arguments": {}},
+            }
+        )
+
+        # setup_tool is converted to env._setup_calls
+        assert len(task.env._setup_calls) == 1
+        assert task.env._setup_calls[0] == ("navigate", {"url": "https://google.com"})
+
+    def test_from_v4_with_evaluate_tool(self) -> None:
+        """Task.from_v4() preserves evaluate_tool via env._evaluate_calls."""
+        task = Task.from_v4(
+            {
+                "prompt": "Check URL",
+                "mcp_config": {"hud": {"url": "https://mcp.hud.ai"}},
+                "evaluate_tool": {"name": "check_url", "arguments": {"expected": "google"}},
+            }
+        )
+
+        # evaluate_tool is converted to env._evaluate_calls
+        assert len(task.env._evaluate_calls) == 1
+        assert task.env._evaluate_calls[0] == ("check_url", {"expected": "google"})
+
+    def test_from_v4_with_invalid_type_raises(self) -> None:
+        """Task.from_v4() raises TypeError for invalid input."""
+        with pytest.raises(TypeError):
+            Task.from_v4(12345)  # type: ignore[arg-type]
+
+    def test_from_v4_with_invalid_json_raises(self) -> None:
+        """Task.from_v4() raises JSONDecodeError for invalid JSON."""
+        import json
+
+        with pytest.raises(json.JSONDecodeError):
+            Task.from_v4("not valid json")
+
+    def test_from_v4_does_not_warn_on_use(self) -> None:
+        """Task.from_v4() suppresses LegacyTask deprecation warning."""
+        import warnings
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            Task.from_v4(
+                {
+                    "prompt": "test",
+                    "mcp_config": {"hud": {}},
+                    "evaluate_tool": {"name": "check", "arguments": {}},
+                }
+            )
+
+        # Should not trigger deprecation warning since we're migrating
+        legacy_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)]
+        assert len(legacy_warnings) == 0

hud/eval/tests/test_manager.py

@@ -0,0 +1,152 @@
+"""Tests for hud.eval.manager module (hud.eval() function)."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from hud.eval.context import EvalContext, get_current_trace_headers
+from hud.eval.manager import run_eval
+
+
+class TestRunEvalNoArgs:
+    """Tests for hud.eval() with no arguments (blank eval)."""
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_creates_context(self) -> None:
+        """hud.eval() with no args creates an EvalContext."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(quiet=True) as ctx:
+                assert isinstance(ctx, EvalContext)
+                assert ctx.eval_name == "eval"
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_generates_trace_id(self) -> None:
+        """hud.eval() with no args generates a trace_id."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(quiet=True) as ctx:
+                assert ctx.trace_id is not None
+                assert len(ctx.trace_id) == 36  # UUID format
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_sets_trace_headers(self) -> None:
+        """hud.eval() sets trace headers in contextvar during context."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            # Before context, no headers
+            assert get_current_trace_headers() is None
+
+            async with run_eval(quiet=True) as ctx:
+                # Inside context, headers are set
+                headers = get_current_trace_headers()
+                assert headers is not None
+                assert headers["Trace-Id"] == ctx.trace_id
+
+            # After context, headers are cleared
+            assert get_current_trace_headers() is None
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_reward_can_be_set(self) -> None:
+        """hud.eval() allows setting reward on context."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(quiet=True) as ctx:
+                assert ctx.reward is None
+                ctx.reward = 0.95
+
+            assert ctx.reward == 0.95
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_reports_reward_on_exit(self) -> None:
+        """hud.eval() reports reward to backend on exit."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock) as mock_exit,
+        ):
+            async with run_eval(quiet=True) as ctx:
+                ctx.reward = 0.85
+
+            # _eval_exit should have been called (with no error)
+            mock_exit.assert_called_once_with(None)
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_empty_variants(self) -> None:
+        """hud.eval() with no args has empty variants dict."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(quiet=True) as ctx:
+                assert ctx.variants == {}
+
+    @pytest.mark.asyncio
+    async def test_blank_eval_has_headers_property(self) -> None:
+        """hud.eval() context has headers property for gateway integration."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(quiet=True) as ctx:
+                headers = ctx.headers
+                assert "Trace-Id" in headers
+                assert headers["Trace-Id"] == ctx.trace_id
+
+
+class TestRunEvalWithApiKey:
+    """Tests for hud.eval() with api_key parameter."""
+
+    @pytest.mark.asyncio
+    async def test_api_key_passed_to_context(self) -> None:
+        """hud.eval(api_key=...) passes api_key to context."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(api_key="test-key", quiet=True) as ctx:
+                assert ctx._eval_api_key == "test-key"
+
+
+class TestRunEvalWithJobId:
+    """Tests for hud.eval() with job_id parameter."""
+
+    @pytest.mark.asyncio
+    async def test_job_id_passed_to_context(self) -> None:
+        """hud.eval(job_id=...) passes job_id to context."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock),
+        ):
+            async with run_eval(job_id="job-123", quiet=True) as ctx:
+                assert ctx.job_id == "job-123"
+
+
+class TestRunEvalErrorHandling:
+    """Tests for hud.eval() error handling."""
+
+    @pytest.mark.asyncio
+    async def test_error_tracked_on_exception(self) -> None:
+        """hud.eval() tracks error when exception occurs."""
+        with (
+            patch.object(EvalContext, "_eval_enter", new_callable=AsyncMock),
+            patch.object(EvalContext, "_eval_exit", new_callable=AsyncMock) as mock_exit,
+        ):
+            with pytest.raises(ValueError):
+                async with run_eval(quiet=True):
+                    raise ValueError("test error")
+
+            # _eval_exit should have been called with error message
+            mock_exit.assert_called_once()
+            error_msg = mock_exit.call_args[0][0]
+            assert error_msg is not None
+            assert "test error" in error_msg