hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/telemetry/tests/test_eval_telemetry.py

@@ -0,0 +1,356 @@
+"""Tests for EvalContext telemetry integration with mock backend."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+import hud
+from hud.environment import Environment
+from hud.eval import Task
+from hud.telemetry.exporter import _pending_futures, _pending_spans
+
+
+@pytest.fixture(autouse=True)
+def clear_pending_state():
+    """Clear pending spans and futures before and after each test."""
+    _pending_spans.clear()
+    _pending_futures.clear()
+    yield
+    _pending_spans.clear()
+    _pending_futures.clear()
+
+
+class TestEvalContextTelemetry:
+    """Tests for EvalContext telemetry integration."""
+
+    @pytest.mark.asyncio
+    async def test_call_tool_records_span(self):
+        """Test that call_tool records a span with correct format."""
+        uploaded_spans: list[dict[str, Any]] = []
+
+        def capture_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded_spans.extend(spans)
+            return True
+
+        # Create environment with a simple tool
+        env = Environment("test-env")
+
+        @env.tool
+        async def greet(name: str) -> str:
+            """Say hello."""
+            return f"Hello, {name}!"
+
+        # Create task from environment
+        task = Task(env=env)
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=capture_upload),
+            patch("hud.eval.context.make_request"),  # Don't send eval enter/exit
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                result = await ctx.call_tool("greet", name="World")
+                # call_tool returns MCPToolResult with formatted content
+                assert "Hello, World!" in str(result)
+                trace_id = ctx.trace_id
+
+            # Wait for thread pool
+            await asyncio.sleep(0.2)
+
+        # Verify span was recorded
+        assert len(uploaded_spans) >= 1
+        span = uploaded_spans[0]
+
+        # Check span structure
+        assert "name" in span
+        assert "trace_id" in span
+        assert "span_id" in span
+        assert "start_time" in span
+        assert "end_time" in span
+        assert "status_code" in span
+        assert "attributes" in span
+
+        # Check attributes
+        attrs = span["attributes"]
+        assert attrs["task_run_id"] == trace_id
+        assert attrs["category"] == "mcp"
+
+    @pytest.mark.asyncio
+    async def test_call_tool_records_error_span(self):
+        """Test that failed call_tool records error span."""
+        uploaded_spans: list[dict[str, Any]] = []
+
+        def capture_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded_spans.extend(spans)
+            return True
+
+        env = Environment("test-env")
+
+        @env.tool
+        async def failing_tool() -> str:
+            """Always fails."""
+            raise ValueError("Tool error")
+
+        task = Task(env=env)
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=capture_upload),
+            patch("hud.eval.context.make_request"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                # Tool errors are wrapped in ToolError
+                with pytest.raises(Exception, match="Tool error"):
+                    await ctx.call_tool("failing_tool")
+
+            await asyncio.sleep(0.2)
+
+        # Should have recorded span with ERROR status
+        assert len(uploaded_spans) >= 1
+        span = uploaded_spans[0]
+        assert span["status_code"] == "ERROR"
+        # Error message contains the original error
+        assert "Tool error" in (span.get("status_message") or "")
+
+    @pytest.mark.asyncio
+    async def test_multiple_call_tools_record_spans(self):
+        """Test that multiple call_tool calls each record a span."""
+        uploaded_spans: list[dict[str, Any]] = []
+
+        def capture_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded_spans.extend(spans)
+            return True
+
+        env = Environment("test-env")
+
+        @env.tool
+        async def add(a: int, b: int) -> int:
+            """Add two numbers."""
+            return a + b
+
+        @env.tool
+        async def multiply(a: int, b: int) -> int:
+            """Multiply two numbers."""
+            return a * b
+
+        task = Task(env=env)
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=capture_upload),
+            patch("hud.eval.context.make_request"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                r1 = await ctx.call_tool("add", a=2, b=3)
+                r2 = await ctx.call_tool("multiply", a=4, b=5)
+                # Results are MCPToolResult objects
+                assert "5" in str(r1)
+                assert "20" in str(r2)
+
+            await asyncio.sleep(0.2)
+
+        # Should have 2 spans
+        assert len(uploaded_spans) >= 2
+
+    @pytest.mark.asyncio
+    async def test_flush_called_on_context_exit(self):
+        """Test that flush is called when context exits."""
+        env = Environment("test-env")
+
+        @env.tool
+        async def simple_tool() -> str:
+            return "done"
+
+        task = Task(env=env)
+
+        with (
+            patch("hud.eval.context.flush") as mock_flush,
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.eval.context.make_request"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                await ctx.call_tool("simple_tool")
+                trace_id = ctx.trace_id
+
+            # Verify flush was called with the trace_id
+            mock_flush.assert_called_once_with(trace_id)
+
+    @pytest.mark.asyncio
+    async def test_telemetry_disabled_no_upload(self):
+        """Test that no upload happens when telemetry is disabled."""
+        upload_called = False
+
+        def should_not_be_called(*args: Any, **kwargs: Any) -> bool:
+            nonlocal upload_called
+            upload_called = True
+            return True
+
+        env = Environment("test-env")
+
+        @env.tool
+        async def test_tool() -> str:
+            return "ok"
+
+        task = Task(env=env)
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=should_not_be_called),
+            patch("hud.eval.context.make_request"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = False  # Disabled!
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                await ctx.call_tool("test_tool")
+
+            await asyncio.sleep(0.1)
+
+        assert upload_called is False
+
+
+class TestSpanFormat:
+    """Tests for the format of recorded spans."""
+
+    @pytest.mark.asyncio
+    async def test_span_has_required_fields(self):
+        """Test that spans have all required HudSpan fields."""
+        uploaded_spans: list[dict[str, Any]] = []
+
+        def capture_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded_spans.extend(spans)
+            return True
+
+        env = Environment("test-env")
+
+        @env.tool
+        async def echo(message: str) -> str:
+            return message
+
+        task = Task(env=env)
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=capture_upload),
+            patch("hud.eval.context.make_request"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                await ctx.call_tool("echo", message="test")
+
+            await asyncio.sleep(0.2)
+
+        assert len(uploaded_spans) >= 1
+        span = uploaded_spans[0]
+
+        # Required fields from HudSpan
+        assert "name" in span
+        assert "trace_id" in span
+        assert len(span["trace_id"]) == 32  # 32-char hex
+        assert "span_id" in span
+        assert len(span["span_id"]) == 16  # 16-char hex
+        assert "start_time" in span
+        assert "end_time" in span
+        assert "status_code" in span
+        assert span["status_code"] in ("OK", "ERROR", "UNSET")
+
+        # Attributes
+        assert "attributes" in span
+        attrs = span["attributes"]
+        assert "task_run_id" in attrs
+        assert "category" in attrs
+
+    @pytest.mark.asyncio
+    async def test_span_timestamps_are_iso(self):
+        """Test that span timestamps are in ISO format."""
+        uploaded_spans: list[dict[str, Any]] = []
+
+        def capture_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded_spans.extend(spans)
+            return True
+
+        env = Environment("test-env")
+
+        @env.tool
+        async def noop() -> None:
+            pass
+
+        task = Task(env=env)
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=capture_upload),
+            patch("hud.eval.context.make_request"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+            mock_settings.hud_api_url = "https://api.hud.ai"
+
+            async with hud.eval(task) as ctx:
+                await ctx.call_tool("noop")
+
+            await asyncio.sleep(0.2)
+
+        span = uploaded_spans[0]
+
+        # ISO format: YYYY-MM-DDTHH:MM:SS.ssssssZ
+        import re
+
+        iso_pattern = r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}"
+        assert re.match(iso_pattern, span["start_time"])
+        assert re.match(iso_pattern, span["end_time"])

hud/telemetry/tests/test_exporter.py

@@ -0,0 +1,258 @@
+"""Tests for telemetry exporter with mock backend."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+from hud.telemetry.exporter import (
+    _do_upload,
+    _pending_futures,
+    _pending_spans,
+    flush,
+    queue_span,
+    shutdown,
+)
+
+
+@pytest.fixture(autouse=True)
+def clear_pending_state():
+    """Clear pending spans and futures before and after each test."""
+    _pending_spans.clear()
+    _pending_futures.clear()
+    yield
+    _pending_spans.clear()
+    _pending_futures.clear()
+
+
+class TestDoUpload:
+    """Tests for _do_upload function."""
+
+    def test_upload_success(self):
+        """Test successful upload."""
+        with patch("hud.telemetry.exporter.make_request_sync") as mock_request:
+            result = _do_upload(
+                task_run_id="test-task-123",
+                spans=[{"name": "test.span", "attributes": {"task_run_id": "test-task-123"}}],
+                telemetry_url="https://api.hud.ai",
+                api_key="test-key",
+            )
+
+            assert result is True
+            mock_request.assert_called_once()
+            call_kwargs = mock_request.call_args.kwargs
+            assert call_kwargs["method"] == "POST"
+            assert "test-task-123" in call_kwargs["url"]
+            assert call_kwargs["api_key"] == "test-key"
+            assert "telemetry" in call_kwargs["json"]
+
+    def test_upload_failure(self):
+        """Test upload failure handling."""
+        with patch("hud.telemetry.exporter.make_request_sync") as mock_request:
+            mock_request.side_effect = Exception("Network error")
+
+            result = _do_upload(
+                task_run_id="test-task-123",
+                spans=[{"name": "test.span"}],
+                telemetry_url="https://api.hud.ai",
+                api_key="test-key",
+            )
+
+            assert result is False
+
+
+class TestQueueSpan:
+    """Tests for queue_span function."""
+
+    def test_queue_span_without_api_key(self):
+        """Test that spans are not queued without API key."""
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = None
+            mock_settings.telemetry_enabled = True
+
+            queue_span({"name": "test", "attributes": {"task_run_id": "123"}})
+
+            assert len(_pending_spans) == 0
+
+    def test_queue_span_without_telemetry_enabled(self):
+        """Test that spans are not queued when telemetry disabled."""
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = False
+
+            queue_span({"name": "test", "attributes": {"task_run_id": "123"}})
+
+            assert len(_pending_spans) == 0
+
+    def test_queue_span_without_task_run_id(self):
+        """Test that spans without task_run_id are ignored."""
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+
+            queue_span({"name": "test", "attributes": {}})
+
+            assert len(_pending_spans) == 0
+
+    def test_queue_span_adds_to_pending(self):
+        """Test that spans are added to pending list."""
+        # Don't mock _do_upload so spans stay in pending
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+
+            # Use a sync context (no event loop) so upload happens sync
+            # But we'll make it fail so span stays in pending
+            with patch("hud.telemetry.exporter._do_upload", return_value=False):
+                span = {"name": "test", "attributes": {"task_run_id": "task-123"}}
+                queue_span(span)
+
+                # Span should be in pending (upload failed so not removed)
+                assert "task-123" in _pending_spans
+                assert span in _pending_spans["task-123"]
+
+    @pytest.mark.asyncio
+    async def test_queue_span_uploads_async(self):
+        """Test that spans are uploaded via thread pool in async context."""
+        uploaded_spans: list[dict[str, Any]] = []
+
+        def mock_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded_spans.extend(spans)
+            return True
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=mock_upload),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+
+            span = {"name": "test.async", "attributes": {"task_run_id": "async-task"}}
+            queue_span(span)
+
+            # Wait for thread pool to complete
+            await asyncio.sleep(0.1)
+
+            assert len(uploaded_spans) == 1
+            assert uploaded_spans[0]["name"] == "test.async"
+
+
+class TestFlush:
+    """Tests for flush function."""
+
+    def test_flush_specific_task(self):
+        """Test flushing spans for specific task."""
+        uploaded: list[tuple[str, list[dict[str, Any]]]] = []
+
+        def mock_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded.append((task_run_id, spans))
+            return True
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=mock_upload),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+
+            # Add spans for two tasks
+            _pending_spans["task-1"].append({"name": "span1"})
+            _pending_spans["task-2"].append({"name": "span2"})
+
+            # Flush only task-1
+            flush("task-1")
+
+            assert len(uploaded) == 1
+            assert uploaded[0][0] == "task-1"
+            assert "task-1" not in _pending_spans
+            assert "task-2" in _pending_spans
+
+    def test_flush_all_tasks(self):
+        """Test flushing all pending spans."""
+        uploaded: list[tuple[str, list[dict[str, Any]]]] = []
+
+        def mock_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded.append((task_run_id, spans))
+            return True
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=mock_upload),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+
+            _pending_spans["task-1"].append({"name": "span1"})
+            _pending_spans["task-2"].append({"name": "span2"})
+
+            flush()
+
+            assert len(uploaded) == 2
+            assert len(_pending_spans) == 0
+
+    def test_flush_clears_without_api_key(self):
+        """Test that flush clears spans when no API key."""
+        with patch("hud.settings.settings") as mock_settings:
+            mock_settings.api_key = None
+            mock_settings.telemetry_enabled = True
+
+            _pending_spans["task-1"].append({"name": "span1"})
+
+            flush()
+
+            assert len(_pending_spans) == 0
+
+
+class TestShutdown:
+    """Tests for shutdown function."""
+
+    def test_shutdown_flushes_pending(self):
+        """Test that shutdown flushes pending spans."""
+        uploaded: list[str] = []
+
+        def mock_upload(
+            task_run_id: str,
+            spans: list[dict[str, Any]],
+            telemetry_url: str,
+            api_key: str,
+        ) -> bool:
+            uploaded.append(task_run_id)
+            return True
+
+        with (
+            patch("hud.settings.settings") as mock_settings,
+            patch("hud.telemetry.exporter._do_upload", side_effect=mock_upload),
+            patch("hud.telemetry.exporter._get_api_key", return_value="test-key"),
+        ):
+            mock_settings.api_key = "test-key"
+            mock_settings.telemetry_enabled = True
+            mock_settings.hud_telemetry_url = "https://api.hud.ai"
+
+            _pending_spans["shutdown-task"].append({"name": "final-span"})
+
+            result = shutdown(timeout=1.0)
+
+            assert result is True
+            assert "shutdown-task" in uploaded