kairo-code 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

kairo/backend/tests/e2e/agents/test_agent_metrics.py

@@ -0,0 +1,587 @@
+"""
+End-to-end tests for Agent Metrics functionality.
+
+Tests cover:
+- Get metrics for different ranges (1h, 24h, 7d)
+- Metrics aggregation
+- Telemetry batch submission
+"""
+
+import pytest
+import uuid
+from datetime import datetime, timedelta, UTC
+from httpx import AsyncClient
+from sqlalchemy import select
+
+from backend.models.agent import Agent, AgentMetrics1m
+
+
+class TestGetMetrics:
+    """Tests for GET /api/agents/{agent_id}/metrics"""
+
+    @pytest.mark.asyncio
+    async def test_get_metrics_empty(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should return empty data for agent with no metrics."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics", headers=auth_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["agent_id"] == test_agent.id
+        assert data["data"] == []
+        assert data["summary"]["total_requests"] == 0
+
+    @pytest.mark.asyncio
+    async def test_get_metrics_with_data(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should return metrics data points."""
+        # Create metrics for the agent
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+        await agent_factory.create_metrics(
+            test_agent.id,
+            bucket_time=now - timedelta(minutes=5),
+            request_count=10,
+            error_count=1,
+            input_tokens=1000,
+            output_tokens=500,
+            total_latency_ms=5000,
+        )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert len(data["data"]) == 1
+        assert data["data"][0]["request_count"] == 10
+        assert data["data"][0]["error_count"] == 1
+
+    @pytest.mark.asyncio
+    async def test_get_metrics_range_1h(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should return metrics for 1 hour range with 1m granularity."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        # Create metrics at different times
+        for i in range(6):
+            await agent_factory.create_metrics(
+                test_agent.id,
+                bucket_time=now - timedelta(minutes=i * 10),
+                request_count=5,
+            )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["range"] == "1h"
+        assert data["granularity"] == "1m"
+        assert len(data["data"]) == 6
+
+    @pytest.mark.asyncio
+    async def test_get_metrics_range_24h(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should return metrics for 24 hour range."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        # Create metrics spanning 24 hours
+        for i in range(24):
+            await agent_factory.create_metrics(
+                test_agent.id,
+                bucket_time=now - timedelta(hours=i),
+                request_count=10,
+                input_tokens=1000,
+            )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=24h", headers=auth_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["range"] == "24h"
+        assert len(data["data"]) == 24
+
+    @pytest.mark.asyncio
+    async def test_get_metrics_range_7d(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should return metrics for 7 day range."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        # Create metrics spanning 7 days
+        for i in range(7):
+            await agent_factory.create_metrics(
+                test_agent.id,
+                bucket_time=now - timedelta(days=i),
+                request_count=100,
+            )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=7d", headers=auth_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["range"] == "7d"
+        assert len(data["data"]) == 7
+
+    @pytest.mark.asyncio
+    async def test_get_metrics_agent_not_found(
+        self, client: AsyncClient, auth_headers: dict
+    ):
+        """Should return 404 for non-existent agent."""
+        fake_id = str(uuid.uuid4())
+
+        response = await client.get(
+            f"/api/agents/{fake_id}/metrics", headers=auth_headers
+        )
+
+        assert response.status_code == 404
+
+
+class TestMetricsSummary:
+    """Tests for metrics summary calculation."""
+
+    @pytest.mark.asyncio
+    async def test_metrics_summary_totals(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should calculate correct totals in summary."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        # Create metrics with known values
+        await agent_factory.create_metrics(
+            test_agent.id,
+            bucket_time=now - timedelta(minutes=5),
+            request_count=100,
+            error_count=10,
+            input_tokens=5000,
+            output_tokens=2500,
+            total_latency_ms=50000,
+        )
+        await agent_factory.create_metrics(
+            test_agent.id,
+            bucket_time=now - timedelta(minutes=10),
+            request_count=50,
+            error_count=5,
+            input_tokens=2500,
+            output_tokens=1250,
+            total_latency_ms=25000,
+        )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        data = response.json()
+        summary = data["summary"]
+
+        assert summary["total_requests"] == 150
+        assert summary["total_errors"] == 15
+        assert summary["total_tokens"] == 11250  # 5000 + 2500 + 2500 + 1250
+        assert summary["error_rate"] == 10.0  # 15/150 * 100
+
+    @pytest.mark.asyncio
+    async def test_metrics_summary_avg_latency(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should calculate average latency correctly."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        await agent_factory.create_metrics(
+            test_agent.id,
+            bucket_time=now - timedelta(minutes=5),
+            request_count=10,
+            total_latency_ms=1000,
+        )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        data = response.json()
+        summary = data["summary"]
+
+        assert summary["avg_latency_ms"] == 100.0  # 1000ms / 10 requests
+
+    @pytest.mark.asyncio
+    async def test_metrics_summary_zero_requests(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should handle zero requests gracefully."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        data = response.json()
+        summary = data["summary"]
+
+        assert summary["total_requests"] == 0
+        assert summary["error_rate"] == 0.0
+        assert summary["avg_latency_ms"] is None
+
+
+class TestMetricsDataPoints:
+    """Tests for individual metric data points."""
+
+    @pytest.mark.asyncio
+    async def test_data_point_fields(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should include all expected fields in data points."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        await agent_factory.create_metrics(
+            test_agent.id,
+            bucket_time=now - timedelta(minutes=5),
+            request_count=20,
+            error_count=2,
+            input_tokens=2000,
+            output_tokens=1000,
+            total_latency_ms=4000,
+            tool_calls=5,
+        )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        data = response.json()
+        point = data["data"][0]
+
+        assert "timestamp" in point
+        assert point["request_count"] == 20
+        assert point["error_count"] == 2
+        assert point["input_tokens"] == 2000
+        assert point["output_tokens"] == 1000
+        assert point["avg_latency_ms"] == 200.0  # 4000 / 20
+        assert point["tool_calls"] == 5
+
+    @pytest.mark.asyncio
+    async def test_data_points_ordered_by_time(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent, agent_factory
+    ):
+        """Should return data points ordered by timestamp."""
+        now = datetime.now(UTC).replace(second=0, microsecond=0)
+
+        # Create metrics in non-chronological order
+        await agent_factory.create_metrics(
+            test_agent.id, bucket_time=now - timedelta(minutes=10)
+        )
+        await agent_factory.create_metrics(
+            test_agent.id, bucket_time=now - timedelta(minutes=30)
+        )
+        await agent_factory.create_metrics(
+            test_agent.id, bucket_time=now - timedelta(minutes=5)
+        )
+
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h", headers=auth_headers
+        )
+
+        data = response.json()
+        timestamps = [point["timestamp"] for point in data["data"]]
+
+        # Should be in ascending order
+        assert timestamps == sorted(timestamps)
+
+
+class TestTelemetryBatch:
+    """Tests for POST /api/agents/telemetry/batch"""
+
+    @pytest.mark.asyncio
+    async def test_submit_telemetry_batch_success(
+        self, client: AsyncClient, api_key_headers: dict, test_agent: Agent
+    ):
+        """Should accept valid telemetry batch."""
+        now = datetime.now(UTC)
+        payload = {
+            "agent_id": test_agent.id,
+            "events": [
+                {
+                    "request_id": str(uuid.uuid4()),
+                    "timestamp_start": (now - timedelta(seconds=30)).isoformat(),
+                    "timestamp_end": (now - timedelta(seconds=29)).isoformat(),
+                    "latency_ms": 1000,
+                    "input_tokens": 100,
+                    "output_tokens": 50,
+                    "model": "nyx",
+                    "status": "success",
+                },
+                {
+                    "request_id": str(uuid.uuid4()),
+                    "timestamp_start": (now - timedelta(seconds=20)).isoformat(),
+                    "timestamp_end": (now - timedelta(seconds=18)).isoformat(),
+                    "latency_ms": 2000,
+                    "input_tokens": 200,
+                    "output_tokens": 100,
+                    "model": "nyx",
+                    "status": "success",
+                },
+            ],
+        }
+
+        response = await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["accepted"] == 2
+        assert data["rejected"] == 0
+        assert data["errors"] == []
+
+    @pytest.mark.asyncio
+    async def test_telemetry_batch_records_metrics(
+        self, client: AsyncClient, api_key_headers: dict, test_agent: Agent, db_session
+    ):
+        """Should record telemetry events as metrics."""
+        now = datetime.now(UTC)
+        payload = {
+            "agent_id": test_agent.id,
+            "events": [
+                {
+                    "request_id": str(uuid.uuid4()),
+                    "timestamp_start": now.isoformat(),
+                    "timestamp_end": (now + timedelta(seconds=1)).isoformat(),
+                    "latency_ms": 500,
+                    "input_tokens": 100,
+                    "output_tokens": 50,
+                    "model": "nyx",
+                    "status": "success",
+                },
+            ],
+        }
+
+        await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        # Check metrics recorded
+        stmt = select(AgentMetrics1m).where(AgentMetrics1m.agent_id == test_agent.id)
+        result = await db_session.execute(stmt)
+        metrics = result.scalar_one_or_none()
+
+        assert metrics is not None
+        assert metrics.request_count == 1
+        assert metrics.input_tokens == 100
+        assert metrics.output_tokens == 50
+
+    @pytest.mark.asyncio
+    async def test_telemetry_batch_error_events(
+        self, client: AsyncClient, api_key_headers: dict, test_agent: Agent, db_session
+    ):
+        """Should count error events correctly."""
+        now = datetime.now(UTC)
+        payload = {
+            "agent_id": test_agent.id,
+            "events": [
+                {
+                    "request_id": str(uuid.uuid4()),
+                    "timestamp_start": now.isoformat(),
+                    "timestamp_end": (now + timedelta(seconds=1)).isoformat(),
+                    "latency_ms": 500,
+                    "input_tokens": 100,
+                    "output_tokens": 0,
+                    "model": "nyx",
+                    "status": "error",
+                    "error_type": "rate_limit",
+                },
+            ],
+        }
+
+        await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        # Check error count
+        stmt = select(AgentMetrics1m).where(AgentMetrics1m.agent_id == test_agent.id)
+        result = await db_session.execute(stmt)
+        metrics = result.scalar_one_or_none()
+
+        assert metrics.error_count == 1
+
+    @pytest.mark.asyncio
+    async def test_telemetry_batch_timeout_events(
+        self, client: AsyncClient, api_key_headers: dict, test_agent: Agent, db_session
+    ):
+        """Should count timeout events correctly."""
+        now = datetime.now(UTC)
+        payload = {
+            "agent_id": test_agent.id,
+            "events": [
+                {
+                    "request_id": str(uuid.uuid4()),
+                    "timestamp_start": now.isoformat(),
+                    "timestamp_end": (now + timedelta(seconds=30)).isoformat(),
+                    "latency_ms": 30000,
+                    "input_tokens": 100,
+                    "output_tokens": 0,
+                    "model": "nyx",
+                    "status": "timeout",
+                },
+            ],
+        }
+
+        await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        stmt = select(AgentMetrics1m).where(AgentMetrics1m.agent_id == test_agent.id)
+        result = await db_session.execute(stmt)
+        metrics = result.scalar_one_or_none()
+
+        assert metrics.timeout_count == 1
+        assert metrics.error_count == 1  # timeout also counts as error
+
+    @pytest.mark.asyncio
+    async def test_telemetry_batch_with_tool_calls(
+        self, client: AsyncClient, api_key_headers: dict, test_agent: Agent, db_session
+    ):
+        """Should record tool call counts."""
+        now = datetime.now(UTC)
+        payload = {
+            "agent_id": test_agent.id,
+            "events": [
+                {
+                    "request_id": str(uuid.uuid4()),
+                    "timestamp_start": now.isoformat(),
+                    "timestamp_end": (now + timedelta(seconds=2)).isoformat(),
+                    "latency_ms": 2000,
+                    "input_tokens": 500,
+                    "output_tokens": 200,
+                    "model": "nyx",
+                    "status": "success",
+                    "tool_calls": [
+                        {"tool": "web_search", "duration_ms": 500},
+                        {"tool": "file_read", "duration_ms": 100},
+                        {"tool": "web_search", "duration_ms": 300},
+                    ],
+                },
+            ],
+        }
+
+        await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        stmt = select(AgentMetrics1m).where(AgentMetrics1m.agent_id == test_agent.id)
+        result = await db_session.execute(stmt)
+        metrics = result.scalar_one_or_none()
+
+        assert metrics.tool_calls == 3
+
+    @pytest.mark.asyncio
+    async def test_telemetry_batch_agent_not_found(
+        self, client: AsyncClient, api_key_headers: dict
+    ):
+        """Should return 404 for non-existent agent."""
+        fake_id = str(uuid.uuid4())
+        payload = {
+            "agent_id": fake_id,
+            "events": [],
+        }
+
+        response = await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        assert response.status_code == 404
+
+    @pytest.mark.asyncio
+    async def test_telemetry_batch_empty_events(
+        self, client: AsyncClient, api_key_headers: dict, test_agent: Agent
+    ):
+        """Should handle empty events list."""
+        payload = {
+            "agent_id": test_agent.id,
+            "events": [],
+        }
+
+        response = await client.post(
+            "/api/agents/telemetry/batch", json=payload, headers=api_key_headers
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["accepted"] == 0
+        assert data["rejected"] == 0
+
+
+class TestMetricsGranularity:
+    """Tests for metrics granularity auto-selection."""
+
+    @pytest.mark.asyncio
+    async def test_auto_granularity_1h_returns_1m(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should use 1m granularity for 1h range."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=1h&granularity=auto",
+            headers=auth_headers,
+        )
+
+        assert response.status_code == 200
+        assert response.json()["granularity"] == "1m"
+
+    @pytest.mark.asyncio
+    async def test_auto_granularity_6h_returns_1m(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should use 1m granularity for 6h range."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=6h&granularity=auto",
+            headers=auth_headers,
+        )
+
+        assert response.status_code == 200
+        assert response.json()["granularity"] == "1m"
+
+    @pytest.mark.asyncio
+    async def test_auto_granularity_24h_returns_1h(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should use 1h granularity for 24h range."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=24h&granularity=auto",
+            headers=auth_headers,
+        )
+
+        assert response.status_code == 200
+        assert response.json()["granularity"] == "1h"
+
+    @pytest.mark.asyncio
+    async def test_auto_granularity_30d_returns_1d(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should use 1d granularity for 30d range."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=30d&granularity=auto",
+            headers=auth_headers,
+        )
+
+        assert response.status_code == 200
+        assert response.json()["granularity"] == "1d"
+
+    @pytest.mark.asyncio
+    async def test_explicit_granularity_override(
+        self, client: AsyncClient, auth_headers: dict, test_agent: Agent
+    ):
+        """Should respect explicit granularity setting."""
+        response = await client.get(
+            f"/api/agents/{test_agent.id}/metrics?range=24h&granularity=1m",
+            headers=auth_headers,
+        )
+
+        assert response.status_code == 200
+        assert response.json()["granularity"] == "1m"