kairo-code 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

kairo/backend/services/agent/agent_metrics_rollup_service.py

@@ -0,0 +1,248 @@
+"""
+Agent metrics rollup and maintenance service.
+
+Handles metrics rollups (1m -> 1h -> daily) and cleanup operations.
+These are background job operations that run periodically.
+"""
+
+import logging
+from datetime import datetime, timedelta, date, UTC
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.models.agent import AgentMetrics1m, AgentMetrics1h, AgentMetricsDaily
+from backend.services.agent.constants import (
+    DEFAULT_METRICS_1M_RETENTION_DAYS,
+    DEFAULT_METRICS_1H_RETENTION_DAYS,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AgentMetricsRollupService:
+    """Service for metrics rollup and maintenance operations."""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+
+    async def rollup_1m_to_1h(self) -> int:
+        """Rollup 1-minute metrics into hourly buckets. Run every hour."""
+        now = datetime.now(UTC)
+        hour_start = (now - timedelta(hours=2)).replace(minute=0, second=0, microsecond=0)
+        hour_end = hour_start + timedelta(hours=1)
+
+        agent_ids = await self._get_agents_with_1m_data(hour_start, hour_end)
+
+        count = 0
+        for agent_id in agent_ids:
+            if await self._hourly_bucket_exists(agent_id, hour_start):
+                continue
+
+            row = await self._aggregate_1m_data(agent_id, hour_start, hour_end)
+
+            if row.request_count and row.request_count > 0:
+                self._create_hourly_bucket(agent_id, hour_start, row)
+                count += 1
+
+        if count > 0:
+            await self.db.commit()
+            logger.info("Rolled up %d hourly metric buckets", count)
+
+        return count
+
+    async def rollup_1h_to_daily(self) -> int:
+        """Rollup hourly metrics into daily buckets. Run once per day."""
+        now = datetime.now(UTC)
+        yesterday = (now - timedelta(days=1)).date()
+        day_start = datetime.combine(yesterday, datetime.min.time()).replace(tzinfo=UTC)
+        day_end = day_start + timedelta(days=1)
+
+        agent_ids = await self._get_agents_with_hourly_data(day_start, day_end)
+
+        count = 0
+        for agent_id in agent_ids:
+            if await self._daily_bucket_exists(agent_id, yesterday):
+                continue
+
+            row = await self._aggregate_1h_data(agent_id, day_start, day_end)
+
+            if row.request_count and row.request_count > 0:
+                self._create_daily_bucket(agent_id, yesterday, row)
+                count += 1
+
+        if count > 0:
+            await self.db.commit()
+            logger.info("Rolled up %d daily metric buckets", count)
+
+        return count
+
+    async def cleanup_old_metrics(
+        self,
+        retention_days_1m: int = DEFAULT_METRICS_1M_RETENTION_DAYS,
+        retention_days_1h: int = DEFAULT_METRICS_1H_RETENTION_DAYS
+    ) -> tuple[int, int]:
+        """Clean up old metrics based on retention policy."""
+        now = datetime.now(UTC)
+
+        deleted_1m = await self._cleanup_metrics_table(
+            AgentMetrics1m,
+            now - timedelta(days=retention_days_1m)
+        )
+
+        deleted_1h = await self._cleanup_metrics_table(
+            AgentMetrics1h,
+            now - timedelta(days=retention_days_1h)
+        )
+
+        if deleted_1m > 0 or deleted_1h > 0:
+            await self.db.commit()
+            logger.info("Cleaned up metrics: %d 1m records, %d 1h records", deleted_1m, deleted_1h)
+
+        return deleted_1m, deleted_1h
+
+    # ─── Private Helpers ───────────────────────────────────────────────────────
+
+    async def _get_agents_with_1m_data(
+        self,
+        start: datetime,
+        end: datetime
+    ) -> list[str]:
+        """Get list of agent IDs with 1m data in the time range."""
+        stmt = (
+            select(AgentMetrics1m.agent_id)
+            .where(AgentMetrics1m.bucket_time >= start)
+            .where(AgentMetrics1m.bucket_time < end)
+            .distinct()
+        )
+        result = await self.db.execute(stmt)
+        return [row[0] for row in result.all()]
+
+    async def _get_agents_with_hourly_data(
+        self,
+        start: datetime,
+        end: datetime
+    ) -> list[str]:
+        """Get list of agent IDs with hourly data."""
+        stmt = (
+            select(AgentMetrics1h.agent_id)
+            .where(AgentMetrics1h.bucket_time >= start)
+            .where(AgentMetrics1h.bucket_time < end)
+            .distinct()
+        )
+        result = await self.db.execute(stmt)
+        return [row[0] for row in result.all()]
+
+    async def _hourly_bucket_exists(self, agent_id: str, hour_start: datetime) -> bool:
+        """Check if hourly bucket already exists."""
+        stmt = select(AgentMetrics1h).where(
+            AgentMetrics1h.agent_id == agent_id,
+            AgentMetrics1h.bucket_time == hour_start
+        )
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none() is not None
+
+    async def _daily_bucket_exists(self, agent_id: str, day: date) -> bool:
+        """Check if daily bucket already exists."""
+        stmt = select(AgentMetricsDaily).where(
+            AgentMetricsDaily.agent_id == agent_id,
+            AgentMetricsDaily.date == day
+        )
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none() is not None
+
+    async def _aggregate_1m_data(self, agent_id: str, start: datetime, end: datetime):
+        """Aggregate 1-minute data for a time range."""
+        stmt = (
+            select(
+                func.sum(AgentMetrics1m.request_count).label("request_count"),
+                func.sum(AgentMetrics1m.error_count).label("error_count"),
+                func.sum(AgentMetrics1m.timeout_count).label("timeout_count"),
+                func.sum(AgentMetrics1m.input_tokens).label("input_tokens"),
+                func.sum(AgentMetrics1m.output_tokens).label("output_tokens"),
+                func.sum(AgentMetrics1m.total_latency_ms).label("total_latency_ms"),
+                func.min(AgentMetrics1m.min_latency_ms).label("min_latency_ms"),
+                func.max(AgentMetrics1m.max_latency_ms).label("max_latency_ms"),
+                func.sum(AgentMetrics1m.tool_calls).label("tool_calls"),
+                func.sum(AgentMetrics1m.tool_errors).label("tool_errors"),
+            )
+            .where(AgentMetrics1m.agent_id == agent_id)
+            .where(AgentMetrics1m.bucket_time >= start)
+            .where(AgentMetrics1m.bucket_time < end)
+        )
+        result = await self.db.execute(stmt)
+        return result.one()
+
+    async def _aggregate_1h_data(self, agent_id: str, start: datetime, end: datetime):
+        """Aggregate hourly data for a time range."""
+        stmt = (
+            select(
+                func.sum(AgentMetrics1h.request_count).label("request_count"),
+                func.sum(AgentMetrics1h.error_count).label("error_count"),
+                func.sum(AgentMetrics1h.timeout_count).label("timeout_count"),
+                func.sum(AgentMetrics1h.input_tokens).label("input_tokens"),
+                func.sum(AgentMetrics1h.output_tokens).label("output_tokens"),
+                func.sum(AgentMetrics1h.total_latency_ms).label("total_latency_ms"),
+                func.min(AgentMetrics1h.min_latency_ms).label("min_latency_ms"),
+                func.max(AgentMetrics1h.max_latency_ms).label("max_latency_ms"),
+                func.sum(AgentMetrics1h.tool_calls).label("tool_calls"),
+                func.sum(AgentMetrics1h.tool_errors).label("tool_errors"),
+            )
+            .where(AgentMetrics1h.agent_id == agent_id)
+            .where(AgentMetrics1h.bucket_time >= start)
+            .where(AgentMetrics1h.bucket_time < end)
+        )
+        result = await self.db.execute(stmt)
+        return result.one()
+
+    def _create_hourly_bucket(self, agent_id: str, hour_start: datetime, row) -> None:
+        """Create an hourly metrics bucket."""
+        hourly = AgentMetrics1h(
+            agent_id=agent_id,
+            bucket_time=hour_start,
+            request_count=row.request_count or 0,
+            error_count=row.error_count or 0,
+            timeout_count=row.timeout_count or 0,
+            input_tokens=row.input_tokens or 0,
+            output_tokens=row.output_tokens or 0,
+            total_latency_ms=row.total_latency_ms or 0,
+            min_latency_ms=row.min_latency_ms,
+            max_latency_ms=row.max_latency_ms,
+            tool_calls=row.tool_calls or 0,
+            tool_errors=row.tool_errors or 0,
+        )
+        self.db.add(hourly)
+
+    def _create_daily_bucket(self, agent_id: str, day: date, row) -> None:
+        """Create a daily metrics bucket."""
+        avg_latency = None
+        if row.request_count and row.request_count > 0 and row.total_latency_ms:
+            avg_latency = int(row.total_latency_ms / row.request_count)
+
+        daily = AgentMetricsDaily(
+            agent_id=agent_id,
+            date=day,
+            request_count=row.request_count or 0,
+            error_count=row.error_count or 0,
+            timeout_count=row.timeout_count or 0,
+            input_tokens=row.input_tokens or 0,
+            output_tokens=row.output_tokens or 0,
+            total_latency_ms=row.total_latency_ms or 0,
+            min_latency_ms=row.min_latency_ms,
+            max_latency_ms=row.max_latency_ms,
+            avg_latency_ms=avg_latency,
+            tool_calls=row.tool_calls or 0,
+            tool_errors=row.tool_errors or 0,
+        )
+        self.db.add(daily)
+
+    async def _cleanup_metrics_table(self, model, cutoff: datetime) -> int:
+        """Delete old records from a metrics table."""
+        stmt = select(model).where(model.bucket_time < cutoff)
+        result = await self.db.execute(stmt)
+        records = list(result.scalars().all())
+
+        for record in records:
+            await self.db.delete(record)
+
+        return len(records)

kairo/backend/services/agent/agent_metrics_service.py

@@ -0,0 +1,259 @@
+"""
+Agent metrics service.
+
+Handles metrics queries and telemetry processing.
+For rollup and cleanup operations, see agent_metrics_rollup_service.py.
+"""
+
+import logging
+from datetime import datetime, timedelta, UTC
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.models.agent import Agent, AgentMetrics1m
+from backend.schemas.agent import TelemetryBatchRequest
+from backend.services.agent.constants import TIME_RANGE_MAP
+from backend.services.agent.agent_metrics_rollup_service import AgentMetricsRollupService
+
+logger = logging.getLogger(__name__)
+
+
+class AgentMetricsService:
+    """Service for agent metrics queries and telemetry processing."""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+        self._rollup = AgentMetricsRollupService(db)
+
+    async def get_metrics(
+        self,
+        agent_id: str,
+        range_str: str = "24h",
+        granularity: str = "auto"
+    ) -> dict:
+        """
+        Get agent metrics for a time range.
+
+        Args:
+            agent_id: The agent ID
+            range_str: Time range (1h, 6h, 24h, 7d, 30d)
+            granularity: Data granularity (auto, 1m, 1h, 1d)
+
+        Returns:
+            Dictionary with metrics data and summary
+        """
+        hours = TIME_RANGE_MAP.get(range_str, 24)
+        delta = timedelta(hours=hours)
+        cutoff = datetime.now(UTC) - delta
+
+        if granularity == "auto":
+            granularity = self._determine_granularity(delta)
+
+        metrics = await self._query_metrics(agent_id, cutoff)
+        return self._build_metrics_response(agent_id, range_str, granularity, metrics)
+
+    def _determine_granularity(self, delta: timedelta) -> str:
+        """Determine the appropriate granularity for a time range."""
+        if delta <= timedelta(hours=6):
+            return "1m"
+        elif delta <= timedelta(days=7):
+            return "1h"
+        return "1d"
+
+    async def _query_metrics(
+        self,
+        agent_id: str,
+        cutoff: datetime
+    ) -> list[AgentMetrics1m]:
+        """Query metrics from the 1m table."""
+        stmt = (
+            select(AgentMetrics1m)
+            .where(AgentMetrics1m.agent_id == agent_id)
+            .where(AgentMetrics1m.bucket_time >= cutoff)
+            .order_by(AgentMetrics1m.bucket_time)
+        )
+        result = await self.db.execute(stmt)
+        return list(result.scalars().all())
+
+    def _build_metrics_response(
+        self,
+        agent_id: str,
+        range_str: str,
+        granularity: str,
+        metrics: list[AgentMetrics1m]
+    ) -> dict:
+        """Build the metrics response dictionary."""
+        data = []
+        totals = {"requests": 0, "errors": 0, "tokens": 0, "latency": 0}
+
+        for m in metrics:
+            data.append(self._format_metric_point(m))
+            totals["requests"] += m.request_count
+            totals["errors"] += m.error_count
+            totals["tokens"] += m.input_tokens + m.output_tokens
+            totals["latency"] += m.total_latency_ms
+
+        return {
+            "agent_id": agent_id,
+            "range": range_str,
+            "granularity": granularity,
+            "data": data,
+            "summary": self._build_summary(totals),
+        }
+
+    def _format_metric_point(self, m: AgentMetrics1m) -> dict:
+        """Format a single metric data point."""
+        avg_latency = m.total_latency_ms / m.request_count if m.request_count > 0 else None
+        return {
+            "timestamp": m.bucket_time,
+            "request_count": m.request_count,
+            "error_count": m.error_count,
+            "input_tokens": m.input_tokens,
+            "output_tokens": m.output_tokens,
+            "avg_latency_ms": avg_latency,
+            "tool_calls": m.tool_calls,
+        }
+
+    def _build_summary(self, totals: dict) -> dict:
+        """Build the summary section of metrics response."""
+        avg_latency = None
+        error_rate = 0.0
+        if totals["requests"] > 0:
+            avg_latency = totals["latency"] / totals["requests"]
+            error_rate = totals["errors"] / totals["requests"] * 100
+
+        return {
+            "total_requests": totals["requests"],
+            "total_errors": totals["errors"],
+            "total_tokens": totals["tokens"],
+            "avg_latency_ms": avg_latency,
+            "error_rate": error_rate,
+        }
+
+    async def get_metrics_for_user(
+        self,
+        user_id: str,
+        agent_id: str,
+        range_str: str = "24h",
+        granularity: str = "auto"
+    ) -> dict | None:
+        """Get metrics for an agent owned by a user. Returns None if not found."""
+        if not await self._verify_agent_ownership(user_id, agent_id):
+            return None
+        return await self.get_metrics(agent_id, range_str, granularity)
+
+    async def _verify_agent_ownership(self, user_id: str, agent_id: str) -> bool:
+        """Verify that a user owns an agent."""
+        stmt = (
+            select(Agent)
+            .where(Agent.id == agent_id, Agent.user_id == user_id)
+            .where(Agent.deleted_at.is_(None))
+        )
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none() is not None
+
+    # ─── Telemetry Processing ──────────────────────────────────────────────────
+
+    async def process_telemetry_batch(
+        self,
+        agent_id: str,
+        req: TelemetryBatchRequest
+    ) -> tuple[int, int, list[str]]:
+        """Process a batch of telemetry events."""
+        accepted = 0
+        rejected = 0
+        errors = []
+
+        for event in req.events:
+            try:
+                await self._process_telemetry_event(agent_id, event)
+                accepted += 1
+            except Exception as e:
+                rejected += 1
+                errors.append(f"{event.request_id}: {str(e)}")
+
+        await self.db.commit()
+        return accepted, rejected, errors
+
+    async def _process_telemetry_event(self, agent_id: str, event) -> None:
+        """Process a single telemetry event."""
+        bucket_time = event.timestamp_start.replace(second=0, microsecond=0)
+        is_error = event.status in ("error", "timeout")
+
+        existing = await self._get_metrics_bucket(agent_id, bucket_time)
+
+        if existing:
+            self._update_metrics_bucket(existing, event, is_error)
+        else:
+            self._create_metrics_bucket(agent_id, bucket_time, event, is_error)
+
+    async def _get_metrics_bucket(
+        self,
+        agent_id: str,
+        bucket_time: datetime
+    ) -> AgentMetrics1m | None:
+        """Get existing metrics bucket."""
+        stmt = select(AgentMetrics1m).where(
+            AgentMetrics1m.agent_id == agent_id,
+            AgentMetrics1m.bucket_time == bucket_time
+        )
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none()
+
+    def _update_metrics_bucket(
+        self,
+        existing: AgentMetrics1m,
+        event,
+        is_error: bool
+    ) -> None:
+        """Update an existing metrics bucket."""
+        existing.request_count += 1
+        existing.error_count += 1 if is_error else 0
+        existing.timeout_count += 1 if event.status == "timeout" else 0
+        existing.input_tokens += event.input_tokens
+        existing.output_tokens += event.output_tokens
+        existing.total_latency_ms += event.latency_ms
+        if event.tool_calls:
+            existing.tool_calls += len(event.tool_calls)
+
+    def _create_metrics_bucket(
+        self,
+        agent_id: str,
+        bucket_time: datetime,
+        event,
+        is_error: bool
+    ) -> None:
+        """Create a new metrics bucket."""
+        new_metrics = AgentMetrics1m(
+            agent_id=agent_id,
+            bucket_time=bucket_time,
+            request_count=1,
+            error_count=1 if is_error else 0,
+            timeout_count=1 if event.status == "timeout" else 0,
+            input_tokens=event.input_tokens,
+            output_tokens=event.output_tokens,
+            total_latency_ms=event.latency_ms,
+            min_latency_ms=event.latency_ms,
+            max_latency_ms=event.latency_ms,
+            tool_calls=len(event.tool_calls) if event.tool_calls else 0,
+        )
+        self.db.add(new_metrics)
+
+    # ─── Delegated Rollup Operations ───────────────────────────────────────────
+
+    async def rollup_1m_to_1h(self) -> int:
+        """Delegate to rollup service."""
+        return await self._rollup.rollup_1m_to_1h()
+
+    async def rollup_1h_to_daily(self) -> int:
+        """Delegate to rollup service."""
+        return await self._rollup.rollup_1h_to_daily()
+
+    async def cleanup_old_metrics(
+        self,
+        retention_days_1m: int = 30,
+        retention_days_1h: int = 365
+    ) -> tuple[int, int]:
+        """Delegate to rollup service."""
+        return await self._rollup.cleanup_old_metrics(retention_days_1m, retention_days_1h)