kairo-code 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

kairo/backend/services/agent/agent_alerts_evaluation_service.py

@@ -0,0 +1,224 @@
+"""
+Agent alerts evaluation service.
+
+Handles alert condition evaluation and triggering.
+This is a background job service that runs periodically.
+"""
+
+import logging
+from datetime import datetime, timedelta, UTC
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.models.agent import Agent, AgentAlertConfig, AgentAlertHistory, AgentMetrics1m
+from backend.services.agent.agent_events_service import AgentEventsService
+
+logger = logging.getLogger(__name__)
+
+
+class AgentAlertsEvaluationService:
+    """Service for evaluating and triggering alerts."""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+        self._events_service = AgentEventsService(db)
+
+    async def evaluate_alerts(self) -> int:
+        """Evaluate all active alert configs and trigger if conditions met."""
+        now = datetime.now(UTC)
+        triggered_count = 0
+
+        configs = await self._get_enabled_configs()
+
+        for config in configs:
+            try:
+                if self._is_in_cooldown(config, now):
+                    continue
+
+                agent = await self._get_agent(config.agent_id)
+                if not agent or agent.deleted_at:
+                    continue
+
+                should_trigger, trigger_value = await self._evaluate_config(config, agent, now)
+
+                if should_trigger:
+                    await self._trigger_alert(config, trigger_value)
+                    triggered_count += 1
+
+            except Exception as e:
+                logger.warning("Error evaluating alert config %s: %s", config.id, e)
+
+        if triggered_count > 0:
+            await self.db.commit()
+            logger.info("Triggered %d alerts", triggered_count)
+
+        return triggered_count
+
+    async def _get_enabled_configs(self) -> list[AgentAlertConfig]:
+        """Get all enabled alert configurations."""
+        stmt = select(AgentAlertConfig).where(AgentAlertConfig.is_enabled == True)
+        result = await self.db.execute(stmt)
+        return list(result.scalars().all())
+
+    def _is_in_cooldown(self, config: AgentAlertConfig, now: datetime) -> bool:
+        """Check if config is still in cooldown period."""
+        if not config.last_triggered_at:
+            return False
+        cooldown_end = config.last_triggered_at + timedelta(seconds=config.cooldown_seconds)
+        return now < cooldown_end
+
+    async def _get_agent(self, agent_id: str) -> Agent | None:
+        """Get agent by ID."""
+        stmt = select(Agent).where(Agent.id == agent_id)
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none()
+
+    async def _evaluate_config(
+        self,
+        config: AgentAlertConfig,
+        agent: Agent,
+        now: datetime
+    ) -> tuple[bool, float | None]:
+        """Evaluate a single alert config. Returns (should_trigger, trigger_value)."""
+        evaluators = {
+            "offline": self._evaluate_offline_alert,
+            "error_rate": self._evaluate_error_rate_alert,
+            "latency": self._evaluate_latency_alert,
+            "token_budget": self._evaluate_token_budget_alert,
+        }
+
+        evaluator = evaluators.get(config.alert_type)
+        if not evaluator:
+            return False, None
+
+        if config.alert_type == "offline":
+            return evaluator(agent)
+        return await evaluator(config, now)
+
+    def _evaluate_offline_alert(self, agent: Agent) -> tuple[bool, float | None]:
+        """Evaluate offline alert condition."""
+        if agent.state in ("offline", "stale"):
+            return True, 1.0
+        return False, None
+
+    async def _evaluate_error_rate_alert(
+        self,
+        config: AgentAlertConfig,
+        now: datetime
+    ) -> tuple[bool, float | None]:
+        """Evaluate error rate alert condition."""
+        window_start = now - timedelta(seconds=config.window_seconds)
+        stmt = (
+            select(
+                func.sum(AgentMetrics1m.request_count).label("requests"),
+                func.sum(AgentMetrics1m.error_count).label("errors"),
+            )
+            .where(AgentMetrics1m.agent_id == config.agent_id)
+            .where(AgentMetrics1m.bucket_time >= window_start)
+        )
+        result = await self.db.execute(stmt)
+        row = result.one_or_none()
+
+        if row and row.requests and row.requests > 0:
+            error_rate = (row.errors or 0) / row.requests * 100
+            should_trigger = self._check_condition(error_rate, config.condition, config.threshold)
+            return should_trigger, error_rate
+        return False, None
+
+    async def _evaluate_latency_alert(
+        self,
+        config: AgentAlertConfig,
+        now: datetime
+    ) -> tuple[bool, float | None]:
+        """Evaluate latency alert condition."""
+        window_start = now - timedelta(seconds=config.window_seconds)
+        stmt = (
+            select(
+                func.sum(AgentMetrics1m.request_count).label("requests"),
+                func.sum(AgentMetrics1m.total_latency_ms).label("latency"),
+            )
+            .where(AgentMetrics1m.agent_id == config.agent_id)
+            .where(AgentMetrics1m.bucket_time >= window_start)
+        )
+        result = await self.db.execute(stmt)
+        row = result.one_or_none()
+
+        if row and row.requests and row.requests > 0:
+            avg_latency = (row.latency or 0) / row.requests
+            should_trigger = self._check_condition(avg_latency, config.condition, config.threshold)
+            return should_trigger, avg_latency
+        return False, None
+
+    async def _evaluate_token_budget_alert(
+        self,
+        config: AgentAlertConfig,
+        now: datetime
+    ) -> tuple[bool, float | None]:
+        """Evaluate token budget alert condition."""
+        window_start = now - timedelta(seconds=config.window_seconds)
+        stmt = (
+            select(
+                func.sum(AgentMetrics1m.input_tokens + AgentMetrics1m.output_tokens).label("tokens"),
+            )
+            .where(AgentMetrics1m.agent_id == config.agent_id)
+            .where(AgentMetrics1m.bucket_time >= window_start)
+        )
+        result = await self.db.execute(stmt)
+        row = result.one_or_none()
+
+        if row and row.tokens:
+            should_trigger = self._check_condition(row.tokens, config.condition, config.threshold)
+            return should_trigger, float(row.tokens)
+        return False, None
+
+    def _check_condition(self, value: float, condition: str, threshold: float) -> bool:
+        """Check if value meets condition against threshold."""
+        conditions = {
+            "gt": lambda v, t: v > t,
+            "gte": lambda v, t: v >= t,
+            "lt": lambda v, t: v < t,
+            "lte": lambda v, t: v <= t,
+            "eq": lambda v, t: v == t,
+        }
+        checker = conditions.get(condition)
+        return checker(value, threshold) if checker else False
+
+    async def _trigger_alert(
+        self,
+        config: AgentAlertConfig,
+        trigger_value: float | None
+    ) -> None:
+        """Trigger an alert and create history record."""
+        now = datetime.now(UTC)
+
+        alert = AgentAlertHistory(
+            config_id=config.id,
+            agent_id=config.agent_id,
+            alert_type=config.alert_type,
+            severity=config.severity,
+            status="triggered",
+            message=f"{config.name}: {config.metric} {config.condition} {config.threshold}",
+            trigger_value=trigger_value,
+            threshold_value=config.threshold,
+        )
+        self.db.add(alert)
+
+        config.last_triggered_at = now
+
+        await self._events_service.log_event(
+            agent_id=config.agent_id,
+            event_type="alert_triggered",
+            event_data={
+                "config_id": config.id,
+                "alert_type": config.alert_type,
+                "severity": config.severity,
+                "trigger_value": trigger_value,
+                "threshold": config.threshold,
+            },
+        )
+
+        logger.info(
+            "Alert triggered: agent=%s type=%s severity=%s value=%s threshold=%s",
+            config.agent_id, config.alert_type, config.severity, trigger_value, config.threshold
+        )

kairo/backend/services/agent/agent_alerts_service.py

@@ -0,0 +1,201 @@
+"""
+Agent alerts service.
+
+Handles alert configuration CRUD and history queries.
+For alert evaluation and triggering, see agent_alerts_evaluation_service.py.
+"""
+
+import logging
+from datetime import datetime, UTC
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.models.agent import Agent, AgentAlertConfig, AgentAlertHistory
+from backend.schemas.agent import CreateAlertConfigRequest, UpdateAlertConfigRequest
+from backend.services.agent.agent_alerts_evaluation_service import AgentAlertsEvaluationService
+
+logger = logging.getLogger(__name__)
+
+
+class AgentAlertsService:
+    """Service for alert configuration management."""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+        self._evaluation = AgentAlertsEvaluationService(db)
+
+    # ─── Alert Config CRUD ─────────────────────────────────────────────────────
+
+    async def create_alert_config(
+        self,
+        agent_id: str,
+        user_id: str,
+        req: CreateAlertConfigRequest
+    ) -> AgentAlertConfig:
+        """Create alert configuration."""
+        config = AgentAlertConfig(
+            agent_id=agent_id,
+            user_id=user_id,
+            name=req.name,
+            alert_type=req.alert_type,
+            metric=req.metric,
+            condition=req.condition,
+            threshold=req.threshold,
+            window_seconds=req.window_seconds,
+            cooldown_seconds=req.cooldown_seconds,
+            severity=req.severity,
+            channels=req.channels,
+        )
+        self.db.add(config)
+        await self.db.commit()
+        await self.db.refresh(config)
+        return config
+
+    async def get_alert_configs(self, agent_id: str) -> list[AgentAlertConfig]:
+        """Get alert configurations for an agent."""
+        stmt = (
+            select(AgentAlertConfig)
+            .where(AgentAlertConfig.agent_id == agent_id)
+            .order_by(AgentAlertConfig.created_at)
+        )
+        result = await self.db.execute(stmt)
+        return list(result.scalars().all())
+
+    async def get_alert_configs_for_user(
+        self,
+        user_id: str,
+        agent_id: str
+    ) -> list[AgentAlertConfig] | None:
+        """Get configs for an agent owned by a user. Returns None if agent not found."""
+        if not await self._verify_agent_ownership(user_id, agent_id):
+            return None
+        return await self.get_alert_configs(agent_id)
+
+    async def update_alert_config(
+        self,
+        agent_id: str,
+        config_id: str,
+        req: UpdateAlertConfigRequest
+    ) -> AgentAlertConfig | None:
+        """Update alert configuration."""
+        config = await self._get_config(agent_id, config_id)
+        if not config:
+            return None
+
+        self._apply_config_updates(config, req)
+        config.updated_at = datetime.now(UTC)
+
+        await self.db.commit()
+        await self.db.refresh(config)
+        return config
+
+    def _apply_config_updates(
+        self,
+        config: AgentAlertConfig,
+        req: UpdateAlertConfigRequest
+    ) -> None:
+        """Apply update request fields to config."""
+        if req.name is not None:
+            config.name = req.name
+        if req.threshold is not None:
+            config.threshold = req.threshold
+        if req.window_seconds is not None:
+            config.window_seconds = req.window_seconds
+        if req.cooldown_seconds is not None:
+            config.cooldown_seconds = req.cooldown_seconds
+        if req.severity is not None:
+            config.severity = req.severity
+        if req.channels is not None:
+            config.channels = req.channels
+        if req.is_enabled is not None:
+            config.is_enabled = req.is_enabled
+
+    async def update_alert_config_for_user(
+        self,
+        user_id: str,
+        agent_id: str,
+        config_id: str,
+        req: UpdateAlertConfigRequest
+    ) -> AgentAlertConfig | None:
+        """Update config for an agent owned by a user. Returns None if not found."""
+        if not await self._verify_agent_ownership(user_id, agent_id):
+            return None
+        return await self.update_alert_config(agent_id, config_id, req)
+
+    async def delete_alert_config(self, agent_id: str, config_id: str) -> bool:
+        """Delete alert configuration."""
+        config = await self._get_config(agent_id, config_id)
+        if not config:
+            return False
+
+        await self.db.delete(config)
+        await self.db.commit()
+        return True
+
+    async def delete_alert_config_for_user(
+        self,
+        user_id: str,
+        agent_id: str,
+        config_id: str
+    ) -> bool:
+        """Delete config for an agent owned by a user."""
+        if not await self._verify_agent_ownership(user_id, agent_id):
+            return False
+        return await self.delete_alert_config(agent_id, config_id)
+
+    # ─── Alert History ─────────────────────────────────────────────────────────
+
+    async def get_alert_history(self, agent_id: str, limit: int = 50) -> list[AgentAlertHistory]:
+        """Get alert history for an agent."""
+        stmt = (
+            select(AgentAlertHistory)
+            .where(AgentAlertHistory.agent_id == agent_id)
+            .order_by(AgentAlertHistory.created_at.desc())
+            .limit(limit)
+        )
+        result = await self.db.execute(stmt)
+        return list(result.scalars().all())
+
+    async def get_alert_history_for_user(
+        self,
+        user_id: str,
+        agent_id: str,
+        limit: int = 50
+    ) -> list[AgentAlertHistory] | None:
+        """Get history for an agent owned by a user. Returns None if not found."""
+        if not await self._verify_agent_ownership(user_id, agent_id):
+            return None
+        return await self.get_alert_history(agent_id, limit)
+
+    # ─── Delegated Evaluation ──────────────────────────────────────────────────
+
+    async def evaluate_alerts(self) -> int:
+        """Evaluate all active alert configs and trigger if conditions met."""
+        return await self._evaluation.evaluate_alerts()
+
+    # ─── Private Helpers ───────────────────────────────────────────────────────
+
+    async def _get_config(
+        self,
+        agent_id: str,
+        config_id: str
+    ) -> AgentAlertConfig | None:
+        """Get a specific alert config."""
+        stmt = (
+            select(AgentAlertConfig)
+            .where(AgentAlertConfig.id == config_id)
+            .where(AgentAlertConfig.agent_id == agent_id)
+        )
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none()
+
+    async def _verify_agent_ownership(self, user_id: str, agent_id: str) -> bool:
+        """Verify that a user owns an agent."""
+        stmt = (
+            select(Agent)
+            .where(Agent.id == agent_id, Agent.user_id == user_id)
+            .where(Agent.deleted_at.is_(None))
+        )
+        result = await self.db.execute(stmt)
+        return result.scalar_one_or_none() is not None

kairo/backend/services/agent/agent_commands_service.py

@@ -0,0 +1,142 @@
+"""
+Agent commands service.
+
+Handles command issuing, signing, dispatch, and acknowledgment.
+"""
+
+import hashlib
+import hmac
+import json
+import logging
+from datetime import datetime, timedelta, UTC
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.models.agent import Agent, AgentCommand
+from backend.schemas.agent import IssueCommandRequest
+from backend.services.agent.constants import COMMAND_SIGNING_KEY, DEFAULT_COMMAND_TTL_MINUTES
+
+logger = logging.getLogger(__name__)
+
+
+class AgentCommandsService:
+    """Service for agent command management."""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+
+    async def issue_command(
+        self,
+        agent: Agent,
+        user_id: str,
+        req: IssueCommandRequest
+    ) -> AgentCommand:
+        """
+        Issue a command to an agent.
+
+        Args:
+            agent: The target agent
+            user_id: The user issuing the command
+            req: The command request data
+
+        Returns:
+            The created command
+        """
+        now = datetime.now(UTC)
+        expires_at = now + timedelta(minutes=DEFAULT_COMMAND_TTL_MINUTES)
+
+        command = AgentCommand(
+            agent_id=agent.id,
+            command_type=req.command_type,
+            payload=req.payload,
+            issued_by=user_id,
+            expires_at=expires_at,
+        )
+        self.db.add(command)
+
+        # If restart, update restart tracking
+        if req.command_type == "restart":
+            agent.restart_count += 1
+            agent.last_restart_at = now
+            agent.last_restart_reason = "user_requested"
+
+        await self.db.commit()
+        await self.db.refresh(command)
+        logger.info("Command issued: agent=%s type=%s", agent.id, req.command_type)
+        return command
+
+    async def get_pending_commands(self, agent_id: str) -> list[dict]:
+        """
+        Get and dispatch pending commands for an agent.
+
+        Returns signed commands ready for dispatch.
+        """
+        now = datetime.now(UTC)
+
+        stmt = (
+            select(AgentCommand)
+            .where(AgentCommand.agent_id == agent_id)
+            .where(AgentCommand.status == "pending")
+            .where(AgentCommand.expires_at > now)
+            .order_by(AgentCommand.created_at)
+        )
+        result = await self.db.execute(stmt)
+        commands = list(result.scalars().all())
+
+        signed_commands = []
+        for cmd in commands:
+            command_data = {
+                "command_id": cmd.id,
+                "type": cmd.command_type,
+                "payload": cmd.payload,
+                "issued_at": cmd.created_at.isoformat(),
+                "expires_at": cmd.expires_at.isoformat(),
+            }
+            signature = self._sign_command(command_data)
+
+            signed_commands.append({
+                "command_id": cmd.id,
+                "type": cmd.command_type,
+                "payload": cmd.payload,
+                "issued_at": cmd.created_at,
+                "expires_at": cmd.expires_at,
+                "signature": signature,
+            })
+
+            # Mark as dispatched
+            cmd.status = "dispatched"
+            cmd.dispatched_at = now
+
+        if commands:
+            await self.db.commit()
+
+        return signed_commands
+
+    async def acknowledge_command(self, agent_id: str, command_id: str) -> bool:
+        """Mark command as acknowledged by agent."""
+        stmt = (
+            select(AgentCommand)
+            .where(AgentCommand.id == command_id)
+            .where(AgentCommand.agent_id == agent_id)
+        )
+        result = await self.db.execute(stmt)
+        command = result.scalar_one_or_none()
+
+        if not command:
+            return False
+
+        command.status = "acknowledged"
+        command.acknowledged_at = datetime.now(UTC)
+        await self.db.commit()
+        return True
+
+    def _sign_command(self, command_data: dict) -> str:
+        """Sign command using HMAC-SHA256."""
+        message = json.dumps(command_data, sort_keys=True, default=str).encode()
+        return hmac.new(COMMAND_SIGNING_KEY, message, hashlib.sha256).hexdigest()
+
+    def verify_command_signature(self, command_data: dict, signature: str) -> bool:
+        """Verify a command signature."""
+        expected = self._sign_command(command_data)
+        return hmac.compare_digest(expected, signature)