nthlayer-workers 1.0.0__py3-none-any.whl

nthlayer_workers/respond/safe_actions/webhook.py

@@ -0,0 +1,194 @@
+"""Webhook dispatcher for safe action execution bindings.
+
+Renders {{variable}} templates, resolves ${ENV_VAR} secrets,
+makes HTTP calls, and optionally verifies results via PromQL.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ExecutionResult:
+    """Result of a safe action execution."""
+
+    success: bool
+    status_code: int | None = None
+    detail: str = ""
+    verified: bool | None = None
+    verification_detail: str | None = None
+
+
+def render_binding_templates(obj: Any, variables: dict[str, str]) -> Any:
+    """Recursively render {{variable}} placeholders in strings."""
+    if isinstance(obj, str):
+        for key, value in variables.items():
+            obj = obj.replace("{{" + key + "}}", str(value))
+            obj = obj.replace("{{ " + key + " }}", str(value))
+        return obj
+    if isinstance(obj, dict):
+        return {k: render_binding_templates(v, variables) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [render_binding_templates(item, variables) for item in obj]
+    return obj
+
+
+def resolve_secrets(obj: Any) -> Any:
+    """Recursively resolve ${ENV_VAR} placeholders from os.environ.
+
+    Raises ValueError if a referenced env var is not set.
+    """
+    if isinstance(obj, str):
+        def _replace(match):
+            var_name = match.group(1)
+            value = os.environ.get(var_name)
+            if value is None:
+                raise ValueError(
+                    f"Secret ${{{var_name}}} not set. "
+                    f"Set the {var_name} environment variable."
+                )
+            return value
+        return re.sub(r'\$\{(\w+)\}', _replace, obj)
+    if isinstance(obj, dict):
+        return {k: resolve_secrets(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [resolve_secrets(item) for item in obj]
+    return obj
+
+
+class WebhookDispatcher:
+    """Execute safe action bindings via HTTP webhooks."""
+
+    async def execute(
+        self, binding: dict | str, variables: dict[str, str]
+    ) -> ExecutionResult:
+        """Render templates, resolve secrets, make HTTP call, verify."""
+        if binding == "stub" or not binding:
+            target = variables.get("service", variables.get("target", "unknown"))
+            return ExecutionResult(
+                success=True,
+                detail=f"Stub execution for {target} (no binding configured).",
+            )
+
+        rendered = render_binding_templates(binding, variables)
+        try:
+            rendered = resolve_secrets(rendered)
+        except ValueError as exc:
+            return ExecutionResult(success=False, detail=str(exc))
+
+        url = rendered.get("url", "")
+        headers = rendered.get("headers", {})
+        body = rendered.get("body")
+        timeout = int(rendered.get("timeout", 30))
+        retry_config = rendered.get("retry", {})
+        verify_config = rendered.get("verify_after")
+
+        result = await self._call_webhook(url, headers, body, timeout, retry_config)
+
+        if verify_config and result.success:
+            verification = await self._verify(verify_config, variables)
+            result.verified = verification.verified
+            result.verification_detail = verification.verification_detail
+
+        return result
+
+    async def _call_webhook(
+        self, url, headers, body, timeout, retry_config
+    ) -> ExecutionResult:
+        """Make HTTP POST with retry logic."""
+        attempts = retry_config.get("attempts", 1)
+        backoff = retry_config.get("backoff", [1])
+        last_error = ""
+        last_status = None
+
+        async with httpx.AsyncClient() as client:
+            for attempt in range(attempts):
+                try:
+                    resp = await client.post(
+                        url, headers=headers, json=body, timeout=timeout
+                    )
+                    last_status = resp.status_code
+                    if resp.is_success:
+                        return ExecutionResult(
+                            success=True,
+                            status_code=resp.status_code,
+                            detail=resp.text[:500],
+                        )
+                    resp.raise_for_status()
+                except httpx.HTTPStatusError as exc:
+                    last_error = f"HTTP {exc.response.status_code}: {exc.response.text[:200]}"
+                    last_status = exc.response.status_code
+                except httpx.TimeoutException:
+                    last_error = f"Timeout after {timeout}s"
+                except Exception as exc:
+                    last_error = str(exc)
+
+                if attempt < attempts - 1:
+                    delay = backoff[min(attempt, len(backoff) - 1)]
+                    await asyncio.sleep(delay)
+
+        return ExecutionResult(
+            success=False, status_code=last_status, detail=last_error
+        )
+
+    async def _verify(self, verify_config, variables) -> ExecutionResult:
+        """Wait, query Prometheus, return verification result."""
+        wait = int(verify_config.get("wait", 30))
+        query = verify_config.get("query", "")
+        description = verify_config.get("description", "")
+        prometheus_url = verify_config.get("prometheus_url") or os.environ.get(
+            "PROMETHEUS_URL", "http://localhost:9090"
+        )
+
+        query = render_binding_templates(query, variables)
+
+        logger.info("Waiting %ds before verification: %s", wait, description)
+        await asyncio.sleep(wait)
+
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(
+                    f"{prometheus_url}/api/v1/query",
+                    params={"query": query},
+                    timeout=10.0,
+                )
+                resp.raise_for_status()
+                data = resp.json()
+                results = data.get("data", {}).get("result", [])
+
+                if not results:
+                    return ExecutionResult(
+                        success=True,
+                        verified=None,
+                        verification_detail=f"No data for query: {description}",
+                    )
+
+                value = float(results[0].get("value", [None, "0"])[1])
+                verified = value == 1.0
+
+                return ExecutionResult(
+                    success=True,
+                    verified=verified,
+                    verification_detail=(
+                        f"Verified: {description}"
+                        if verified
+                        else f"Verification failed: {description} (value={value})"
+                    ),
+                )
+
+        except Exception as exc:
+            logger.warning("Verification query failed: %s", exc)
+            return ExecutionResult(
+                success=True,
+                verified=None,
+                verification_detail=f"Verification unavailable: {exc}",
+            )

nthlayer_workers/respond/server.py

@@ -0,0 +1,357 @@
+"""HTTP server for incident approval workflows.
+
+Starlette ASGI app with routes for approve, reject, status, and
+Slack interaction callbacks. Embedded in `nthlayer-respond serve`.
+"""
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import logging
+import time as _time
+from collections.abc import AsyncGenerator
+from datetime import datetime, timezone
+from typing import Any
+
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+from starlette.routing import Route
+
+from nthlayer_common.slack_web import SlackWebClient
+from nthlayer_workers.respond.config import RespondConfig
+from nthlayer_workers.respond.metrics import VerdictMetricsCollector
+from nthlayer_workers.respond.types import IncidentState
+
+logger = logging.getLogger(__name__)
+
+
+class ApprovalServer:
+    """HTTP server for incident approval workflows."""
+
+    def __init__(
+        self,
+        coordinator: Any,
+        context_store: Any,
+        config: RespondConfig,
+        verdict_store: Any = None,
+    ) -> None:
+        self._coordinator = coordinator
+        self._context_store = context_store
+        self._config = config
+        self._timeouts: dict[str, asyncio.Task] = {}
+        self._locks: dict[str, asyncio.Lock] = {}
+        self._metrics = VerdictMetricsCollector(verdict_store) if verdict_store else None
+
+    def _get_lock(self, incident_id: str) -> asyncio.Lock:
+        """Get or create a per-incident lock to serialize approve/reject/timeout."""
+        if incident_id not in self._locks:
+            self._locks[incident_id] = asyncio.Lock()
+        return self._locks[incident_id]
+
+    @contextlib.asynccontextmanager
+    async def _lifespan(self, app: Starlette) -> AsyncGenerator[None, None]:
+        """Recover pending approval timeouts on startup."""
+        await self.recover_pending_approvals()
+        yield
+
+    def build_app(self) -> Starlette:
+        """Build the Starlette ASGI application."""
+        routes = [
+            Route(
+                "/api/v1/incidents/{incident_id}/approve",
+                self.handle_approve,
+                methods=["POST"],
+            ),
+            Route(
+                "/api/v1/incidents/{incident_id}/reject",
+                self.handle_reject,
+                methods=["POST"],
+            ),
+            Route(
+                "/api/v1/incidents/{incident_id}",
+                self.handle_status,
+                methods=["GET"],
+            ),
+            Route(
+                "/api/v1/slack/interactions",
+                self.handle_slack_interaction,
+                methods=["POST"],
+            ),
+            Route("/metrics", self.handle_metrics, methods=["GET"]),
+        ]
+        return Starlette(routes=routes, lifespan=self._lifespan)
+
+    async def handle_approve(self, request: Request) -> JSONResponse:
+        """POST /api/v1/incidents/{id}/approve"""
+        incident_id = request.path_params["incident_id"]
+        try:
+            body = await request.json() if await request.body() else {}
+        except (json.JSONDecodeError, ValueError):
+            return JSONResponse({"error": "Invalid JSON body"}, status_code=400)
+        approved_by = body.get("approved_by")
+
+        async with self._get_lock(incident_id):
+            try:
+                ctx = await self._coordinator.approve(
+                    incident_id, approved_by=approved_by
+                )
+            except ValueError as exc:
+                msg = str(exc)
+                if "not found" in msg.lower():
+                    return JSONResponse({"error": msg}, status_code=404)
+                return JSONResponse({"error": msg}, status_code=409)
+
+            self.cancel_timeout(incident_id)
+
+        return JSONResponse({
+            "incident_id": ctx.id,
+            "state": ctx.state.value,
+            "action": ctx.remediation.proposed_action if ctx.remediation else None,
+            "target": ctx.remediation.target if ctx.remediation else None,
+            "approved_by": approved_by,
+            "execution_result": ctx.remediation.execution_result if ctx.remediation else None,
+            "verdict_id": ctx.verdict_chain[-1] if ctx.verdict_chain else None,
+        })
+
+    async def handle_reject(self, request: Request) -> JSONResponse:
+        """POST /api/v1/incidents/{id}/reject"""
+        incident_id = request.path_params["incident_id"]
+        try:
+            body = await request.json() if await request.body() else {}
+        except (json.JSONDecodeError, ValueError):
+            return JSONResponse({"error": "Invalid JSON body"}, status_code=400)
+        reason = body.get("reason")
+        rejected_by = body.get("rejected_by")
+
+        if not reason:
+            return JSONResponse(
+                {"error": "reason is required"}, status_code=400
+            )
+
+        async with self._get_lock(incident_id):
+            try:
+                ctx = await self._coordinator.reject(
+                    incident_id, reason, rejected_by=rejected_by
+                )
+            except ValueError as exc:
+                msg = str(exc)
+                if "not found" in msg.lower():
+                    return JSONResponse({"error": msg}, status_code=404)
+                return JSONResponse({"error": msg}, status_code=409)
+
+            self.cancel_timeout(incident_id)
+
+        return JSONResponse({
+            "incident_id": ctx.id,
+            "state": ctx.state.value,
+            "rejected_by": rejected_by,
+            "reason": reason,
+        })
+
+    async def handle_status(self, request: Request) -> JSONResponse:
+        """GET /api/v1/incidents/{id}"""
+        incident_id = request.path_params["incident_id"]
+        ctx = self._context_store.load(incident_id)
+
+        if ctx is None:
+            return JSONResponse(
+                {"error": f"Incident {incident_id!r} not found"}, status_code=404
+            )
+
+        result: dict[str, Any] = {
+            "incident_id": ctx.id,
+            "state": ctx.state.value,
+            "created_at": ctx.created_at,
+            "updated_at": ctx.updated_at,
+            "trigger_source": ctx.trigger_source,
+        }
+        if ctx.remediation:
+            result["proposed_action"] = ctx.remediation.proposed_action
+            result["target"] = ctx.remediation.target
+            result["requires_human_approval"] = ctx.remediation.requires_human_approval
+            result["executed"] = ctx.remediation.executed
+        if ctx.triage:
+            result["severity"] = ctx.triage.severity
+        return JSONResponse(result)
+
+    async def handle_slack_interaction(self, request: Request) -> Response:
+        """POST /api/v1/slack/interactions — Slack callback endpoint."""
+        signing_secret = self._config.slack_signing_secret
+        if not signing_secret:
+            return Response(
+                content=json.dumps({"error": "Slack signing secret not configured"}),
+                status_code=403,
+                media_type="application/json",
+            )
+
+        body = await request.body()
+        timestamp = request.headers.get("X-Slack-Request-Timestamp", "")
+        signature = request.headers.get("X-Slack-Signature", "")
+
+        if not SlackWebClient.verify_signature(signing_secret, timestamp, body, signature):
+            return Response(status_code=401)
+
+        # Slack sends payload as form-encoded
+        form = await request.form()
+        payload_str = form.get("payload", "")
+        try:
+            payload = json.loads(payload_str)
+        except (json.JSONDecodeError, TypeError):
+            return Response(status_code=400)
+
+        actions = payload.get("actions", [])
+        if not actions:
+            return Response(status_code=200)
+
+        action = actions[0]
+        action_id = action.get("action_id")
+        incident_id = action.get("value")
+        if not incident_id:
+            return Response(status_code=200)
+
+        user = payload.get("user") or {}
+        user_name = user.get("name", "unknown") if isinstance(user, dict) else "unknown"
+        channel = payload.get("channel") or {}
+        channel_id = channel.get("id") if isinstance(channel, dict) else None
+        message = payload.get("message") or {}
+        message_ts = message.get("ts") if isinstance(message, dict) else None
+
+        async with self._get_lock(incident_id):
+            try:
+                if action_id == "approve":
+                    ctx = await self._coordinator.approve(
+                        incident_id, approved_by=user_name
+                    )
+                elif action_id == "reject":
+                    ctx = await self._coordinator.reject(
+                        incident_id,
+                        f"Rejected via Slack by {user_name}",
+                        rejected_by=user_name,
+                    )
+                else:
+                    return Response(status_code=200)
+            except ValueError as exc:
+                logger.warning("Slack interaction failed: %s", exc)
+                return Response(status_code=200)  # Slack expects 200
+
+            self.cancel_timeout(incident_id)
+
+        # Update original message to remove buttons (fire-and-forget)
+        if self._config.slack_bot_token and channel_id and message_ts:
+            asyncio.create_task(
+                self._update_slack_message(
+                    channel_id, message_ts, action_id, user_name, ctx
+                )
+            )
+
+        return Response(status_code=200)
+
+    async def handle_metrics(self, request: Request) -> Response:
+        """GET /metrics — Prometheus text exposition."""
+        if self._metrics is None:
+            return Response(content="", media_type="text/plain")
+        text = self._metrics.collect()
+        return Response(
+            content=text,
+            media_type="text/plain; version=0.0.4; charset=utf-8",
+        )
+
+    async def _update_slack_message(
+        self,
+        channel_id: str,
+        message_ts: str,
+        action_id: str,
+        user_name: str,
+        ctx: Any,
+    ) -> None:
+        """Replace buttons with confirmation text in the original Slack message."""
+        client = SlackWebClient(self._config.slack_bot_token)
+
+        if action_id == "approve":
+            status_text = f"\u2705 Approved by @{user_name}"
+        else:
+            status_text = f"\u274c Rejected by @{user_name}"
+
+        blocks = [
+            {"type": "section", "text": {"type": "mrkdwn", "text": f"*{status_text}*"}},
+            {"type": "context", "elements": [
+                {"type": "mrkdwn", "text": f"State: {ctx.state.value} \u00b7 nthlayer-respond"},
+            ]},
+        ]
+
+        try:
+            await client.update_message(channel_id, message_ts, blocks, status_text)
+        except Exception as exc:
+            logger.warning("Slack message update failed: %s", exc)
+
+    def start_timeout(self, incident_id: str) -> None:
+        """Start a background timeout task for an incident."""
+        self.cancel_timeout(incident_id)
+        task = asyncio.create_task(self._timeout_task(incident_id))
+        self._timeouts[incident_id] = task
+
+    def cancel_timeout(self, incident_id: str) -> None:
+        """Cancel the timeout task for an incident if active."""
+        task = self._timeouts.pop(incident_id, None)
+        if task and not task.done():
+            task.cancel()
+        self._locks.pop(incident_id, None)
+
+    async def _timeout_task(self, incident_id: str, delay: float | None = None) -> None:
+        """Wait for delay seconds, then auto-reject if still awaiting approval.
+
+        If delay is None, uses approval_timeout_seconds from config.
+        """
+        wait = delay if delay is not None else self._config.approval_timeout_seconds
+        try:
+            await asyncio.sleep(wait)
+        except asyncio.CancelledError:
+            return
+
+        async with self._get_lock(incident_id):
+            ctx = self._context_store.load(incident_id)
+            if ctx is None or ctx.state != IncidentState.AWAITING_APPROVAL:
+                return
+
+            try:
+                await self._coordinator.reject(
+                    incident_id,
+                    f"Approval timed out after {self._config.approval_timeout_seconds}s",
+                    rejected_by="system/timeout",
+                )
+                logger.info("Approval timed out", extra={"incident_id": incident_id})
+            except Exception as exc:
+                logger.warning("Timeout reject failed: %s", exc)
+            finally:
+                self._timeouts.pop(incident_id, None)
+
+    async def recover_pending_approvals(self) -> None:
+        """On startup, scan for AWAITING_APPROVAL incidents and start timeouts."""
+        active = self._context_store.list_active()
+        for incident_id in active:
+            ctx = self._context_store.load(incident_id)
+            if ctx is None or ctx.state != IncidentState.AWAITING_APPROVAL:
+                continue
+
+            try:
+                updated = datetime.fromisoformat(ctx.updated_at)
+                elapsed = _time.time() - updated.replace(tzinfo=timezone.utc).timestamp()
+                remaining = self._config.approval_timeout_seconds - elapsed
+            except (ValueError, TypeError):
+                remaining = self._config.approval_timeout_seconds
+
+            if remaining <= 0:
+                try:
+                    await self._coordinator.reject(
+                        incident_id,
+                        "Approval timed out (expired during server downtime)",
+                        rejected_by="system/timeout",
+                    )
+                except Exception as exc:
+                    logger.warning("Timeout recovery reject failed: %s", exc)
+            else:
+                self.cancel_timeout(incident_id)
+                task = asyncio.create_task(self._timeout_task(incident_id, delay=remaining))
+                self._timeouts[incident_id] = task

nthlayer_workers/respond/sre/__init__.py

@@ -0,0 +1 @@
+"""SRE experience features — templates over verdicts."""