nthlayer-workers 1.0.0__py3-none-any.whl

nthlayer_workers/correlate/ingestion/protocol.py

@@ -0,0 +1,10 @@
+"""Ingester protocol — the ingestion contract for SitRep."""
+from __future__ import annotations
+from typing import Awaitable, Callable, Protocol, Union
+from nthlayer_workers.correlate.types import SitRepEvent
+
+
+class Ingester(Protocol):
+    async def start(self) -> None: ...
+    async def stop(self) -> None: ...
+    def on_event(self, handler: Callable[[SitRepEvent], Union[Awaitable[None], None]]) -> None: ...

nthlayer_workers/correlate/ingestion/severity.py

@@ -0,0 +1,18 @@
+"""Severity pre-scoring from SLO targets. Pure arithmetic, no judgment."""
+from __future__ import annotations
+from nthlayer_workers.correlate.types import SitRepEvent
+
+
+def pre_score(event: SitRepEvent, slo_targets: dict | None) -> float:
+    """Pre-score severity using SLO targets.
+
+    severity = min(1.0, max(0.0, (current_value - target_value) / target_value))
+    Returns event.severity unchanged if no SLO context available.
+    """
+    if slo_targets is None or event.service not in slo_targets:
+        return event.severity
+    value = event.payload.get("value")
+    threshold = event.payload.get("threshold")
+    if value is None or threshold is None or threshold == 0:
+        return event.severity
+    return min(1.0, max(0.0, (value - threshold) / threshold))

nthlayer_workers/correlate/ingestion/webhook.py

@@ -0,0 +1,197 @@
+"""WebhookIngester — raw asyncio TCP HTTP server for event ingestion."""
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+from dataclasses import replace
+from datetime import datetime, timezone
+from typing import Callable, Awaitable
+
+from nthlayer_workers.correlate.ingestion import severity as _severity
+from nthlayer_workers.correlate.types import EventType, SitRepEvent
+
+_MAX_HEADER_SIZE = 65_536   # 64 KB
+_MAX_BODY_SIZE = 10 * 1024 * 1024  # 10 MB
+_CONNECTION_TIMEOUT = 30  # seconds — per-connection deadline to prevent slowloris
+
+_REQUIRED_FIELDS = {"source", "type", "service", "payload"}
+
+
+class WebhookIngester:
+    """Accepts HTTP POST requests and dispatches SitRepEvents to a registered handler."""
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 8081,
+        slo_targets: dict | None = None,
+    ) -> None:
+        self._host = host
+        self._port = port
+        self._slo_targets = slo_targets
+        self._handler: Callable[[SitRepEvent], Awaitable[None]] | None = None
+        self._server: asyncio.Server | None = None
+
+    def on_event(self, handler: Callable[[SitRepEvent], Awaitable[None]]) -> None:
+        """Register the async event handler called for each valid incoming event."""
+        self._handler = handler
+
+    async def start(self) -> None:
+        """Start listening for connections."""
+        self._server = await asyncio.start_server(
+            self._handle_connection, self._host, self._port
+        )
+
+    async def stop(self) -> None:
+        """Stop the server and wait for it to close."""
+        if self._server:
+            self._server.close()
+            await self._server.wait_closed()
+            self._server = None
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _parse_body(self, body: bytes) -> SitRepEvent:
+        """Parse, validate, and construct a SitRepEvent from raw JSON bytes."""
+        data = json.loads(body)
+        missing = _REQUIRED_FIELDS - set(data.keys())
+        if missing:
+            raise ValueError(f"Missing required fields: {sorted(missing)}")
+
+        # Normalise event type
+        raw_type = data["type"]
+        try:
+            event_type = EventType(raw_type)
+        except ValueError:
+            raise ValueError(f"Unknown event type: {raw_type!r}")
+
+        # Auto-generate id and timestamp when absent
+        event_id = data.get("id") or str(uuid.uuid4())
+        timestamp = data.get("timestamp") or datetime.now(timezone.utc).isoformat()
+
+        event = SitRepEvent(
+            id=event_id,
+            timestamp=timestamp,
+            source=data["source"],
+            type=event_type,
+            service=data["service"],
+            environment=data.get("environment", ""),
+            severity=max(0.0, min(1.0, float(data.get("severity", 0.5)))),
+            payload=data["payload"],
+            dependencies=data.get("dependencies", []),
+            dependents=data.get("dependents", []),
+            ttl=int(data.get("ttl", 86400)),
+        )
+
+        # Apply arithmetic severity pre-scoring if SLO targets are configured
+        scored = _severity.pre_score(event, self._slo_targets)
+        if scored != event.severity:
+            event = replace(event, severity=scored)
+
+        return event
+
+    async def _handle_connection(
+        self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+    ) -> None:
+        """Handle a single HTTP connection with per-connection timeout."""
+        try:
+            await asyncio.wait_for(
+                self._handle_request(reader, writer), timeout=_CONNECTION_TIMEOUT
+            )
+        except asyncio.TimeoutError:
+            writer.close()
+            try:
+                await writer.wait_closed()
+            except Exception:
+                pass
+
+    async def _handle_request(
+        self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+    ) -> None:
+        """Inner request handler, called within a per-connection timeout."""
+        try:
+            # --- Read headers until \r\n\r\n ---
+            header_data = b""
+            while b"\r\n\r\n" not in header_data:
+                chunk = await reader.read(4096)
+                if not chunk:
+                    return
+                header_data += chunk
+                if len(header_data) > _MAX_HEADER_SIZE:
+                    writer.write(
+                        b"HTTP/1.1 431 Request Header Fields Too Large\r\nContent-Length: 0\r\n\r\n"
+                    )
+                    await writer.drain()
+                    return
+
+            header_part, _, body_start = header_data.partition(b"\r\n\r\n")
+            headers_text = header_part.decode("utf-8", errors="replace")
+            lines = headers_text.split("\r\n")
+            request_line = lines[0] if lines else ""
+
+            # --- Parse Content-Length ---
+            content_length = 0
+            for line in lines[1:]:
+                if line.lower().startswith("content-length:"):
+                    content_length = int(line.split(":", 1)[1].strip())
+                    break
+
+            if content_length > _MAX_BODY_SIZE:
+                writer.write(
+                    b"HTTP/1.1 413 Payload Too Large\r\nContent-Length: 0\r\n\r\n"
+                )
+                await writer.drain()
+                return
+
+            # --- Read remaining body bytes ---
+            body = body_start
+            while len(body) < content_length:
+                chunk = await reader.read(content_length - len(body))
+                if not chunk:
+                    break
+                body += chunk
+
+            # --- Method check ---
+            if not request_line.startswith("POST"):
+                writer.write(
+                    b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\n\r\n"
+                )
+                await writer.drain()
+                return
+
+            # --- Parse, validate, and dispatch ---
+            try:
+                event = self._parse_body(body)
+            except (json.JSONDecodeError, ValueError, KeyError) as exc:
+                error_body = json.dumps({"error": str(exc)}).encode()
+                response = (
+                    f"HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\n"
+                    f"Content-Length: {len(error_body)}\r\n\r\n"
+                ).encode() + error_body
+                writer.write(response)
+                await writer.drain()
+                return
+
+            # Call the registered handler (async or sync)
+            if self._handler is not None:
+                result = self._handler(event)
+                if asyncio.iscoroutine(result):
+                    await result
+
+            ok_body = b'{"status":"ok"}'
+            response = (
+                f"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"
+                f"Content-Length: {len(ok_body)}\r\n\r\n"
+            ).encode() + ok_body
+            writer.write(response)
+            await writer.drain()
+
+        finally:
+            writer.close()
+            try:
+                await writer.wait_closed()
+            except Exception:
+                pass

nthlayer_workers/correlate/notifications.py

@@ -0,0 +1,85 @@
+"""Slack block builders for nthlayer-correlate verdicts."""
+from __future__ import annotations
+
+
+def build_correlation_blocks(verdict) -> tuple[list[dict], str]:
+    """Build Slack blocks for root cause identification.
+
+    Returns (blocks, fallback_text).
+    """
+    custom = getattr(verdict.metadata, "custom", {}) or {}
+    service = verdict.subject.ref or "unknown"
+    root_causes = custom.get("root_causes", [])
+    blast_radius = custom.get("blast_radius", [])
+    confidence = verdict.judgment.confidence
+
+    rc_text = root_causes[0].get("service", "unknown") if root_causes else "under investigation"
+    blast_count = len(blast_radius)
+    blast_services = ", ".join(
+        b.get("service", b) if isinstance(b, dict) else b
+        for b in blast_radius[:5]
+    )
+
+    text = f"\U0001f50d Root cause: {rc_text} \u2014 {blast_count} services in blast radius"
+
+    blocks = [
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": f"*\U0001f50d ROOT CAUSE IDENTIFIED \u00b7 {service}*",
+            },
+        },
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": (
+                    f"*Root cause:* {rc_text}\n"
+                    f"*Blast radius:* {blast_count} services \u2014 {blast_services}\n"
+                    "NthLayer correlated the breach with the service dependency graph."
+                ),
+            },
+        },
+        {
+            "type": "context",
+            "elements": [
+                {
+                    "type": "mrkdwn",
+                    "text": f"nthlayer-correlate \u00b7 confidence {confidence:.2f} \u00b7 {verdict.id}",
+                },
+            ],
+        },
+    ]
+
+    return blocks, text
+
+
+def find_slack_thread_ts(verdict_store, verdict_ids: list[str]) -> str | None:
+    """Walk verdict lineage to find slack_thread_ts from the earliest verdict.
+
+    Returns None if no thread_ts found in lineage.
+    """
+    for vid in verdict_ids:
+        try:
+            v = verdict_store.get(vid)
+            if v is None:
+                continue
+            custom = getattr(v.metadata, "custom", {}) or {}
+            ts = custom.get("slack_thread_ts")
+            if ts:
+                return ts
+            # Walk up lineage
+            for ctx_id in (v.lineage.context or []):
+                try:
+                    ctx_v = verdict_store.get(ctx_id)
+                    if ctx_v:
+                        ctx_custom = getattr(ctx_v.metadata, "custom", {}) or {}
+                        ts = ctx_custom.get("slack_thread_ts")
+                        if ts:
+                            return ts
+                except Exception:
+                    pass
+        except Exception:
+            pass
+    return None

nthlayer_workers/correlate/prometheus.py

@@ -0,0 +1,234 @@
+"""Prometheus query helpers for live correlation."""
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+import httpx
+import structlog
+
+from nthlayer_workers.correlate.types import EventType, SitRepEvent
+
+logger = structlog.get_logger(__name__)
+
+
+async def fetch_alerts(
+    client: httpx.AsyncClient,
+    prometheus_url: str,
+    services: set[str],
+) -> list[SitRepEvent]:
+    """Fetch currently firing alerts from Prometheus, filtered to given services."""
+    events: list[SitRepEvent] = []
+    try:
+        resp = await client.get(f"{prometheus_url}/api/v1/alerts", timeout=10.0)
+        resp.raise_for_status()
+        alerts = resp.json().get("data", {}).get("alerts", [])
+        for alert in alerts:
+            if alert.get("state") != "firing":
+                continue
+            labels = alert.get("labels", {})
+            service = labels.get("service", "")
+            if service not in services:
+                continue
+            events.append(SitRepEvent(
+                id=f"prom-alert-{uuid.uuid4().hex[:8]}",
+                timestamp=alert.get("activeAt", datetime.now(timezone.utc).isoformat()),
+                source="prometheus",
+                type=EventType.ALERT,
+                service=service,
+                environment=labels.get("environment", "production"),
+                severity=_alert_severity(labels.get("severity", "warning")),
+                payload={
+                    "alert_name": labels.get("alertname", "unknown"),
+                    "labels": labels,
+                    "annotations": alert.get("annotations", {}),
+                },
+            ))
+    except (httpx.HTTPError, ValueError, KeyError) as exc:
+        logger.debug("Failed to fetch Prometheus alerts", error=str(exc))
+    return events
+
+
+async def fetch_metric_breaches(
+    client: httpx.AsyncClient,
+    prometheus_url: str,
+    services: set[str],
+    window_minutes: int = 30,
+) -> list[SitRepEvent]:
+    """Query Prometheus for SLO metric breaches on given services."""
+    events: list[SitRepEvent] = []
+    now = datetime.now(timezone.utc)
+
+    for service in services:
+        # Check error budget
+        budget = await _query_instant(
+            client, prometheus_url,
+            f'slo:error_budget:ratio{{service="{service}"}}',
+        )
+        if budget is not None and budget < 0.0:
+            events.append(SitRepEvent(
+                id=f"prom-metric-{uuid.uuid4().hex[:8]}",
+                timestamp=now.isoformat(),
+                source="prometheus",
+                type=EventType.METRIC_BREACH,
+                service=service,
+                environment="production",
+                severity=min(1.0, abs(budget) * 5),  # 20% deficit → severity 1.0
+                payload={
+                    "metric": "slo:error_budget:ratio",
+                    "value": budget,
+                    "breach": "error_budget_exhausted",
+                },
+            ))
+
+        # Check p99 latency
+        p99 = await _query_instant(
+            client, prometheus_url,
+            f'slo:http_request_duration_seconds:p99{{service="{service}"}}',
+        )
+        if p99 is not None and p99 > 0.5:  # >500ms is concerning
+            events.append(SitRepEvent(
+                id=f"prom-metric-{uuid.uuid4().hex[:8]}",
+                timestamp=now.isoformat(),
+                source="prometheus",
+                type=EventType.METRIC_BREACH,
+                service=service,
+                environment="production",
+                severity=min(1.0, p99 / 2.0),  # 2s p99 → severity 1.0
+                payload={
+                    "metric": "slo:http_request_duration_seconds:p99",
+                    "value": p99,
+                    "breach": "latency_exceeded",
+                },
+            ))
+
+        # Check error rate
+        error_rate = await _query_instant(
+            client, prometheus_url,
+            f'service:http_errors:rate5m{{service="{service}"}}',
+        )
+        if error_rate is not None and error_rate > 0.01:  # >1% error rate
+            events.append(SitRepEvent(
+                id=f"prom-metric-{uuid.uuid4().hex[:8]}",
+                timestamp=now.isoformat(),
+                source="prometheus",
+                type=EventType.METRIC_BREACH,
+                service=service,
+                environment="production",
+                severity=min(1.0, error_rate * 10),  # 10% error rate → severity 1.0
+                payload={
+                    "metric": "service:http_errors:rate5m",
+                    "value": error_rate,
+                    "breach": "error_rate_elevated",
+                },
+            ))
+
+    return events
+
+
+def verdict_to_event(verdict) -> SitRepEvent:
+    """Convert a verdict from the verdict store into a SitRepEvent."""
+    custom = getattr(verdict.metadata, "custom", {}) or {}
+    return SitRepEvent(
+        id=f"verdict-{verdict.id}",
+        timestamp=verdict.timestamp.isoformat() if hasattr(verdict.timestamp, "isoformat") else str(verdict.timestamp),
+        source=verdict.producer.system,
+        type=EventType.VERDICT,
+        service=verdict.subject.ref or verdict.subject.service or "unknown",
+        environment="production",
+        severity=verdict.judgment.confidence if verdict.judgment.action == "flag" else 0.2,
+        payload={
+            "verdict_id": verdict.id,
+            "action": verdict.judgment.action,
+            "confidence": verdict.judgment.confidence,
+            "slo_name": custom.get("slo_name"),
+            "slo_type": custom.get("slo_type"),
+            "breach": custom.get("breach"),
+        },
+    )
+
+
+def load_dependency_graph(specs_dir: str) -> dict[str, dict]:
+    """Load service dependency graph from OpenSRM specs directory.
+
+    Returns dict mapping service name → {tier, dependencies, dependents}.
+    """
+    import yaml
+    from pathlib import Path
+
+    graph: dict[str, dict] = {}
+    specs_path = Path(specs_dir)
+    if not specs_path.is_dir():
+        return graph
+
+    for spec_file in sorted(specs_path.glob("*.yaml")):
+        try:
+            raw = yaml.safe_load(spec_file.read_text())
+        except Exception:
+            continue
+        if not isinstance(raw, dict):
+            continue
+
+        metadata = raw.get("metadata", {})
+        service = metadata.get("name", spec_file.stem)
+        spec = raw.get("spec", {})
+        tier = metadata.get("tier", "standard")
+        deps = [d["name"] for d in spec.get("dependencies", []) if isinstance(d, dict)]
+
+        graph[service] = {"tier": tier, "dependencies": deps, "dependents": []}
+
+    # Build reverse dependencies
+    for svc, info in graph.items():
+        for dep in info["dependencies"]:
+            if dep in graph:
+                graph[dep]["dependents"].append(svc)
+
+    return graph
+
+
+def blast_radius_services(
+    trigger_service: str,
+    dependency_graph: dict[str, dict],
+) -> set[str]:
+    """Compute blast radius: trigger service + dependents (upstream consumers) + dependencies (downstream)."""
+    affected = {trigger_service}
+    # Walk dependents (who depends on the trigger service?)
+    to_visit = list(dependency_graph.get(trigger_service, {}).get("dependents", []))
+    while to_visit:
+        svc = to_visit.pop(0)
+        if svc not in affected:
+            affected.add(svc)
+            to_visit.extend(dependency_graph.get(svc, {}).get("dependents", []))
+    # Also include dependencies (downstream services)
+    for dep in dependency_graph.get(trigger_service, {}).get("dependencies", []):
+        affected.add(dep)
+    return affected
+
+
+async def _query_instant(
+    client: httpx.AsyncClient,
+    prometheus_url: str,
+    query: str,
+) -> float | None:
+    """Execute a PromQL instant query and return the scalar value."""
+    try:
+        resp = await client.get(
+            f"{prometheus_url}/api/v1/query",
+            params={"query": query},
+            timeout=10.0,
+        )
+        resp.raise_for_status()
+        results = resp.json().get("data", {}).get("result", [])
+        if not results:
+            return None
+        val = float(results[0].get("value", [None, None])[1])
+        if val != val:  # NaN
+            return None
+        return val
+    except (httpx.HTTPError, ValueError, KeyError, IndexError, TypeError):
+        return None
+
+
+def _alert_severity(severity_label: str) -> float:
+    """Map Prometheus alert severity label to 0.0-1.0."""
+    mapping = {"critical": 0.95, "warning": 0.6, "info": 0.3}
+    return mapping.get(severity_label, 0.5)