nthlayer-workers 1.0.0__py3-none-any.whl

nthlayer_workers/correlate/config.py

@@ -0,0 +1,111 @@
+"""SitRep configuration."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import yaml
+
+
+@dataclass
+class SitRepConfig:
+    store_path: str = "sitrep-events.db"
+    ingestion_host: str = "127.0.0.1"
+    ingestion_port: int = 8081
+    correlation_window_minutes: int = 5
+    dedup_key_fields: list[str] = field(
+        default_factory=lambda: ["source", "service", "type", "environment"]
+    )
+    token_budget: int = 4000
+    cache_ttl_minutes: int = 15
+    model_name: str = "claude-sonnet-4-20250514"
+    model_max_tokens: int = 4096
+    verdict_store_path: str = "verdicts.db"
+    manifests_dir: str | None = None
+    watching_interval: int = 300
+    alert_interval: int = 60
+    incident_interval: int = 30
+    degraded_interval: int = 120
+    # Trace backend configuration
+    trace_backend: str | None = None
+    trace_detail: str = "full"
+    trace_baseline_window: str = "1h"
+    tempo_endpoint: str = "http://localhost:3200"
+    tempo_org_id: str = ""
+    tempo_timeout: int = 30
+    tempo_use_service_graphs: bool = True
+
+
+def load_config(path: str | None = None) -> SitRepConfig:
+    """Load config from YAML file. Missing fields use defaults."""
+    if path is None:
+        return SitRepConfig()
+
+    with open(path) as f:
+        data = yaml.safe_load(f) or {}
+
+    kwargs: dict[str, Any] = {}
+
+    # Flatten nested YAML structure to flat config
+    store = data.get("store", {})
+    if "path" in store:
+        kwargs["store_path"] = store["path"]
+
+    ingestion = data.get("ingestion", {})
+    if "host" in ingestion:
+        kwargs["ingestion_host"] = ingestion["host"]
+    if "port" in ingestion:
+        kwargs["ingestion_port"] = ingestion["port"]
+
+    correlation = data.get("correlation", {})
+    if "window_minutes" in correlation:
+        kwargs["correlation_window_minutes"] = correlation["window_minutes"]
+    if "dedup_key_fields" in correlation:
+        kwargs["dedup_key_fields"] = correlation["dedup_key_fields"]
+
+    snapshot = data.get("snapshot", {})
+    if "token_budget" in snapshot:
+        kwargs["token_budget"] = snapshot["token_budget"]
+    if "cache_ttl_minutes" in snapshot:
+        kwargs["cache_ttl_minutes"] = snapshot["cache_ttl_minutes"]
+
+    model = data.get("model", {})
+    if "model" in model:
+        kwargs["model_name"] = model["model"]
+    if "max_tokens" in model:
+        kwargs["model_max_tokens"] = model["max_tokens"]
+
+    verdict = data.get("verdict", {}).get("store", {})
+    if "path" in verdict:
+        kwargs["verdict_store_path"] = verdict["path"]
+
+    topology = data.get("topology", {})
+    if "manifests_dir" in topology:
+        kwargs["manifests_dir"] = topology["manifests_dir"]
+
+    state = data.get("state", {})
+    for key in ["watching_interval_seconds", "alert_interval_seconds",
+                "incident_interval_seconds", "degraded_interval_seconds"]:
+        short_key = key.replace("_seconds", "")
+        if key in state:
+            kwargs[short_key] = state[key]
+
+    traces = data.get("traces", {})
+    if "backend" in traces:
+        kwargs["trace_backend"] = traces["backend"]
+    if "detail" in traces:
+        kwargs["trace_detail"] = traces["detail"]
+    if "baseline_window" in traces:
+        kwargs["trace_baseline_window"] = traces["baseline_window"]
+
+    tempo = traces.get("tempo", {})
+    if "endpoint" in tempo:
+        kwargs["tempo_endpoint"] = tempo["endpoint"]
+    if "org_id" in tempo:
+        kwargs["tempo_org_id"] = tempo["org_id"]
+    if "timeout_seconds" in tempo:
+        kwargs["tempo_timeout"] = tempo["timeout_seconds"]
+    if "use_service_graphs" in tempo:
+        kwargs["tempo_use_service_graphs"] = tempo["use_service_graphs"]
+
+    return SitRepConfig(**kwargs)

nthlayer_workers/correlate/correlation/__init__.py

@@ -0,0 +1 @@
+"""Signal correlation for SitRep. All operations are deterministic transport."""

nthlayer_workers/correlate/correlation/changes.py

@@ -0,0 +1,87 @@
+"""Change candidate indexing. Deterministic transport."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from nthlayer_workers.correlate.store.protocol import EventStore
+from nthlayer_workers.correlate.types import ChangeCandidate, TemporalGroup
+
+
+def _parse_ts(ts: str) -> datetime:
+    """Parse ISO 8601 timestamp. Always returns timezone-aware (UTC default)."""
+    dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def find_change_candidates(
+    store: EventStore,
+    temporal_groups: list[TemporalGroup],
+    topology: dict | None = None,
+    window_minutes: int = 30,
+) -> dict[str, list[ChangeCandidate]]:
+    """For each temporal group, find recent changes on the same service or dependencies.
+
+    Returns mapping from service name to list of change candidates,
+    sorted by temporal proximity (closest first).
+    """
+    result: dict[str, list[ChangeCandidate]] = {}
+
+    for group in temporal_groups:
+        service = group.service
+        candidates: list[ChangeCandidate] = []
+
+        # Parse group's first event timestamp for proximity calculation
+        group_first_ts = _parse_ts(group.time_window[0])
+
+        # Use the group's first event as reference time for "recent" lookback.
+        # This is essential for replay with historical timestamps — without it,
+        # get_recent_changes uses datetime('now') and finds nothing.
+        reference_time = group.time_window[0]
+
+        # Check for changes on the same service
+        same_service_changes = store.get_recent_changes(
+            service, window_minutes, reference_time=reference_time
+        )
+        for change in same_service_changes:
+            change_ts = _parse_ts(change.timestamp)
+            proximity = abs((group_first_ts - change_ts).total_seconds())
+            candidates.append(
+                ChangeCandidate(
+                    change=change,
+                    affected_service=service,
+                    temporal_proximity_seconds=proximity,
+                    same_service=True,
+                    dependency_related=False,
+                )
+            )
+
+        # Check for changes on dependency services
+        if topology is not None:
+            svc_info = topology.get(service, {})
+            dependencies = svc_info.get("dependencies", [])
+            for dep_service in dependencies:
+                dep_changes = store.get_recent_changes(
+                    dep_service, window_minutes, reference_time=reference_time
+                )
+                for change in dep_changes:
+                    change_ts = _parse_ts(change.timestamp)
+                    proximity = abs((group_first_ts - change_ts).total_seconds())
+                    candidates.append(
+                        ChangeCandidate(
+                            change=change,
+                            affected_service=service,
+                            temporal_proximity_seconds=proximity,
+                            same_service=False,
+                            dependency_related=True,
+                        )
+                    )
+
+        # Sort by temporal proximity (closest first)
+        candidates.sort(key=lambda c: c.temporal_proximity_seconds)
+
+        if candidates:
+            result[service] = candidates
+
+    return result

nthlayer_workers/correlate/correlation/dedup.py

@@ -0,0 +1,62 @@
+"""Signal deduplication. Deterministic transport."""
+from __future__ import annotations
+
+from dataclasses import replace
+
+from nthlayer_workers.correlate.types import SitRepEvent
+
+
+def _dedup_key(event: SitRepEvent) -> str:
+    """Build dedup key from event fields."""
+    parts = [event.source, event.service, event.type.value, event.environment]
+    # Add alert_name or metric from payload if present
+    alert_name = event.payload.get("alert_name") or event.payload.get("metric")
+    if alert_name:
+        parts.append(str(alert_name))
+    return "|".join(parts)
+
+
+def deduplicate(events: list[SitRepEvent]) -> list[SitRepEvent]:
+    """Collapse events with the same dedup key.
+
+    Returns deduplicated list. The first occurrence of each key is kept,
+    with _dedup_count and _dedup_duration_seconds added to its payload.
+    """
+    seen: dict[str, SitRepEvent] = {}
+    counts: dict[str, int] = {}
+    first_ts: dict[str, str] = {}
+    last_ts: dict[str, str] = {}
+
+    for event in events:
+        key = _dedup_key(event)
+        if key not in seen:
+            seen[key] = event
+            counts[key] = 1
+            first_ts[key] = event.timestamp
+            last_ts[key] = event.timestamp
+        else:
+            counts[key] += 1
+            if event.timestamp < first_ts[key]:
+                first_ts[key] = event.timestamp
+            if event.timestamp > last_ts[key]:
+                last_ts[key] = event.timestamp
+
+    result = []
+    for key, event in seen.items():
+        if counts[key] > 1:
+            # Add dedup metadata to payload (don't mutate original)
+            from datetime import datetime
+            try:
+                start = datetime.fromisoformat(first_ts[key].replace("Z", "+00:00"))
+                end = datetime.fromisoformat(last_ts[key].replace("Z", "+00:00"))
+                duration = (end - start).total_seconds()
+            except (ValueError, TypeError):
+                duration = 0.0
+
+            new_payload = dict(event.payload)
+            new_payload["_dedup_count"] = counts[key]
+            new_payload["_dedup_duration_seconds"] = duration
+            event = replace(event, payload=new_payload)
+        result.append(event)
+
+    return result

nthlayer_workers/correlate/correlation/engine.py

@@ -0,0 +1,244 @@
+"""Pre-correlation engine. Orchestrates all correlation steps."""
+from __future__ import annotations
+
+import uuid
+from dataclasses import replace
+from datetime import datetime, timedelta, timezone
+
+from nthlayer_workers.correlate.correlation.changes import find_change_candidates
+from nthlayer_workers.correlate.correlation.dedup import deduplicate
+from nthlayer_workers.correlate.correlation.temporal import group_temporal
+from nthlayer_workers.correlate.correlation.topology import group_topology
+from nthlayer_workers.correlate.ingestion.severity import pre_score
+from nthlayer_workers.correlate.store.protocol import EventStore
+from nthlayer_workers.correlate.types import ChangeCandidate, CorrelationGroup, TemporalGroup
+
+
+class CorrelationEngine:
+    """Orchestrates the full pre-correlation pipeline. All deterministic transport."""
+
+    def correlate(
+        self,
+        store: EventStore,
+        window_minutes: int = 5,
+        topology: dict | None = None,
+        slo_targets: dict | None = None,
+    ) -> list[CorrelationGroup]:
+        """Run full pre-correlation pipeline. All deterministic transport.
+
+        Steps:
+        1. Query store for events in window
+        2. Deduplicate
+        3. Severity enrichment (second-pass for events without SLO context)
+        4. Temporal grouping
+        5. Topology-aware grouping
+        6. Change candidate indexing
+        7. Assemble CorrelationGroups with priority scoring
+        """
+        # Step 1: Query store for events in window
+        now = datetime.now(timezone.utc)
+        start = (now - timedelta(minutes=window_minutes)).isoformat()
+        end = now.isoformat()
+
+        events = store.get_by_time_window(start, end)
+        if not events:
+            return []
+
+        # Step 2: Deduplicate
+        deduped = deduplicate(events)
+
+        # Step 3: Severity enrichment
+        enriched = []
+        for event in deduped:
+            new_severity = pre_score(event, slo_targets)
+            if new_severity != event.severity:
+                event = replace(event, severity=new_severity)
+            enriched.append(event)
+
+        # Step 4: Temporal grouping
+        temporal_groups = group_temporal(enriched, window_minutes=window_minutes)
+        if not temporal_groups:
+            return []
+
+        # Step 5: Topology-aware grouping
+        topology_correlations = group_topology(temporal_groups, topology)
+
+        # Step 6: Change candidate indexing
+        change_candidates_map = find_change_candidates(
+            store, temporal_groups, topology=topology, window_minutes=max(window_minutes, 30)
+        )
+
+        # Step 7: Assemble CorrelationGroups with priority scoring
+        return self.assemble_groups(
+            temporal_groups, topology_correlations, change_candidates_map, topology
+        )
+
+    def assemble_groups(
+        self,
+        temporal_groups: list[TemporalGroup],
+        topology_correlations: list,
+        change_candidates_map: dict[str, list[ChangeCandidate]],
+        topology: dict | None,
+    ) -> list[CorrelationGroup]:
+        """Assemble CorrelationGroups from pre-computed correlation sub-step outputs.
+
+        Two-pass assembly: topology-linked groups first, then unassigned temporal groups.
+        Used by both correlate() and CLI replay.
+        """
+        assigned: set[int] = set()
+        correlation_groups: list[CorrelationGroup] = []
+
+        # First pass: create groups from topology correlations
+        for tc in topology_correlations:
+            involved_services = {tc.primary_service}
+            for rs in tc.related_services:
+                involved_services.add(rs["service"])
+
+            signals: list[TemporalGroup] = []
+            for gi, tg in enumerate(temporal_groups):
+                if tg.service in involved_services and gi not in assigned:
+                    signals.append(tg)
+                    assigned.add(gi)
+
+            if not signals:
+                continue
+
+            all_changes: list[ChangeCandidate] = []
+            for svc in involved_services:
+                all_changes.extend(change_candidates_map.get(svc, []))
+
+            peak_severity = max(s.peak_severity for s in signals)
+            priority = self._compute_priority(
+                peak_severity, list(involved_services), topology, topology_correlations
+            )
+
+            total_events = sum(s.count for s in signals)
+            primary_type = self._dominant_event_type(signals)
+            services_str = ", ".join(sorted(involved_services))
+            summary = (
+                f"{total_events} {primary_type}(s) on {services_str} "
+                f"with {len(all_changes)} recent change(s)"
+            )
+
+            all_timestamps = []
+            for s in signals:
+                all_timestamps.append(s.time_window[0])
+                all_timestamps.append(s.time_window[1])
+            first_seen = min(all_timestamps)
+            last_updated = max(all_timestamps)
+
+            group_id = f"cg-{uuid.uuid4().hex[:8]}"
+            correlation_groups.append(
+                CorrelationGroup(
+                    id=group_id,
+                    priority=priority,
+                    summary=summary,
+                    services=sorted(involved_services),
+                    signals=signals,
+                    topology=tc,
+                    change_candidates=all_changes,
+                    first_seen=first_seen,
+                    last_updated=last_updated,
+                    event_count=total_events,
+                )
+            )
+
+        # Second pass: create groups for unassigned temporal groups
+        for gi, tg in enumerate(temporal_groups):
+            if gi in assigned:
+                continue
+
+            service = tg.service
+            changes = change_candidates_map.get(service, [])
+            peak_severity = tg.peak_severity
+            priority = self._compute_priority(
+                peak_severity, [service], topology, topology_correlations
+            )
+
+            primary_type = self._dominant_event_type([tg])
+            summary = (
+                f"{tg.count} {primary_type}(s) on {service} "
+                f"with {len(changes)} recent change(s)"
+            )
+
+            group_id = f"cg-{uuid.uuid4().hex[:8]}"
+            correlation_groups.append(
+                CorrelationGroup(
+                    id=group_id,
+                    priority=priority,
+                    summary=summary,
+                    services=[service],
+                    signals=[tg],
+                    topology=None,
+                    change_candidates=changes,
+                    first_seen=tg.time_window[0],
+                    last_updated=tg.time_window[1],
+                    event_count=tg.count,
+                )
+            )
+
+        correlation_groups.sort(
+            key=lambda g: (g.priority, -max(s.peak_severity for s in g.signals))
+        )
+        return correlation_groups
+
+    def _compute_priority(
+        self,
+        peak_severity: float,
+        services: list[str],
+        topology: dict | None,
+        topology_correlations: list,
+    ) -> int:
+        """Compute priority tier for a correlation group.
+
+        P0: peak_severity > 0.8 AND service tier is "critical"
+        P1: peak_severity > 0.6 OR has topology correlation with a P0 group
+        P2: peak_severity > 0.3
+        P3: everything else
+        """
+        # Check if any service is critical tier
+        has_critical = False
+        if topology is not None:
+            for svc in services:
+                svc_info = topology.get(svc, {})
+                if svc_info.get("tier") == "critical":
+                    has_critical = True
+                    break
+
+        if peak_severity > 0.8 and has_critical:
+            return 0  # P0
+
+        if peak_severity > 0.6:
+            return 1  # P1
+
+        # P1 if has topology correlation with a P0-eligible group
+        if topology is not None and topology_correlations:
+            for tc in topology_correlations:
+                linked_services = {tc.primary_service}
+                for rs in tc.related_services:
+                    linked_services.add(rs["service"])
+                if linked_services & set(services):
+                    # Check if any linked service is P0-eligible
+                    for svc in linked_services:
+                        svc_info = topology.get(svc, {})
+                        if svc_info.get("tier") == "critical":
+                            return 1  # P1 — topology link to critical service
+
+        if peak_severity > 0.3:
+            return 2  # P2
+
+        return 3  # P3
+
+    @staticmethod
+    def _dominant_event_type(signals: list[TemporalGroup]) -> str:
+        """Find the most common event type across all signals."""
+        type_counts: dict[str, int] = {}
+        for signal in signals:
+            for event in signal.events:
+                t = event.type.value
+                type_counts[t] = type_counts.get(t, 0) + 1
+
+        if not type_counts:
+            return "event"
+
+        return max(type_counts, key=type_counts.get)  # type: ignore[arg-type]

nthlayer_workers/correlate/correlation/temporal.py

@@ -0,0 +1,79 @@
+"""Temporal grouping — windowed aggregation by service. Deterministic transport."""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from nthlayer_workers.correlate.types import SitRepEvent, TemporalGroup
+
+
+def _parse_ts(ts: str) -> datetime:
+    """Parse ISO 8601 timestamp. Always returns timezone-aware (UTC default)."""
+    dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def group_temporal(
+    events: list[SitRepEvent], window_minutes: int = 5
+) -> list[TemporalGroup]:
+    """Group events by service within time windows.
+
+    Events on the same service within window_minutes of the earliest
+    event in the group are grouped together.
+    """
+    if not events:
+        return []
+
+    # Sort by timestamp
+    sorted_events = sorted(events, key=lambda e: e.timestamp)
+
+    # Group by service
+    by_service: dict[str, list[SitRepEvent]] = {}
+    for event in sorted_events:
+        by_service.setdefault(event.service, []).append(event)
+
+    groups: list[TemporalGroup] = []
+    window_seconds = window_minutes * 60
+
+    for service, svc_events in by_service.items():
+        # Split into windows
+        current_window: list[SitRepEvent] = [svc_events[0]]
+        window_start = _parse_ts(svc_events[0].timestamp)
+
+        for event in svc_events[1:]:
+            event_ts = _parse_ts(event.timestamp)
+            if (event_ts - window_start).total_seconds() <= window_seconds:
+                current_window.append(event)
+            else:
+                # Emit current window, start new one
+                groups.append(_make_group(service, current_window))
+                current_window = [event]
+                window_start = event_ts
+
+        # Emit last window
+        if current_window:
+            groups.append(_make_group(service, current_window))
+
+    return groups
+
+
+def _make_group(service: str, events: list[SitRepEvent]) -> TemporalGroup:
+    """Create a TemporalGroup from a list of events."""
+    first_ts = events[0].timestamp
+    last_ts = events[-1].timestamp
+    try:
+        start = _parse_ts(first_ts)
+        end = _parse_ts(last_ts)
+        duration = (end - start).total_seconds()
+    except (ValueError, TypeError):
+        duration = 0.0
+
+    return TemporalGroup(
+        service=service,
+        time_window=(first_ts, last_ts),
+        events=events,
+        count=len(events),
+        peak_severity=max(e.severity for e in events),
+        duration_seconds=duration,
+    )

nthlayer_workers/correlate/correlation/topology.py

@@ -0,0 +1,104 @@
+"""Topology-aware grouping. Deterministic transport."""
+from __future__ import annotations
+
+from nthlayer_workers.correlate.types import TemporalGroup, TopologyCorrelation
+
+
+def _get_tier_rank(topology: dict, service: str) -> int:
+    """Return numeric rank for service tier. Higher = more critical."""
+    tier_ranks = {"critical": 3, "standard": 2, "low": 1}
+    svc_info = topology.get(service, {})
+    tier = svc_info.get("tier", "standard")
+    return tier_ranks.get(tier, 1)
+
+
+def _depends_on(topology: dict, service_a: str, service_b: str) -> bool:
+    """Check if service_a depends on service_b."""
+    svc_info = topology.get(service_a, {})
+    return service_b in svc_info.get("dependencies", [])
+
+
+def group_topology(
+    temporal_groups: list[TemporalGroup],
+    topology: dict | None,
+) -> list[TopologyCorrelation]:
+    """Link temporal groups for services that have dependency relationships.
+
+    Args:
+        temporal_groups: Groups from temporal.group_temporal()
+        topology: Dict mapping service names to {dependencies: [...], dependents: [...], tier: ...}
+                  Typically from OpenSRM manifests. None = skip topology grouping.
+
+    Returns:
+        List of TopologyCorrelations linking related service groups.
+    """
+    if topology is None:
+        return []
+
+    if len(temporal_groups) < 2:
+        return []
+
+    # Track which pairs have been linked to avoid duplicates
+    linked: set[tuple[str, str]] = set()
+    results: list[TopologyCorrelation] = []
+
+    for i, group_a in enumerate(temporal_groups):
+        for j, group_b in enumerate(temporal_groups):
+            if i >= j:
+                continue
+            if group_a.service == group_b.service:
+                continue
+
+            pair_key = tuple(sorted((group_a.service, group_b.service)))
+            if pair_key in linked:
+                continue
+
+            a_depends_on_b = _depends_on(topology, group_a.service, group_b.service)
+            b_depends_on_a = _depends_on(topology, group_b.service, group_a.service)
+
+            if not a_depends_on_b and not b_depends_on_a:
+                continue
+
+            linked.add(pair_key)
+
+            # Determine primary: higher severity wins, then higher tier
+            a_severity = group_a.peak_severity
+            b_severity = group_b.peak_severity
+            a_tier = _get_tier_rank(topology, group_a.service)
+            b_tier = _get_tier_rank(topology, group_b.service)
+
+            if a_severity > b_severity or (a_severity == b_severity and a_tier >= b_tier):
+                primary_service = group_a.service
+                secondary_service = group_b.service
+                secondary_group = group_b
+            else:
+                primary_service = group_b.service
+                secondary_service = group_a.service
+                secondary_group = group_a
+
+            # Determine relationship: from primary's perspective toward secondary
+            if _depends_on(topology, primary_service, secondary_service):
+                relationship = "depends_on"
+            else:
+                relationship = "depended_by"
+
+            related_services = [
+                {
+                    "service": secondary_service,
+                    "relationship": relationship,
+                    "events": secondary_group.events,
+                }
+            ]
+
+            # Build topology path
+            topology_path = [primary_service, secondary_service]
+
+            results.append(
+                TopologyCorrelation(
+                    primary_service=primary_service,
+                    related_services=related_services,
+                    topology_path=topology_path,
+                )
+            )
+
+    return results

nthlayer_workers/correlate/ingestion/__init__.py

@@ -0,0 +1 @@
+"""Event ingestion for SitRep."""