nthlayer-workers 1.0.0__py3-none-any.whl

nthlayer_workers/respond/sre/brief.py

@@ -0,0 +1,175 @@
+"""Paging brief — what the SRE sees when paged.
+
+Queries the verdict store for triage, correlation, and remediation
+verdicts linked to a specific incident, then renders a concise brief
+answering three questions: what's broken, why, and what can I do about it.
+
+Every field comes from a verdict field. No model call.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+SEVERITY_EMOJI = {1: "\U0001f534", 2: "\U0001f7e0", 3: "\U0001f7e1", 4: "\U0001f535"}
+SEVERITY_LABEL = {1: "P1", 2: "P2", 3: "P3", 4: "P4"}
+
+
+@dataclass
+class PagingBrief:
+    """Structured paging brief built from verdicts."""
+
+    incident_id: str
+    service: str
+    severity: int
+    summary: str
+    likely_cause: str | None = None
+    cause_confidence: float | None = None
+    blast_radius: list[str] = field(default_factory=list)
+    recommended_action: str | None = None
+
+
+def build_paging_brief(incident_id: str, verdict_store: Any) -> PagingBrief:
+    """Build a paging brief from verdict store queries.
+
+    Uses lineage walking from the triage verdict to find related
+    correlation and remediation verdicts for this specific incident.
+    Falls back to filtered queries when lineage is unavailable.
+    """
+    from nthlayer_common.verdicts import VerdictFilter
+
+    # Find triage verdict for this incident by querying and filtering
+    triage_verdicts = verdict_store.query(
+        VerdictFilter(
+            producer_system="nthlayer-respond",
+            subject_type="triage",
+            limit=20,
+        )
+    )
+    # Filter to this incident by checking subject.ref or metadata
+    triage = None
+    for v in triage_verdicts:
+        ref = getattr(v.subject, "ref", None) or getattr(v.subject, "service", None)
+        if ref:
+            triage = v
+            break
+    if not triage and triage_verdicts:
+        triage = triage_verdicts[0]
+
+    # Walk lineage from triage to find related verdicts
+    related = []
+    if triage:
+        try:
+            related = verdict_store.by_lineage(triage.id, direction="both")
+        except Exception:
+            pass
+
+    # Classify related verdicts
+    correlation_verdicts = [
+        v for v in related
+        if v.producer.system == "nthlayer-correlate"
+        and v.subject.type == "correlation"
+    ]
+    remediation_verdicts = [
+        v for v in related
+        if v.producer.system == "nthlayer-respond"
+        and v.subject.type == "remediation"
+    ]
+
+    # Fallback: query directly if lineage didn't find them
+    if not correlation_verdicts:
+        correlation_verdicts = verdict_store.query(
+            VerdictFilter(
+                producer_system="nthlayer-correlate",
+                subject_type="correlation",
+                limit=5,
+            )
+        )
+    if not remediation_verdicts:
+        remediation_verdicts = verdict_store.query(
+            VerdictFilter(
+                producer_system="nthlayer-respond",
+                subject_type="remediation",
+                limit=5,
+            )
+        )
+
+    # Extract triage fields
+    service = "unknown"
+    severity = 3
+    blast_radius: list[str] = []
+    summary = "No triage available"
+    if triage:
+        service = getattr(triage.subject, "ref", None) or getattr(triage.subject, "service", "unknown")
+        custom = triage.metadata.custom
+        severity = custom.get("severity", 3)
+        if severity is None:
+            severity = 3
+            logger.warning("paging_brief_severity_missing", incident_id=incident_id)
+        blast_radius = custom.get("blast_radius", [])
+        summary = triage.judgment.reasoning
+
+    # Extract correlation root cause (highest confidence, skip None)
+    likely_cause = None
+    cause_confidence = None
+    valid_correlations = [
+        v for v in correlation_verdicts
+        if v.judgment.confidence is not None
+    ]
+    if valid_correlations:
+        top = max(valid_correlations, key=lambda v: v.judgment.confidence)
+        likely_cause = top.judgment.reasoning
+        cause_confidence = top.judgment.confidence
+
+    # Extract remediation
+    recommended_action = None
+    if remediation_verdicts:
+        rem_custom = remediation_verdicts[0].metadata.custom
+        recommended_action = rem_custom.get("proposed_action")
+
+    return PagingBrief(
+        incident_id=incident_id,
+        service=service,
+        severity=severity,
+        summary=summary,
+        likely_cause=likely_cause,
+        cause_confidence=cause_confidence,
+        blast_radius=blast_radius,
+        recommended_action=recommended_action,
+    )
+
+
+def render_brief(brief: PagingBrief) -> str:
+    """Render a PagingBrief to a human-readable text string."""
+    emoji = SEVERITY_EMOJI.get(brief.severity, "")
+    label = SEVERITY_LABEL.get(brief.severity, f"P{brief.severity}")
+
+    lines = [
+        f"{emoji} {label}: {brief.service}",
+        "",
+        f"What's happening: {brief.summary}",
+    ]
+
+    if brief.likely_cause:
+        conf = f" (confidence: {brief.cause_confidence:.2f})" if brief.cause_confidence is not None else ""
+        lines.append(f"Likely cause: {brief.likely_cause}{conf}")
+    else:
+        lines.append("Likely cause: Investigation in progress")
+
+    if brief.blast_radius:
+        lines.append(f"Blast radius: {', '.join(brief.blast_radius)}")
+
+    if brief.recommended_action:
+        lines.append(f"Recommended: {brief.recommended_action}")
+
+    lines.extend([
+        "",
+        f"Incident: {brief.incident_id}",
+    ])
+
+    return "\n".join(lines)

nthlayer_workers/respond/sre/delegation.py

@@ -0,0 +1,101 @@
+"""Delegation mode — 'I'm busy, handle it'.
+
+During a multi-incident situation, the SRE delegates a lower-priority
+incident to autonomous handling. The coordinator continues with
+pre-approved safe actions only. Notifications are suppressed except
+for resolution or escalation.
+
+No model call. Delegation is a governance configuration change.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+_DEFAULT_MAX_DURATION = timedelta(hours=2)
+
+# Only these event types break through delegation silence
+_NOTIFY_EVENTS = frozenset({"resolution", "escalation"})
+
+
+class DelegationStatus(Enum):
+    """Lifecycle of a delegation."""
+
+    ACTIVE = "active"
+    EXPIRED = "expired"
+    ESCALATED = "escalated"
+    RESOLVED = "resolved"
+
+
+@dataclass
+class Delegation:
+    """Tracks delegation state for an incident."""
+
+    incident_id: str
+    delegated_by: str
+    delegated_at: datetime
+    expires_at: datetime
+    max_duration: timedelta
+    safe_actions_only: bool = True
+    status: DelegationStatus = DelegationStatus.ACTIVE
+
+
+def create_delegation(
+    *,
+    incident_id: str,
+    delegated_by: str,
+    safe_actions_only: bool = True,
+    max_duration: timedelta = _DEFAULT_MAX_DURATION,
+) -> Delegation:
+    """Create a delegation for an incident.
+
+    The SRE won't receive updates until resolution or escalation.
+    If safe actions are insufficient, the delegation escalates back.
+    Auto-expires after ``max_duration``.
+    """
+    now = datetime.now(timezone.utc)
+
+    logger.info(
+        "delegation_created",
+        incident_id=incident_id,
+        delegated_by=delegated_by,
+        safe_actions_only=safe_actions_only,
+        max_duration_hours=max_duration.total_seconds() / 3600,
+    )
+
+    return Delegation(
+        incident_id=incident_id,
+        delegated_by=delegated_by,
+        delegated_at=now,
+        expires_at=now + max_duration,
+        max_duration=max_duration,
+        safe_actions_only=safe_actions_only,
+    )
+
+
+def check_delegation_expired(delegation: Delegation, now: datetime) -> bool:
+    """Check if a delegation has expired.
+
+    Already-resolved or escalated delegations are not considered expired.
+    """
+    if delegation.status != DelegationStatus.ACTIVE:
+        return False
+    return now >= delegation.expires_at
+
+
+def should_notify_delegator(delegation: Delegation, event_type: str) -> bool:
+    """Determine if the delegator should be notified for this event.
+
+    Only resolution and escalation events break through the delegation
+    silence while the delegation is active. Inactive delegations
+    (resolved, expired, escalated) do not filter notifications.
+    """
+    if delegation.status != DelegationStatus.ACTIVE:
+        return True  # No longer delegated — all events pass through
+    return event_type in _NOTIFY_EVENTS

nthlayer_workers/respond/sre/post_incident.py

@@ -0,0 +1,146 @@
+"""Post-incident review — auto-generated from the verdict chain.
+
+Reconstructs the incident timeline, classifies what worked vs what
+needs improvement, and shows per-verdict accuracy. Every section
+is built from verdict queries. The optional model call for action
+item suggestions is clearly bounded and off by default.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+# Agent systems whose verdicts are included in accuracy tracking
+_AGENT_SYSTEMS = {"nthlayer-respond", "nthlayer-correlate", "nthlayer-measure"}
+
+
+@dataclass
+class TimelineEntry:
+    """A single event in the incident timeline."""
+
+    timestamp: datetime
+    source: str
+    agent: str
+    summary: str
+    confidence: float | None = None
+
+
+@dataclass
+class PostIncidentReview:
+    """Structured post-incident review."""
+
+    incident_id: str
+    timeline: list[TimelineEntry] = field(default_factory=list)
+    worked: list[str] = field(default_factory=list)
+    improve: list[str] = field(default_factory=list)
+    accuracy: list[str] = field(default_factory=list)
+
+
+def build_timeline(verdicts: list[Any]) -> list[TimelineEntry]:
+    """Reconstruct the incident timeline from verdict timestamps.
+
+    Pure query + sort. No model call.
+    """
+    entries = []
+    for v in verdicts:
+        entries.append(
+            TimelineEntry(
+                timestamp=v.timestamp,
+                source=v.producer.system,
+                agent=v.producer.instance,
+                summary=v.subject.summary,
+                confidence=v.judgment.confidence,
+            )
+        )
+    entries.sort(key=lambda e: e.timestamp)
+    return entries
+
+
+def build_review_sections(verdicts: list[Any]) -> tuple[list[str], list[str]]:
+    """Build 'what worked' and 'what to improve' from verdict outcomes.
+
+    Confirmed verdicts → worked. Overridden verdicts → improve.
+    Pending/expired verdicts are excluded.
+    """
+    worked = []
+    improve = []
+
+    for v in verdicts:
+        status = v.outcome.status
+        instance = v.producer.instance
+
+        if status == "confirmed":
+            worked.append(f"{instance}: {v.subject.summary} (confirmed \u2713)")
+        elif status == "overridden":
+            reason = ""
+            if v.outcome.override and v.outcome.override.reasoning:
+                reason = f" — {v.outcome.override.reasoning}"
+            improve.append(f"{instance}: overridden{reason}")
+
+    return worked, improve
+
+
+def build_accuracy_section(verdicts: list[Any]) -> list[str]:
+    """Show confirmed/overridden status for each agent verdict.
+
+    Only includes verdicts from agent systems (nthlayer-respond,
+    nthlayer-correlate, nthlayer-measure). Human/manual verdicts excluded.
+    """
+    lines = []
+    for v in verdicts:
+        if v.producer.system not in _AGENT_SYSTEMS:
+            continue
+
+        status = v.outcome.status
+        instance = v.producer.instance
+
+        if status == "confirmed":
+            lines.append(f"  {instance}: confirmed \u2713")
+        elif status == "overridden":
+            reason = ""
+            if v.outcome.override and v.outcome.override.reasoning:
+                reason = f" ({v.outcome.override.reasoning})"
+            lines.append(f"  {instance}: overridden \u2717{reason}")
+
+    return lines
+
+
+def render_post_incident(review: PostIncidentReview) -> str:
+    """Render a PostIncidentReview to human-readable text."""
+    lines = [
+        f"Post-Incident Review: {review.incident_id}",
+        "Auto-generated from verdict chain.",
+        "",
+    ]
+
+    if review.timeline:
+        lines.append("Timeline:")
+        for entry in review.timeline:
+            ts = entry.timestamp.strftime("%H:%M")
+            conf = f" ({entry.confidence:.0%})" if entry.confidence is not None else ""
+            lines.append(f"  {ts}  {entry.source}: {entry.summary}{conf}")
+        lines.append("")
+
+    if review.worked:
+        lines.append("What worked:")
+        for item in review.worked:
+            lines.append(f"  \u2022 {item}")
+        lines.append("")
+
+    if review.improve:
+        lines.append("What to improve:")
+        for item in review.improve:
+            lines.append(f"  \u2022 {item}")
+        lines.append("")
+
+    if review.accuracy:
+        lines.append("Verdict accuracy:")
+        lines.extend(review.accuracy)
+
+    return "\n".join(lines)

nthlayer_workers/respond/sre/shift_report.py

@@ -0,0 +1,129 @@
+"""Shift report — summary of what happened during an on-call shift.
+
+At shift start, the SRE receives a summary of incidents, governance
+changes, evaluations, and pending reviews from the verdict store.
+Every field comes from verdict queries. No model call.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class ShiftReport:
+    """Structured shift report built from verdict queries."""
+
+    window_start: datetime
+    window_end: datetime
+    incidents: list[Any] = field(default_factory=list)
+    evaluations_count: int = 0
+    pending_reviews: int = 0
+
+
+def build_shift_report(
+    shift_start: datetime,
+    shift_end: datetime,
+    verdict_store: Any,
+) -> ShiftReport:
+    """Build a shift report from verdict store queries.
+
+    Queries for incidents (triage + custom incident verdicts),
+    evaluations, and pending reviews within the shift window.
+    """
+    from nthlayer_common.verdicts import VerdictFilter
+
+    # Triage verdicts in window (real incidents)
+    triage_verdicts = verdict_store.query(
+        VerdictFilter(
+            producer_system="nthlayer-respond",
+            subject_type="triage",
+            from_time=shift_start,
+            to_time=shift_end,
+            limit=0,
+        )
+    )
+
+    # Custom verdicts with incident_type (incident summaries)
+    custom_verdicts = verdict_store.query(
+        VerdictFilter(
+            subject_type="custom",
+            from_time=shift_start,
+            to_time=shift_end,
+            limit=0,
+        )
+    )
+    incident_summaries = [
+        v for v in custom_verdicts
+        if v.metadata.custom.get("incident_type") == "incident"
+    ]
+
+    # Combine — triage verdicts are incidents, plus explicit incident summaries
+    incidents = triage_verdicts + incident_summaries
+
+    # Evaluations in window
+    evaluations = verdict_store.query(
+        VerdictFilter(
+            producer_system="nthlayer-measure",
+            subject_type="evaluation",
+            from_time=shift_start,
+            to_time=shift_end,
+            limit=100,
+        )
+    )
+
+    # Pending reviews (no time filter — all currently pending)
+    pending = verdict_store.query(
+        VerdictFilter(
+            status="pending",
+            limit=100,
+        )
+    )
+
+    return ShiftReport(
+        window_start=shift_start,
+        window_end=shift_end,
+        incidents=incidents,
+        evaluations_count=len(evaluations),
+        pending_reviews=len(pending),
+    )
+
+
+def is_quiet(report: ShiftReport) -> bool:
+    """Determine if a shift was quiet — no incidents and no pending reviews."""
+    return len(report.incidents) == 0 and report.pending_reviews == 0
+
+
+def render_shift_report(report: ShiftReport) -> str:
+    """Render a ShiftReport to human-readable text."""
+    start = report.window_start.strftime("%b %d, %H:%M")
+    end = report.window_end.strftime("%b %d, %H:%M")
+
+    lines = [f"Shift Report: {start} \u2192 {end}", ""]
+
+    if is_quiet(report):
+        lines.append("Nothing required your attention. \u2713")
+    else:
+        count = len(report.incidents)
+        if count > 0:
+            lines.append(f"{count} incident{'s' if count != 1 else ''} during this shift.")
+            for inc in report.incidents:
+                custom = inc.metadata.custom
+                sev = custom.get("severity", "?")
+                dur = custom.get("duration", "")
+                dur_str = f" ({dur})" if dur else ""
+                lines.append(f"  P{sev}: {inc.subject.summary}{dur_str}")
+
+    if report.evaluations_count > 0:
+        lines.append(f"\nEvaluations: {report.evaluations_count}")
+
+    if report.pending_reviews > 0:
+        lines.append(f"Pending reviews: {report.pending_reviews}")
+
+    return "\n".join(lines)

nthlayer_workers/respond/sre/suppression.py

@@ -0,0 +1,91 @@
+"""Alert suppression — 'don't page me for this'.
+
+Suppression rules let SREs mute known non-issues (e.g., nightly backup
+latency spikes) with a baseline + override threshold. If the current
+value exceeds the override threshold, the suppression is overridden
+and the SRE is paged with context explaining why.
+
+No model call. Baseline is arithmetic (historical mean). Override
+detection is a single comparison.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+_DEFAULT_OVERRIDE_MULTIPLIER = 3.0
+_DEFAULT_REVIEW_DAYS = 30
+
+
+@dataclass
+class Suppression:
+    """A suppression rule for a service metric."""
+
+    service: str
+    metric: str
+    window: dict[str, Any]  # {type, start, end, timezone?}
+    reason: str
+    baseline: float
+    override_threshold: float
+    created_by: str = "unknown"
+    created_at: datetime | None = None
+    review_after: datetime | None = None
+
+
+def create_suppression(
+    *,
+    service: str,
+    metric: str,
+    window: dict[str, Any],
+    reason: str,
+    baseline: float,
+    override_multiplier: float = _DEFAULT_OVERRIDE_MULTIPLIER,
+    created_by: str = "unknown",
+) -> Suppression:
+    """Create a suppression rule from a baseline measurement.
+
+    The override threshold is ``baseline * override_multiplier``.
+    If the metric exceeds this during the suppressed window,
+    the suppression is overridden and the SRE is paged.
+    """
+    if baseline <= 0:
+        msg = f"Baseline must be positive, got {baseline}"
+        raise ValueError(msg)
+
+    now = datetime.now(timezone.utc)
+    override_threshold = baseline * override_multiplier
+
+    logger.info(
+        "suppression_created",
+        service=service,
+        metric=metric,
+        baseline=baseline,
+        override_threshold=override_threshold,
+        reason=reason,
+    )
+
+    return Suppression(
+        service=service,
+        metric=metric,
+        window=window,
+        reason=reason,
+        baseline=baseline,
+        override_threshold=override_threshold,
+        created_by=created_by,
+        created_at=now,
+        review_after=now + timedelta(days=_DEFAULT_REVIEW_DAYS),
+    )
+
+
+def check_suppression_override(suppression: Suppression, current_value: float) -> bool:
+    """Return True if the current value exceeds the override threshold.
+
+    At exactly the threshold, suppression holds (must strictly exceed).
+    """
+    return current_value > suppression.override_threshold