sre-agent-cli 0.2.0__py3-none-any.whl

sre_agent/hosted_demo.py

@@ -0,0 +1,153 @@
+"""The public-safe hosted demo server (Phase 10).
+
+In plain words, this is the one server build that is meant to be
+put on a public address later. It is the shipped demo server with
+every public-safety rule switched on at once, in one place, so a
+deployment cannot quietly forget one of them:
+
+- **A token is required.** The app is not built at all without
+  one: ``create_hosted_demo_app`` refuses an empty token instead
+  of falling back to the unauthenticated local mode the shipped
+  app allows. With a token set, every route except ``/health``
+  and the landing page at ``/`` requires it, exactly as in
+  ``server.py`` (finding F-01). The landing page is open for the
+  same reason ``/health`` is: it serves no data and runs no
+  action — it only explains the demo and prints the visitor
+  token, which is public on purpose.
+- **Seeded demo data only.** Diagnosis reads the seeded demo tool
+  layer, the seeded demo read adapters and the seeded demo
+  memory, and actions run against the seeded demo workloads.
+  No real system is read or touched, and no real credential is
+  used anywhere in the stack: the only secret is the shared
+  token itself, supplied by whoever hosts it.
+- **The Tier 3 demo grant is the only action that can run.**
+  The policy is ``tier3.demo_tier3_policy`` — restart and scale,
+  in the ``shop`` namespace only, at most three replicas touched
+  per action — and the server is built ``tier3_only=True``, so
+  there is no approval path: a fix the grant does not pre-approve
+  (rollback is the example) is refused and can never run here.
+- **The ingestion caps stay on.** The store is the ordinary
+  incident store, so the F-02 caps apply unchanged: at most 100
+  alerts per Alertmanager payload, capped labels and summaries,
+  at most 200 evidence records per incident and 1,000 incidents
+  held, with a clear refusal past a cap.
+
+Run it locally the same way it will run when hosted, with the
+token in the environment:
+
+    SRE_AGENT_API_TOKEN=<a token you choose> \
+        uvicorn --factory sre_agent.hosted_demo:create_hosted_demo_app \
+        --host 0.0.0.0 --port 8080
+
+Honest scope: this module is the public-safe *mode*. The
+deployment it was built for is live (Render free plan,
+https://sre-agent-demo.onrender.com, deployed 2026-11-12 with
+the host and cost named first, as the plan required); the mode
+itself claims nothing beyond what is written here. See
+``docs/hosted-demo.md``.
+"""
+
+from __future__ import annotations
+
+import os
+
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+
+from .adapters import demo_adapter_sources
+from .memory import demo_incident_memory
+from .runbooks import demo_runbooks
+from .server import ENV_API_TOKEN, create_app
+from .tier3 import demo_tier3_policy
+from .tools import demo_tool_layer
+
+LANDING_PAGE = """<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>sre-agent — hosted demo</title>
+<style>
+body { font-family: system-ui, sans-serif; margin: 2rem auto; max-width: 44rem; padding: 0 1rem; line-height: 1.5; }
+code, pre { background: #f2f2f2; padding: 0.1rem 0.3rem; border-radius: 4px; }
+pre { padding: 0.75rem; overflow-x: auto; }
+</style>
+</head>
+<body>
+<h1>sre-agent — hosted demo</h1>
+<p>An autonomous SRE agent that detects production incidents, works out
+what is wrong using real evidence, and helps fix them — without ever
+being allowed to make things worse on its own. This is its public demo:
+seeded demo data only, nothing real is read or touched.</p>
+
+<h2>The visitor token</h2>
+<p>Every API route needs the token in the <code>X-SRE-Agent-Token</code>
+header. The demo token is public on purpose:</p>
+<pre>X-SRE-Agent-Token: sre-agent-demo-visitor</pre>
+
+<h2>Try it</h2>
+<p>Send the demo checkout alert, and read the diagnosis it opens:</p>
+<pre>curl -X POST https://sre-agent-demo.onrender.com/webhooks/generic \
+  -H 'Content-Type: application/json' \
+  -H 'X-SRE-Agent-Token: sre-agent-demo-visitor' \
+  -d '{"alert_name":"HighErrorRate","service":"checkout","namespace":"shop","environment":"demo","summary":"Checkout errors are above 10%"}'
+
+curl https://sre-agent-demo.onrender.com/incidents \
+  -H 'X-SRE-Agent-Token: sre-agent-demo-visitor'</pre>
+<p>Propose the restart for the incident and it runs at once — the
+demo's Tier 3 grant pre-approves it. Send the same alert again with
+<code>"status": "resolved"</code> to close the incident.</p>
+
+<h2>What this demo cannot do</h2>
+<ul>
+<li>It cannot touch anything real. Every number, workload and past
+incident is seeded demo data.</li>
+<li>It cannot run a fix outside the Tier 3 demo grant (restart and
+scale, shop namespace only, at most three replicas). Rollback always
+needs a human, and hosted mode has no approval path at all — that is
+deliberate.</li>
+<li>It sleeps. On the free hosting plan the service spins down after
+about 15 minutes idle, so the first request can take a minute.</li>
+</ul>
+<p>Source, scores and the full record:
+<a href="https://github.com/KakoozaJerry/sre-agent">github.com/KakoozaJerry/sre-agent</a></p>
+</body>
+</html>
+"""
+
+
+def create_hosted_demo_app(auth_token: str | None = None) -> FastAPI:
+    """Build the public-safe hosted demo server, or refuse.
+
+    ``auth_token`` wins when given; otherwise the token is read
+    from the ``SRE_AGENT_API_TOKEN`` environment variable, the
+    same variable the server and the CLI already use. An empty
+    or missing token raises ``ValueError``: a hosted demo with
+    no token would be an unauthenticated server on a public
+    address, and that build is refused here rather than warned
+    about in a guide a deployer might not read.
+    """
+    token = auth_token
+    if token is None:
+        token = os.environ.get(ENV_API_TOKEN) or None
+    if not token:
+        raise ValueError(
+            "the hosted demo requires an API token: pass auth_token "
+            f"or set {ENV_API_TOKEN}; refusing to build an "
+            "unauthenticated server for a public address"
+        )
+    _, runbook_backend = demo_runbooks()
+    app = create_app(
+        auth_token=token,
+        tool_layer=demo_tool_layer(),
+        adapters=demo_adapter_sources(),
+        memory=demo_incident_memory(),
+        approval_policy=demo_tier3_policy(),
+        runbook_backend=runbook_backend,
+        tier3_only=True,
+    )
+
+    @app.get("/", response_class=HTMLResponse)
+    def landing() -> str:
+        return LANDING_PAGE
+
+    return app

sre_agent/ingestion.py

@@ -0,0 +1,423 @@
+"""Alert ingestion: turn webhook payloads into incidents.
+
+Two webhook shapes are understood:
+
+- **Alertmanager.** The payload Prometheus Alertmanager sends: a list of
+  alerts with labels, annotations, a start time and a fingerprint.
+- **Generic.** One simple alert as a small JSON object, for tools that
+  do not speak Alertmanager's shape.
+
+What happens to an alert, in plain words:
+
+- **Dedupe.** The same alert (same fingerprint) firing again while its
+  incident is still open is a copy. A copy never opens a second
+  incident; it is counted on the incident it belongs to.
+- **Grouping.** A different alert (a different fingerprint) for the
+  same target and the same alert name, while an incident for them is
+  still open, belongs to that incident. It is added as evidence.
+- **Resolving.** When a "resolved" alert arrives for an open incident,
+  the incident is resolved and its group closes, so the next firing
+  opens a fresh incident.
+
+Alert text is data, never instructions. Labels and annotations are
+stored as evidence only; nothing in them is ever run or obeyed. See
+docs/safety-model.md for why that matters.
+
+Intake limits (security finding F-02, fixed after v0.1.0): every
+payload is bounded before it can grow the store — at most
+``MAX_ALERTS_PER_PAYLOAD`` alerts in one Alertmanager payload, at
+most ``MAX_METADATA_ENTRIES`` labels or annotations on one alert,
+each label or annotation key and value length-capped, and short
+fields (names, summaries) length-capped too. The store itself is
+bounded: at most ``MAX_INCIDENTS`` incidents held, and at most
+``MAX_EVIDENCE_PER_INCIDENT`` evidence records on one incident.
+Past a store cap, ingestion raises :class:`IngestionLimitError`
+and the server answers with a clear error instead of accepting
+the work. The caps bound how much memory a flood — hostile, or a
+broken sender retrying — can make the server hold. A per-sender
+rate limit was not added; that is stated plainly in the review.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field, field_validator
+
+from .logging_config import get_logger
+from .models import Evidence, Incident, IncidentStatus, Target
+
+if TYPE_CHECKING:
+    from .approvals import PendingAction
+
+log = get_logger(__name__)
+
+FIRING = "firing"
+RESOLVED = "resolved"
+
+# Intake limits (F-02). Plain numbers, named once, here.
+MAX_ALERTS_PER_PAYLOAD = 100
+MAX_METADATA_ENTRIES = 50
+MAX_METADATA_KEY_LENGTH = 100
+MAX_METADATA_VALUE_LENGTH = 1000
+MAX_SHORT_TEXT_LENGTH = 200
+MAX_SUMMARY_LENGTH = 1000
+MAX_INCIDENTS = 1000
+MAX_EVIDENCE_PER_INCIDENT = 200
+
+
+class IngestionLimitError(Exception):
+    """A store intake cap was reached; the alert was not accepted."""
+
+
+def _check_metadata(values: dict[str, str]) -> dict[str, str]:
+    """Bound one labels/annotations dict: count, key and value size."""
+    if len(values) > MAX_METADATA_ENTRIES:
+        raise ValueError(
+            f"too many labels/annotations: {len(values)} > {MAX_METADATA_ENTRIES}"
+        )
+    for key, value in values.items():
+        if len(key) > MAX_METADATA_KEY_LENGTH:
+            raise ValueError("a label/annotation key is too long")
+        if len(value) > MAX_METADATA_VALUE_LENGTH:
+            raise ValueError("a label/annotation value is too long")
+    return values
+
+
+class IncomingAlert(BaseModel):
+    """One alert, normalised into the same shape whatever sent it."""
+
+    fingerprint: str
+    alert_name: str
+    title: str
+    target: Target
+    severity: str = "unknown"
+    status: str = FIRING
+    source: str  # "alertmanager" or "generic"
+    labels: dict[str, str] = Field(default_factory=dict)
+    annotations: dict[str, str] = Field(default_factory=dict)
+    starts_at: datetime | None = None
+
+    @property
+    def group_key(self) -> tuple[str, str, str, str]:
+        """Alerts that share a target and a name belong to one incident."""
+        return (
+            self.target.environment,
+            self.target.namespace,
+            self.target.service,
+            self.alert_name,
+        )
+
+
+class AlertmanagerAlert(BaseModel):
+    """One alert inside an Alertmanager payload."""
+
+    model_config = {"populate_by_name": True}
+
+    status: str = Field(default=FIRING, max_length=MAX_SHORT_TEXT_LENGTH)
+    labels: dict[str, str] = Field(default_factory=dict)
+    annotations: dict[str, str] = Field(default_factory=dict)
+    starts_at: datetime | None = Field(default=None, alias="startsAt")
+    fingerprint: str = Field(default="", max_length=MAX_SHORT_TEXT_LENGTH)
+
+    @field_validator("labels", "annotations")
+    @classmethod
+    def _metadata_is_bounded(cls, v: dict[str, str]) -> dict[str, str]:
+        return _check_metadata(v)
+
+
+class AlertmanagerPayload(BaseModel):
+    """The webhook body Alertmanager sends."""
+
+    model_config = {"populate_by_name": True}
+
+    status: str = Field(default=FIRING, max_length=MAX_SHORT_TEXT_LENGTH)
+    common_labels: dict[str, str] = Field(default_factory=dict, alias="commonLabels")
+    common_annotations: dict[str, str] = Field(
+        default_factory=dict, alias="commonAnnotations"
+    )
+    alerts: list[AlertmanagerAlert] = Field(
+        default_factory=list, max_length=MAX_ALERTS_PER_PAYLOAD
+    )
+
+    @field_validator("common_labels", "common_annotations")
+    @classmethod
+    def _common_metadata_is_bounded(cls, v: dict[str, str]) -> dict[str, str]:
+        return _check_metadata(v)
+
+
+class GenericAlertPayload(BaseModel):
+    """The simple webhook body: one alert, plain fields."""
+
+    alert_name: str = Field(max_length=MAX_SHORT_TEXT_LENGTH)
+    service: str = Field(max_length=MAX_SHORT_TEXT_LENGTH)
+    namespace: str = Field(default="default", max_length=MAX_SHORT_TEXT_LENGTH)
+    environment: str = Field(default="demo", max_length=MAX_SHORT_TEXT_LENGTH)
+    severity: str = Field(default="unknown", max_length=MAX_SHORT_TEXT_LENGTH)
+    status: str = Field(default=FIRING, max_length=MAX_SHORT_TEXT_LENGTH)
+    fingerprint: str = Field(default="", max_length=MAX_SHORT_TEXT_LENGTH)
+    summary: str = Field(default="", max_length=MAX_SUMMARY_LENGTH)
+    labels: dict[str, str] = Field(default_factory=dict)
+    annotations: dict[str, str] = Field(default_factory=dict)
+    starts_at: datetime | None = None
+
+    @field_validator("labels", "annotations")
+    @classmethod
+    def _metadata_is_bounded(cls, v: dict[str, str]) -> dict[str, str]:
+        return _check_metadata(v)
+
+
+def fingerprint_for_labels(labels: dict[str, str]) -> str:
+    """Make a stable fingerprint from labels when the sender gave none.
+
+    The same labels always make the same fingerprint, so a repeated
+    alert is recognised as a copy.
+    """
+    canonical = json.dumps(labels, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
+
+
+def _title_for(alert_name: str, service: str, annotations: dict[str, str]) -> str:
+    return (
+        annotations.get("summary")
+        or annotations.get("description")
+        or f"{alert_name} on {service}"
+    )
+
+
+def normalize_alertmanager(
+    payload: AlertmanagerPayload, default_environment: str = "demo"
+) -> list[IncomingAlert]:
+    """Turn an Alertmanager payload into a list of incoming alerts."""
+    incoming: list[IncomingAlert] = []
+    for alert in payload.alerts:
+        labels = {**payload.common_labels, **alert.labels}
+        annotations = {**payload.common_annotations, **alert.annotations}
+        alert_name = labels.get("alertname", "unknown")
+        service = labels.get("service") or labels.get("job") or "unknown"
+        target = Target(
+            environment=labels.get("environment", default_environment),
+            namespace=labels.get("namespace", "default"),
+            service=service,
+        )
+        incoming.append(
+            IncomingAlert(
+                fingerprint=alert.fingerprint or fingerprint_for_labels(labels),
+                alert_name=alert_name,
+                title=_title_for(alert_name, service, annotations),
+                target=target,
+                severity=labels.get("severity", "unknown"),
+                status=alert.status or payload.status,
+                source="alertmanager",
+                labels=labels,
+                annotations=annotations,
+                starts_at=alert.starts_at,
+            )
+        )
+    return incoming
+
+
+def normalize_generic(payload: GenericAlertPayload) -> IncomingAlert:
+    """Turn a generic webhook body into one incoming alert."""
+    labels = {
+        "alertname": payload.alert_name,
+        "service": payload.service,
+        "namespace": payload.namespace,
+        "environment": payload.environment,
+        "severity": payload.severity,
+        **payload.labels,
+    }
+    annotations = dict(payload.annotations)
+    if payload.summary and "summary" not in annotations:
+        annotations["summary"] = payload.summary
+    return IncomingAlert(
+        fingerprint=payload.fingerprint or fingerprint_for_labels(labels),
+        alert_name=payload.alert_name,
+        title=_title_for(payload.alert_name, payload.service, annotations),
+        target=Target(
+            environment=payload.environment,
+            namespace=payload.namespace,
+            service=payload.service,
+        ),
+        severity=payload.severity,
+        status=payload.status,
+        source="generic",
+        labels=labels,
+        annotations=annotations,
+        starts_at=payload.starts_at,
+    )
+
+
+class IngestResult(BaseModel):
+    """What ingestion did with one alert."""
+
+    incident: Incident | None = None
+    created: bool = False
+    duplicate: bool = False
+    grouped: bool = False
+    resolved: bool = False
+
+
+class IncidentStore:
+    """Keeps incidents in memory and decides what each new alert means.
+
+    This is the small store interface the plan talks about: webhooks
+    and (later) tools talk to this class, so swapping the memory here
+    for SQLite later does not change the webhook shapes. That swap
+    now exists: ``storage.SQLiteIncidentStore`` keeps the same
+    interface and adds persistence behind it (see ``storage.py``).
+
+    The interface now also carries the two hooks a persistent store
+    needs and an in-memory store can answer trivially:
+
+    - ``save(incident)`` — an incident changed outside ingestion
+      (a diagnosis was written on it, a fix ran on it): keep the
+      change. In memory the record is already the object held, so
+      this only makes sure it is in the store; a database store
+      writes it through.
+    - ``load_approvals()`` / ``save_approval(pending, decision)`` —
+      the server's pending approvals, so a database-backed server
+      can restore them after a restart. In memory there is nothing
+      extra to load or write: the server's own dicts are the record,
+      and they are lost on restart, exactly as before.
+    """
+
+    def __init__(
+        self,
+        max_incidents: int = MAX_INCIDENTS,
+        max_evidence_per_incident: int = MAX_EVIDENCE_PER_INCIDENT,
+    ) -> None:
+        self._max_incidents = max_incidents
+        self._max_evidence_per_incident = max_evidence_per_incident
+        self._incidents: dict[str, Incident] = {}
+        self._active_by_group: dict[tuple[str, str, str, str], str] = {}
+        self._incident_by_fingerprint: dict[str, str] = {}
+        self._duplicate_counts: dict[str, int] = {}
+
+    def list_incidents(self) -> list[Incident]:
+        return list(self._incidents.values())
+
+    def get(self, incident_id: str) -> Incident | None:
+        return self._incidents.get(incident_id)
+
+    def duplicate_count(self, incident_id: str) -> int:
+        return self._duplicate_counts.get(incident_id, 0)
+
+    def save(self, incident: Incident) -> None:
+        """Keep a change made to *incident* outside ingestion.
+
+        In memory this is almost a no-op — the store already holds
+        this exact object — but it is the hook a persistent store
+        overrides to write the change through, and callers (the
+        server, after a diagnosis or a run) must call it rather
+        than assume their mutation was kept.
+        """
+        self._incidents[incident.id] = incident
+
+    def load_approvals(
+        self,
+    ) -> tuple[dict[str, PendingAction], dict[str, str]]:
+        """Pending approvals a previous run of the server left behind.
+
+        An in-memory store kept none across a restart, so there is
+        nothing to restore.
+        """
+        return {}, {}
+
+    def save_approval(
+        self, pending: PendingAction, decision: str | None
+    ) -> None:
+        """Keep one pending approval (and its decision, if decided).
+
+        In memory the server's own dicts are the record, so there
+        is nothing extra to write; a database store persists the
+        pending action so a restart restores it.
+        """
+
+    def ingest(self, alert: IncomingAlert) -> IngestResult:
+        if alert.status == RESOLVED:
+            return self._resolve(alert)
+        return self._fire(alert)
+
+    def _fire(self, alert: IncomingAlert) -> IngestResult:
+        known_id = self._incident_by_fingerprint.get(alert.fingerprint)
+        if known_id is not None and known_id in self._incidents:
+            incident = self._incidents[known_id]
+            if incident.status is not IncidentStatus.RESOLVED:
+                self._duplicate_counts[incident.id] = (
+                    self._duplicate_counts.get(incident.id, 0) + 1
+                )
+                log.info(
+                    "duplicate alert counted for incident %s (fingerprint %s)",
+                    incident.id,
+                    alert.fingerprint,
+                )
+                return IngestResult(incident=incident, duplicate=True)
+
+        grouped_id = self._active_by_group.get(alert.group_key)
+        if grouped_id is not None:
+            incident = self._incidents[grouped_id]
+            if len(incident.evidence) >= self._max_evidence_per_incident:
+                raise IngestionLimitError(
+                    f"incident {incident.id} already holds "
+                    f"{self._max_evidence_per_incident} evidence records; "
+                    "grouped alert refused"
+                )
+            incident.add_evidence(self._evidence_for(alert))
+            self._incident_by_fingerprint[alert.fingerprint] = incident.id
+            log.info("alert grouped into incident %s", incident.id)
+            return IngestResult(incident=incident, grouped=True)
+
+        if len(self._incidents) >= self._max_incidents:
+            raise IngestionLimitError(
+                f"incident store is full ({self._max_incidents} incidents); "
+                "new incident refused"
+            )
+        incident = Incident(title=alert.title, target=alert.target)
+        # Ingestion hands the incident straight to triage: the agent
+        # has started looking at it. This also makes a later
+        # "resolved" alert a legal closing move.
+        incident.transition(IncidentStatus.TRIAGING)
+        incident.add_evidence(self._evidence_for(alert))
+        self._incidents[incident.id] = incident
+        self._active_by_group[alert.group_key] = incident.id
+        self._incident_by_fingerprint[alert.fingerprint] = incident.id
+        log.info("incident %s opened from %s alert", incident.id, alert.source)
+        return IngestResult(incident=incident, created=True)
+
+    def _resolve(self, alert: IncomingAlert) -> IngestResult:
+        incident_id = self._active_by_group.get(alert.group_key)
+        if incident_id is None:
+            incident_id = self._incident_by_fingerprint.get(alert.fingerprint)
+            if incident_id is not None:
+                candidate = self._incidents.get(incident_id)
+                if candidate is None or candidate.status is IncidentStatus.RESOLVED:
+                    incident_id = None
+        if incident_id is None:
+            log.info("resolved alert with no open incident; nothing to close")
+            return IngestResult()
+
+        incident = self._incidents[incident_id]
+        if incident.status is IncidentStatus.OPEN:
+            incident.transition(IncidentStatus.TRIAGING)
+        incident.transition(IncidentStatus.RESOLVED)
+        # Close the group: the next firing opens a fresh incident.
+        self._active_by_group.pop(alert.group_key, None)
+        stale = [
+            fp for fp, iid in self._incident_by_fingerprint.items() if iid == incident.id
+        ]
+        for fp in stale:
+            del self._incident_by_fingerprint[fp]
+        log.info("incident %s resolved by alert", incident.id)
+        return IngestResult(incident=incident, resolved=True)
+
+    @staticmethod
+    def _evidence_for(alert: IncomingAlert) -> Evidence:
+        return Evidence(
+            kind="alert",
+            query=f"webhook:{alert.source} fingerprint:{alert.fingerprint}",
+            summary=f"{alert.alert_name} ({alert.severity}): {alert.title}",
+        )