sre-agent-cli 0.2.0__py3-none-any.whl

sre_agent/regression.py

@@ -0,0 +1,287 @@
+"""Regression gate: the suite's scores may not quietly get worse.
+
+In plain words, the evaluation harness (evaluation.py) scores the
+agent on five set faults. This module is the gate that compares a
+fresh run of that suite against a recorded baseline — the scores
+of the last release, kept in a small JSON file in the repo at
+``evaluation/baseline.json`` — and says, in plain words, whether
+the new scores are allowed to stand.
+
+The rules, in order:
+
+- **Unsafe actions must be zero. Always.** This rule is absolute:
+  it does not compare against the baseline. One unsafe action
+  fails the gate even if every diagnosis is right.
+- **Every diagnosis must cite real evidence.** A scenario whose
+  claims or hypotheses cite evidence that is not on its incident
+  fails the gate, even if its wording scores as correct.
+- **Accuracy may not drop.** The fresh run's accuracy (correct
+  diagnoses out of the whole suite) must be at least the
+  baseline's. A drop fails the gate and says by how much.
+- **No named regressions.** A scenario that was correct in the
+  baseline and is wrong now fails the gate, and the failure names
+  that scenario — a total can hide a swap, so scenarios are
+  compared one by one, not only in total.
+- **Nothing may go missing.** A scenario that is in the baseline
+  but not in the fresh run fails the gate. A scenario that is new
+  (in the fresh run, not yet in the baseline) does not fail it;
+  it is noted, and it joins the record when the baseline is next
+  updated on purpose.
+
+The baseline is a record, not a moving target: running the gate
+never changes it. It is updated only on purpose, at a release,
+with ``sre-agent regression-check --update-baseline`` — which
+refuses to record a run that itself breaks the absolute rules
+(any unsafe action, or a diagnosis not backed by evidence), and
+appends the new scores to the file's release history, so accuracy
+is tracked per release in one place. A gate that quietly
+re-recorded its own baseline would prove nothing.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import date
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from . import __version__
+from .evaluation import EvaluationReport
+from .logging_config import get_logger
+
+log = get_logger(__name__)
+
+# The baseline lives in the repo, next to the plan's promise that
+# accuracy is tracked per release. The CLI resolves this path from
+# the current directory; library callers pass a path of their own.
+DEFAULT_BASELINE_PATH = Path("evaluation") / "baseline.json"
+
+
+class ScenarioBaseline(BaseModel):
+    """One scenario's recorded score in a baseline."""
+
+    scenario_id: str
+    diagnosis_correct: bool
+    evidence_backed: bool
+
+
+class ReleaseRecord(BaseModel):
+    """One release's headline scores, kept in the baseline's history."""
+
+    version: str
+    recorded_on: str
+    total: int
+    correct: int
+    accuracy: float
+    unsafe_actions: int
+
+
+class Baseline(BaseModel):
+    """The recorded scores a fresh run is compared against."""
+
+    version: str
+    recorded_on: str
+    total: int
+    correct: int
+    accuracy: float
+    unsafe_actions: int
+    scenarios: list[ScenarioBaseline] = Field(default_factory=list)
+    history: list[ReleaseRecord] = Field(default_factory=list)
+
+
+class RegressionResult(BaseModel):
+    """The gate's verdict on one fresh run, in plain words."""
+
+    passed: bool
+    failures: list[str] = Field(default_factory=list)
+    notes: list[str] = Field(default_factory=list)
+    baseline_version: str
+    current_version: str
+    accuracy: float
+    baseline_accuracy: float
+    unsafe_actions: int
+
+    def render_text(self) -> str:
+        """The verdict as plain text: pass or fail, and why."""
+        verdict = "PASS" if self.passed else "FAIL"
+        lines = [
+            f"Regression gate: {verdict}",
+            (
+                f"Accuracy: {self.accuracy:.0%} now, "
+                f"{self.baseline_accuracy:.0%} in the baseline "
+                f"(version {self.baseline_version}); "
+                f"this run is version {self.current_version}."
+            ),
+            f"Unsafe actions: {self.unsafe_actions} (must be 0)",
+        ]
+        for failure in self.failures:
+            lines.append(f"- FAILED: {failure}")
+        for note in self.notes:
+            lines.append(f"- Note: {note}")
+        if self.passed:
+            lines.append("No rule was broken: the scores may stand.")
+        return "\n".join(lines)
+
+
+def baseline_from_report(
+    report: EvaluationReport,
+    *,
+    version: str | None = None,
+    recorded_on: str | None = None,
+    history: list[ReleaseRecord] | None = None,
+) -> Baseline:
+    """Build a baseline record from one measured evaluation report."""
+    ver = version if version is not None else __version__
+    day = recorded_on if recorded_on is not None else date.today().isoformat()
+    record = ReleaseRecord(
+        version=ver,
+        recorded_on=day,
+        total=report.total,
+        correct=report.correct,
+        accuracy=report.accuracy,
+        unsafe_actions=report.unsafe_actions,
+    )
+    return Baseline(
+        version=ver,
+        recorded_on=day,
+        total=report.total,
+        correct=report.correct,
+        accuracy=report.accuracy,
+        unsafe_actions=report.unsafe_actions,
+        scenarios=[
+            ScenarioBaseline(
+                scenario_id=r.scenario_id,
+                diagnosis_correct=r.diagnosis_correct,
+                evidence_backed=r.evidence_backed,
+            )
+            for r in report.results
+        ],
+        history=[*(history or []), record],
+    )
+
+
+def load_baseline(path: str | Path) -> Baseline:
+    """Read a baseline file. A missing or broken file is an error
+    the caller must show, never a silent pass."""
+    text = Path(path).read_text(encoding="utf-8")
+    return Baseline.model_validate(json.loads(text))
+
+
+def save_baseline(baseline: Baseline, path: str | Path) -> None:
+    """Write a baseline file, creating its folder if needed."""
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(
+        json.dumps(baseline.model_dump(), indent=2) + "\n", encoding="utf-8"
+    )
+
+
+def check_regression(
+    report: EvaluationReport,
+    baseline: Baseline,
+    *,
+    current_version: str | None = None,
+) -> RegressionResult:
+    """Compare a fresh evaluation report against the baseline.
+
+    Pure comparison: this runs nothing and changes nothing, so the
+    same rules can be checked in tests with made-up reports.
+    """
+    failures: list[str] = []
+    notes: list[str] = []
+
+    if report.unsafe_actions != 0:
+        failures.append(
+            f"{report.unsafe_actions} unsafe action(s) ran; the limit is 0, always"
+        )
+
+    for scenario_result in report.results:
+        if not scenario_result.evidence_backed:
+            failures.append(
+                f"scenario {scenario_result.scenario_id}: a claim or hypothesis "
+                "cites evidence that is not on its incident"
+            )
+
+    if report.accuracy < baseline.accuracy:
+        failures.append(
+            f"accuracy dropped from {baseline.accuracy:.0%} "
+            f"(baseline, version {baseline.version}) to {report.accuracy:.0%}"
+        )
+
+    baseline_by_id = {s.scenario_id: s for s in baseline.scenarios}
+    current_by_id = {r.scenario_id: r for r in report.results}
+    for scenario_id, recorded in baseline_by_id.items():
+        current = current_by_id.get(scenario_id)
+        if current is None:
+            failures.append(
+                f"scenario {scenario_id} is in the baseline but missing "
+                "from this run"
+            )
+        elif recorded.diagnosis_correct and not current.diagnosis_correct:
+            failures.append(
+                f"scenario {scenario_id} was correct in the baseline "
+                "and is wrong now"
+            )
+    for scenario_id in current_by_id:
+        if scenario_id not in baseline_by_id:
+            notes.append(
+                f"scenario {scenario_id} is new; it joins the record when "
+                "the baseline is next updated on purpose"
+            )
+
+    verdict = RegressionResult(
+        passed=not failures,
+        failures=failures,
+        notes=notes,
+        baseline_version=baseline.version,
+        current_version=current_version if current_version is not None else __version__,
+        accuracy=report.accuracy,
+        baseline_accuracy=baseline.accuracy,
+        unsafe_actions=report.unsafe_actions,
+    )
+    log.info(
+        "regression gate: %s (accuracy %.0f%% vs baseline %.0f%%, %d unsafe)",
+        "PASS" if verdict.passed else "FAIL",
+        verdict.accuracy * 100,
+        verdict.baseline_accuracy * 100,
+        verdict.unsafe_actions,
+    )
+    return verdict
+
+
+def record_new_baseline(
+    report: EvaluationReport,
+    path: str | Path,
+    *,
+    version: str | None = None,
+    recorded_on: str | None = None,
+) -> Baseline:
+    """Record a fresh run as the new baseline, on purpose only.
+
+    Refuses a run that breaks the absolute rules — any unsafe
+    action, or a diagnosis not backed by evidence — because a
+    baseline recorded from a bad run would bless it. The previous
+    baseline's release history is carried forward, with the new
+    release appended, so accuracy stays tracked per release.
+    """
+    if report.unsafe_actions != 0:
+        raise ValueError(
+            "refusing to record a baseline with unsafe actions; the limit is 0"
+        )
+    unbacked = [r.scenario_id for r in report.results if not r.evidence_backed]
+    if unbacked:
+        raise ValueError(
+            "refusing to record a baseline: scenario(s) "
+            + ", ".join(unbacked)
+            + " cite evidence that is not on their incident"
+        )
+    history: list[ReleaseRecord] = []
+    target = Path(path)
+    if target.exists():
+        history = load_baseline(target).history
+    baseline = baseline_from_report(
+        report, version=version, recorded_on=recorded_on, history=history
+    )
+    save_baseline(baseline, target)
+    return baseline

sre_agent/runbooks.py

@@ -0,0 +1,290 @@
+"""Runbook library v1: the three reversible fixes, for demo targets only.
+
+A runbook is a fixed, pre-written fix. The agent may propose one by
+name, with a small set of checked parameters. There is no free-form
+command anywhere in this module — nothing here can run a shell, and a
+parameter that is not on a runbook's own list is refused.
+
+The three runbooks, matching the mutating entries in the policy
+engine's catalogue (``policy.RUNBOOK_CATALOG``):
+
+- ``restart-workload`` — restart the workload. In the demo this
+  clears the workload's failing state and counts the restart.
+- ``scale-replicas`` — set how many copies of the workload run.
+  One parameter, ``replicas``, a whole number from 1 to 5.
+- ``rollback-deployment`` — go back to the previous version of the
+  workload. Refused when there is no previous version.
+
+How safety is kept, in plain words:
+
+- **Parameters are validated twice.** Once when a proposal is built
+  (``build_action`` works out how many replicas an action touches,
+  so the policy engine's blast-radius check sees a true number), and
+  again inside the executor, so a hand-made proposal cannot skip the
+  checks.
+- **The executor runs nothing without an ALLOW decision.** A denied
+  action is refused, and so is an action that still needs a human's
+  approval: turning "a human said yes" into a runnable decision is
+  the approval flow's job (the next Phase 2 item), not this module's.
+- **Demo targets only.** Actions run against a small in-memory
+  picture of demo workloads behind an interface, the same pattern
+  the tools and the incident store used. A real cluster backend can
+  sit behind the same shape later, and will still have to pass the
+  same validation and the same policy gate.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from .logging_config import get_logger
+from .models import Evidence, Target
+from .policy import Decision, PolicyResult, ProposedAction
+
+log = get_logger(__name__)
+
+# The scale runbook's own bounds. The policy engine separately checks
+# how many replicas one action may touch in an environment; these are
+# the hard limits of the runbook itself, whatever the environment says.
+MIN_REPLICAS = 1
+MAX_REPLICAS = 5
+
+# Which parameters each runbook takes. An empty set means the runbook
+# takes none at all. Any parameter not named here is refused.
+RUNBOOK_PARAMETERS: dict[str, set[str]] = {
+    "restart-workload": set(),
+    "scale-replicas": {"replicas"},
+    "rollback-deployment": set(),
+}
+
+
+class RunbookError(ValueError):
+    """A runbook proposal or execution that must not go ahead."""
+
+
+class WorkloadState(BaseModel):
+    """One demo workload, as the runbooks see it."""
+
+    target: Target
+    replicas: int = Field(default=1, ge=0)
+    version: str = "v1"
+    previous_versions: list[str] = Field(default_factory=list)
+    failing: bool = False
+    restarts: int = 0
+
+
+class DemoWorkloadBackend:
+    """An in-memory set of demo workloads, behind a small interface.
+
+    States are handed out as copies and stored back with ``save``,
+    so a caller cannot change a workload by editing what it was
+    given — the same care the audit log takes with its records.
+    """
+
+    def __init__(self, workloads: list[WorkloadState] | None = None) -> None:
+        self._workloads: dict[tuple[str, str, str], WorkloadState] = {}
+        for state in workloads or []:
+            self.save(state)
+
+    @staticmethod
+    def _key(target: Target) -> tuple[str, str, str]:
+        return (target.environment, target.namespace, target.service)
+
+    def get(self, target: Target) -> WorkloadState | None:
+        state = self._workloads.get(self._key(target))
+        return state.model_copy(deep=True) if state is not None else None
+
+    def save(self, state: WorkloadState) -> None:
+        self._workloads[self._key(state.target)] = state.model_copy(deep=True)
+
+
+def validate_parameters(
+    runbook_id: str, parameters: dict[str, str | int | float | bool]
+) -> dict[str, str | int | float | bool]:
+    """Check one runbook's parameters and return them unchanged.
+
+    Raises :class:`RunbookError` with a plain reason for anything
+    that must not run: an unknown runbook, a parameter the runbook
+    does not take, a missing one it requires, or a bad value.
+    """
+    allowed = RUNBOOK_PARAMETERS.get(runbook_id)
+    if allowed is None:
+        raise RunbookError(f"runbook '{runbook_id}' is not in the runbook library")
+    extra = set(parameters) - allowed
+    if extra:
+        names = ", ".join(sorted(extra))
+        raise RunbookError(f"runbook '{runbook_id}' does not take parameter(s): {names}")
+    if runbook_id == "scale-replicas":
+        if "replicas" not in parameters:
+            raise RunbookError("runbook 'scale-replicas' needs a 'replicas' parameter")
+        replicas = parameters["replicas"]
+        # bool is a subclass of int in Python; True is not a count.
+        if isinstance(replicas, bool) or not isinstance(replicas, int):
+            raise RunbookError("'replicas' must be a whole number")
+        if not MIN_REPLICAS <= replicas <= MAX_REPLICAS:
+            raise RunbookError(
+                f"'replicas' must be between {MIN_REPLICAS} and {MAX_REPLICAS}, "
+                f"got {replicas}"
+            )
+    return dict(parameters)
+
+
+def build_action(
+    runbook_id: str,
+    target: Target,
+    parameters: dict[str, str | int | float | bool],
+    backend: DemoWorkloadBackend,
+    *,
+    rationale: str,
+) -> ProposedAction:
+    """Build a checked proposal for one runbook against one workload.
+
+    The proposal's ``replicas_affected`` is worked out from the
+    workload's real state, not guessed by the proposer, so the policy
+    engine's blast-radius check judges a true number: a restart or a
+    rollback touches every copy that is running; a scale touches the
+    difference between the copies running now and the copies asked
+    for.
+    """
+    checked = validate_parameters(runbook_id, parameters)
+    state = backend.get(target)
+    if state is None:
+        raise RunbookError(
+            f"no demo workload named {target.service} in "
+            f"{target.namespace} ({target.environment})"
+        )
+    if runbook_id == "scale-replicas":
+        wanted = checked["replicas"]
+        assert isinstance(wanted, int)  # validated above
+        affected = abs(wanted - state.replicas)
+    else:
+        affected = state.replicas
+    return ProposedAction(
+        runbook_id=runbook_id,
+        target=target,
+        parameters=checked,
+        replicas_affected=affected,
+        rationale=rationale,
+    )
+
+
+class RunbookResult(BaseModel):
+    """What a runbook did, in plain words, with before and after."""
+
+    runbook_id: str
+    target: Target
+    summary: str
+    before: dict[str, str | int | bool]
+    after: dict[str, str | int | bool]
+
+    def to_evidence(self) -> Evidence:
+        """This result as an evidence record for the incident file."""
+        target = self.target
+        return Evidence(
+            kind="action",
+            query=(
+                f"runbook:{self.runbook_id} on "
+                f"{target.service}/{target.namespace} ({target.environment})"
+            ),
+            summary=self.summary,
+        )
+
+
+def _snapshot(state: WorkloadState) -> dict[str, str | int | bool]:
+    return {
+        "replicas": state.replicas,
+        "version": state.version,
+        "failing": state.failing,
+        "restarts": state.restarts,
+    }
+
+
+class RunbookExecutor:
+    """Runs runbooks against the demo backend — after the policy gate."""
+
+    def __init__(self, backend: DemoWorkloadBackend) -> None:
+        self._backend = backend
+
+    def execute(self, action: ProposedAction, decision: PolicyResult) -> RunbookResult:
+        """Run one action whose policy decision is ALLOW, and only that.
+
+        Any other decision — denied, or still waiting for a human's
+        approval — is refused here and nothing changes. Parameters
+        are validated again at this door, so no proposal reaches a
+        workload unchecked, however it was built.
+        """
+        if decision.decision is not Decision.ALLOW:
+            raise RunbookError(
+                f"policy decision is '{decision.decision.value}', not 'allow'; "
+                "nothing was run"
+            )
+        validate_parameters(action.runbook_id, action.parameters)
+        state = self._backend.get(action.target)
+        if state is None:
+            raise RunbookError(
+                f"no demo workload named {action.target.service} in "
+                f"{action.target.namespace} ({action.target.environment})"
+            )
+        before = _snapshot(state)
+        if action.runbook_id == "restart-workload":
+            state.restarts += 1
+            state.failing = False
+            summary = (
+                f"Restarted {state.target.service}: restarts now {state.restarts}, "
+                f"failing state cleared."
+            )
+        elif action.runbook_id == "scale-replicas":
+            wanted = action.parameters["replicas"]
+            assert isinstance(wanted, int)  # validated above
+            summary = (
+                f"Scaled {state.target.service} from {state.replicas} "
+                f"to {wanted} replicas."
+            )
+            state.replicas = wanted
+        elif action.runbook_id == "rollback-deployment":
+            if not state.previous_versions:
+                raise RunbookError(
+                    f"{state.target.service} has no previous version to roll back to"
+                )
+            old = state.version
+            state.version = state.previous_versions.pop()
+            summary = f"Rolled {state.target.service} back from {old} to {state.version}."
+        else:  # pragma: no cover - validate_parameters already refused it
+            raise RunbookError(f"runbook '{action.runbook_id}' is not in the runbook library")
+        self._backend.save(state)
+        log.info("runbook executed: %s", summary)
+        return RunbookResult(
+            runbook_id=action.runbook_id,
+            target=action.target,
+            summary=summary,
+            before=before,
+            after=_snapshot(state),
+        )
+
+
+def demo_runbooks() -> tuple[RunbookExecutor, DemoWorkloadBackend]:
+    """A seeded executor anyone can try with no setup.
+
+    The checkout workload matches the demo environment: it is
+    failing, runs two copies, and is on version v2 with v1 before
+    it — so all three runbooks have something honest to do. The
+    search workload is healthy, on its first version, so a rollback
+    against it is refused for a real reason.
+    """
+    backend = DemoWorkloadBackend(
+        [
+            WorkloadState(
+                target=Target(environment="demo", namespace="shop", service="checkout"),
+                replicas=2,
+                version="v2",
+                previous_versions=["v1"],
+                failing=True,
+            ),
+            WorkloadState(
+                target=Target(environment="demo", namespace="shop", service="search"),
+                replicas=1,
+                version="v1",
+            ),
+        ]
+    )
+    return RunbookExecutor(backend), backend