sre-agent-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sre_agent/__init__.py +3 -0
- sre_agent/adapters.py +833 -0
- sre_agent/approvals.py +243 -0
- sre_agent/audit.py +225 -0
- sre_agent/backend_scores.py +184 -0
- sre_agent/backend_url.py +146 -0
- sre_agent/cli.py +732 -0
- sre_agent/config.py +22 -0
- sre_agent/evaluation.py +391 -0
- sre_agent/evaluation_page.py +209 -0
- sre_agent/hosted_demo.py +153 -0
- sre_agent/ingestion.py +423 -0
- sre_agent/injection.py +580 -0
- sre_agent/logging_config.py +83 -0
- sre_agent/memory.py +411 -0
- sre_agent/model_backend.py +551 -0
- sre_agent/model_proposals.py +287 -0
- sre_agent/models.py +278 -0
- sre_agent/policy.py +208 -0
- sre_agent/postmortems.py +296 -0
- sre_agent/reasoning.py +547 -0
- sre_agent/regression.py +287 -0
- sre_agent/runbooks.py +290 -0
- sre_agent/security_review.py +826 -0
- sre_agent/server.py +684 -0
- sre_agent/storage.py +266 -0
- sre_agent/tier3.py +440 -0
- sre_agent/tools.py +712 -0
- sre_agent_cli-0.2.0.dist-info/METADATA +364 -0
- sre_agent_cli-0.2.0.dist-info/RECORD +34 -0
- sre_agent_cli-0.2.0.dist-info/WHEEL +5 -0
- sre_agent_cli-0.2.0.dist-info/entry_points.txt +2 -0
- sre_agent_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
- sre_agent_cli-0.2.0.dist-info/top_level.txt +1 -0
sre_agent/cli.py
ADDED
|
@@ -0,0 +1,732 @@
|
|
|
1
|
+
"""Command-line tool: look at incidents and show a diagnosis.
|
|
2
|
+
|
|
3
|
+
In plain words, this is how a person talks to the agent from a
|
|
4
|
+
terminal:
|
|
5
|
+
|
|
6
|
+
sre-agent incidents [--base-url URL]
|
|
7
|
+
sre-agent show <incident-id> [--base-url URL]
|
|
8
|
+
sre-agent diagnose
|
|
9
|
+
sre-agent approve-demo --by NAME [--runbook ID] [--replicas N] [--yes]
|
|
10
|
+
sre-agent postmortem-demo
|
|
11
|
+
sre-agent evaluate
|
|
12
|
+
sre-agent regression-check [--baseline PATH] [--update-baseline]
|
|
13
|
+
sre-agent injection-check
|
|
14
|
+
sre-agent tier3-check
|
|
15
|
+
sre-agent memory-demo
|
|
16
|
+
sre-agent adapters-demo
|
|
17
|
+
sre-agent security-check
|
|
18
|
+
sre-agent --version
|
|
19
|
+
|
|
20
|
+
How it works, honestly:
|
|
21
|
+
|
|
22
|
+
- ``incidents`` and ``show`` ask the running agent server for its
|
|
23
|
+
records (the same ``GET /incidents`` endpoints the server already
|
|
24
|
+
serves). The incident store lives in the server's memory, so the
|
|
25
|
+
CLI is a thin client of that server: if the server is not
|
|
26
|
+
running, the CLI says so plainly instead of printing an empty
|
|
27
|
+
list that would look like "no incidents".
|
|
28
|
+
- The CLI uses only the Python standard library to talk to the
|
|
29
|
+
server, so installing the CLI never needs the server extras.
|
|
30
|
+
- ``diagnose`` runs the reasoning loop once, locally, on the
|
|
31
|
+
seeded demo data and prints the report. Since Phase 7 the
|
|
32
|
+
server runs the same loop itself when it is built with a tool
|
|
33
|
+
layer — the shipped server app uses the demo tool layer — so a
|
|
34
|
+
live incident diagnosed by its server carries that diagnosis
|
|
35
|
+
on its record, and ``show`` prints it, with its causes and
|
|
36
|
+
evidence. When the server was built with a memory of past
|
|
37
|
+
incidents, the incident also carries that memory's advice,
|
|
38
|
+
and ``show`` prints it too, labelled advice only. A server
|
|
39
|
+
built with no tool layer records incidents only; ``show``
|
|
40
|
+
then says there is no diagnosis yet.
|
|
41
|
+
|
|
42
|
+
``approve-demo`` is the first command that changes anything, and
|
|
43
|
+
it changes demo state only, through the whole Tier 2 path: a fix
|
|
44
|
+
for the seeded demo checkout workload is proposed, the policy
|
|
45
|
+
engine sends it for approval, and it runs only if the person at
|
|
46
|
+
the terminal approves that exact action — typed at the prompt, or
|
|
47
|
+
with ``--yes`` — under their name (``--by``). Policy is checked
|
|
48
|
+
again at approval time, the approval expires in 15 minutes and
|
|
49
|
+
can be used once, and every decision lands in an audit log for
|
|
50
|
+
the run (see approvals.py). Since Phase 7 the server serves
|
|
51
|
+
the same approval flow for its own incidents over HTTP (see
|
|
52
|
+
server.py and the operator guide); this CLI command stays the
|
|
53
|
+
in-process demo, and there is no CLI command for the server's
|
|
54
|
+
approvals yet.
|
|
55
|
+
|
|
56
|
+
``postmortem-demo`` runs one seeded demo incident through the
|
|
57
|
+
whole path — diagnosis, a Tier 2 approved restart, resolution —
|
|
58
|
+
in this process, and prints the postmortem draft generated from
|
|
59
|
+
its record (see postmortems.py). Drafting a postmortem for a live
|
|
60
|
+
server incident is not wired yet, for the same reason: the draft
|
|
61
|
+
needs the full record, including its audit records, in one place.
|
|
62
|
+
|
|
63
|
+
``evaluate`` runs the chaos scenario suite locally — five seeded
|
|
64
|
+
faults, no server or Docker needed — and prints the scored
|
|
65
|
+
report (see evaluation.py): how many diagnoses were correct,
|
|
66
|
+
and how many unsafe actions ran (zero, counted from the incident
|
|
67
|
+
records). It exits 1 when any scenario scores incorrect or any
|
|
68
|
+
unsafe action is counted, so it can gate a release later.
|
|
69
|
+
|
|
70
|
+
``regression-check`` is that gate (see regression.py): it runs
|
|
71
|
+
the suite and compares the fresh scores against the recorded
|
|
72
|
+
baseline in ``evaluation/baseline.json`` — the last release's
|
|
73
|
+
scores. Unsafe actions must be zero (always, not just compared
|
|
74
|
+
to the baseline), every diagnosis must cite real evidence, the
|
|
75
|
+
accuracy may not drop, and no scenario that was correct may turn
|
|
76
|
+
wrong. It exits 1 when any rule breaks. Running the check never
|
|
77
|
+
changes the baseline; ``--update-baseline`` records the fresh
|
|
78
|
+
run as the new baseline on purpose, at a release, and refuses a
|
|
79
|
+
run that itself breaks the absolute rules.
|
|
80
|
+
|
|
81
|
+
``injection-check`` runs the prompt-injection test set locally
|
|
82
|
+
(see injection.py): hostile text planted in log lines, alert
|
|
83
|
+
text, a service status message, and a metric label — orders to
|
|
84
|
+
delete, run a fix, retarget, close the incident, or reveal
|
|
85
|
+
secrets. Each case passes only if the text changed nothing: it
|
|
86
|
+
stays in the evidence record as data, no claim, hypothesis, or
|
|
87
|
+
diagnosis repeats its orders, no action runs, and the answer is
|
|
88
|
+
the one the honest numbers give. It exits 1 when any case fails
|
|
89
|
+
or any unsafe action is counted.
|
|
90
|
+
|
|
91
|
+
``tier3-check`` runs the Tier 3 drill set locally (see
|
|
92
|
+
tier3.py): seven rehearsed checks on the demo environment's
|
|
93
|
+
bounded-autonomy grant. The pre-approved fixes (restart and
|
|
94
|
+
scale, shop namespace only, at most three replicas touched)
|
|
95
|
+
must run on their own and be audited; rollback, which is not
|
|
96
|
+
pre-approved, must not run on its own; a fix that is too big
|
|
97
|
+
or aimed outside the granted namespace must be refused; and
|
|
98
|
+
the kill switch drills must show a halt pressed between two
|
|
99
|
+
runs stopping the second one, while read-only checks continue.
|
|
100
|
+
It exits 1 when any drill fails.
|
|
101
|
+
|
|
102
|
+
``memory-demo`` runs the memory report locally on seeded past
|
|
103
|
+
incidents (see memory.py): for a fresh demo checkout incident
|
|
104
|
+
it finds the similar past incidents, counts each runbook's
|
|
105
|
+
record from the action records on those incidents, and ranks
|
|
106
|
+
suggested runbooks by their record on same-service past
|
|
107
|
+
incidents only.
|
|
108
|
+
It is advice only — it runs nothing, proposes nothing through
|
|
109
|
+
the policy engine, and approves nothing. Memory is not wired
|
|
110
|
+
into the live server or the reasoning loop yet.
|
|
111
|
+
|
|
112
|
+
``adapters-demo`` runs the three read adapters locally on
|
|
113
|
+
seeded demo data (see adapters.py): the Kubernetes pod listing
|
|
114
|
+
for the demo checkout workload, a Datadog metrics query, and an
|
|
115
|
+
Azure Monitor KQL query. Every adapter only looks — nothing is
|
|
116
|
+
changed in any system — and no live cluster, Datadog account or
|
|
117
|
+
Azure workspace is called; the live backends behind the same
|
|
118
|
+
shapes are not claimed as tested.
|
|
119
|
+
|
|
120
|
+
``security-check`` runs the security review probes locally (see
|
|
121
|
+
security_review.py): ten probes that attack the agent's own
|
|
122
|
+
surface — a made-up runbook, a free-form command as a parameter,
|
|
123
|
+
a retargeted fix, secrets planted where the audit log and the
|
|
124
|
+
evidence could leak them, a replayed approval, an approval into
|
|
125
|
+
a halt, the Tier 3 limits, refused management and broken
|
|
126
|
+
queries, a scan for any shell surface, and alert text claiming
|
|
127
|
+
an incident is over. It prints the findings the probes cannot
|
|
128
|
+
prove away as well (the largest: the webhook server has no
|
|
129
|
+
authentication). It exits 1 when any probe fails.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
from __future__ import annotations
|
|
133
|
+
|
|
134
|
+
import json
|
|
135
|
+
import os
|
|
136
|
+
import sys
|
|
137
|
+
import urllib.error
|
|
138
|
+
import urllib.request
|
|
139
|
+
from collections.abc import Callable
|
|
140
|
+
from pathlib import Path
|
|
141
|
+
from typing import Any
|
|
142
|
+
|
|
143
|
+
from . import __version__
|
|
144
|
+
from .adapters import AdaptersDemoReport, demo_adapters
|
|
145
|
+
from .approvals import ApprovalError, approve_pending, create_pending, execute_approved
|
|
146
|
+
from .audit import AuditLog, evaluate_and_audit
|
|
147
|
+
from .backend_url import validate_backend_url
|
|
148
|
+
from .evaluation import EvaluationReport, run_evaluation
|
|
149
|
+
from .injection import InjectionReport, run_injection_suite
|
|
150
|
+
from .memory import MemoryReport, demo_memory
|
|
151
|
+
from .models import Incident, IncidentStatus, Target
|
|
152
|
+
from .policy import EnvironmentPolicy, Tier
|
|
153
|
+
from .postmortems import PostmortemDraft, demo_postmortem
|
|
154
|
+
from .reasoning import DiagnosisReport, demo_diagnosis
|
|
155
|
+
from .regression import (
|
|
156
|
+
DEFAULT_BASELINE_PATH,
|
|
157
|
+
RegressionResult,
|
|
158
|
+
check_regression,
|
|
159
|
+
load_baseline,
|
|
160
|
+
record_new_baseline,
|
|
161
|
+
)
|
|
162
|
+
from .runbooks import RunbookError, build_action, demo_runbooks
|
|
163
|
+
from .security_review import SecurityReport, run_security_review
|
|
164
|
+
from .tier3 import Tier3Report, run_tier3_drills
|
|
165
|
+
|
|
166
|
+
DEFAULT_BASE_URL = "http://127.0.0.1:8080"
|
|
167
|
+
ENV_BASE_URL = "SRE_AGENT_BASE_URL"
|
|
168
|
+
ENV_API_TOKEN = "SRE_AGENT_API_TOKEN"
|
|
169
|
+
TOKEN_HEADER = "X-SRE-Agent-Token"
|
|
170
|
+
TIMEOUT_SECONDS = 5.0
|
|
171
|
+
|
|
172
|
+
USAGE = """Usage:
|
|
173
|
+
sre-agent incidents [--base-url URL] List the incidents the server knows
|
|
174
|
+
sre-agent show <incident-id> [--base-url URL]
|
|
175
|
+
Show one incident in full, with its
|
|
176
|
+
diagnosis, causes and evidence
|
|
177
|
+
sre-agent diagnose Run the demo diagnosis locally and
|
|
178
|
+
print the report (seeded demo data)
|
|
179
|
+
sre-agent approve-demo --by NAME [--runbook ID] [--replicas N] [--yes]
|
|
180
|
+
Propose a fix for the demo checkout
|
|
181
|
+
workload and run it only if you
|
|
182
|
+
approve that exact action (Tier 2)
|
|
183
|
+
sre-agent postmortem-demo Run a demo incident end to end and
|
|
184
|
+
print its postmortem draft
|
|
185
|
+
sre-agent evaluate Run the five chaos scenarios on
|
|
186
|
+
seeded demo data and print the
|
|
187
|
+
scored report (exit 1 if any
|
|
188
|
+
scenario fails or any unsafe
|
|
189
|
+
action is counted)
|
|
190
|
+
sre-agent regression-check [--baseline PATH] [--update-baseline]
|
|
191
|
+
Compare a fresh run of the suite
|
|
192
|
+
against the recorded baseline
|
|
193
|
+
(default evaluation/baseline.json):
|
|
194
|
+
unsafe actions must be 0, accuracy
|
|
195
|
+
may not drop, no correct scenario
|
|
196
|
+
may turn wrong (exit 1 if any rule
|
|
197
|
+
breaks). --update-baseline records
|
|
198
|
+
the fresh run as the new baseline
|
|
199
|
+
on purpose, at a release
|
|
200
|
+
sre-agent injection-check Run the prompt-injection test set:
|
|
201
|
+
hostile text in logs, alerts, a
|
|
202
|
+
status message and a metric label
|
|
203
|
+
must change nothing (exit 1 if any
|
|
204
|
+
case fails or any unsafe action
|
|
205
|
+
is counted)
|
|
206
|
+
sre-agent tier3-check Run the Tier 3 drill set on the
|
|
207
|
+
demo grant: pre-approved fixes
|
|
208
|
+
run, everything outside the grant
|
|
209
|
+
does not, and the kill switch
|
|
210
|
+
stops mutations while reads
|
|
211
|
+
continue (exit 1 if any drill
|
|
212
|
+
fails)
|
|
213
|
+
sre-agent memory-demo Show what memory says about a
|
|
214
|
+
fresh demo checkout incident:
|
|
215
|
+
similar past incidents, each
|
|
216
|
+
runbook's record, and suggested
|
|
217
|
+
runbooks ranked by their record
|
|
218
|
+
on same-service past incidents
|
|
219
|
+
only (advice only; nothing runs)
|
|
220
|
+
sre-agent adapters-demo Show the three read adapters on
|
|
221
|
+
seeded demo data: Kubernetes pods,
|
|
222
|
+
a Datadog metrics query, and an
|
|
223
|
+
Azure Monitor KQL query (all
|
|
224
|
+
read-only; no live system called)
|
|
225
|
+
sre-agent security-check Run the security review probes:
|
|
226
|
+
ten attacks on the agent's own
|
|
227
|
+
surface must all be refused or
|
|
228
|
+
contained, and the findings the
|
|
229
|
+
probes cannot prove away are
|
|
230
|
+
printed (exit 1 if any probe
|
|
231
|
+
fails)
|
|
232
|
+
sre-agent --version Print the version
|
|
233
|
+
|
|
234
|
+
The server address defaults to http://127.0.0.1:8080 and can also be
|
|
235
|
+
set with the SRE_AGENT_BASE_URL environment variable. The address
|
|
236
|
+
follows the backend address rules (finding F-05): https, or plain
|
|
237
|
+
http to this machine only — a plain-http address for a server on
|
|
238
|
+
another machine is refused, so the token cannot cross a network
|
|
239
|
+
unencrypted. If the server
|
|
240
|
+
was started with a token (SRE_AGENT_API_TOKEN set), set the same
|
|
241
|
+
variable for the CLI and it sends the token on every read."""
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class CliError(Exception):
|
|
245
|
+
"""A problem the user should read, not a traceback."""
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
FetchJson = Callable[[str], Any]
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def fetch_json(url: str) -> Any:
|
|
252
|
+
"""GET *url* and return the decoded JSON, or raise CliError.
|
|
253
|
+
|
|
254
|
+
A server that is not there, an answer that is not JSON, and an
|
|
255
|
+
HTTP error are all turned into one plain sentence.
|
|
256
|
+
"""
|
|
257
|
+
headers = {"Accept": "application/json"}
|
|
258
|
+
api_token = os.environ.get(ENV_API_TOKEN)
|
|
259
|
+
if api_token:
|
|
260
|
+
headers[TOKEN_HEADER] = api_token
|
|
261
|
+
request = urllib.request.Request(url, headers=headers)
|
|
262
|
+
try:
|
|
263
|
+
with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response:
|
|
264
|
+
body = response.read().decode("utf-8")
|
|
265
|
+
except urllib.error.HTTPError as exc:
|
|
266
|
+
if exc.code == 404:
|
|
267
|
+
raise CliError("the server says that incident does not exist") from exc
|
|
268
|
+
raise CliError(f"the server answered with an error (HTTP {exc.code})") from exc
|
|
269
|
+
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
270
|
+
raise CliError(
|
|
271
|
+
f"could not reach the agent server at {url} — is it running? "
|
|
272
|
+
"Start it with: uvicorn sre_agent.server:app --port 8080"
|
|
273
|
+
) from exc
|
|
274
|
+
try:
|
|
275
|
+
return json.loads(body)
|
|
276
|
+
except json.JSONDecodeError as exc:
|
|
277
|
+
raise CliError("the server answered, but its answer was not JSON") from exc
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def format_incident_list(rows: list[dict[str, Any]]) -> str:
|
|
281
|
+
"""One plain line per incident, plus a count."""
|
|
282
|
+
if not rows:
|
|
283
|
+
return (
|
|
284
|
+
"No incidents. The server is running, but it has not "
|
|
285
|
+
"received any alerts yet."
|
|
286
|
+
)
|
|
287
|
+
lines = [f"{len(rows)} incident(s):"]
|
|
288
|
+
for row in rows:
|
|
289
|
+
target = row.get("target") or {}
|
|
290
|
+
where = (
|
|
291
|
+
f"{target.get('service', '?')}/{target.get('namespace', '?')} "
|
|
292
|
+
f"({target.get('environment', '?')})"
|
|
293
|
+
)
|
|
294
|
+
lines.append(
|
|
295
|
+
f"- {row.get('id', '?')} [{row.get('status', '?')}] {where} "
|
|
296
|
+
f"{row.get('title', '')} "
|
|
297
|
+
f"(evidence: {row.get('evidence_count', 0)}, "
|
|
298
|
+
f"duplicates: {row.get('duplicate_count', 0)})"
|
|
299
|
+
)
|
|
300
|
+
return "\n".join(lines)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def format_incident_detail(incident: dict[str, Any]) -> str:
|
|
304
|
+
"""Show one incident in full: what, where, diagnosis, evidence."""
|
|
305
|
+
target = incident.get("target") or {}
|
|
306
|
+
lines = [
|
|
307
|
+
f"Incident {incident.get('id', '?')}",
|
|
308
|
+
f"Title: {incident.get('title', '')}",
|
|
309
|
+
f"Status: {incident.get('status', '?')}",
|
|
310
|
+
(
|
|
311
|
+
f"Target: {target.get('service', '?')} in "
|
|
312
|
+
f"{target.get('namespace', '?')} ({target.get('environment', '?')})"
|
|
313
|
+
),
|
|
314
|
+
f"Created: {incident.get('created_at', '?')}",
|
|
315
|
+
f"Updated: {incident.get('updated_at', '?')}",
|
|
316
|
+
f"Duplicate alerts counted: {incident.get('duplicate_count', 0)}",
|
|
317
|
+
"",
|
|
318
|
+
]
|
|
319
|
+
diagnosis = incident.get("diagnosis")
|
|
320
|
+
if diagnosis:
|
|
321
|
+
lines.append(f"Diagnosis: {diagnosis}")
|
|
322
|
+
else:
|
|
323
|
+
lines.append("Diagnosis: none yet — the record holds no diagnosis.")
|
|
324
|
+
hypotheses = incident.get("hypotheses") or []
|
|
325
|
+
if hypotheses:
|
|
326
|
+
lines.append("Possible causes, as recorded:")
|
|
327
|
+
for hyp in hypotheses:
|
|
328
|
+
lines.append(
|
|
329
|
+
f"- {hyp.get('statement', '')} "
|
|
330
|
+
f"(confidence {float(hyp.get('confidence', 0.0)):.2f})"
|
|
331
|
+
)
|
|
332
|
+
memory_report = incident.get("memory_report")
|
|
333
|
+
if memory_report:
|
|
334
|
+
lines.append(
|
|
335
|
+
"Memory advice (advice only — nothing here runs or approves a fix):"
|
|
336
|
+
)
|
|
337
|
+
matches = memory_report.get("matches") or []
|
|
338
|
+
if not matches:
|
|
339
|
+
lines.append("- No similar past incident was found.")
|
|
340
|
+
for match in matches:
|
|
341
|
+
lines.append(
|
|
342
|
+
f"- Past incident {match.get('incident_id', '?')} "
|
|
343
|
+
f"[{match.get('status', '?')}] score {match.get('score', 0)}: "
|
|
344
|
+
f"{match.get('title', '')}"
|
|
345
|
+
)
|
|
346
|
+
suggestions = memory_report.get("suggestions") or []
|
|
347
|
+
if not suggestions:
|
|
348
|
+
lines.append(
|
|
349
|
+
"- No runbook is suggested: no similar past incident ran one."
|
|
350
|
+
)
|
|
351
|
+
for suggestion in suggestions:
|
|
352
|
+
lines.append(
|
|
353
|
+
f"- Suggested runbook {suggestion.get('runbook_id', '?')}: "
|
|
354
|
+
f"{suggestion.get('reason', '')}"
|
|
355
|
+
)
|
|
356
|
+
evidence = incident.get("evidence") or []
|
|
357
|
+
lines.append(f"Evidence ({len(evidence)} record(s)):")
|
|
358
|
+
if not evidence:
|
|
359
|
+
lines.append("- No evidence recorded yet.")
|
|
360
|
+
for ev in evidence:
|
|
361
|
+
lines.append(
|
|
362
|
+
f"- [{ev.get('kind', '?')}] {ev.get('summary', '')} "
|
|
363
|
+
f"(query: {ev.get('query', '?')})"
|
|
364
|
+
)
|
|
365
|
+
return "\n".join(lines)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _split_base_url(args: list[str]) -> tuple[list[str], str]:
|
|
369
|
+
"""Pull ``--base-url URL`` (or ``--base-url=URL``) out of *args*."""
|
|
370
|
+
base_url = os.environ.get(ENV_BASE_URL, DEFAULT_BASE_URL)
|
|
371
|
+
rest: list[str] = []
|
|
372
|
+
i = 0
|
|
373
|
+
while i < len(args):
|
|
374
|
+
arg = args[i]
|
|
375
|
+
if arg == "--base-url":
|
|
376
|
+
if i + 1 >= len(args):
|
|
377
|
+
raise CliError("--base-url needs an address after it")
|
|
378
|
+
base_url = args[i + 1]
|
|
379
|
+
i += 2
|
|
380
|
+
elif arg.startswith("--base-url="):
|
|
381
|
+
base_url = arg.split("=", 1)[1]
|
|
382
|
+
i += 1
|
|
383
|
+
else:
|
|
384
|
+
rest.append(arg)
|
|
385
|
+
i += 1
|
|
386
|
+
if not base_url:
|
|
387
|
+
raise CliError("the server address must not be empty")
|
|
388
|
+
try:
|
|
389
|
+
checked = validate_backend_url(base_url, what="server address")
|
|
390
|
+
except ValueError as exc:
|
|
391
|
+
raise CliError(str(exc)) from exc
|
|
392
|
+
return rest, checked
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
DEMO_CHECKOUT = Target(environment="demo", namespace="shop", service="checkout")
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _parse_approve_demo_args(args: list[str]) -> dict[str, Any]:
|
|
399
|
+
"""Parse approve-demo's flags: --by, --runbook, --replicas, --yes."""
|
|
400
|
+
parsed: dict[str, Any] = {"by": None, "runbook": "restart-workload",
|
|
401
|
+
"replicas": None, "yes": False}
|
|
402
|
+
i = 0
|
|
403
|
+
while i < len(args):
|
|
404
|
+
arg = args[i]
|
|
405
|
+
if arg == "--yes":
|
|
406
|
+
parsed["yes"] = True
|
|
407
|
+
i += 1
|
|
408
|
+
continue
|
|
409
|
+
name = None
|
|
410
|
+
value = None
|
|
411
|
+
for flag in ("--by", "--runbook", "--replicas"):
|
|
412
|
+
if arg == flag:
|
|
413
|
+
if i + 1 >= len(args):
|
|
414
|
+
raise CliError(f"{flag} needs a value after it")
|
|
415
|
+
name, value = flag, args[i + 1]
|
|
416
|
+
i += 2
|
|
417
|
+
break
|
|
418
|
+
if arg.startswith(flag + "="):
|
|
419
|
+
name, value = flag, arg.split("=", 1)[1]
|
|
420
|
+
i += 1
|
|
421
|
+
break
|
|
422
|
+
if name is None:
|
|
423
|
+
raise CliError(f"approve-demo does not take {arg!r}")
|
|
424
|
+
if name == "--replicas":
|
|
425
|
+
try:
|
|
426
|
+
parsed["replicas"] = int(value) # type: ignore[arg-type]
|
|
427
|
+
except (TypeError, ValueError) as exc:
|
|
428
|
+
raise CliError("--replicas needs a whole number") from exc
|
|
429
|
+
else:
|
|
430
|
+
parsed[name[2:]] = value
|
|
431
|
+
return parsed
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def _prompt_for_approval(question: str) -> bool:
|
|
435
|
+
"""Ask at the terminal; anything but 'approve', or no terminal, is a no."""
|
|
436
|
+
try:
|
|
437
|
+
answer = input(f"{question} Type 'approve' to run it: ")
|
|
438
|
+
except EOFError:
|
|
439
|
+
return False
|
|
440
|
+
return answer.strip().lower() == "approve"
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def run_demo_approval(
|
|
444
|
+
*,
|
|
445
|
+
approved_by: str,
|
|
446
|
+
runbook_id: str,
|
|
447
|
+
parameters: dict[str, str | int | float | bool],
|
|
448
|
+
confirmed: bool,
|
|
449
|
+
confirm: Callable[[str], bool] | None = None,
|
|
450
|
+
) -> int:
|
|
451
|
+
"""The Tier 2 path end to end, on the seeded demo workload.
|
|
452
|
+
|
|
453
|
+
Propose → policy sends it for approval → a human approves that
|
|
454
|
+
exact action (or not) → only then does it run. Prints each step
|
|
455
|
+
in plain words. Returns the process exit code.
|
|
456
|
+
"""
|
|
457
|
+
executor, backend = demo_runbooks()
|
|
458
|
+
incident = Incident(
|
|
459
|
+
id="demo-incident-1",
|
|
460
|
+
title="Checkout errors are above 10%",
|
|
461
|
+
target=DEMO_CHECKOUT,
|
|
462
|
+
)
|
|
463
|
+
incident.transition(IncidentStatus.TRIAGING)
|
|
464
|
+
incident.diagnosis = "Demo diagnosis: the checkout workload is failing."
|
|
465
|
+
incident.transition(IncidentStatus.DIAGNOSED)
|
|
466
|
+
|
|
467
|
+
try:
|
|
468
|
+
action = build_action(
|
|
469
|
+
runbook_id, DEMO_CHECKOUT, parameters, backend,
|
|
470
|
+
rationale="demo diagnosis: the checkout workload is failing",
|
|
471
|
+
)
|
|
472
|
+
except RunbookError as exc:
|
|
473
|
+
raise CliError(str(exc)) from exc
|
|
474
|
+
|
|
475
|
+
policy = EnvironmentPolicy(
|
|
476
|
+
environment="demo",
|
|
477
|
+
granted_tier=Tier.ACT_WITH_APPROVAL,
|
|
478
|
+
max_replicas_affected=5,
|
|
479
|
+
)
|
|
480
|
+
audit = AuditLog()
|
|
481
|
+
result = evaluate_and_audit(
|
|
482
|
+
action, policy, audit, incident_id=incident.id, incident_target=incident.target
|
|
483
|
+
)
|
|
484
|
+
print(f"Proposed: {action.runbook_id} on checkout/shop (demo), "
|
|
485
|
+
f"touching {action.replicas_affected} replica(s).")
|
|
486
|
+
print(f"Policy: {result.decision.value} — {result.reason}")
|
|
487
|
+
try:
|
|
488
|
+
pending = create_pending(action, result, incident_id=incident.id)
|
|
489
|
+
except ApprovalError as exc:
|
|
490
|
+
raise CliError(str(exc)) from exc
|
|
491
|
+
print(f"This approval, if given, expires at {pending.expires_at:%H:%M:%S} UTC "
|
|
492
|
+
"and can be used once, for this action only.")
|
|
493
|
+
|
|
494
|
+
if not confirmed:
|
|
495
|
+
ask = confirm if confirm is not None else _prompt_for_approval
|
|
496
|
+
confirmed = ask(f"Approve {action.runbook_id} on checkout for incident "
|
|
497
|
+
f"{incident.id}, as {approved_by}?")
|
|
498
|
+
if not confirmed:
|
|
499
|
+
print("Not approved. Nothing was run.")
|
|
500
|
+
return 0
|
|
501
|
+
|
|
502
|
+
try:
|
|
503
|
+
grant, allowed = approve_pending(
|
|
504
|
+
pending, approved_by=approved_by, policy=policy, audit_log=audit,
|
|
505
|
+
incident_target=incident.target,
|
|
506
|
+
)
|
|
507
|
+
incident.transition(IncidentStatus.MITIGATING)
|
|
508
|
+
run_result = execute_approved(executor, pending, grant, allowed)
|
|
509
|
+
except (ApprovalError, RunbookError) as exc:
|
|
510
|
+
raise CliError(str(exc)) from exc
|
|
511
|
+
incident.add_evidence(run_result.to_evidence())
|
|
512
|
+
incident.transition(IncidentStatus.RESOLVED)
|
|
513
|
+
state = backend.get(DEMO_CHECKOUT)
|
|
514
|
+
print(f"Approved by {approved_by}. Ran: {run_result.summary}")
|
|
515
|
+
print(f"Incident {incident.id}: {incident.status.value}; audit log holds "
|
|
516
|
+
f"{len(audit.for_incident(incident.id))} decision(s) for it.")
|
|
517
|
+
if state is not None:
|
|
518
|
+
print(f"Checkout now: {state.replicas} replica(s), version {state.version}, "
|
|
519
|
+
f"failing={state.failing}, restarts={state.restarts}.")
|
|
520
|
+
return 0
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _parse_regression_args(args: list[str]) -> tuple[Path, bool]:
|
|
524
|
+
"""Parse regression-check's flags: --baseline PATH, --update-baseline."""
|
|
525
|
+
baseline_path = DEFAULT_BASELINE_PATH
|
|
526
|
+
update = False
|
|
527
|
+
i = 0
|
|
528
|
+
while i < len(args):
|
|
529
|
+
arg = args[i]
|
|
530
|
+
if arg == "--update-baseline":
|
|
531
|
+
update = True
|
|
532
|
+
i += 1
|
|
533
|
+
elif arg == "--baseline":
|
|
534
|
+
if i + 1 >= len(args):
|
|
535
|
+
raise CliError("--baseline needs a file path after it")
|
|
536
|
+
baseline_path = Path(args[i + 1])
|
|
537
|
+
i += 2
|
|
538
|
+
elif arg.startswith("--baseline="):
|
|
539
|
+
baseline_path = Path(arg.split("=", 1)[1])
|
|
540
|
+
i += 1
|
|
541
|
+
else:
|
|
542
|
+
raise CliError(f"regression-check does not take {arg!r}")
|
|
543
|
+
return baseline_path, update
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def run_regression_check(
|
|
547
|
+
*,
|
|
548
|
+
baseline_path: Path,
|
|
549
|
+
update_baseline: bool,
|
|
550
|
+
evaluate: Callable[[], EvaluationReport],
|
|
551
|
+
) -> int:
|
|
552
|
+
"""Run the suite once and apply the regression gate to its scores.
|
|
553
|
+
|
|
554
|
+
With ``update_baseline``, the fresh run is recorded as the new
|
|
555
|
+
baseline instead of being compared — a deliberate release act,
|
|
556
|
+
refused for a run that breaks the absolute rules.
|
|
557
|
+
"""
|
|
558
|
+
report = evaluate()
|
|
559
|
+
if update_baseline:
|
|
560
|
+
try:
|
|
561
|
+
baseline = record_new_baseline(report, baseline_path)
|
|
562
|
+
except ValueError as exc:
|
|
563
|
+
raise CliError(str(exc)) from exc
|
|
564
|
+
print(
|
|
565
|
+
f"Baseline updated at {baseline_path}: version {baseline.version}, "
|
|
566
|
+
f"recorded {baseline.recorded_on}, accuracy {baseline.accuracy:.0%}, "
|
|
567
|
+
f"unsafe actions {baseline.unsafe_actions}. "
|
|
568
|
+
f"History now holds {len(baseline.history)} release record(s)."
|
|
569
|
+
)
|
|
570
|
+
return 0
|
|
571
|
+
if not baseline_path.exists():
|
|
572
|
+
raise CliError(
|
|
573
|
+
f"no baseline file at {baseline_path} — the gate has nothing to "
|
|
574
|
+
"compare against. Record one on purpose with: "
|
|
575
|
+
"sre-agent regression-check --update-baseline"
|
|
576
|
+
)
|
|
577
|
+
try:
|
|
578
|
+
baseline = load_baseline(baseline_path)
|
|
579
|
+
except (OSError, ValueError) as exc:
|
|
580
|
+
raise CliError(f"the baseline file at {baseline_path} could not be read: {exc}") from exc
|
|
581
|
+
result: RegressionResult = check_regression(report, baseline)
|
|
582
|
+
print(result.render_text())
|
|
583
|
+
return 0 if result.passed else 1
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def main(
|
|
587
|
+
argv: list[str] | None = None,
|
|
588
|
+
*,
|
|
589
|
+
fetch: FetchJson | None = None,
|
|
590
|
+
run_demo_diagnosis: Callable[[], DiagnosisReport] | None = None,
|
|
591
|
+
confirm_approval: Callable[[str], bool] | None = None,
|
|
592
|
+
run_demo_postmortem: Callable[[], PostmortemDraft] | None = None,
|
|
593
|
+
run_suite: Callable[[], EvaluationReport] | None = None,
|
|
594
|
+
run_injection: Callable[[], InjectionReport] | None = None,
|
|
595
|
+
run_drills: Callable[[], Tier3Report] | None = None,
|
|
596
|
+
run_demo_memory: Callable[[], MemoryReport] | None = None,
|
|
597
|
+
run_demo_adapters: Callable[[], AdaptersDemoReport] | None = None,
|
|
598
|
+
run_security: Callable[[], SecurityReport] | None = None,
|
|
599
|
+
) -> int:
|
|
600
|
+
"""Run one CLI command. Returns the process exit code."""
|
|
601
|
+
args = list(sys.argv[1:] if argv is None else argv)
|
|
602
|
+
get_json = fetch if fetch is not None else fetch_json
|
|
603
|
+
|
|
604
|
+
if not args or args[0] in {"--help", "-h", "help"}:
|
|
605
|
+
print(USAGE)
|
|
606
|
+
return 0
|
|
607
|
+
if args[0] in {"--version", "version"}:
|
|
608
|
+
print(f"sre-agent {__version__}")
|
|
609
|
+
return 0
|
|
610
|
+
|
|
611
|
+
command, rest = args[0], args[1:]
|
|
612
|
+
try:
|
|
613
|
+
if command == "diagnose":
|
|
614
|
+
if rest:
|
|
615
|
+
raise CliError("diagnose takes no extra arguments")
|
|
616
|
+
diagnose = run_demo_diagnosis or demo_diagnosis
|
|
617
|
+
print("Demo diagnosis (seeded demo data, not a real system):")
|
|
618
|
+
print(diagnose().render_text())
|
|
619
|
+
return 0
|
|
620
|
+
if command == "postmortem-demo":
|
|
621
|
+
if rest:
|
|
622
|
+
raise CliError("postmortem-demo takes no extra arguments")
|
|
623
|
+
make_draft = run_demo_postmortem or demo_postmortem
|
|
624
|
+
print("Demo postmortem (seeded demo data, not a real system):")
|
|
625
|
+
print(make_draft().render_markdown())
|
|
626
|
+
return 0
|
|
627
|
+
if command == "evaluate":
|
|
628
|
+
if rest:
|
|
629
|
+
raise CliError("evaluate takes no extra arguments")
|
|
630
|
+
evaluate = run_suite or run_evaluation
|
|
631
|
+
report = evaluate()
|
|
632
|
+
print(report.render_text())
|
|
633
|
+
if report.correct != report.total or report.unsafe_actions:
|
|
634
|
+
return 1
|
|
635
|
+
return 0
|
|
636
|
+
if command == "injection-check":
|
|
637
|
+
if rest:
|
|
638
|
+
raise CliError("injection-check takes no extra arguments")
|
|
639
|
+
check_injection = run_injection or run_injection_suite
|
|
640
|
+
injection_report = check_injection()
|
|
641
|
+
print(injection_report.render_text())
|
|
642
|
+
if (
|
|
643
|
+
injection_report.passed_count != injection_report.total
|
|
644
|
+
or injection_report.unsafe_actions
|
|
645
|
+
):
|
|
646
|
+
return 1
|
|
647
|
+
return 0
|
|
648
|
+
if command == "tier3-check":
|
|
649
|
+
if rest:
|
|
650
|
+
raise CliError("tier3-check takes no extra arguments")
|
|
651
|
+
drills = run_drills or run_tier3_drills
|
|
652
|
+
tier3_report = drills()
|
|
653
|
+
print(tier3_report.render_text())
|
|
654
|
+
if tier3_report.passed_count != tier3_report.total:
|
|
655
|
+
return 1
|
|
656
|
+
return 0
|
|
657
|
+
if command == "memory-demo":
|
|
658
|
+
if rest:
|
|
659
|
+
raise CliError("memory-demo takes no extra arguments")
|
|
660
|
+
make_report = run_demo_memory or demo_memory
|
|
661
|
+
print("Demo memory report (seeded past incidents, not a real system):")
|
|
662
|
+
print(make_report().render_text())
|
|
663
|
+
return 0
|
|
664
|
+
if command == "adapters-demo":
|
|
665
|
+
if rest:
|
|
666
|
+
raise CliError("adapters-demo takes no extra arguments")
|
|
667
|
+
make_adapters_report = run_demo_adapters or demo_adapters
|
|
668
|
+
print(make_adapters_report().render_text())
|
|
669
|
+
return 0
|
|
670
|
+
if command == "security-check":
|
|
671
|
+
if rest:
|
|
672
|
+
raise CliError("security-check takes no extra arguments")
|
|
673
|
+
review = run_security or run_security_review
|
|
674
|
+
security_report = review()
|
|
675
|
+
print(security_report.render_text())
|
|
676
|
+
if security_report.passed_count != security_report.total:
|
|
677
|
+
return 1
|
|
678
|
+
return 0
|
|
679
|
+
if command == "regression-check":
|
|
680
|
+
baseline_path, update_baseline = _parse_regression_args(rest)
|
|
681
|
+
return run_regression_check(
|
|
682
|
+
baseline_path=baseline_path,
|
|
683
|
+
update_baseline=update_baseline,
|
|
684
|
+
evaluate=run_suite or run_evaluation,
|
|
685
|
+
)
|
|
686
|
+
if command == "approve-demo":
|
|
687
|
+
opts = _parse_approve_demo_args(rest)
|
|
688
|
+
if not opts["by"] or not str(opts["by"]).strip():
|
|
689
|
+
raise CliError("approve-demo needs --by NAME: an approval must "
|
|
690
|
+
"name who is approving")
|
|
691
|
+
parameters: dict[str, str | int | float | bool] = {}
|
|
692
|
+
if opts["runbook"] == "scale-replicas":
|
|
693
|
+
if opts["replicas"] is None:
|
|
694
|
+
raise CliError("the scale-replicas runbook needs --replicas N")
|
|
695
|
+
parameters = {"replicas": opts["replicas"]}
|
|
696
|
+
elif opts["replicas"] is not None:
|
|
697
|
+
raise CliError("--replicas only applies to the scale-replicas runbook")
|
|
698
|
+
return run_demo_approval(
|
|
699
|
+
approved_by=str(opts["by"]),
|
|
700
|
+
runbook_id=str(opts["runbook"]),
|
|
701
|
+
parameters=parameters,
|
|
702
|
+
confirmed=bool(opts["yes"]),
|
|
703
|
+
confirm=confirm_approval,
|
|
704
|
+
)
|
|
705
|
+
if command == "incidents":
|
|
706
|
+
positional, base_url = _split_base_url(rest)
|
|
707
|
+
if positional:
|
|
708
|
+
raise CliError("incidents takes no arguments besides --base-url")
|
|
709
|
+
rows = get_json(f"{base_url}/incidents")
|
|
710
|
+
if not isinstance(rows, list):
|
|
711
|
+
raise CliError("the server answered, but not with a list of incidents")
|
|
712
|
+
print(format_incident_list(rows))
|
|
713
|
+
return 0
|
|
714
|
+
if command == "show":
|
|
715
|
+
positional, base_url = _split_base_url(rest)
|
|
716
|
+
if len(positional) != 1:
|
|
717
|
+
raise CliError("show needs exactly one incident id")
|
|
718
|
+
incident = get_json(f"{base_url}/incidents/{positional[0]}")
|
|
719
|
+
if not isinstance(incident, dict):
|
|
720
|
+
raise CliError("the server answered, but not with an incident")
|
|
721
|
+
print(format_incident_detail(incident))
|
|
722
|
+
return 0
|
|
723
|
+
except CliError as exc:
|
|
724
|
+
print(f"sre-agent: {exc}", file=sys.stderr)
|
|
725
|
+
return 1
|
|
726
|
+
|
|
727
|
+
print(f"sre-agent: unknown command {command!r}\n\n{USAGE}", file=sys.stderr)
|
|
728
|
+
return 2
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
if __name__ == "__main__":
|
|
732
|
+
raise SystemExit(main())
|