sre-agent-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sre_agent/__init__.py +3 -0
- sre_agent/adapters.py +833 -0
- sre_agent/approvals.py +243 -0
- sre_agent/audit.py +225 -0
- sre_agent/backend_scores.py +184 -0
- sre_agent/backend_url.py +146 -0
- sre_agent/cli.py +732 -0
- sre_agent/config.py +22 -0
- sre_agent/evaluation.py +391 -0
- sre_agent/evaluation_page.py +209 -0
- sre_agent/hosted_demo.py +153 -0
- sre_agent/ingestion.py +423 -0
- sre_agent/injection.py +580 -0
- sre_agent/logging_config.py +83 -0
- sre_agent/memory.py +411 -0
- sre_agent/model_backend.py +551 -0
- sre_agent/model_proposals.py +287 -0
- sre_agent/models.py +278 -0
- sre_agent/policy.py +208 -0
- sre_agent/postmortems.py +296 -0
- sre_agent/reasoning.py +547 -0
- sre_agent/regression.py +287 -0
- sre_agent/runbooks.py +290 -0
- sre_agent/security_review.py +826 -0
- sre_agent/server.py +684 -0
- sre_agent/storage.py +266 -0
- sre_agent/tier3.py +440 -0
- sre_agent/tools.py +712 -0
- sre_agent_cli-0.2.0.dist-info/METADATA +364 -0
- sre_agent_cli-0.2.0.dist-info/RECORD +34 -0
- sre_agent_cli-0.2.0.dist-info/WHEEL +5 -0
- sre_agent_cli-0.2.0.dist-info/entry_points.txt +2 -0
- sre_agent_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
- sre_agent_cli-0.2.0.dist-info/top_level.txt +1 -0
sre_agent/adapters.py
ADDED
|
@@ -0,0 +1,833 @@
|
|
|
1
|
+
"""Read adapters: Kubernetes, Datadog and Azure Monitor (KQL).
|
|
2
|
+
|
|
3
|
+
In plain words, this module is how the agent looks at three more
|
|
4
|
+
places an SRE already uses — still only looking, never changing:
|
|
5
|
+
|
|
6
|
+
- **Kubernetes.** List the pods (the running copies of a workload)
|
|
7
|
+
in one namespace, with each pod's phase, how many of its
|
|
8
|
+
containers are ready, and how many times they have restarted.
|
|
9
|
+
This is the read a person does first with ``kubectl get pods``.
|
|
10
|
+
- **Datadog.** Ask a metrics question, in the same shape the
|
|
11
|
+
existing PromQL tool already returns, so a Datadog series can
|
|
12
|
+
become evidence exactly like a Prometheus one.
|
|
13
|
+
- **Azure Monitor.** Run a KQL query (Kusto Query Language, the
|
|
14
|
+
query language of Azure Log Analytics) against a workspace and
|
|
15
|
+
get rows back as evidence.
|
|
16
|
+
|
|
17
|
+
How safety is kept, the same rules as the tool layer:
|
|
18
|
+
|
|
19
|
+
- **Read-only by construction.** Every backend here only fetches.
|
|
20
|
+
There is no method that creates, scales, restarts or deletes
|
|
21
|
+
anything, in any of the three systems. The KQL check also
|
|
22
|
+
refuses management commands (the ones that start with a dot,
|
|
23
|
+
such as ``.drop``), because those change a workspace, they do
|
|
24
|
+
not read it.
|
|
25
|
+
- **Inputs are checked and outputs are bounded.** Namespace
|
|
26
|
+
names, query lengths and result counts all have hard limits,
|
|
27
|
+
anything cut off is flagged, and every text value passes the
|
|
28
|
+
tool layer's secret cleaning before it can be stored or shown.
|
|
29
|
+
- **Secrets are passed in, never written down.** The live
|
|
30
|
+
backends take their API keys or token as arguments from the
|
|
31
|
+
caller — nothing here reads a key from a file, logs one, or
|
|
32
|
+
has one in its code.
|
|
33
|
+
- **Backend addresses are checked.** Every live backend's
|
|
34
|
+
address must pass the rules in ``backend_url.py`` (finding
|
|
35
|
+
F-05) before a credential can be sent to it: https, or plain
|
|
36
|
+
http to this machine only, no credentials inside the address,
|
|
37
|
+
and an optional host allow-list per backend.
|
|
38
|
+
- **Replies are parsed by pure functions.** Each system's reply
|
|
39
|
+
shape is turned into this project's own models by a function
|
|
40
|
+
that takes the decoded reply and nothing else, so the parsing
|
|
41
|
+
is tested against recorded reply shapes with no live account,
|
|
42
|
+
no cluster and no network — the same two-layer honesty the
|
|
43
|
+
demo environment uses.
|
|
44
|
+
|
|
45
|
+
Scope, stated honestly: the demo backends and the parsers are
|
|
46
|
+
built and tested here. The live backends are thin standard-
|
|
47
|
+
library HTTP clients behind the same shapes, but no live
|
|
48
|
+
Kubernetes cluster (kind or otherwise), Datadog account or Azure
|
|
49
|
+
workspace was available in the build environment, so no live
|
|
50
|
+
call is claimed as tested; running against a real kind cluster
|
|
51
|
+
is a human check, the same status the Docker demo had at first.
|
|
52
|
+
|
|
53
|
+
Feeding a diagnosis (Phase 7): an operator may select any of
|
|
54
|
+
these adapters as extra evidence sources for the reasoning loop
|
|
55
|
+
by handing an :class:`AdapterSources` bundle to it (or to the
|
|
56
|
+
server, which passes it on). That selection is the only wiring:
|
|
57
|
+
the loop asks each selected adapter one fixed question, built
|
|
58
|
+
only from the incident's validated target names, and the answers
|
|
59
|
+
become evidence records on the incident like any other. An
|
|
60
|
+
adapter that is not selected is never called, and a selected
|
|
61
|
+
adapter that fails is skipped — it never takes the diagnosis
|
|
62
|
+
down with it.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
from __future__ import annotations
|
|
66
|
+
|
|
67
|
+
import json
|
|
68
|
+
import re
|
|
69
|
+
import urllib.parse
|
|
70
|
+
import urllib.request
|
|
71
|
+
from dataclasses import dataclass
|
|
72
|
+
from datetime import UTC, datetime
|
|
73
|
+
|
|
74
|
+
from pydantic import BaseModel, Field, field_validator
|
|
75
|
+
|
|
76
|
+
from .backend_url import validate_backend_url
|
|
77
|
+
from .logging_config import get_logger
|
|
78
|
+
from .models import Evidence
|
|
79
|
+
from .tools import (
|
|
80
|
+
MAX_EVIDENCE_SUMMARY_CHARS,
|
|
81
|
+
MetricSample,
|
|
82
|
+
MetricSeries,
|
|
83
|
+
QueryMetricsRequest,
|
|
84
|
+
QueryMetricsResult,
|
|
85
|
+
metric_name_from_query,
|
|
86
|
+
redact_secrets,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
log = get_logger(__name__)
|
|
90
|
+
|
|
91
|
+
MAX_PODS = 50
|
|
92
|
+
MAX_AZURE_ROWS = 50
|
|
93
|
+
MAX_KQL_LENGTH = 1000
|
|
94
|
+
MAX_NAMESPACE_LENGTH = 63
|
|
95
|
+
|
|
96
|
+
_NAMESPACE_RE = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$")
|
|
97
|
+
# Azure management commands change a workspace; a read adapter
|
|
98
|
+
# must never send one. They all start with a dot.
|
|
99
|
+
_MANAGEMENT_RE = re.compile(r"(?i)(^|[;\s])\.(drop|delete|set|clear|ingest|create|alter)\b")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _bound(text: str, limit: int) -> str:
|
|
103
|
+
if len(text) <= limit:
|
|
104
|
+
return text
|
|
105
|
+
return text[:limit] + "…[truncated]"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
# Kubernetes: list pods (read-only)
|
|
110
|
+
# ---------------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class PodStatus(BaseModel):
|
|
114
|
+
"""One pod, as a person scanning ``kubectl get pods`` sees it."""
|
|
115
|
+
|
|
116
|
+
name: str
|
|
117
|
+
namespace: str
|
|
118
|
+
phase: str # Running, Pending, Failed, Succeeded, Unknown, …
|
|
119
|
+
ready_containers: int = Field(default=0, ge=0)
|
|
120
|
+
total_containers: int = Field(default=0, ge=0)
|
|
121
|
+
restarts: int = Field(default=0, ge=0)
|
|
122
|
+
service: str = "" # the app label, if the pod carries one
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def parse_kubernetes_pods(payload: dict[str, object]) -> list[PodStatus]:
|
|
126
|
+
"""Turn a Kubernetes ``PodList`` reply into pod records.
|
|
127
|
+
|
|
128
|
+
Pure function, tested against a recorded reply shape. A reply
|
|
129
|
+
that is not a list of items gives back no pods. Container
|
|
130
|
+
readiness and restarts come from ``status.containerStatuses``,
|
|
131
|
+
exactly where the API puts them; a pod with no container
|
|
132
|
+
statuses yet (still being created) reads as 0 of however many
|
|
133
|
+
containers its spec asks for.
|
|
134
|
+
"""
|
|
135
|
+
items = payload.get("items")
|
|
136
|
+
if not isinstance(items, list):
|
|
137
|
+
return []
|
|
138
|
+
pods: list[PodStatus] = []
|
|
139
|
+
for item in items:
|
|
140
|
+
if not isinstance(item, dict):
|
|
141
|
+
continue
|
|
142
|
+
metadata = item.get("metadata")
|
|
143
|
+
if not isinstance(metadata, dict):
|
|
144
|
+
continue
|
|
145
|
+
labels = metadata.get("labels")
|
|
146
|
+
service = ""
|
|
147
|
+
if isinstance(labels, dict):
|
|
148
|
+
service = str(
|
|
149
|
+
labels.get("app") or labels.get("app.kubernetes.io/name") or ""
|
|
150
|
+
)
|
|
151
|
+
spec = item.get("spec")
|
|
152
|
+
total = 0
|
|
153
|
+
if isinstance(spec, dict) and isinstance(spec.get("containers"), list):
|
|
154
|
+
total = len(spec["containers"])
|
|
155
|
+
status = item.get("status")
|
|
156
|
+
phase = "Unknown"
|
|
157
|
+
ready = 0
|
|
158
|
+
restarts = 0
|
|
159
|
+
if isinstance(status, dict):
|
|
160
|
+
phase = str(status.get("phase", "Unknown"))
|
|
161
|
+
container_statuses = status.get("containerStatuses")
|
|
162
|
+
if isinstance(container_statuses, list):
|
|
163
|
+
total = total or len(container_statuses)
|
|
164
|
+
for entry in container_statuses:
|
|
165
|
+
if not isinstance(entry, dict):
|
|
166
|
+
continue
|
|
167
|
+
if entry.get("ready") is True:
|
|
168
|
+
ready += 1
|
|
169
|
+
try:
|
|
170
|
+
restarts += int(entry.get("restartCount", 0))
|
|
171
|
+
except (TypeError, ValueError):
|
|
172
|
+
continue
|
|
173
|
+
pods.append(
|
|
174
|
+
PodStatus(
|
|
175
|
+
name=str(metadata.get("name", "unknown")),
|
|
176
|
+
namespace=str(metadata.get("namespace", "default")),
|
|
177
|
+
phase=phase,
|
|
178
|
+
ready_containers=ready,
|
|
179
|
+
total_containers=total,
|
|
180
|
+
restarts=restarts,
|
|
181
|
+
service=service,
|
|
182
|
+
)
|
|
183
|
+
)
|
|
184
|
+
return pods
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class ListPodsRequest(BaseModel):
|
|
188
|
+
"""A pod listing. Checked before it is allowed to run."""
|
|
189
|
+
|
|
190
|
+
namespace: str
|
|
191
|
+
service: str | None = None
|
|
192
|
+
limit: int = Field(default=20, ge=1, le=MAX_PODS)
|
|
193
|
+
|
|
194
|
+
@field_validator("namespace")
|
|
195
|
+
@classmethod
|
|
196
|
+
def _namespace_must_be_a_kubernetes_name(cls, v: str) -> str:
|
|
197
|
+
name = v.strip()
|
|
198
|
+
if not name or len(name) > MAX_NAMESPACE_LENGTH or not _NAMESPACE_RE.match(name):
|
|
199
|
+
raise ValueError(f"not a valid Kubernetes namespace name: {v!r}")
|
|
200
|
+
return name
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class KubernetesPodsResult(BaseModel):
|
|
204
|
+
namespace: str
|
|
205
|
+
service: str | None = None
|
|
206
|
+
pods: list[PodStatus] = Field(default_factory=list)
|
|
207
|
+
total_found: int = 0
|
|
208
|
+
truncated: bool = False
|
|
209
|
+
|
|
210
|
+
def to_evidence(self) -> Evidence:
|
|
211
|
+
"""Turn this listing into an evidence record for an incident."""
|
|
212
|
+
query = f"kubernetes pods: namespace={self.namespace}"
|
|
213
|
+
if self.service:
|
|
214
|
+
query += f" app={self.service}"
|
|
215
|
+
if not self.pods:
|
|
216
|
+
summary = f"No pods found in namespace {self.namespace}"
|
|
217
|
+
if self.service:
|
|
218
|
+
summary += f" for app {self.service}"
|
|
219
|
+
else:
|
|
220
|
+
parts = [
|
|
221
|
+
f"{p.name} {p.phase} {p.ready_containers}/{p.total_containers} ready, "
|
|
222
|
+
f"{p.restarts} restart(s)"
|
|
223
|
+
for p in self.pods[:3]
|
|
224
|
+
]
|
|
225
|
+
summary = "; ".join(parts)
|
|
226
|
+
if self.total_found > len(self.pods):
|
|
227
|
+
summary += f"; +{self.total_found - len(self.pods)} more pod(s)"
|
|
228
|
+
if self.truncated:
|
|
229
|
+
summary += " [truncated]"
|
|
230
|
+
return Evidence(
|
|
231
|
+
kind="kubernetes",
|
|
232
|
+
query=query,
|
|
233
|
+
summary=_bound(redact_secrets(summary), MAX_EVIDENCE_SUMMARY_CHARS),
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class InMemoryKubernetesBackend:
|
|
238
|
+
"""Demo Kubernetes backend: pods held in memory.
|
|
239
|
+
|
|
240
|
+
``add_pod`` exists only so a demo or a test can seed data.
|
|
241
|
+
Listing is read-only.
|
|
242
|
+
"""
|
|
243
|
+
|
|
244
|
+
def __init__(self, pods: list[PodStatus] | None = None) -> None:
|
|
245
|
+
self._pods: list[PodStatus] = list(pods or [])
|
|
246
|
+
|
|
247
|
+
def add_pod(self, pod: PodStatus) -> None:
|
|
248
|
+
self._pods.append(pod)
|
|
249
|
+
|
|
250
|
+
def fetch(self, namespace: str, service: str | None) -> list[PodStatus]:
|
|
251
|
+
return [
|
|
252
|
+
p
|
|
253
|
+
for p in self._pods
|
|
254
|
+
if p.namespace == namespace and (service is None or p.service == service)
|
|
255
|
+
]
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class KubernetesApiBackend:
|
|
259
|
+
"""Kubernetes backend that asks a real cluster's API server.
|
|
260
|
+
|
|
261
|
+
Uses only the standard library. The bearer token, when the
|
|
262
|
+
cluster needs one, is passed in by the caller and is only
|
|
263
|
+
ever put in the request header — never logged, never stored
|
|
264
|
+
on a result. Listing pods is a read; this backend builds no
|
|
265
|
+
other kind of request.
|
|
266
|
+
"""
|
|
267
|
+
|
|
268
|
+
def __init__(
|
|
269
|
+
self,
|
|
270
|
+
base_url: str,
|
|
271
|
+
token: str | None = None,
|
|
272
|
+
timeout_seconds: float = 5.0,
|
|
273
|
+
allowed_hosts: list[str] | None = None,
|
|
274
|
+
) -> None:
|
|
275
|
+
self._base_url = validate_backend_url(
|
|
276
|
+
base_url, allowed_hosts=allowed_hosts, what="Kubernetes API address"
|
|
277
|
+
)
|
|
278
|
+
self._token = token
|
|
279
|
+
self._timeout = timeout_seconds
|
|
280
|
+
|
|
281
|
+
def fetch(self, namespace: str, service: str | None) -> list[PodStatus]:
|
|
282
|
+
path = f"/api/v1/namespaces/{urllib.parse.quote(namespace)}/pods"
|
|
283
|
+
if service:
|
|
284
|
+
selector = urllib.parse.urlencode({"labelSelector": f"app={service}"})
|
|
285
|
+
path += f"?{selector}"
|
|
286
|
+
request = urllib.request.Request(
|
|
287
|
+
f"{self._base_url}{path}", headers={"Accept": "application/json"}
|
|
288
|
+
)
|
|
289
|
+
if self._token:
|
|
290
|
+
request.add_header("Authorization", f"Bearer {self._token}")
|
|
291
|
+
with urllib.request.urlopen(request, timeout=self._timeout) as resp:
|
|
292
|
+
payload = json.loads(resp.read().decode("utf-8"))
|
|
293
|
+
if not isinstance(payload, dict):
|
|
294
|
+
return []
|
|
295
|
+
return parse_kubernetes_pods(payload)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
class KubernetesReader:
|
|
299
|
+
"""The Kubernetes read tool, with bounds applied in one place."""
|
|
300
|
+
|
|
301
|
+
def __init__(
|
|
302
|
+
self, backend: InMemoryKubernetesBackend | KubernetesApiBackend | None = None
|
|
303
|
+
) -> None:
|
|
304
|
+
self._backend = backend if backend is not None else InMemoryKubernetesBackend()
|
|
305
|
+
|
|
306
|
+
def list_pods(self, request: ListPodsRequest) -> KubernetesPodsResult:
|
|
307
|
+
found = self._backend.fetch(request.namespace, request.service)
|
|
308
|
+
truncated = len(found) > request.limit
|
|
309
|
+
kept = found[: request.limit]
|
|
310
|
+
log.info(
|
|
311
|
+
"list_pods namespace=%s service=%s -> %d pod(s)",
|
|
312
|
+
request.namespace,
|
|
313
|
+
request.service,
|
|
314
|
+
len(kept),
|
|
315
|
+
)
|
|
316
|
+
return KubernetesPodsResult(
|
|
317
|
+
namespace=request.namespace,
|
|
318
|
+
service=request.service,
|
|
319
|
+
pods=kept,
|
|
320
|
+
total_found=len(found),
|
|
321
|
+
truncated=truncated,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
# ---------------------------------------------------------------------------
|
|
326
|
+
# Datadog: metrics query (read-only)
|
|
327
|
+
# ---------------------------------------------------------------------------
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def parse_datadog_response(payload: dict[str, object]) -> list[MetricSeries]:
|
|
331
|
+
"""Turn a Datadog metrics query reply into series.
|
|
332
|
+
|
|
333
|
+
Pure function, tested against a recorded reply shape. Datadog
|
|
334
|
+
answers with a ``series`` list; each entry names its metric,
|
|
335
|
+
carries its tags as ``"key:value"`` strings, and holds its
|
|
336
|
+
points as ``[timestamp-milliseconds, value]`` pairs. A reply
|
|
337
|
+
whose status is not ``ok`` gives back no series.
|
|
338
|
+
"""
|
|
339
|
+
if payload.get("status") != "ok":
|
|
340
|
+
return []
|
|
341
|
+
raw_series = payload.get("series")
|
|
342
|
+
if not isinstance(raw_series, list):
|
|
343
|
+
return []
|
|
344
|
+
series: list[MetricSeries] = []
|
|
345
|
+
for item in raw_series:
|
|
346
|
+
if not isinstance(item, dict):
|
|
347
|
+
continue
|
|
348
|
+
labels: dict[str, str] = {}
|
|
349
|
+
tags = item.get("tags")
|
|
350
|
+
if isinstance(tags, list):
|
|
351
|
+
for tag in tags:
|
|
352
|
+
if isinstance(tag, str) and ":" in tag:
|
|
353
|
+
key, value = tag.split(":", 1)
|
|
354
|
+
labels[key] = value
|
|
355
|
+
scope = item.get("scope")
|
|
356
|
+
if isinstance(scope, str) and ":" in scope:
|
|
357
|
+
key, value = scope.split(":", 1)
|
|
358
|
+
labels.setdefault(key, value)
|
|
359
|
+
samples: list[MetricSample] = []
|
|
360
|
+
pointlist = item.get("pointlist")
|
|
361
|
+
if isinstance(pointlist, list):
|
|
362
|
+
for point in pointlist:
|
|
363
|
+
if not (isinstance(point, list) and len(point) == 2):
|
|
364
|
+
continue
|
|
365
|
+
try:
|
|
366
|
+
ts = datetime.fromtimestamp(float(point[0]) / 1000.0, tz=UTC)
|
|
367
|
+
samples.append(MetricSample(timestamp=ts, value=float(point[1])))
|
|
368
|
+
except (TypeError, ValueError):
|
|
369
|
+
continue
|
|
370
|
+
series.append(
|
|
371
|
+
MetricSeries(
|
|
372
|
+
metric=str(item.get("metric", "unknown")), labels=labels, samples=samples
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
return series
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
class InMemoryDatadogBackend:
|
|
379
|
+
"""Demo Datadog backend: series held in memory, seeded by tests."""
|
|
380
|
+
|
|
381
|
+
def __init__(self) -> None:
|
|
382
|
+
self._series: list[MetricSeries] = []
|
|
383
|
+
|
|
384
|
+
def add_series(
|
|
385
|
+
self, metric: str, labels: dict[str, str], samples: list[MetricSample]
|
|
386
|
+
) -> None:
|
|
387
|
+
self._series.append(
|
|
388
|
+
MetricSeries(metric=metric, labels=dict(labels), samples=list(samples))
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
def fetch(self, query: str) -> list[MetricSeries]:
|
|
392
|
+
# Datadog queries open with an aggregator — "sum:metric{...}".
|
|
393
|
+
# The metric name follows it; without this step the whole
|
|
394
|
+
# "sum:metric" string would be mistaken for the name.
|
|
395
|
+
head = query.split("{", 1)[0]
|
|
396
|
+
prefix, _, rest = head.partition(":")
|
|
397
|
+
candidate = rest if rest and prefix in {"sum", "avg", "max", "min", "count"} else query
|
|
398
|
+
name = metric_name_from_query(candidate)
|
|
399
|
+
if name is None:
|
|
400
|
+
return []
|
|
401
|
+
return [s for s in self._series if s.metric == name]
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class DatadogMetricsBackend:
|
|
405
|
+
"""Datadog backend that asks the real Datadog API.
|
|
406
|
+
|
|
407
|
+
The API key and application key are passed in by the caller
|
|
408
|
+
and sent only as request headers — never logged, never put
|
|
409
|
+
on a result. The query window is the last hour by default.
|
|
410
|
+
"""
|
|
411
|
+
|
|
412
|
+
def __init__(
|
|
413
|
+
self,
|
|
414
|
+
api_key: str,
|
|
415
|
+
app_key: str,
|
|
416
|
+
base_url: str = "https://api.datadoghq.com",
|
|
417
|
+
window_seconds: int = 3600,
|
|
418
|
+
timeout_seconds: float = 5.0,
|
|
419
|
+
allowed_hosts: list[str] | None = None,
|
|
420
|
+
) -> None:
|
|
421
|
+
self._api_key = api_key
|
|
422
|
+
self._app_key = app_key
|
|
423
|
+
self._base_url = validate_backend_url(
|
|
424
|
+
base_url, allowed_hosts=allowed_hosts, what="Datadog address"
|
|
425
|
+
)
|
|
426
|
+
self._window_seconds = window_seconds
|
|
427
|
+
self._timeout = timeout_seconds
|
|
428
|
+
|
|
429
|
+
def fetch(self, query: str) -> list[MetricSeries]:
|
|
430
|
+
now = int(datetime.now(UTC).timestamp())
|
|
431
|
+
qs = urllib.parse.urlencode(
|
|
432
|
+
{"query": query, "from": now - self._window_seconds, "to": now}
|
|
433
|
+
)
|
|
434
|
+
request = urllib.request.Request(
|
|
435
|
+
f"{self._base_url}/api/v1/query?{qs}",
|
|
436
|
+
headers={
|
|
437
|
+
"Accept": "application/json",
|
|
438
|
+
"DD-API-KEY": self._api_key,
|
|
439
|
+
"DD-APPLICATION-KEY": self._app_key,
|
|
440
|
+
},
|
|
441
|
+
)
|
|
442
|
+
with urllib.request.urlopen(request, timeout=self._timeout) as resp:
|
|
443
|
+
payload = json.loads(resp.read().decode("utf-8"))
|
|
444
|
+
if not isinstance(payload, dict):
|
|
445
|
+
return []
|
|
446
|
+
return parse_datadog_response(payload)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
class DatadogAdapter:
|
|
450
|
+
"""The Datadog metrics read, returning the tool layer's shape.
|
|
451
|
+
|
|
452
|
+
The result is a plain ``QueryMetricsResult``, so a Datadog
|
|
453
|
+
series becomes evidence through the same ``to_evidence`` the
|
|
454
|
+
Prometheus tool uses — the query on the record says it came
|
|
455
|
+
from Datadog, so the source is never blurred.
|
|
456
|
+
"""
|
|
457
|
+
|
|
458
|
+
def __init__(
|
|
459
|
+
self, backend: InMemoryDatadogBackend | DatadogMetricsBackend | None = None
|
|
460
|
+
) -> None:
|
|
461
|
+
self._backend = backend if backend is not None else InMemoryDatadogBackend()
|
|
462
|
+
|
|
463
|
+
def query_metrics(self, request: QueryMetricsRequest) -> QueryMetricsResult:
|
|
464
|
+
found = self._backend.fetch(request.query)
|
|
465
|
+
truncated = len(found) > request.limit_series
|
|
466
|
+
kept: list[MetricSeries] = []
|
|
467
|
+
for series in found[: request.limit_series]:
|
|
468
|
+
samples = series.samples
|
|
469
|
+
if len(samples) > request.limit_points_per_series:
|
|
470
|
+
truncated = True
|
|
471
|
+
samples = samples[-request.limit_points_per_series :]
|
|
472
|
+
kept.append(
|
|
473
|
+
MetricSeries(
|
|
474
|
+
metric=series.metric,
|
|
475
|
+
labels={k: redact_secrets(v) for k, v in series.labels.items()},
|
|
476
|
+
samples=samples,
|
|
477
|
+
)
|
|
478
|
+
)
|
|
479
|
+
log.info("datadog query_metrics %r -> %d series", request.query, len(kept))
|
|
480
|
+
return QueryMetricsResult(
|
|
481
|
+
query=f"datadog: {request.query}",
|
|
482
|
+
series=kept,
|
|
483
|
+
total_series_found=len(found),
|
|
484
|
+
truncated=truncated,
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
# ---------------------------------------------------------------------------
|
|
489
|
+
# Azure Monitor: KQL query against Log Analytics (read-only)
|
|
490
|
+
# ---------------------------------------------------------------------------
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
class KqlQueryRequest(BaseModel):
|
|
494
|
+
"""A KQL question. Checked before it is allowed to run.
|
|
495
|
+
|
|
496
|
+
KQL itself is a read language, but a Log Analytics workspace
|
|
497
|
+
also accepts management commands — those start with a dot and
|
|
498
|
+
change things, so any query carrying one is refused here.
|
|
499
|
+
"""
|
|
500
|
+
|
|
501
|
+
query: str
|
|
502
|
+
limit_rows: int = Field(default=20, ge=1, le=MAX_AZURE_ROWS)
|
|
503
|
+
|
|
504
|
+
@field_validator("query")
|
|
505
|
+
@classmethod
|
|
506
|
+
def _query_must_be_a_safe_read(cls, v: str) -> str:
|
|
507
|
+
q = v.strip()
|
|
508
|
+
if not q:
|
|
509
|
+
raise ValueError("KQL query must not be empty")
|
|
510
|
+
if len(q) > MAX_KQL_LENGTH:
|
|
511
|
+
raise ValueError(f"KQL query is too long (limit {MAX_KQL_LENGTH})")
|
|
512
|
+
if q.startswith(".") or _MANAGEMENT_RE.search(q):
|
|
513
|
+
raise ValueError("KQL management commands are not reads and are refused")
|
|
514
|
+
return q
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def parse_azure_response(
|
|
518
|
+
payload: dict[str, object],
|
|
519
|
+
) -> tuple[list[str], list[dict[str, str]]]:
|
|
520
|
+
"""Turn an Azure Log Analytics query reply into columns and rows.
|
|
521
|
+
|
|
522
|
+
Pure function, tested against a recorded reply shape. Azure
|
|
523
|
+
answers with ``tables``; the first table names its ``columns``
|
|
524
|
+
and holds each result as a ``rows`` list in the same order.
|
|
525
|
+
Values are returned as text — that is all an evidence summary
|
|
526
|
+
needs, and it avoids guessing at a type the reply's column
|
|
527
|
+
types only loosely pin down. A reply with no tables gives
|
|
528
|
+
back no columns and no rows.
|
|
529
|
+
"""
|
|
530
|
+
tables = payload.get("tables")
|
|
531
|
+
if not isinstance(tables, list) or not tables or not isinstance(tables[0], dict):
|
|
532
|
+
return [], []
|
|
533
|
+
table = tables[0]
|
|
534
|
+
columns: list[str] = []
|
|
535
|
+
raw_columns = table.get("columns")
|
|
536
|
+
if isinstance(raw_columns, list):
|
|
537
|
+
for column in raw_columns:
|
|
538
|
+
if isinstance(column, dict) and "name" in column:
|
|
539
|
+
columns.append(str(column["name"]))
|
|
540
|
+
rows: list[dict[str, str]] = []
|
|
541
|
+
raw_rows = table.get("rows")
|
|
542
|
+
if isinstance(raw_rows, list):
|
|
543
|
+
for raw_row in raw_rows:
|
|
544
|
+
if not isinstance(raw_row, list):
|
|
545
|
+
continue
|
|
546
|
+
row: dict[str, str] = {}
|
|
547
|
+
for idx, column in enumerate(columns):
|
|
548
|
+
value = raw_row[idx] if idx < len(raw_row) else ""
|
|
549
|
+
row[column] = "" if value is None else str(value)
|
|
550
|
+
rows.append(row)
|
|
551
|
+
return columns, rows
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
class AzureQueryResult(BaseModel):
|
|
555
|
+
query: str
|
|
556
|
+
columns: list[str] = Field(default_factory=list)
|
|
557
|
+
rows: list[dict[str, str]] = Field(default_factory=list)
|
|
558
|
+
total_rows_found: int = 0
|
|
559
|
+
truncated: bool = False
|
|
560
|
+
|
|
561
|
+
def to_evidence(self) -> Evidence:
|
|
562
|
+
if not self.rows:
|
|
563
|
+
summary = f"No rows matched KQL query: {self.query}"
|
|
564
|
+
else:
|
|
565
|
+
first = self.rows[0]
|
|
566
|
+
shown = ", ".join(f"{k}={v}" for k, v in list(first.items())[:4])
|
|
567
|
+
summary = f"{self.total_rows_found} row(s) matched; first: {shown}"
|
|
568
|
+
if self.truncated:
|
|
569
|
+
summary += " [truncated]"
|
|
570
|
+
return Evidence(
|
|
571
|
+
kind="azure-log",
|
|
572
|
+
query=f"kql: {self.query}",
|
|
573
|
+
summary=_bound(redact_secrets(summary), MAX_EVIDENCE_SUMMARY_CHARS),
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
class InMemoryAzureBackend:
|
|
578
|
+
"""Demo Azure backend: rows held in memory, seeded by tests."""
|
|
579
|
+
|
|
580
|
+
def __init__(self, rows: list[dict[str, str]] | None = None) -> None:
|
|
581
|
+
self._rows: list[dict[str, str]] = [dict(r) for r in rows or []]
|
|
582
|
+
|
|
583
|
+
def fetch(self, query: str) -> tuple[list[str], list[dict[str, str]]]:
|
|
584
|
+
columns = list(self._rows[0]) if self._rows else []
|
|
585
|
+
return columns, [dict(r) for r in self._rows]
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
class AzureLogAnalyticsBackend:
|
|
589
|
+
"""Azure backend that asks a real Log Analytics workspace.
|
|
590
|
+
|
|
591
|
+
The bearer token is passed in by the caller (in real use it
|
|
592
|
+
comes from the caller's own Azure sign-in) and is only ever
|
|
593
|
+
put in the request header. The query endpoint is a read.
|
|
594
|
+
"""
|
|
595
|
+
|
|
596
|
+
def __init__(
|
|
597
|
+
self,
|
|
598
|
+
workspace_id: str,
|
|
599
|
+
token: str,
|
|
600
|
+
base_url: str = "https://api.loganalytics.azure.com",
|
|
601
|
+
timespan: str = "PT1H",
|
|
602
|
+
timeout_seconds: float = 5.0,
|
|
603
|
+
allowed_hosts: list[str] | None = None,
|
|
604
|
+
) -> None:
|
|
605
|
+
self._workspace_id = workspace_id
|
|
606
|
+
self._token = token
|
|
607
|
+
self._base_url = validate_backend_url(
|
|
608
|
+
base_url, allowed_hosts=allowed_hosts, what="Azure Monitor address"
|
|
609
|
+
)
|
|
610
|
+
self._timespan = timespan
|
|
611
|
+
self._timeout = timeout_seconds
|
|
612
|
+
|
|
613
|
+
def fetch(self, query: str) -> tuple[list[str], list[dict[str, str]]]:
|
|
614
|
+
url = (
|
|
615
|
+
f"{self._base_url}/v1/workspaces/"
|
|
616
|
+
f"{urllib.parse.quote(self._workspace_id)}/query"
|
|
617
|
+
)
|
|
618
|
+
body = json.dumps({"query": query, "timespan": self._timespan}).encode("utf-8")
|
|
619
|
+
request = urllib.request.Request(
|
|
620
|
+
url,
|
|
621
|
+
data=body,
|
|
622
|
+
headers={
|
|
623
|
+
"Accept": "application/json",
|
|
624
|
+
"Content-Type": "application/json",
|
|
625
|
+
"Authorization": f"Bearer {self._token}",
|
|
626
|
+
},
|
|
627
|
+
method="POST",
|
|
628
|
+
)
|
|
629
|
+
with urllib.request.urlopen(request, timeout=self._timeout) as resp:
|
|
630
|
+
payload = json.loads(resp.read().decode("utf-8"))
|
|
631
|
+
if not isinstance(payload, dict):
|
|
632
|
+
return [], []
|
|
633
|
+
return parse_azure_response(payload)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
class AzureAdapter:
|
|
637
|
+
"""The Azure Monitor KQL read, with bounds and cleaning applied."""
|
|
638
|
+
|
|
639
|
+
def __init__(
|
|
640
|
+
self, backend: InMemoryAzureBackend | AzureLogAnalyticsBackend | None = None
|
|
641
|
+
) -> None:
|
|
642
|
+
self._backend = backend if backend is not None else InMemoryAzureBackend()
|
|
643
|
+
|
|
644
|
+
def query(self, request: KqlQueryRequest) -> AzureQueryResult:
|
|
645
|
+
columns, rows = self._backend.fetch(request.query)
|
|
646
|
+
truncated = len(rows) > request.limit_rows
|
|
647
|
+
kept = [
|
|
648
|
+
{k: redact_secrets(v) for k, v in row.items()}
|
|
649
|
+
for row in rows[: request.limit_rows]
|
|
650
|
+
]
|
|
651
|
+
log.info("azure kql query -> %d row(s)", len(kept))
|
|
652
|
+
return AzureQueryResult(
|
|
653
|
+
query=request.query,
|
|
654
|
+
columns=columns,
|
|
655
|
+
rows=kept,
|
|
656
|
+
total_rows_found=len(rows),
|
|
657
|
+
truncated=truncated,
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# ---------------------------------------------------------------------------
|
|
662
|
+
# Selecting adapters as evidence sources for a diagnosis
|
|
663
|
+
# ---------------------------------------------------------------------------
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@dataclass
|
|
667
|
+
class AdapterSources:
|
|
668
|
+
"""The read adapters selected to feed a diagnosis.
|
|
669
|
+
|
|
670
|
+
Every field is optional: a source left as ``None`` is never
|
|
671
|
+
called. This bundle is how an operator selects evidence
|
|
672
|
+
sources — for the reasoning loop directly, or for a server,
|
|
673
|
+
which passes the bundle to its loop. The live adapters inside
|
|
674
|
+
a bundle were built by the operator's own code, so their
|
|
675
|
+
addresses already passed the backend address rules
|
|
676
|
+
(``backend_url.py``) when they were constructed; nothing in
|
|
677
|
+
an alert, a log line or any other telemetry can add a source
|
|
678
|
+
or change an address.
|
|
679
|
+
"""
|
|
680
|
+
|
|
681
|
+
kubernetes: KubernetesReader | None = None
|
|
682
|
+
datadog: DatadogAdapter | None = None
|
|
683
|
+
azure: AzureAdapter | None = None
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _demo_kubernetes_reader() -> KubernetesReader:
|
|
687
|
+
return KubernetesReader(
|
|
688
|
+
InMemoryKubernetesBackend(
|
|
689
|
+
[
|
|
690
|
+
PodStatus(
|
|
691
|
+
name="checkout-7d9f-a", namespace="shop", phase="Running",
|
|
692
|
+
ready_containers=1, total_containers=1, restarts=2,
|
|
693
|
+
service="checkout",
|
|
694
|
+
),
|
|
695
|
+
PodStatus(
|
|
696
|
+
name="checkout-7d9f-b", namespace="shop", phase="Running",
|
|
697
|
+
ready_containers=1, total_containers=1, restarts=0,
|
|
698
|
+
service="checkout",
|
|
699
|
+
),
|
|
700
|
+
PodStatus(
|
|
701
|
+
name="checkout-7d9f-c", namespace="shop", phase="Pending",
|
|
702
|
+
ready_containers=0, total_containers=1, restarts=5,
|
|
703
|
+
service="checkout",
|
|
704
|
+
),
|
|
705
|
+
PodStatus(
|
|
706
|
+
name="search-5c1-a", namespace="shop", phase="Running",
|
|
707
|
+
ready_containers=1, total_containers=1, restarts=0,
|
|
708
|
+
service="search",
|
|
709
|
+
),
|
|
710
|
+
]
|
|
711
|
+
)
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def _demo_datadog_adapter() -> DatadogAdapter:
|
|
716
|
+
now = datetime.now(UTC)
|
|
717
|
+
backend = InMemoryDatadogBackend()
|
|
718
|
+
backend.add_series(
|
|
719
|
+
"http_request_errors_total",
|
|
720
|
+
{"service": "checkout", "namespace": "shop"},
|
|
721
|
+
[MetricSample(timestamp=now, value=162.0)],
|
|
722
|
+
)
|
|
723
|
+
return DatadogAdapter(backend)
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def _demo_azure_adapter() -> AzureAdapter:
|
|
727
|
+
now = datetime.now(UTC)
|
|
728
|
+
return AzureAdapter(
|
|
729
|
+
InMemoryAzureBackend(
|
|
730
|
+
[
|
|
731
|
+
{
|
|
732
|
+
"TimeGenerated": now.isoformat(),
|
|
733
|
+
"Service": "checkout",
|
|
734
|
+
"Level": "Error",
|
|
735
|
+
"Message": "payment gateway timeout after 2000ms",
|
|
736
|
+
},
|
|
737
|
+
{
|
|
738
|
+
"TimeGenerated": now.isoformat(),
|
|
739
|
+
"Service": "checkout",
|
|
740
|
+
"Level": "Error",
|
|
741
|
+
"Message": "checkout request failed with status 502",
|
|
742
|
+
},
|
|
743
|
+
]
|
|
744
|
+
)
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def demo_adapter_sources() -> AdapterSources:
|
|
749
|
+
"""The three demo adapters as one selectable bundle.
|
|
750
|
+
|
|
751
|
+
The seeded picture is the same one ``demo_adapters`` reports:
|
|
752
|
+
checkout in the shop namespace, degraded. It claims nothing
|
|
753
|
+
about any real system.
|
|
754
|
+
"""
|
|
755
|
+
return AdapterSources(
|
|
756
|
+
kubernetes=_demo_kubernetes_reader(),
|
|
757
|
+
datadog=_demo_datadog_adapter(),
|
|
758
|
+
azure=_demo_azure_adapter(),
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
# ---------------------------------------------------------------------------
|
|
763
|
+
# Seeded demo: all three adapters, no live system needed
|
|
764
|
+
# ---------------------------------------------------------------------------
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
class AdaptersDemoReport(BaseModel):
|
|
768
|
+
"""What the three read adapters say about the demo, in one report."""
|
|
769
|
+
|
|
770
|
+
pods: KubernetesPodsResult
|
|
771
|
+
datadog: QueryMetricsResult
|
|
772
|
+
azure: AzureQueryResult
|
|
773
|
+
|
|
774
|
+
def render_text(self) -> str:
|
|
775
|
+
lines = [
|
|
776
|
+
(
|
|
777
|
+
"Read adapters demo (seeded demo data, not a real system; "
|
|
778
|
+
"every adapter here only looks, nothing is changed)"
|
|
779
|
+
),
|
|
780
|
+
"Kubernetes — pods for checkout in namespace shop:",
|
|
781
|
+
]
|
|
782
|
+
if not self.pods.pods:
|
|
783
|
+
lines.append("- None found.")
|
|
784
|
+
for pod in self.pods.pods:
|
|
785
|
+
lines.append(
|
|
786
|
+
f"- {pod.name}: {pod.phase}, {pod.ready_containers}/"
|
|
787
|
+
f"{pod.total_containers} containers ready, {pod.restarts} restart(s)"
|
|
788
|
+
)
|
|
789
|
+
lines.append("Datadog — metrics query result:")
|
|
790
|
+
if not self.datadog.series:
|
|
791
|
+
lines.append("- No series matched.")
|
|
792
|
+
for series in self.datadog.series:
|
|
793
|
+
last = series.samples[-1].value if series.samples else 0.0
|
|
794
|
+
lines.append(f"- {series.metric}{series.labels} last value {last:g}")
|
|
795
|
+
lines.append("Azure Monitor — KQL query result:")
|
|
796
|
+
if not self.azure.rows:
|
|
797
|
+
lines.append("- No rows matched.")
|
|
798
|
+
for row in self.azure.rows:
|
|
799
|
+
shown = ", ".join(f"{k}={v}" for k, v in list(row.items())[:4])
|
|
800
|
+
lines.append(f"- {shown}")
|
|
801
|
+
lines.append(
|
|
802
|
+
"Note: these are the demo backends and the reply parsers, "
|
|
803
|
+
"tested with recorded reply shapes. No live cluster, Datadog "
|
|
804
|
+
"account or Azure workspace was called."
|
|
805
|
+
)
|
|
806
|
+
return "\n".join(lines)
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def demo_adapters() -> AdaptersDemoReport:
|
|
810
|
+
"""A seeded report from all three adapters, with no setup.
|
|
811
|
+
|
|
812
|
+
The demo picture agrees with the rest of the demo on purpose:
|
|
813
|
+
checkout in the shop namespace is degraded — two pods ready,
|
|
814
|
+
one still pending after several restarts — its Datadog error
|
|
815
|
+
series is high, and its Azure log rows say why.
|
|
816
|
+
"""
|
|
817
|
+
sources = demo_adapter_sources()
|
|
818
|
+
assert sources.kubernetes is not None
|
|
819
|
+
assert sources.datadog is not None
|
|
820
|
+
assert sources.azure is not None
|
|
821
|
+
return AdaptersDemoReport(
|
|
822
|
+
pods=sources.kubernetes.list_pods(
|
|
823
|
+
ListPodsRequest(namespace="shop", service="checkout")
|
|
824
|
+
),
|
|
825
|
+
datadog=sources.datadog.query_metrics(
|
|
826
|
+
QueryMetricsRequest(query="sum:http_request_errors_total{service:checkout}")
|
|
827
|
+
),
|
|
828
|
+
azure=sources.azure.query(
|
|
829
|
+
KqlQueryRequest(
|
|
830
|
+
query="AppTraces | where Service == 'checkout' and Level == 'Error'"
|
|
831
|
+
)
|
|
832
|
+
),
|
|
833
|
+
)
|