kubesift 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift/__init__.py +3 -0
- kubesift/analysis/__init__.py +3 -0
- kubesift/analysis/changes.py +165 -0
- kubesift/analysis/correlate.py +160 -0
- kubesift/analysis/describe.py +83 -0
- kubesift/analysis/engine.py +362 -0
- kubesift/analysis/llm.py +152 -0
- kubesift/checks/__init__.py +17 -0
- kubesift/checks/base.py +150 -0
- kubesift/checks/config_refs.py +84 -0
- kubesift/checks/events.py +77 -0
- kubesift/checks/hygiene.py +264 -0
- kubesift/checks/lifecycle.py +202 -0
- kubesift/checks/network.py +325 -0
- kubesift/checks/nodes.py +183 -0
- kubesift/checks/operations.py +319 -0
- kubesift/checks/pods.py +485 -0
- kubesift/checks/storage.py +72 -0
- kubesift/checks/workloads.py +201 -0
- kubesift/cli.py +874 -0
- kubesift/cluster/__init__.py +4 -0
- kubesift/cluster/client.py +141 -0
- kubesift/cluster/snapshot.py +513 -0
- kubesift/config.py +134 -0
- kubesift/cost.py +150 -0
- kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
- kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
- kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
- kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
- kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
- kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
- kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
- kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
- kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
- kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
- kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
- kubesift/drill_scenarios/__init__.py +1 -0
- kubesift/drills.py +122 -0
- kubesift/knowledge.py +51 -0
- kubesift/leader.py +145 -0
- kubesift/lint.py +89 -0
- kubesift/mcp_server.py +193 -0
- kubesift/models.py +144 -0
- kubesift/notify.py +71 -0
- kubesift/postmortem.py +152 -0
- kubesift/preflight.py +435 -0
- kubesift/redact.py +38 -0
- kubesift/remediate.py +99 -0
- kubesift/report.py +193 -0
- kubesift/runbooks.py +58 -0
- kubesift/server/__init__.py +0 -0
- kubesift/server/app.py +480 -0
- kubesift/server/static/index.html +152 -0
- kubesift/slack.py +106 -0
- kubesift/store.py +219 -0
- kubesift/timetravel.py +156 -0
- kubesift/upgrade.py +51 -0
- kubesift/util.py +174 -0
- kubesift-2.0.0.dist-info/METADATA +192 -0
- kubesift-2.0.0.dist-info/RECORD +63 -0
- kubesift-2.0.0.dist-info/WHEEL +4 -0
- kubesift-2.0.0.dist-info/entry_points.txt +3 -0
- kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""The investigation pipeline: collect -> check -> correlate -> explain."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import re
|
|
8
|
+
from datetime import timedelta
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from kubesift import runbooks
|
|
12
|
+
from kubesift.analysis.changes import detect_changes
|
|
13
|
+
from kubesift.analysis.correlate import build_incidents, link_changes, pod_counts
|
|
14
|
+
from kubesift.analysis.describe import ALIASES, summarize
|
|
15
|
+
from kubesift.analysis.llm import LLMError, get_provider
|
|
16
|
+
from kubesift.checks.base import run_checks
|
|
17
|
+
from kubesift.cluster.client import KubeClient
|
|
18
|
+
from kubesift.cluster.snapshot import KIND_TO_KEY, Snapshot, collect
|
|
19
|
+
from kubesift.config import Settings
|
|
20
|
+
from kubesift.models import Category, Diagnosis, Finding, Report, RootCause
|
|
21
|
+
from kubesift.redact import redact
|
|
22
|
+
from kubesift.store import Store
|
|
23
|
+
from kubesift.util import meta
|
|
24
|
+
|
|
25
|
+
log = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
SYSTEM_PROMPT = """You are KubeSift, a senior Kubernetes SRE helping an on-call engineer.
|
|
28
|
+
You receive findings that deterministic checks proved from live cluster data, recent changes, and resource summaries.
|
|
29
|
+
|
|
30
|
+
Rules:
|
|
31
|
+
- Base every conclusion on the provided evidence. Cite the fingerprints of the findings you rely on in `evidence`.
|
|
32
|
+
- Never invent resource names, versions, or log lines. If the data does not show the cause, say what is missing and how to get it.
|
|
33
|
+
- Separate the root cause from its symptoms: a bad rollout, a config edit or a broken node often explains several findings.
|
|
34
|
+
- When a recent change lines up with the start of a problem, say so and give the rollback command if one is provided.
|
|
35
|
+
- Prefer read-only kubectl commands. Mark anything that changes the cluster with "(changes the cluster)".
|
|
36
|
+
- Confidence: 80-100 only when evidence directly proves the cause; 50-79 when it is the most likely explanation; below 50 when guessing.
|
|
37
|
+
- Be concise and concrete. Write for someone who is tired and needs the next step."""
|
|
38
|
+
|
|
39
|
+
DIAGNOSIS_SCHEMA: dict[str, Any] = {
|
|
40
|
+
"type": "object",
|
|
41
|
+
"additionalProperties": False,
|
|
42
|
+
"required": ["summary", "root_causes", "next_checks"],
|
|
43
|
+
"properties": {
|
|
44
|
+
"summary": {"type": "string", "description": "Two or three sentences: what is broken and why."},
|
|
45
|
+
"root_causes": {
|
|
46
|
+
"type": "array",
|
|
47
|
+
"items": {
|
|
48
|
+
"type": "object",
|
|
49
|
+
"additionalProperties": False,
|
|
50
|
+
"required": ["title", "explanation", "confidence", "evidence", "fix_steps", "commands"],
|
|
51
|
+
"properties": {
|
|
52
|
+
"title": {"type": "string"},
|
|
53
|
+
"explanation": {"type": "string"},
|
|
54
|
+
"confidence": {"type": "integer"},
|
|
55
|
+
"evidence": {"type": "array", "items": {"type": "string"}},
|
|
56
|
+
"fix_steps": {"type": "array", "items": {"type": "string"}},
|
|
57
|
+
"commands": {"type": "array", "items": {"type": "string"}},
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
"next_checks": {"type": "array", "items": {"type": "string"}},
|
|
62
|
+
},
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
ANSWER_SCHEMA: dict[str, Any] = {
|
|
66
|
+
"type": "object",
|
|
67
|
+
"additionalProperties": False,
|
|
68
|
+
"required": ["answer", "confidence", "evidence", "commands", "follow_up"],
|
|
69
|
+
"properties": {
|
|
70
|
+
"answer": {"type": "string"},
|
|
71
|
+
"confidence": {"type": "integer"},
|
|
72
|
+
"evidence": {"type": "array", "items": {"type": "string"}},
|
|
73
|
+
"commands": {"type": "array", "items": {"type": "string"}},
|
|
74
|
+
"follow_up": {"type": "array", "items": {"type": "string"}},
|
|
75
|
+
},
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
_MAX_CONTEXT = 60_000
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def snapshot_for(settings: Settings, namespace: str | None, context: str | None = None,
|
|
82
|
+
with_logs: bool = True, kinds: set[str] | None = None) -> Snapshot:
|
|
83
|
+
client = KubeClient(kubeconfig=settings.kubeconfig, context=context or settings.context,
|
|
84
|
+
timeout=settings.request_timeout)
|
|
85
|
+
return collect(client, namespace=namespace, max_items=settings.max_items_per_kind,
|
|
86
|
+
with_logs=with_logs, kinds=kinds)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def investigate(
|
|
90
|
+
settings: Settings,
|
|
91
|
+
snap: Snapshot,
|
|
92
|
+
*,
|
|
93
|
+
use_llm: bool = True,
|
|
94
|
+
window: timedelta = timedelta(hours=6),
|
|
95
|
+
categories: list[Category] | None = None,
|
|
96
|
+
only: list[str] | None = None,
|
|
97
|
+
include_risks: bool = True,
|
|
98
|
+
store: Store | None = None,
|
|
99
|
+
) -> Report:
|
|
100
|
+
findings = run_checks(snap, settings, only=only, categories=categories)
|
|
101
|
+
if not include_risks:
|
|
102
|
+
findings = [f for f in findings if f.kind == "problem"]
|
|
103
|
+
changes = detect_changes(snap, window)
|
|
104
|
+
link_changes(snap, findings, changes)
|
|
105
|
+
if store:
|
|
106
|
+
store.attach_notes(findings)
|
|
107
|
+
runbooks.attach(runbooks.load(settings.runbooks_dir), findings)
|
|
108
|
+
report = Report(
|
|
109
|
+
cluster=snap.context, namespace=snap.namespace, collected_at=snap.collected_at,
|
|
110
|
+
server_version=snap.server_version, findings=findings, incidents=build_incidents(snap, findings),
|
|
111
|
+
changes=changes, warnings=list(snap.warnings),
|
|
112
|
+
)
|
|
113
|
+
report.diagnosis = explain(settings, report, use_llm=use_llm)
|
|
114
|
+
return report
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ----------------------------------------------------------------- diagnosis
|
|
118
|
+
def rules_diagnosis(report: Report) -> Diagnosis:
|
|
119
|
+
problems = [i for i in report.incidents]
|
|
120
|
+
if not problems:
|
|
121
|
+
risks = [f for f in report.findings if f.kind == "risk"]
|
|
122
|
+
msg = "No active problems found."
|
|
123
|
+
if risks:
|
|
124
|
+
msg += f" {len(risks)} configuration risk(s) worth fixing before they cause an incident."
|
|
125
|
+
return Diagnosis(summary=msg, generated_by="rules")
|
|
126
|
+
causes = []
|
|
127
|
+
for inc in problems[:5]:
|
|
128
|
+
top = inc.findings[0]
|
|
129
|
+
steps = [top.remediation] if top.remediation else []
|
|
130
|
+
cmds = list(top.commands)
|
|
131
|
+
conf = 85 if top.severity.rank >= 3 else 70
|
|
132
|
+
explanation = top.message
|
|
133
|
+
if inc.likely_trigger:
|
|
134
|
+
t = inc.likely_trigger
|
|
135
|
+
extra = [d for d in t.details[1:4]]
|
|
136
|
+
explanation += f" This started after: {t.summary}" + (f" ({'; '.join(extra)})." if extra else ".")
|
|
137
|
+
if inc.likely_trigger.rollback_command:
|
|
138
|
+
steps.insert(0, "If the change caused it, roll back first, then investigate.")
|
|
139
|
+
cmds.insert(0, inc.likely_trigger.rollback_command + " # (changes the cluster)")
|
|
140
|
+
causes.append(RootCause(title=inc.title, explanation=explanation, confidence=conf,
|
|
141
|
+
evidence=[f.fingerprint for f in inc.findings], fix_steps=steps, commands=cmds[:4]))
|
|
142
|
+
worst = problems[0]
|
|
143
|
+
summary = f"{len(problems)} incident(s). Most severe: {worst.title}."
|
|
144
|
+
if worst.likely_trigger:
|
|
145
|
+
summary += f" It lines up with a recent change: {worst.likely_trigger.summary}."
|
|
146
|
+
return Diagnosis(summary=summary, root_causes=causes, generated_by="rules")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _finding_payload(f: Finding, logs: bool = True) -> dict[str, Any]:
|
|
150
|
+
evidence = [e for e in f.evidence if logs or not e.startswith("log ")]
|
|
151
|
+
return {
|
|
152
|
+
"fingerprint": f.fingerprint, "check": f.check, "severity": f.severity.value, "resource": str(f.resource),
|
|
153
|
+
"owner": str(f.owner) if f.owner else None, "node": f.node, "message": f.message,
|
|
154
|
+
"evidence": evidence[:8], "suggested_fix": f.remediation, "commands": f.commands[:3],
|
|
155
|
+
"started": f.since.isoformat() if f.since else None,
|
|
156
|
+
"related_changes": [c.summary + (f" [rollback: {c.rollback_command}]" if c.rollback_command else "")
|
|
157
|
+
for c in f.related_changes],
|
|
158
|
+
"team_notes": f.notes[:3],
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _context_json(report: Report, extra: dict[str, Any] | None = None, logs: bool = True) -> str:
|
|
163
|
+
payload: dict[str, Any] = {
|
|
164
|
+
"cluster": report.cluster, "namespace": report.namespace or "all", "kubernetes": report.server_version,
|
|
165
|
+
"incidents": [{"title": i.title, "layer": i.layer, "severity": i.severity.value,
|
|
166
|
+
"findings": [_finding_payload(f, logs) for f in i.findings[:6]]} for i in report.incidents[:12]],
|
|
167
|
+
"risks": [{"fingerprint": f.fingerprint, "check": f.check, "resource": str(f.resource), "message": f.message}
|
|
168
|
+
for f in report.findings if f.kind == "risk"][:25],
|
|
169
|
+
"recent_changes": [{"time": c.time.isoformat(), "summary": c.summary, "details": c.details[:5],
|
|
170
|
+
"rollback": c.rollback_command} for c in report.changes[:15]],
|
|
171
|
+
"data_gaps": report.warnings[:10],
|
|
172
|
+
}
|
|
173
|
+
if extra:
|
|
174
|
+
payload.update(extra)
|
|
175
|
+
text = redact(json.dumps(payload, default=str))
|
|
176
|
+
if len(text) > _MAX_CONTEXT:
|
|
177
|
+
payload["risks"] = payload["risks"][:5]
|
|
178
|
+
payload["incidents"] = payload["incidents"][:6]
|
|
179
|
+
text = redact(json.dumps(payload, default=str))[:_MAX_CONTEXT]
|
|
180
|
+
return text
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def explain(settings: Settings, report: Report, use_llm: bool = True) -> Diagnosis:
|
|
184
|
+
base = rules_diagnosis(report)
|
|
185
|
+
if not use_llm or not report.incidents:
|
|
186
|
+
return base
|
|
187
|
+
try:
|
|
188
|
+
provider = get_provider(settings)
|
|
189
|
+
except LLMError as exc:
|
|
190
|
+
report.warnings.append(f"LLM disabled: {exc}")
|
|
191
|
+
return base
|
|
192
|
+
if provider is None:
|
|
193
|
+
return base
|
|
194
|
+
try:
|
|
195
|
+
data = provider.complete_json(SYSTEM_PROMPT, "Diagnose this cluster state.\n\n"
|
|
196
|
+
+ _context_json(report, logs=settings.send_logs_to_llm),
|
|
197
|
+
DIAGNOSIS_SCHEMA)
|
|
198
|
+
return _validated(data, report, f"{provider.name}:{settings.resolved_model()}")
|
|
199
|
+
except (LLMError, KeyError, TypeError, ValueError) as exc:
|
|
200
|
+
report.warnings.append(f"LLM analysis failed, showing rule-based diagnosis: {exc}")
|
|
201
|
+
return base
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _validated(data: dict[str, Any], report: Report, by: str) -> Diagnosis:
|
|
205
|
+
known = {f.fingerprint for f in report.findings}
|
|
206
|
+
causes = []
|
|
207
|
+
for rc in data.get("root_causes") or []:
|
|
208
|
+
evidence = [e for e in rc.get("evidence") or [] if e in known]
|
|
209
|
+
conf = max(0, min(100, int(rc.get("confidence", 0))))
|
|
210
|
+
if not evidence:
|
|
211
|
+
conf = min(conf, 40) # unsupported claims cannot be confident
|
|
212
|
+
causes.append(RootCause(title=str(rc.get("title", ""))[:200], explanation=str(rc.get("explanation", "")),
|
|
213
|
+
confidence=conf, evidence=evidence,
|
|
214
|
+
fix_steps=[str(s) for s in rc.get("fix_steps") or []][:8],
|
|
215
|
+
commands=[str(c) for c in rc.get("commands") or []][:6]))
|
|
216
|
+
causes.sort(key=lambda c: -c.confidence)
|
|
217
|
+
return Diagnosis(summary=str(data.get("summary", "")), root_causes=causes,
|
|
218
|
+
next_checks=[str(x) for x in data.get("next_checks") or []][:6], generated_by=by)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# ------------------------------------------------------------------------ ask
|
|
222
|
+
_TOPICS: list[tuple[re.Pattern[str], list[Category]]] = [
|
|
223
|
+
(re.compile(r"crash|restart|exit|oom|memory|killed|error|fail|down|broken|not working", re.I),
|
|
224
|
+
[Category.WORKLOAD, Category.CONFIG, Category.RESOURCES]),
|
|
225
|
+
(re.compile(r"pending|schedul|capacity|node|taint", re.I), [Category.SCHEDULING, Category.NODE, Category.RESOURCES]),
|
|
226
|
+
(re.compile(r"network|dns|service|endpoint|ingress|gateway|503|502|504|timeout|connect|traffic|route", re.I),
|
|
227
|
+
[Category.NETWORK, Category.PLATFORM]),
|
|
228
|
+
(re.compile(r"volume|pvc|disk|storage|mount", re.I), [Category.STORAGE]),
|
|
229
|
+
(re.compile(r"secur|rbac|privileg|root|secret|cert|tls", re.I), [Category.SECURITY]),
|
|
230
|
+
(re.compile(r"cost|money|spend|waste|idle|gpu|right-?siz|over-?provision", re.I), [Category.COST, Category.RESOURCES]),
|
|
231
|
+
(re.compile(r"upgrade|deprecat|version|removed|eol|retire", re.I), [Category.UPGRADE]),
|
|
232
|
+
(re.compile(r"ready for prod|production ready|best practice|reliab|readiness|probe", re.I),
|
|
233
|
+
[Category.RELIABILITY, Category.SECURITY, Category.RESOURCES]),
|
|
234
|
+
]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def mentioned_resources(snap: Snapshot, question: str) -> list[dict[str, Any]]:
|
|
238
|
+
"""Objects whose name appears in the question (e.g. 'why is checkout-api failing?')."""
|
|
239
|
+
words = set(re.findall(r"[a-z0-9][a-z0-9.-]{2,}", question.lower()))
|
|
240
|
+
hits = []
|
|
241
|
+
for key in ("deployments", "statefulsets", "daemonsets", "cronjobs", "jobs", "services", "ingresses", "pods",
|
|
242
|
+
"nodes", "pvcs"):
|
|
243
|
+
for obj in snap.all(key):
|
|
244
|
+
name = (meta(obj).get("name") or "").lower()
|
|
245
|
+
if name in words:
|
|
246
|
+
hits.append(obj)
|
|
247
|
+
# pods matched only by name prefix of a mentioned workload are noise; keep the top-level objects first
|
|
248
|
+
hits.sort(key=lambda o: o.get("kind") == "Pod")
|
|
249
|
+
return hits[:6]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def mentioned_namespace(question: str, namespaces: list[str]) -> str | None:
|
|
253
|
+
m = re.search(r"(?:namespace|ns|-n)\s+([a-z0-9-]+)", question, re.I)
|
|
254
|
+
if m and m.group(1) in namespaces:
|
|
255
|
+
return m.group(1)
|
|
256
|
+
for ns in namespaces:
|
|
257
|
+
if re.search(rf"\b{re.escape(ns)}\b", question) and ns not in ("default",):
|
|
258
|
+
return ns
|
|
259
|
+
return None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def ask(settings: Settings, snap: Snapshot, question: str, use_llm: bool = True,
|
|
263
|
+
store: Store | None = None) -> tuple[Report, dict[str, Any]]:
|
|
264
|
+
"""Answer a free-form question about the cluster. Returns (report, answer)."""
|
|
265
|
+
targets = mentioned_resources(snap, question)
|
|
266
|
+
cats: list[Category] = []
|
|
267
|
+
for pattern, c in _TOPICS:
|
|
268
|
+
if pattern.search(question):
|
|
269
|
+
cats.extend(x for x in c if x not in cats)
|
|
270
|
+
report = investigate(settings, snap, use_llm=False, store=store)
|
|
271
|
+
report.question = question
|
|
272
|
+
|
|
273
|
+
relevant = report.findings
|
|
274
|
+
if targets:
|
|
275
|
+
names = {(meta(t).get("namespace"), meta(t).get("name")) for t in targets}
|
|
276
|
+
relevant = [f for f in report.findings
|
|
277
|
+
if (f.resource.namespace, f.resource.name) in names
|
|
278
|
+
or (f.owner and (f.owner.namespace, f.owner.name) in names)]
|
|
279
|
+
elif cats:
|
|
280
|
+
relevant = [f for f in report.findings if f.category in cats] or report.findings
|
|
281
|
+
|
|
282
|
+
summaries = [summarize(snap, t) for t in targets]
|
|
283
|
+
answer = _rules_answer(question, relevant, summaries, targets)
|
|
284
|
+
provider = None
|
|
285
|
+
if use_llm:
|
|
286
|
+
try:
|
|
287
|
+
provider = get_provider(settings)
|
|
288
|
+
except LLMError as exc:
|
|
289
|
+
report.warnings.append(f"LLM disabled: {exc}")
|
|
290
|
+
if provider:
|
|
291
|
+
logs = settings.send_logs_to_llm
|
|
292
|
+
sent = [{k: v for k, v in s.items() if logs or k != "logs"} for s in summaries]
|
|
293
|
+
ctx = _context_json(report, {"question": question,
|
|
294
|
+
"relevant_findings": [_finding_payload(f, logs) for f in relevant[:20]],
|
|
295
|
+
"resources_mentioned": sent}, logs=logs)
|
|
296
|
+
try:
|
|
297
|
+
data = provider.complete_json(
|
|
298
|
+
SYSTEM_PROMPT + "\nAnswer the engineer's question directly first, then the supporting detail.",
|
|
299
|
+
f"Question: {question}\n\nCluster data:\n{ctx}", ANSWER_SCHEMA)
|
|
300
|
+
known = {f.fingerprint for f in report.findings}
|
|
301
|
+
answer = {
|
|
302
|
+
"answer": str(data.get("answer", "")),
|
|
303
|
+
"confidence": max(0, min(100, int(data.get("confidence", 0)))),
|
|
304
|
+
"evidence": [e for e in data.get("evidence") or [] if e in known],
|
|
305
|
+
"commands": [str(c) for c in data.get("commands") or []][:6],
|
|
306
|
+
"follow_up": [str(c) for c in data.get("follow_up") or []][:5],
|
|
307
|
+
"generated_by": f"{provider.name}:{settings.resolved_model()}",
|
|
308
|
+
}
|
|
309
|
+
except (LLMError, KeyError, TypeError, ValueError) as exc:
|
|
310
|
+
report.warnings.append(f"LLM answer failed, showing rule-based answer: {exc}")
|
|
311
|
+
report.findings = relevant
|
|
312
|
+
report.incidents = [i for i in report.incidents if any(f in relevant for f in i.findings)]
|
|
313
|
+
report.diagnosis = rules_diagnosis(report)
|
|
314
|
+
return report, answer
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _rules_answer(question: str, relevant: list[Finding], summaries: list[dict[str, Any]],
|
|
318
|
+
targets: list[dict[str, Any]]) -> dict[str, Any]:
|
|
319
|
+
problems = [f for f in relevant if f.kind == "problem"]
|
|
320
|
+
if problems:
|
|
321
|
+
top = problems[0]
|
|
322
|
+
text = f"{top.resource}: {top.message}"
|
|
323
|
+
if top.related_changes:
|
|
324
|
+
text += f" This began after: {top.related_changes[0].summary}."
|
|
325
|
+
if len(problems) > 1:
|
|
326
|
+
text += f" {len(problems) - 1} more problem(s) are listed below."
|
|
327
|
+
return {"answer": text, "confidence": 80, "evidence": [f.fingerprint for f in problems[:5]],
|
|
328
|
+
"commands": top.commands[:3], "follow_up": [top.remediation] if top.remediation else [],
|
|
329
|
+
"generated_by": "rules"}
|
|
330
|
+
if targets:
|
|
331
|
+
names = ", ".join(f"{s['kind']}/{s['name']}" for s in summaries)
|
|
332
|
+
risks = [f for f in relevant if f.kind == "risk"]
|
|
333
|
+
text = f"No active problems found on {names}."
|
|
334
|
+
if risks:
|
|
335
|
+
text += f" {len(risks)} configuration risk(s) found (listed below)."
|
|
336
|
+
return {"answer": text, "confidence": 70, "evidence": [f.fingerprint for f in risks[:5]], "commands": [],
|
|
337
|
+
"follow_up": [], "generated_by": "rules"}
|
|
338
|
+
return {"answer": "No active problems match this question." + (
|
|
339
|
+
f" {len(relevant)} related risk(s) listed below." if relevant else ""),
|
|
340
|
+
"confidence": 60, "evidence": [], "commands": [], "follow_up": [
|
|
341
|
+
"Name the workload in your question (e.g. 'why is checkout-api failing in shop?') for a focused answer."],
|
|
342
|
+
"generated_by": "rules"}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def explain_resource(settings: Settings, snap: Snapshot, kind: str, namespace: str | None, name: str) -> dict[str, Any]:
|
|
346
|
+
key = KIND_TO_KEY.get(ALIASES.get(kind.lower(), kind))
|
|
347
|
+
obj = snap.get(key, namespace, name) if key else None
|
|
348
|
+
if obj is None:
|
|
349
|
+
return {"error": f"{kind}/{name} not found" + (f" in namespace {namespace}" if namespace else "")}
|
|
350
|
+
report = investigate(settings, snap, use_llm=False)
|
|
351
|
+
related = [f for f in report.findings
|
|
352
|
+
if (f.resource.namespace, f.resource.name) == (namespace, name)
|
|
353
|
+
or (f.owner and (f.owner.namespace, f.owner.name, f.owner.kind) == (namespace, name, obj.get("kind")))]
|
|
354
|
+
pods = []
|
|
355
|
+
if obj.get("kind") in ("Deployment", "StatefulSet", "DaemonSet", "Job"):
|
|
356
|
+
pods = [summarize(snap, p, events=4) for p in snap.in_namespace("pods", namespace)
|
|
357
|
+
if str(snap.owner_of(p)) == f"{obj.get('kind')}/{namespace}/{name}"][:5]
|
|
358
|
+
return {"resource": summarize(snap, obj), "pods": pods,
|
|
359
|
+
"findings": [f.model_dump(mode="json") for f in related],
|
|
360
|
+
"changes": [c.model_dump(mode="json") for c in report.changes
|
|
361
|
+
if c.resource.name == name and c.resource.namespace == namespace],
|
|
362
|
+
"counts": pod_counts(snap)}
|
kubesift/analysis/llm.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""LLM providers. KubeSift works without any; an LLM only explains and ranks evidence the checks found.
|
|
2
|
+
|
|
3
|
+
Providers:
|
|
4
|
+
anthropic Claude through the official Anthropic SDK (pip install 'kubesift[anthropic]')
|
|
5
|
+
openai any OpenAI-compatible endpoint: OpenAI, OpenRouter, Ollama, vLLM, LiteLLM, Azure gateways
|
|
6
|
+
none rules only
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import re
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
|
|
18
|
+
from kubesift.config import Settings
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class LLMError(Exception):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Provider:
|
|
28
|
+
name = "none"
|
|
29
|
+
|
|
30
|
+
def complete_json(self, system: str, user: str, schema: dict[str, Any]) -> dict[str, Any]:
|
|
31
|
+
raise NotImplementedError
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class AnthropicProvider(Provider):
|
|
35
|
+
name = "anthropic"
|
|
36
|
+
|
|
37
|
+
def __init__(self, settings: Settings):
|
|
38
|
+
try:
|
|
39
|
+
import anthropic
|
|
40
|
+
except ImportError as exc:
|
|
41
|
+
raise LLMError("Install the Anthropic SDK: pip install 'kubesift[anthropic]'") from exc
|
|
42
|
+
self._anthropic = anthropic
|
|
43
|
+
kwargs: dict[str, Any] = {"timeout": 180.0}
|
|
44
|
+
if settings.llm_api_key:
|
|
45
|
+
kwargs["api_key"] = settings.llm_api_key
|
|
46
|
+
if settings.llm_base_url:
|
|
47
|
+
kwargs["base_url"] = settings.llm_base_url
|
|
48
|
+
self.client = anthropic.Anthropic(**kwargs)
|
|
49
|
+
self.model = settings.resolved_model()
|
|
50
|
+
self.effort = settings.llm_effort
|
|
51
|
+
|
|
52
|
+
def complete_json(self, system: str, user: str, schema: dict[str, Any]) -> dict[str, Any]:
|
|
53
|
+
a = self._anthropic
|
|
54
|
+
request = {
|
|
55
|
+
"model": self.model,
|
|
56
|
+
"max_tokens": 16000,
|
|
57
|
+
"system": system,
|
|
58
|
+
"messages": [{"role": "user", "content": user}],
|
|
59
|
+
"output_config": {"effort": self.effort, "format": {"type": "json_schema", "schema": schema}},
|
|
60
|
+
}
|
|
61
|
+
try:
|
|
62
|
+
# Server-side fallback re-runs a declined request on another model inside the same call.
|
|
63
|
+
response = self.client.beta.messages.create(
|
|
64
|
+
**request, betas=["server-side-fallback-2026-07-01"], fallbacks="default")
|
|
65
|
+
except (a.BadRequestError, TypeError):
|
|
66
|
+
# Older SDKs (TypeError) or endpoints (400) without server-side fallback: send a plain request.
|
|
67
|
+
try:
|
|
68
|
+
response = self.client.messages.create(**request)
|
|
69
|
+
except a.APIStatusError as exc:
|
|
70
|
+
raise LLMError(f"Anthropic API error {exc.status_code}: {exc.message}") from exc
|
|
71
|
+
except a.APIConnectionError as exc:
|
|
72
|
+
raise LLMError(f"Cannot reach the Anthropic API: {exc}") from exc
|
|
73
|
+
except a.AuthenticationError as exc:
|
|
74
|
+
raise LLMError("Anthropic rejected the API key (set ANTHROPIC_API_KEY).") from exc
|
|
75
|
+
except a.RateLimitError as exc:
|
|
76
|
+
raise LLMError("Anthropic rate limit reached; try again shortly.") from exc
|
|
77
|
+
except a.APIStatusError as exc:
|
|
78
|
+
raise LLMError(f"Anthropic API error {exc.status_code}: {exc.message}") from exc
|
|
79
|
+
except a.APIConnectionError as exc:
|
|
80
|
+
raise LLMError(f"Cannot reach the Anthropic API: {exc}") from exc
|
|
81
|
+
if response.stop_reason == "refusal":
|
|
82
|
+
raise LLMError("The model declined this request.")
|
|
83
|
+
text = "".join(b.text for b in response.content if getattr(b, "type", "") == "text")
|
|
84
|
+
return parse_json(text)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class OpenAICompatibleProvider(Provider):
|
|
88
|
+
name = "openai"
|
|
89
|
+
|
|
90
|
+
def __init__(self, settings: Settings):
|
|
91
|
+
self.base_url = (settings.resolved_base_url() or "https://api.openai.com/v1").rstrip("/")
|
|
92
|
+
self.model = settings.resolved_model()
|
|
93
|
+
self.api_key = settings.resolved_api_key()
|
|
94
|
+
self.provider = settings.resolved_provider()
|
|
95
|
+
if self.provider != "ollama" and not self.api_key and "localhost" not in self.base_url:
|
|
96
|
+
raise LLMError(f"No API key for {self.provider}; set {self.provider.upper()}_API_KEY or KUBESIFT_LLM_API_KEY.")
|
|
97
|
+
|
|
98
|
+
def complete_json(self, system: str, user: str, schema: dict[str, Any]) -> dict[str, Any]:
|
|
99
|
+
headers = {"Content-Type": "application/json"}
|
|
100
|
+
if self.api_key:
|
|
101
|
+
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
102
|
+
if self.provider == "openrouter":
|
|
103
|
+
headers["X-Title"] = "KubeSift"
|
|
104
|
+
body: dict[str, Any] = {
|
|
105
|
+
"model": self.model,
|
|
106
|
+
"messages": [
|
|
107
|
+
{"role": "system", "content": system + "\n\nReply with one JSON object matching this schema:\n"
|
|
108
|
+
+ json.dumps(schema)},
|
|
109
|
+
{"role": "user", "content": user},
|
|
110
|
+
],
|
|
111
|
+
"response_format": {"type": "json_object"},
|
|
112
|
+
}
|
|
113
|
+
with httpx.Client(timeout=180.0) as http:
|
|
114
|
+
for attempt in range(3):
|
|
115
|
+
try:
|
|
116
|
+
r = http.post(f"{self.base_url}/chat/completions", headers=headers, json=body)
|
|
117
|
+
except httpx.HTTPError as exc:
|
|
118
|
+
if attempt == 2:
|
|
119
|
+
raise LLMError(f"Cannot reach {self.base_url}: {exc}") from exc
|
|
120
|
+
continue
|
|
121
|
+
if r.status_code == 400 and "response_format" in body:
|
|
122
|
+
body.pop("response_format") # some servers don't support JSON mode
|
|
123
|
+
continue
|
|
124
|
+
if r.status_code in (429, 500, 502, 503, 504) and attempt < 2:
|
|
125
|
+
continue
|
|
126
|
+
if r.status_code >= 400:
|
|
127
|
+
raise LLMError(f"{self.provider} returned HTTP {r.status_code}: {r.text[:300]}")
|
|
128
|
+
data = r.json()
|
|
129
|
+
return parse_json(data["choices"][0]["message"]["content"] or "")
|
|
130
|
+
raise LLMError("LLM request failed after retries")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def parse_json(text: str) -> dict[str, Any]:
|
|
134
|
+
text = re.sub(r"```(?:json)?", "", text).strip()
|
|
135
|
+
start, end = text.find("{"), text.rfind("}")
|
|
136
|
+
if start == -1 or end == -1:
|
|
137
|
+
raise LLMError("The model did not return JSON.")
|
|
138
|
+
try:
|
|
139
|
+
return json.loads(text[start:end + 1])
|
|
140
|
+
except json.JSONDecodeError as exc:
|
|
141
|
+
raise LLMError(f"The model returned invalid JSON: {exc}") from exc
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def get_provider(settings: Settings) -> Provider | None:
|
|
145
|
+
name = settings.resolved_provider()
|
|
146
|
+
if name == "none":
|
|
147
|
+
return None
|
|
148
|
+
if name == "anthropic":
|
|
149
|
+
return AnthropicProvider(settings)
|
|
150
|
+
if name in ("openai", "openrouter", "ollama"):
|
|
151
|
+
return OpenAICompatibleProvider(settings)
|
|
152
|
+
raise LLMError(f"Unknown LLM provider {name!r}; use anthropic, openai, openrouter, ollama or none.")
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Importing this package registers every check."""
|
|
2
|
+
|
|
3
|
+
from kubesift.checks import ( # noqa: F401
|
|
4
|
+
config_refs,
|
|
5
|
+
events,
|
|
6
|
+
hygiene,
|
|
7
|
+
lifecycle,
|
|
8
|
+
network,
|
|
9
|
+
nodes,
|
|
10
|
+
operations,
|
|
11
|
+
pods,
|
|
12
|
+
storage,
|
|
13
|
+
workloads,
|
|
14
|
+
)
|
|
15
|
+
from kubesift.checks.base import REGISTRY, Context, run_checks
|
|
16
|
+
|
|
17
|
+
__all__ = ["REGISTRY", "Context", "run_checks"]
|
kubesift/checks/base.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Check registry. A check is a function Snapshot -> iterable of Findings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import Callable, Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from kubesift.cluster.snapshot import Snapshot
|
|
11
|
+
from kubesift.config import Settings
|
|
12
|
+
from kubesift.models import Category, Finding, ResourceRef, Severity
|
|
13
|
+
from kubesift.util import meta
|
|
14
|
+
|
|
15
|
+
log = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
CheckFn = Callable[["Context"], Iterable[Finding]]
|
|
18
|
+
DOCS_URL = "https://github.com/OpeyemiAdeniji/KubeSift/blob/main/docs/checks.md"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class CheckInfo:
|
|
23
|
+
id: str
|
|
24
|
+
title: str
|
|
25
|
+
category: Category
|
|
26
|
+
severity: Severity
|
|
27
|
+
live_only: bool
|
|
28
|
+
description: str
|
|
29
|
+
fn: CheckFn
|
|
30
|
+
risk: bool = False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
REGISTRY: dict[str, CheckInfo] = {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def check(
|
|
37
|
+
id: str,
|
|
38
|
+
title: str,
|
|
39
|
+
category: Category,
|
|
40
|
+
severity: Severity,
|
|
41
|
+
live_only: bool = True,
|
|
42
|
+
description: str = "",
|
|
43
|
+
risk: bool = False,
|
|
44
|
+
) -> Callable[[CheckFn], CheckFn]:
|
|
45
|
+
"""Register a check. `live_only` checks need runtime status and are skipped when linting files."""
|
|
46
|
+
|
|
47
|
+
def wrap(fn: CheckFn) -> CheckFn:
|
|
48
|
+
REGISTRY[id] = CheckInfo(id, title, category, severity, live_only,
|
|
49
|
+
description or (fn.__doc__ or "").strip(), fn, risk)
|
|
50
|
+
return fn
|
|
51
|
+
|
|
52
|
+
return wrap
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class Context:
|
|
57
|
+
snap: Snapshot
|
|
58
|
+
settings: Settings
|
|
59
|
+
target_version: str | None = None
|
|
60
|
+
|
|
61
|
+
def is_system(self, namespace: str | None) -> bool:
|
|
62
|
+
return namespace in self.settings.system_namespaces
|
|
63
|
+
|
|
64
|
+
def finding(
|
|
65
|
+
self,
|
|
66
|
+
check_id: str,
|
|
67
|
+
obj: dict[str, Any] | ResourceRef,
|
|
68
|
+
message: str,
|
|
69
|
+
*,
|
|
70
|
+
severity: Severity | None = None,
|
|
71
|
+
title: str | None = None,
|
|
72
|
+
evidence: list[str] | None = None,
|
|
73
|
+
remediation: str = "",
|
|
74
|
+
commands: list[str] | None = None,
|
|
75
|
+
docs: str | None = None,
|
|
76
|
+
**extra: Any,
|
|
77
|
+
) -> Finding:
|
|
78
|
+
info = REGISTRY[check_id]
|
|
79
|
+
if isinstance(obj, ResourceRef):
|
|
80
|
+
ref = obj
|
|
81
|
+
owner = obj
|
|
82
|
+
source = None
|
|
83
|
+
else:
|
|
84
|
+
m = meta(obj)
|
|
85
|
+
ref = ResourceRef(kind=obj.get("kind", "?"), name=m.get("name", "?"), namespace=m.get("namespace"))
|
|
86
|
+
owner = self.snap.owner_of(obj) if self.snap.live else ref
|
|
87
|
+
source = (m.get("annotations") or {}).get("kubesift.dev/source")
|
|
88
|
+
return Finding(
|
|
89
|
+
check=check_id,
|
|
90
|
+
title=title or info.title,
|
|
91
|
+
severity=severity or info.severity,
|
|
92
|
+
category=info.category,
|
|
93
|
+
kind="risk" if info.risk else "problem",
|
|
94
|
+
resource=ref,
|
|
95
|
+
owner=owner,
|
|
96
|
+
message=message,
|
|
97
|
+
evidence=evidence or [],
|
|
98
|
+
remediation=remediation,
|
|
99
|
+
commands=commands or [],
|
|
100
|
+
docs=docs or f"{DOCS_URL}#{check_id}",
|
|
101
|
+
source=source,
|
|
102
|
+
**extra,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def run_checks(
|
|
107
|
+
snap: Snapshot,
|
|
108
|
+
settings: Settings,
|
|
109
|
+
only: Iterable[str] | None = None,
|
|
110
|
+
categories: Iterable[Category] | None = None,
|
|
111
|
+
target_version: str | None = None,
|
|
112
|
+
) -> list[Finding]:
|
|
113
|
+
from kubesift import checks as _ # noqa: F401 (registers all checks)
|
|
114
|
+
|
|
115
|
+
ctx = Context(snap=snap, settings=settings, target_version=target_version)
|
|
116
|
+
only_set = set(only) if only else None
|
|
117
|
+
cat_set = set(categories) if categories else None
|
|
118
|
+
findings: list[Finding] = []
|
|
119
|
+
for info in REGISTRY.values():
|
|
120
|
+
if only_set is not None and info.id not in only_set:
|
|
121
|
+
continue
|
|
122
|
+
if cat_set is not None and info.category not in cat_set:
|
|
123
|
+
continue
|
|
124
|
+
if info.live_only and not snap.live:
|
|
125
|
+
continue
|
|
126
|
+
try:
|
|
127
|
+
findings.extend(info.fn(ctx))
|
|
128
|
+
except Exception as exc: # one broken check must never break the scan
|
|
129
|
+
log.warning("check %s failed: %s", info.id, exc, exc_info=True)
|
|
130
|
+
snap.warnings.append(f"Check {info.id} failed: {exc}")
|
|
131
|
+
if snap.namespace:
|
|
132
|
+
findings = [f for f in findings if f.resource.namespace in (snap.namespace, None)]
|
|
133
|
+
return dedupe(findings)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def dedupe(findings: list[Finding]) -> list[Finding]:
|
|
137
|
+
"""Collapse identical findings on pods of the same workload into one, keeping the count."""
|
|
138
|
+
seen: dict[tuple[str, str], Finding] = {}
|
|
139
|
+
for f in findings:
|
|
140
|
+
key = (f.check, str(f.owner or f.resource))
|
|
141
|
+
prev = seen.get(key)
|
|
142
|
+
if prev is None:
|
|
143
|
+
seen[key] = f
|
|
144
|
+
continue
|
|
145
|
+
keep, drop = (f, prev) if f.severity.rank > prev.severity.rank else (prev, f)
|
|
146
|
+
note = f"Also affects {drop.resource}"
|
|
147
|
+
if str(drop.resource) != str(keep.resource) and note not in keep.evidence and len(keep.evidence) < 12:
|
|
148
|
+
keep.evidence.append(note)
|
|
149
|
+
seen[key] = keep
|
|
150
|
+
return sorted(seen.values(), key=lambda f: (-f.severity.rank, f.category.value, str(f.resource)))
|