kubesift 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. kubesift/__init__.py +3 -0
  2. kubesift/analysis/__init__.py +3 -0
  3. kubesift/analysis/changes.py +165 -0
  4. kubesift/analysis/correlate.py +160 -0
  5. kubesift/analysis/describe.py +83 -0
  6. kubesift/analysis/engine.py +362 -0
  7. kubesift/analysis/llm.py +152 -0
  8. kubesift/checks/__init__.py +17 -0
  9. kubesift/checks/base.py +150 -0
  10. kubesift/checks/config_refs.py +84 -0
  11. kubesift/checks/events.py +77 -0
  12. kubesift/checks/hygiene.py +264 -0
  13. kubesift/checks/lifecycle.py +202 -0
  14. kubesift/checks/network.py +325 -0
  15. kubesift/checks/nodes.py +183 -0
  16. kubesift/checks/operations.py +319 -0
  17. kubesift/checks/pods.py +485 -0
  18. kubesift/checks/storage.py +72 -0
  19. kubesift/checks/workloads.py +201 -0
  20. kubesift/cli.py +874 -0
  21. kubesift/cluster/__init__.py +4 -0
  22. kubesift/cluster/client.py +141 -0
  23. kubesift/cluster/snapshot.py +513 -0
  24. kubesift/config.py +134 -0
  25. kubesift/cost.py +150 -0
  26. kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
  27. kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
  28. kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
  29. kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
  30. kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
  31. kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
  32. kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
  33. kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
  34. kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
  35. kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
  36. kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
  37. kubesift/drill_scenarios/__init__.py +1 -0
  38. kubesift/drills.py +122 -0
  39. kubesift/knowledge.py +51 -0
  40. kubesift/leader.py +145 -0
  41. kubesift/lint.py +89 -0
  42. kubesift/mcp_server.py +193 -0
  43. kubesift/models.py +144 -0
  44. kubesift/notify.py +71 -0
  45. kubesift/postmortem.py +152 -0
  46. kubesift/preflight.py +435 -0
  47. kubesift/redact.py +38 -0
  48. kubesift/remediate.py +99 -0
  49. kubesift/report.py +193 -0
  50. kubesift/runbooks.py +58 -0
  51. kubesift/server/__init__.py +0 -0
  52. kubesift/server/app.py +480 -0
  53. kubesift/server/static/index.html +152 -0
  54. kubesift/slack.py +106 -0
  55. kubesift/store.py +219 -0
  56. kubesift/timetravel.py +156 -0
  57. kubesift/upgrade.py +51 -0
  58. kubesift/util.py +174 -0
  59. kubesift-2.0.0.dist-info/METADATA +192 -0
  60. kubesift-2.0.0.dist-info/RECORD +63 -0
  61. kubesift-2.0.0.dist-info/WHEEL +4 -0
  62. kubesift-2.0.0.dist-info/entry_points.txt +3 -0
  63. kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,319 @@
1
+ """Day-to-day operational failures from public postmortems: rollout 502s, stale config, stuck namespaces,
2
+ fragile DNS, single-node replicas, autoscaler thrashing, Karpenter, and baseline security gaps."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import re
7
+ from collections import Counter, defaultdict
8
+ from collections.abc import Iterator
9
+ from datetime import timedelta
10
+ from typing import Any
11
+
12
+ from kubesift.analysis.changes import last_user_update
13
+ from kubesift.checks.base import Context, check
14
+ from kubesift.checks.hygiene import templates
15
+ from kubesift.cluster.snapshot import event_time
16
+ from kubesift.models import Category, Finding, ResourceRef, Severity
17
+ from kubesift.util import (
18
+ age,
19
+ containers_of,
20
+ human_duration,
21
+ meta,
22
+ parse_time,
23
+ selector_matches,
24
+ )
25
+
26
+
27
+ def _prestop_sleep(container: dict[str, Any]) -> int | None:
28
+ pre = ((container.get("lifecycle") or {}).get("preStop")) or {}
29
+ if not pre:
30
+ return None
31
+ if pre.get("sleep"):
32
+ return int(pre["sleep"].get("seconds", 0))
33
+ cmd = " ".join(str(x) for x in ((pre.get("exec") or {}).get("command") or []))
34
+ m = re.search(r"sleep\s+(\d+)", cmd)
35
+ return int(m.group(1)) if m else 0
36
+
37
+
38
+ @check("rollout-drops-traffic", "Rollouts can drop live traffic (502s)", Category.RELIABILITY, Severity.MEDIUM,
39
+ live_only=False, risk=True)
40
+ def rollout_drops_traffic(ctx: Context) -> Iterator[Finding]:
41
+ """Pods behind a Service with no preStop delay stop before load balancers stop routing to them,
42
+ so every deploy returns 502s / connection resets for a few seconds."""
43
+ for obj, spec in templates(ctx):
44
+ if obj.get("kind") not in ("Deployment", "StatefulSet"):
45
+ continue
46
+ labels = (((obj.get("spec") or {}).get("template") or {}).get("metadata") or {}).get("labels") or {}
47
+ ns = meta(obj).get("namespace")
48
+ served = any(selector_matches((s.get("spec") or {}).get("selector"), labels)
49
+ for s in ctx.snap.in_namespace("services", ns))
50
+ if not served:
51
+ continue
52
+ grace = spec.get("terminationGracePeriodSeconds", 30)
53
+ serving = [c for c in containers_of(spec) if c.get("ports")]
54
+ for c in serving:
55
+ sleep = _prestop_sleep(c)
56
+ if sleep is None:
57
+ yield ctx.finding(
58
+ "rollout-drops-traffic", obj,
59
+ f"Container '{c.get('name')}' has no preStop hook. During rollouts and scale-downs it gets SIGTERM "
60
+ "while load balancers and kube-proxy still send it traffic, so some requests fail.",
61
+ remediation="Add `lifecycle: {preStop: {sleep: {seconds: 10}}}` (or `exec: [sleep, '10']` on clusters "
62
+ "before 1.30), make the app drain on SIGTERM, and keep terminationGracePeriodSeconds above "
63
+ "the sleep plus the app's shutdown time.")
64
+ break
65
+ if sleep and grace <= sleep:
66
+ yield ctx.finding(
67
+ "rollout-drops-traffic", obj,
68
+ f"terminationGracePeriodSeconds ({grace}s) is not longer than the preStop sleep ({sleep}s): the pod "
69
+ "is killed before the app can finish in-flight requests.",
70
+ severity=Severity.HIGH,
71
+ remediation=f"Set terminationGracePeriodSeconds to at least {sleep + 20}.")
72
+ break
73
+
74
+
75
+ @check("config-stale-in-pod", "Pods run with old config values", Category.CONFIG, Severity.MEDIUM)
76
+ def config_stale(ctx: Context) -> Iterator[Finding]:
77
+ """A ConfigMap/Secret used through env vars changed after the pod started; env vars are only read at start."""
78
+ for pod in ctx.snap.all("pods"):
79
+ st = pod.get("status") or {}
80
+ if st.get("phase") != "Running":
81
+ continue
82
+ started = parse_time(st.get("startTime"))
83
+ if not started:
84
+ continue
85
+ ns = meta(pod).get("namespace")
86
+ stale = []
87
+ for c in containers_of(pod.get("spec"), include_init=True):
88
+ refs: set[tuple[str, str]] = set()
89
+ for ef in c.get("envFrom") or []:
90
+ if ef.get("configMapRef"):
91
+ refs.add(("configmaps", ef["configMapRef"].get("name", "")))
92
+ if ef.get("secretRef"):
93
+ refs.add(("secrets", ef["secretRef"].get("name", "")))
94
+ for env in c.get("env") or []:
95
+ vf = env.get("valueFrom") or {}
96
+ if vf.get("configMapKeyRef"):
97
+ refs.add(("configmaps", vf["configMapKeyRef"].get("name", "")))
98
+ if vf.get("secretKeyRef"):
99
+ refs.add(("secrets", vf["secretKeyRef"].get("name", "")))
100
+ for key, name in refs:
101
+ obj = ctx.snap.get(key, ns, name)
102
+ changed, who = last_user_update(obj) if obj else (None, None)
103
+ if changed and changed > started + timedelta(seconds=30):
104
+ stale.append((obj.get("kind"), name, changed, who))
105
+ if stale:
106
+ kind, name, changed, who = stale[0]
107
+ yield ctx.finding(
108
+ "config-stale-in-pod", pod,
109
+ f"{kind} '{name}' was changed at {changed:%Y-%m-%d %H:%M} UTC" + (f" by {who}" if who else "")
110
+ + ", after this pod started. It still uses the old values (env vars are read only at start).",
111
+ evidence=[f"pod started {st.get('startTime')}"] + [f"{k} {n} updated {t:%H:%M}" for k, n, t, _ in stale[:4]],
112
+ remediation="Restart the workload to pick up the change, or add a config checksum annotation to the pod "
113
+ "template (or use a reloader) so changes roll out automatically.",
114
+ commands=[f"kubectl rollout restart {ctx.snap.owner_of(pod).kind.lower()}/{ctx.snap.owner_of(pod).name} -n {ns}"]
115
+ if ctx.snap.owner_of(pod).kind in ("Deployment", "StatefulSet", "DaemonSet") else [],
116
+ )
117
+
118
+
119
+ @check("namespace-stuck-terminating", "Namespace stuck deleting", Category.PLATFORM, Severity.HIGH)
120
+ def namespace_stuck(ctx: Context) -> Iterator[Finding]:
121
+ """A namespace has been Terminating for 10+ minutes, usually a finalizer or an unavailable APIService."""
122
+ for ns in ctx.snap.all("namespaces"):
123
+ if (ns.get("status") or {}).get("phase") != "Terminating":
124
+ continue
125
+ a = age(meta(ns).get("deletionTimestamp"))
126
+ if a is not None and a < timedelta(minutes=10):
127
+ continue
128
+ conds = [c for c in (ns.get("status") or {}).get("conditions") or [] if c.get("status") == "True"]
129
+ reason = conds[0].get("message", "") if conds else "resources with finalizers are still present"
130
+ hint = "Remove what blocks it; force-removing finalizers can leave cloud resources behind."
131
+ if any(c.get("type") in ("NamespaceDeletionDiscoveryFailure", "NamespaceDeletionGroupVersionParsingFailure")
132
+ for c in conds):
133
+ hint = "An aggregated API (APIService) is unavailable, so Kubernetes cannot list everything to delete. " \
134
+ "Fix or delete the broken APIService (see the apiservice-unavailable finding)."
135
+ yield ctx.finding(
136
+ "namespace-stuck-terminating", ns, f"Terminating for {human_duration(a)}: {reason[:250]}",
137
+ evidence=[f"{c.get('type')}: {c.get('message', '')[:200]}" for c in conds[:4]],
138
+ remediation=hint,
139
+ commands=[f"kubectl api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl get --show-kind "
140
+ f"--ignore-not-found -n {meta(ns).get('name')}"],
141
+ )
142
+
143
+
144
+ @check("apiservice-unavailable", "Extension API is unavailable", Category.PLATFORM, Severity.HIGH)
145
+ def apiservice_unavailable(ctx: Context) -> Iterator[Finding]:
146
+ """An aggregated APIService (metrics-server, custom metrics, ...) is down: it breaks HPAs, `kubectl top`,
147
+ API discovery and namespace deletion."""
148
+ for svc in ctx.snap.all("apiservices"):
149
+ if not (svc.get("spec") or {}).get("service"):
150
+ continue # built-in, served by the API server itself
151
+ cond = next((c for c in (svc.get("status") or {}).get("conditions") or [] if c.get("type") == "Available"), {})
152
+ if cond.get("status") != "False":
153
+ continue
154
+ backend = (svc.get("spec") or {}).get("service") or {}
155
+ name = meta(svc).get("name", "")
156
+ impact = "HPAs and `kubectl top` stop working" if "metrics" in name else "API discovery and namespace deletion fail"
157
+ yield ctx.finding(
158
+ "apiservice-unavailable", svc,
159
+ f"{name} is unavailable ({cond.get('reason')}): {impact}.",
160
+ evidence=[cond.get("message", "")[:300], f"backend: {backend.get('namespace')}/{backend.get('name')}"],
161
+ remediation="Fix the backing Service/pods, or delete the APIService if its add-on was uninstalled.",
162
+ commands=[f"kubectl get apiservice {name} -o yaml", f"kubectl -n {backend.get('namespace')} get pods"],
163
+ )
164
+
165
+
166
+ @check("dns-fragile", "Cluster DNS is a single point of failure", Category.PLATFORM, Severity.MEDIUM, risk=True)
167
+ def dns_fragile(ctx: Context) -> Iterator[Finding]:
168
+ """CoreDNS with one replica, or with a CPU limit (throttling made DNS fail cluster-wide in public postmortems)."""
169
+ for d in ctx.snap.in_namespace("deployments", "kube-system"):
170
+ labels = (((d.get("spec") or {}).get("template") or {}).get("metadata") or {}).get("labels") or {}
171
+ if labels.get("k8s-app") != "kube-dns" and meta(d).get("name") != "coredns":
172
+ continue
173
+ replicas = (d.get("spec") or {}).get("replicas", 1)
174
+ if replicas < 2:
175
+ yield ctx.finding("dns-fragile", d, f"CoreDNS runs {replicas} replica: if that pod or its node fails, every "
176
+ "service-name lookup in the cluster fails.",
177
+ remediation="Run at least 2 replicas spread across nodes (or enable the dns-autoscaler).",
178
+ commands=["kubectl -n kube-system scale deployment coredns --replicas=2"])
179
+ for c in containers_of((d.get("spec") or {}).get("template", {}).get("spec")):
180
+ if ((c.get("resources") or {}).get("limits") or {}).get("cpu"):
181
+ yield ctx.finding("dns-fragile", d, "CoreDNS has a CPU limit. Under load it gets CPU-throttled, lookups "
182
+ "time out and apps retry, which makes it worse.",
183
+ severity=Severity.LOW,
184
+ remediation="Remove the CPU limit (keep a CPU request), consider NodeLocal DNSCache, and "
185
+ "lower ndots for apps that call external hosts.")
186
+ break
187
+
188
+
189
+ @check("replicas-single-node", "All replicas on one node or zone", Category.RELIABILITY, Severity.MEDIUM, risk=True)
190
+ def replicas_single_node(ctx: Context) -> Iterator[Finding]:
191
+ """Several replicas but every pod is on the same node (or zone): one failure takes the whole app down."""
192
+ zone_of = {meta(n).get("name"): (meta(n).get("labels") or {}).get("topology.kubernetes.io/zone")
193
+ for n in ctx.snap.all("nodes")}
194
+ if len(zone_of) < 2:
195
+ return
196
+ zones = {z for z in zone_of.values() if z}
197
+ by_owner: dict[str, list[dict[str, Any]]] = defaultdict(list)
198
+ for p in ctx.snap.all("pods"):
199
+ if (p.get("status") or {}).get("phase") == "Running" and (p.get("spec") or {}).get("nodeName") \
200
+ and not ctx.is_system(meta(p).get("namespace")):
201
+ by_owner[str(ctx.snap.owner_of(p))].append(p)
202
+ for owner, pods in by_owner.items():
203
+ if len(pods) < 2 or not owner.startswith(("Deployment/", "StatefulSet/")):
204
+ continue
205
+ nodes = {(p.get("spec") or {}).get("nodeName") for p in pods}
206
+ pod_zones = {zone_of.get(n) for n in nodes}
207
+ kind, ns, name = owner.split("/", 2)
208
+ ref = ResourceRef(kind=kind, namespace=ns, name=name)
209
+ if len(nodes) == 1:
210
+ yield ctx.finding("replicas-single-node", ref,
211
+ f"All {len(pods)} replicas run on node {next(iter(nodes))}: losing that node is a full outage.",
212
+ severity=Severity.HIGH,
213
+ remediation="Add topologySpreadConstraints on kubernetes.io/hostname (whenUnsatisfiable: "
214
+ "ScheduleAnyway) or pod anti-affinity.")
215
+ elif len(zones) > 1 and len(pod_zones) == 1 and None not in pod_zones:
216
+ yield ctx.finding("replicas-single-node", ref,
217
+ f"All {len(pods)} replicas run in zone {next(iter(pod_zones))}: a zone outage takes it down.",
218
+ remediation="Spread across zones with topologySpreadConstraints on topology.kubernetes.io/zone.")
219
+
220
+
221
+ @check("hpa-flapping", "Autoscaler keeps scaling up and down", Category.RESOURCES, Severity.MEDIUM)
222
+ def hpa_flapping(ctx: Context) -> Iterator[Finding]:
223
+ """Frequent alternating rescales in the last hour: pods start and stop constantly, hurting latency and cost."""
224
+ rescales: dict[tuple[str | None, str], list[dict[str, Any]]] = defaultdict(list)
225
+ for e in ctx.snap.all("events"):
226
+ inv = e.get("involvedObject") or {}
227
+ if e.get("reason") != "SuccessfulRescale" or inv.get("kind") != "HorizontalPodAutoscaler":
228
+ continue
229
+ a = age(event_time(e))
230
+ if a is not None and a <= timedelta(hours=1):
231
+ rescales[(inv.get("namespace"), inv.get("name", ""))].append(e)
232
+ for (ns, name), events in rescales.items():
233
+ total = sum(e.get("count", 1) or 1 for e in events)
234
+ text = " ".join(e.get("message", "") for e in events)
235
+ if total >= 6 and "above target" in text and "below target" in text:
236
+ hpa = ctx.snap.get("hpas", ns, name)
237
+ yield ctx.finding(
238
+ "hpa-flapping", hpa or ResourceRef(kind="HorizontalPodAutoscaler", namespace=ns, name=name),
239
+ f"Rescaled {total} times in the last hour, both up and down.",
240
+ evidence=[e.get("message", "")[:200] for e in events[:3]],
241
+ remediation="Add a scale-down stabilization window (behavior.scaleDown.stabilizationWindowSeconds: 300), "
242
+ "check that requests match real usage, and avoid targets near normal load.",
243
+ )
244
+
245
+
246
+ @check("karpenter-nodeclaim-failed", "Karpenter cannot launch nodes", Category.NODE, Severity.HIGH)
247
+ def karpenter_failed(ctx: Context) -> Iterator[Finding]:
248
+ """A Karpenter NodeClaim has not become Ready: capacity errors, launch template or IAM problems."""
249
+ for nc in ctx.snap.all("nodeclaims"):
250
+ a = age(meta(nc).get("creationTimestamp"))
251
+ if a is not None and a < timedelta(minutes=10):
252
+ continue
253
+ bad = next((c for c in (nc.get("status") or {}).get("conditions") or []
254
+ if c.get("type") in ("Launched", "Registered", "Initialized", "Ready") and c.get("status") == "False"),
255
+ None)
256
+ if not bad:
257
+ continue
258
+ msg = bad.get("message", "")
259
+ hint = "Check the Karpenter controller logs and the NodePool/EC2NodeClass."
260
+ if re.search(r"InsufficientInstanceCapacity|insufficient capacity", msg, re.I):
261
+ hint = "The cloud has no capacity for the allowed instance types: allow more types/zones or use on-demand."
262
+ elif re.search(r"UnauthorizedOperation|AccessDenied|not authorized", msg, re.I):
263
+ hint = "The Karpenter or node IAM role lacks a permission named in the message."
264
+ yield ctx.finding("karpenter-nodeclaim-failed", nc,
265
+ f"NodeClaim {bad.get('type')}=False for {human_duration(a)} ({bad.get('reason')}): {msg[:200]}",
266
+ remediation=hint, commands=["kubectl -n kube-system logs deploy/karpenter --tail=100"])
267
+
268
+
269
+ @check("pvc-stuck-terminating", "Volume claim stuck deleting", Category.STORAGE, Severity.MEDIUM)
270
+ def pvc_stuck(ctx: Context) -> Iterator[Finding]:
271
+ """PVC deletion is blocked by kubernetes.io/pvc-protection because a pod still mounts it."""
272
+ for pvc in ctx.snap.all("pvcs"):
273
+ deleted = meta(pvc).get("deletionTimestamp")
274
+ if not deleted or (age(deleted) or timedelta()) < timedelta(minutes=10):
275
+ continue
276
+ ns, name = meta(pvc).get("namespace"), meta(pvc).get("name")
277
+ users = [meta(p).get("name") for p in ctx.snap.in_namespace("pods", ns)
278
+ if any((v.get("persistentVolumeClaim") or {}).get("claimName") == name
279
+ for v in (p.get("spec") or {}).get("volumes") or [])]
280
+ yield ctx.finding("pvc-stuck-terminating", pvc,
281
+ f"Deletion requested {human_duration(age(deleted))} ago but blocked"
282
+ + (f": still mounted by {', '.join(users[:3])}." if users else f" by finalizers {meta(pvc).get('finalizers')}."),
283
+ remediation="Delete or scale down the pods using it; the claim is removed automatically afterwards.")
284
+
285
+
286
+ def _app_namespaces(ctx: Context) -> list[str]:
287
+ counts = Counter(meta(p).get("namespace") for p in ctx.snap.all("pods"))
288
+ return sorted(ns for ns in counts if ns and not ctx.is_system(ns) and ns not in ("ingress-nginx", "cert-manager"))
289
+
290
+
291
+ @check("security-pss-not-enforced", "Pod Security Standards not enforced", Category.SECURITY, Severity.LOW, risk=True)
292
+ def pss_not_enforced(ctx: Context) -> Iterator[Finding]:
293
+ """Application namespaces without a pod-security.kubernetes.io/enforce label accept privileged pods."""
294
+ if not ctx.snap.can_read("namespaces"):
295
+ return
296
+ labels = {meta(n).get("name"): meta(n).get("labels") or {} for n in ctx.snap.all("namespaces")}
297
+ missing = [ns for ns in _app_namespaces(ctx) if "pod-security.kubernetes.io/enforce" not in labels.get(ns, {})]
298
+ if missing:
299
+ yield ctx.finding("security-pss-not-enforced", ResourceRef(kind="Namespace", name=missing[0]),
300
+ f"{len(missing)} application namespace(s) don't enforce a Pod Security level: "
301
+ f"{', '.join(missing[:6])}{'…' if len(missing) > 6 else ''}.",
302
+ remediation="Label them with pod-security.kubernetes.io/enforce=baseline (or restricted); start "
303
+ "with the warn label to see what would break.",
304
+ commands=[f"kubectl label namespace {missing[0]} pod-security.kubernetes.io/warn=restricted"])
305
+
306
+
307
+ @check("security-no-network-policy", "Namespaces with no NetworkPolicy", Category.SECURITY, Severity.LOW, risk=True)
308
+ def no_network_policy(ctx: Context) -> Iterator[Finding]:
309
+ """Without any NetworkPolicy every pod can reach every other pod: one compromised pod reaches all of them."""
310
+ if not ctx.snap.can_read("networkpolicies"):
311
+ return
312
+ covered = {meta(n).get("namespace") for n in ctx.snap.all("networkpolicies")}
313
+ missing = [ns for ns in _app_namespaces(ctx) if ns not in covered]
314
+ if missing:
315
+ yield ctx.finding("security-no-network-policy", ResourceRef(kind="Namespace", name=missing[0]),
316
+ f"{len(missing)} application namespace(s) have no NetworkPolicy: {', '.join(missing[:6])}"
317
+ f"{'…' if len(missing) > 6 else ''}.",
318
+ remediation="Start with a default-deny ingress policy per namespace plus explicit allows "
319
+ "(remember to allow DNS egress if you also restrict egress).")