kubesift 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. kubesift/__init__.py +3 -0
  2. kubesift/analysis/__init__.py +3 -0
  3. kubesift/analysis/changes.py +165 -0
  4. kubesift/analysis/correlate.py +160 -0
  5. kubesift/analysis/describe.py +83 -0
  6. kubesift/analysis/engine.py +362 -0
  7. kubesift/analysis/llm.py +152 -0
  8. kubesift/checks/__init__.py +17 -0
  9. kubesift/checks/base.py +150 -0
  10. kubesift/checks/config_refs.py +84 -0
  11. kubesift/checks/events.py +77 -0
  12. kubesift/checks/hygiene.py +264 -0
  13. kubesift/checks/lifecycle.py +202 -0
  14. kubesift/checks/network.py +325 -0
  15. kubesift/checks/nodes.py +183 -0
  16. kubesift/checks/operations.py +319 -0
  17. kubesift/checks/pods.py +485 -0
  18. kubesift/checks/storage.py +72 -0
  19. kubesift/checks/workloads.py +201 -0
  20. kubesift/cli.py +874 -0
  21. kubesift/cluster/__init__.py +4 -0
  22. kubesift/cluster/client.py +141 -0
  23. kubesift/cluster/snapshot.py +513 -0
  24. kubesift/config.py +134 -0
  25. kubesift/cost.py +150 -0
  26. kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
  27. kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
  28. kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
  29. kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
  30. kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
  31. kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
  32. kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
  33. kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
  34. kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
  35. kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
  36. kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
  37. kubesift/drill_scenarios/__init__.py +1 -0
  38. kubesift/drills.py +122 -0
  39. kubesift/knowledge.py +51 -0
  40. kubesift/leader.py +145 -0
  41. kubesift/lint.py +89 -0
  42. kubesift/mcp_server.py +193 -0
  43. kubesift/models.py +144 -0
  44. kubesift/notify.py +71 -0
  45. kubesift/postmortem.py +152 -0
  46. kubesift/preflight.py +435 -0
  47. kubesift/redact.py +38 -0
  48. kubesift/remediate.py +99 -0
  49. kubesift/report.py +193 -0
  50. kubesift/runbooks.py +58 -0
  51. kubesift/server/__init__.py +0 -0
  52. kubesift/server/app.py +480 -0
  53. kubesift/server/static/index.html +152 -0
  54. kubesift/slack.py +106 -0
  55. kubesift/store.py +219 -0
  56. kubesift/timetravel.py +156 -0
  57. kubesift/upgrade.py +51 -0
  58. kubesift/util.py +174 -0
  59. kubesift-2.0.0.dist-info/METADATA +192 -0
  60. kubesift-2.0.0.dist-info/RECORD +63 -0
  61. kubesift-2.0.0.dist-info/WHEEL +4 -0
  62. kubesift-2.0.0.dist-info/entry_points.txt +3 -0
  63. kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,325 @@
1
+ """Services, endpoints, Ingress, Gateway API, DNS and NetworkPolicies."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterator
6
+ from datetime import timedelta
7
+ from typing import Any
8
+
9
+ from kubesift.checks.base import Context, check
10
+ from kubesift.models import Category, Finding, ResourceRef, Severity
11
+ from kubesift.util import age, containers_of, human_duration, label_selector_matches, meta, selector_matches
12
+
13
+
14
+ def _pods_for(ctx: Context, ns: str | None, selector: dict[str, str]) -> list[dict[str, Any]]:
15
+ return [p for p in ctx.snap.in_namespace("pods", ns) if selector_matches(selector, meta(p).get("labels"))
16
+ and (p.get("status") or {}).get("phase") not in ("Succeeded", "Failed")]
17
+
18
+
19
+ def _ready(pod: dict[str, Any]) -> bool:
20
+ return any(c.get("type") == "Ready" and c.get("status") == "True"
21
+ for c in (pod.get("status") or {}).get("conditions") or [])
22
+
23
+
24
+ def _ready_endpoints(ctx: Context, svc: dict[str, Any]) -> int | None:
25
+ if not ctx.snap.can_read("endpointslices"):
26
+ return None
27
+ ns, name = meta(svc).get("namespace"), meta(svc).get("name")
28
+ count = 0
29
+ for es in ctx.snap.in_namespace("endpointslices", ns):
30
+ if (meta(es).get("labels") or {}).get("kubernetes.io/service-name") != name:
31
+ continue
32
+ for ep in es.get("endpoints") or []:
33
+ if (ep.get("conditions") or {}).get("ready") is not False:
34
+ count += 1
35
+ return count
36
+
37
+
38
+ @check("service-no-endpoints", "Service has no ready backends", Category.NETWORK, Severity.HIGH)
39
+ def service_no_endpoints(ctx: Context) -> Iterator[Finding]:
40
+ """Traffic to the Service fails: its selector matches no ready pods."""
41
+ for svc in ctx.snap.all("services"):
42
+ spec = svc.get("spec") or {}
43
+ selector = spec.get("selector")
44
+ if not selector or spec.get("type") == "ExternalName":
45
+ continue
46
+ if (a := age(meta(svc).get("creationTimestamp"))) is not None and a < timedelta(minutes=2):
47
+ continue
48
+ ns, name = meta(svc).get("namespace"), meta(svc).get("name", "")
49
+ ready_eps = _ready_endpoints(ctx, svc)
50
+ pods = _pods_for(ctx, ns, selector)
51
+ if ready_eps is None:
52
+ ready_eps = sum(1 for p in pods if _ready(p))
53
+ if ready_eps > 0:
54
+ continue
55
+ sel = ",".join(f"{k}={v}" for k, v in selector.items())
56
+ if not pods:
57
+ near = _closest_labels(ctx, ns, selector)
58
+ msg = f"Selector {sel} matches no pods in {ns}, so the Service has no endpoints."
59
+ evidence = [f"selector: {sel}"] + ([f"pods with similar labels: {near}"] if near else [])
60
+ fix = "Make the Service selector equal the pod template labels (a typo or a renamed label is the usual cause)."
61
+ sev = Severity.CRITICAL
62
+ else:
63
+ msg = f"{len(pods)} pod(s) match {sel} but none is ready, so the Service sends no traffic."
64
+ evidence = [f"{meta(p).get('name')}: phase={(p.get('status') or {}).get('phase')}" for p in pods[:5]]
65
+ fix = "Fix the pods' readiness (see the pod findings)."
66
+ sev = Severity.HIGH
67
+ yield ctx.finding(
68
+ "service-no-endpoints", svc, msg, severity=sev, evidence=evidence, remediation=fix,
69
+ commands=[f"kubectl get endpointslices -n {ns} -l kubernetes.io/service-name={name}",
70
+ f"kubectl get pods -n {ns} --show-labels"],
71
+ )
72
+
73
+
74
+ def _closest_labels(ctx: Context, ns: str | None, selector: dict[str, str]) -> str:
75
+ from difflib import SequenceMatcher
76
+
77
+ best, best_score = None, 0.5
78
+ for p in ctx.snap.in_namespace("pods", ns):
79
+ labels = meta(p).get("labels") or {}
80
+ score = sum(SequenceMatcher(None, str(v), str(labels[k])).ratio() for k, v in selector.items() if k in labels)
81
+ score /= max(len(selector), 1)
82
+ if score > best_score:
83
+ best, best_score = labels, score
84
+ if not best:
85
+ return ""
86
+ return ",".join(f"{k}={v}" for k, v in list(best.items())[:6])
87
+
88
+
89
+ @check("service-port-mismatch", "Service targetPort does not match any container port", Category.NETWORK,
90
+ Severity.HIGH, live_only=False)
91
+ def service_port_mismatch(ctx: Context) -> Iterator[Finding]:
92
+ """targetPort refers to a named or numbered port the selected pods do not expose."""
93
+ for svc in ctx.snap.all("services"):
94
+ selector = (svc.get("spec") or {}).get("selector")
95
+ if not selector:
96
+ continue
97
+ ns = meta(svc).get("namespace")
98
+ specs = [p.get("spec") or {} for p in _pods_for(ctx, ns, selector)] if ctx.snap.live else []
99
+ if not specs:
100
+ for w in ctx.snap.workloads():
101
+ tmpl = ((w.get("spec") or {}).get("template") or {})
102
+ if meta(w).get("namespace") == ns and selector_matches(selector, (tmpl.get("metadata") or {}).get("labels")):
103
+ specs.append(tmpl.get("spec") or {})
104
+ if not specs:
105
+ continue
106
+ names, numbers = set(), set()
107
+ for s in specs:
108
+ for c in containers_of(s):
109
+ for p in c.get("ports") or []:
110
+ if p.get("name"):
111
+ names.add(p["name"])
112
+ numbers.add(p.get("containerPort"))
113
+ if not numbers:
114
+ continue # ports not declared; nothing to compare
115
+ for port in (svc.get("spec") or {}).get("ports") or []:
116
+ target = port.get("targetPort", port.get("port"))
117
+ bad = (isinstance(target, str) and not target.isdigit() and target not in names) or (
118
+ (isinstance(target, int) or str(target).isdigit()) and int(target) not in numbers)
119
+ if bad:
120
+ yield ctx.finding(
121
+ "service-port-mismatch", svc,
122
+ f"Port {port.get('port')} forwards to targetPort {target}, but the pods expose "
123
+ f"{sorted(n for n in numbers if n)}{' / ' + ', '.join(sorted(names)) if names else ''}.",
124
+ remediation="Set targetPort to the port the container actually listens on.",
125
+ evidence=[f"service port {port.get('name') or ''} {port.get('port')} -> {target}"],
126
+ )
127
+
128
+
129
+ @check("ingress-backend-missing", "Ingress points to a missing Service or port", Category.NETWORK, Severity.HIGH)
130
+ def ingress_backend(ctx: Context) -> Iterator[Finding]:
131
+ """An Ingress rule routes to a Service (or port) that does not exist, which returns 503/404."""
132
+ for ing in ctx.snap.all("ingresses"):
133
+ ns = meta(ing).get("namespace")
134
+ spec = ing.get("spec") or {}
135
+ backends = []
136
+ if spec.get("defaultBackend"):
137
+ backends.append(("default", spec["defaultBackend"]))
138
+ for rule in spec.get("rules") or []:
139
+ for path in (rule.get("http") or {}).get("paths") or []:
140
+ backends.append((f"{rule.get('host', '*')}{path.get('path', '/')}", path.get("backend") or {}))
141
+ problems = []
142
+ for where, b in backends:
143
+ svc_ref = b.get("service")
144
+ if not svc_ref:
145
+ continue
146
+ svc = ctx.snap.get("services", ns, svc_ref.get("name", ""))
147
+ if svc is None:
148
+ problems.append(f"{where} -> Service '{svc_ref.get('name')}' does not exist")
149
+ continue
150
+ port = svc_ref.get("port") or {}
151
+ ports = (svc.get("spec") or {}).get("ports") or []
152
+ if port.get("number") and all(p.get("port") != port["number"] for p in ports):
153
+ problems.append(f"{where} -> Service '{svc_ref.get('name')}' has no port {port['number']}")
154
+ if port.get("name") and all(p.get("name") != port["name"] for p in ports):
155
+ problems.append(f"{where} -> Service '{svc_ref.get('name')}' has no port named {port['name']}")
156
+ for tls in spec.get("tls") or []:
157
+ secret = tls.get("secretName")
158
+ if secret and ctx.snap.can_read("secrets") and ctx.snap.get("secrets", ns, secret) is None:
159
+ problems.append(f"TLS secret '{secret}' for {', '.join(tls.get('hosts') or [])} does not exist")
160
+ if problems:
161
+ yield ctx.finding(
162
+ "ingress-backend-missing", ing, problems[0] + ".", evidence=problems[:6],
163
+ remediation="Point the Ingress at an existing Service and port in the same namespace "
164
+ "(or create the TLS Secret, e.g. via cert-manager).",
165
+ )
166
+
167
+
168
+ INGRESS_NGINX_IMAGES = ("ingress-nginx/controller", "k8s.gcr.io/ingress-nginx", "registry.k8s.io/ingress-nginx")
169
+
170
+
171
+ @check("ingress-nginx-retired", "ingress-nginx controller is retired", Category.UPGRADE, Severity.HIGH, risk=True)
172
+ def ingress_nginx_retired(ctx: Context) -> Iterator[Finding]:
173
+ """The community ingress-nginx project was retired in March 2026: no more fixes or security patches."""
174
+ controllers = []
175
+ for pod in ctx.snap.all("pods"):
176
+ for c in containers_of(pod.get("spec")):
177
+ if any(s in c.get("image", "") for s in INGRESS_NGINX_IMAGES):
178
+ controllers.append((pod, c.get("image")))
179
+ break
180
+ classes = [ic for ic in ctx.snap.all("ingressclasses")
181
+ if (ic.get("spec") or {}).get("controller") == "k8s.io/ingress-nginx"]
182
+ if not controllers and not classes:
183
+ return
184
+ class_names = {meta(ic).get("name") for ic in classes} or {"nginx"}
185
+ using = [i for i in ctx.snap.all("ingresses")
186
+ if (i.get("spec") or {}).get("ingressClassName") in class_names
187
+ or (meta(i).get("annotations") or {}).get("kubernetes.io/ingress.class") in class_names]
188
+ annotated = sum(1 for i in using if any(k.startswith("nginx.ingress.kubernetes.io/")
189
+ for k in (meta(i).get("annotations") or {})))
190
+ image = controllers[0][1] if controllers else "unknown"
191
+ ref = ResourceRef(kind="IngressClass", name=sorted(class_names)[0]) if classes else ResourceRef(
192
+ kind="Pod", name=meta(controllers[0][0]).get("name", ""), namespace=meta(controllers[0][0]).get("namespace"))
193
+ yield ctx.finding(
194
+ "ingress-nginx-retired", ref,
195
+ f"The cluster runs the community ingress-nginx controller ({image}), which was retired in March 2026 "
196
+ f"and gets no security fixes. {len(using)} Ingress object(s) use it, {annotated} with nginx-specific annotations "
197
+ "that need translating.",
198
+ evidence=[f"controller image: {image}", f"ingresses using it: {len(using)}",
199
+ f"with nginx.ingress.kubernetes.io/* annotations: {annotated}"],
200
+ remediation="Plan a migration to Gateway API (or another maintained controller). Convert manifests with "
201
+ "ingress2gateway, run both paths side by side, then shift traffic gradually.",
202
+ commands=["kubectl get ingress -A -o wide"],
203
+ docs="https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/",
204
+ )
205
+
206
+
207
+ @check("ingress-class-missing", "Ingress uses a class with no controller", Category.NETWORK, Severity.HIGH)
208
+ def ingress_class_missing(ctx: Context) -> Iterator[Finding]:
209
+ """ingressClassName names an IngressClass that does not exist, so no controller serves it."""
210
+ if not ctx.snap.can_read("ingressclasses") or not ctx.snap.all("ingressclasses"):
211
+ return
212
+ names = {meta(ic).get("name") for ic in ctx.snap.all("ingressclasses")}
213
+ for ing in ctx.snap.all("ingresses"):
214
+ cls = (ing.get("spec") or {}).get("ingressClassName")
215
+ if cls and cls not in names:
216
+ yield ctx.finding(
217
+ "ingress-class-missing", ing,
218
+ f"ingressClassName '{cls}' does not exist (available: {', '.join(sorted(n for n in names if n))}). "
219
+ "No controller will route this Ingress.",
220
+ remediation="Use one of the installed IngressClasses or install the matching controller.",
221
+ )
222
+
223
+
224
+ @check("loadbalancer-pending", "LoadBalancer has no external address", Category.NETWORK, Severity.HIGH)
225
+ def lb_pending(ctx: Context) -> Iterator[Finding]:
226
+ """A type=LoadBalancer Service has waited 5+ minutes without an IP/hostname."""
227
+ for svc in ctx.snap.all("services"):
228
+ spec = svc.get("spec") or {}
229
+ if spec.get("type") != "LoadBalancer":
230
+ continue
231
+ if ((svc.get("status") or {}).get("loadBalancer") or {}).get("ingress"):
232
+ continue
233
+ a = age(meta(svc).get("creationTimestamp"))
234
+ if a is not None and a < timedelta(minutes=5):
235
+ continue
236
+ events = [e.get("message", "") for e in ctx.snap.events_for("Service", meta(svc).get("namespace"),
237
+ meta(svc).get("name", ""))
238
+ if e.get("type") == "Warning"]
239
+ yield ctx.finding(
240
+ "loadbalancer-pending", svc,
241
+ f"No external address after {human_duration(a)}." + (f" Cloud controller says: {events[0][:200]}" if events else
242
+ " No cloud load-balancer controller may be installed (common on kind/bare metal)."),
243
+ evidence=events[:3],
244
+ remediation="Check the cloud controller/load-balancer controller logs, subnet tags and quotas; on bare metal install MetalLB or use an Ingress.",
245
+ )
246
+
247
+
248
+ @check("dns-unhealthy", "Cluster DNS is unhealthy", Category.PLATFORM, Severity.CRITICAL)
249
+ def dns_unhealthy(ctx: Context) -> Iterator[Finding]:
250
+ """CoreDNS/kube-dns pods are not ready: every service-name lookup in the cluster may fail."""
251
+ pods = [p for p in ctx.snap.in_namespace("pods", "kube-system")
252
+ if (meta(p).get("labels") or {}).get("k8s-app") == "kube-dns"]
253
+ if not pods:
254
+ return
255
+ ready = [p for p in pods if _ready(p)]
256
+ if len(ready) == len(pods):
257
+ return
258
+ yield ctx.finding(
259
+ "dns-unhealthy", ResourceRef(kind="Deployment", name="coredns", namespace="kube-system"),
260
+ f"{len(ready)}/{len(pods)} DNS pods are ready." + (" DNS is down for the whole cluster." if not ready else
261
+ " Lookups may intermittently fail."),
262
+ severity=Severity.CRITICAL if not ready else Severity.HIGH,
263
+ evidence=[f"{meta(p).get('name')}: {(p.get('status') or {}).get('phase')}" for p in pods],
264
+ commands=["kubectl -n kube-system logs -l k8s-app=kube-dns --tail=50",
265
+ "kubectl -n kube-system get configmap coredns -o yaml"],
266
+ remediation="Check the CoreDNS logs (loop detection, upstream resolver errors, OOM) and the coredns ConfigMap.",
267
+ )
268
+
269
+
270
+ @check("netpol-blocks-dns", "NetworkPolicy blocks DNS", Category.NETWORK, Severity.HIGH, live_only=False)
271
+ def netpol_blocks_dns(ctx: Context) -> Iterator[Finding]:
272
+ """An egress NetworkPolicy selects pods but does not allow port 53, so name lookups time out."""
273
+ for np in ctx.snap.all("networkpolicies"):
274
+ spec = np.get("spec") or {}
275
+ if "Egress" not in (spec.get("policyTypes") or []):
276
+ continue
277
+ egress = spec.get("egress") or []
278
+ allows_dns = False
279
+ for rule in egress:
280
+ ports, peers = rule.get("ports"), rule.get("to")
281
+ if ports and any(str(p.get("port")) in ("53", "dns", "dns-tcp") for p in ports) or not ports and (not peers or any(_peer_maybe_dns(peer) for peer in peers)):
282
+ allows_dns = True
283
+ if allows_dns:
284
+ continue
285
+ ns = meta(np).get("namespace")
286
+ selected = [p for p in ctx.snap.in_namespace("pods", ns)
287
+ if label_selector_matches(spec.get("podSelector") or {}, meta(p).get("labels"))]
288
+ if ctx.snap.live and not selected:
289
+ continue
290
+ yield ctx.finding(
291
+ "netpol-blocks-dns", np,
292
+ f"This policy restricts egress for {len(selected) if ctx.snap.live else 'its'} pod(s) but allows no traffic to port 53; "
293
+ "DNS lookups from those pods will fail.",
294
+ remediation="Add an egress rule to kube-system pods with label k8s-app=kube-dns on UDP and TCP port 53.",
295
+ evidence=[f"podSelector: {spec.get('podSelector')}"],
296
+ )
297
+
298
+
299
+ def _peer_maybe_dns(peer: dict[str, Any]) -> bool:
300
+ if peer.get("ipBlock"):
301
+ return True
302
+ ns_sel = (peer.get("namespaceSelector") or {}).get("matchLabels") or {}
303
+ return peer.get("namespaceSelector") == {} or ns_sel.get("kubernetes.io/metadata.name") == "kube-system"
304
+
305
+
306
+ @check("gateway-not-accepted", "Gateway or HTTPRoute not accepted", Category.NETWORK, Severity.HIGH)
307
+ def gateway_status(ctx: Context) -> Iterator[Finding]:
308
+ """Gateway API objects whose Accepted/Programmed/ResolvedRefs condition is False."""
309
+ for gw in ctx.snap.all("gateways"):
310
+ for c in (gw.get("status") or {}).get("conditions") or []:
311
+ if c.get("type") in ("Accepted", "Programmed") and c.get("status") == "False":
312
+ yield ctx.finding("gateway-not-accepted", gw,
313
+ f"Gateway {c.get('type')}=False ({c.get('reason')}): {c.get('message', '')[:200]}")
314
+ break
315
+ for route in ctx.snap.all("httproutes"):
316
+ for parent in (route.get("status") or {}).get("parents") or []:
317
+ bad = next((c for c in parent.get("conditions") or []
318
+ if c.get("type") in ("Accepted", "ResolvedRefs") and c.get("status") == "False"), None)
319
+ if bad:
320
+ yield ctx.finding("gateway-not-accepted", route,
321
+ f"HTTPRoute {bad.get('type')}=False on {(parent.get('parentRef') or {}).get('name')} "
322
+ f"({bad.get('reason')}): {bad.get('message', '')[:200]}",
323
+ remediation="Check the parentRef name/namespace, allowedRoutes on the Gateway listener, "
324
+ "and that backendRefs exist (cross-namespace refs need a ReferenceGrant).")
325
+ break
@@ -0,0 +1,183 @@
1
+ """Node health, capacity and version skew."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Iterator
7
+ from typing import Any
8
+
9
+ from kubesift.checks.base import Context, check
10
+ from kubesift.models import Category, Finding, Severity
11
+ from kubesift.util import age, containers_of, human_duration, meta, parse_quantity, parse_time
12
+
13
+
14
+ def minor(version: str | None) -> int | None:
15
+ m = re.match(r"v?1\.(\d+)", version or "")
16
+ return int(m.group(1)) if m else None
17
+
18
+
19
+ def _cond(node: dict[str, Any], ctype: str) -> dict[str, Any]:
20
+ return next((c for c in (node.get("status") or {}).get("conditions") or [] if c.get("type") == ctype), {})
21
+
22
+
23
+ def pods_on(ctx: Context, node: str) -> list[dict[str, Any]]:
24
+ return [p for p in ctx.snap.all("pods") if (p.get("spec") or {}).get("nodeName") == node
25
+ and (p.get("status") or {}).get("phase") not in ("Succeeded", "Failed")]
26
+
27
+
28
+ @check("node-not-ready", "Node is not ready", Category.NODE, Severity.CRITICAL)
29
+ def node_not_ready(ctx: Context) -> Iterator[Finding]:
30
+ """The kubelet stopped reporting or reports itself unhealthy; pods on it are being lost."""
31
+ for node in ctx.snap.all("nodes"):
32
+ ready = _cond(node, "Ready")
33
+ if not ready or ready.get("status") == "True":
34
+ continue
35
+ name = meta(node).get("name", "")
36
+ affected = pods_on(ctx, name)
37
+ yield ctx.finding(
38
+ "node-not-ready", node,
39
+ f"Node has been NotReady for {human_duration(age(ready.get('lastTransitionTime')))} "
40
+ f"({ready.get('reason')}: {ready.get('message', '')[:160]}). {len(affected)} pod(s) run on it.",
41
+ evidence=[f"Ready={ready.get('status')} reason={ready.get('reason')}"] + [
42
+ f"{c.get('type')}={c.get('status')}" for c in (node.get("status") or {}).get("conditions") or []
43
+ if c.get("type") != "Ready" and c.get("status") == "True"],
44
+ remediation="Check the node in your cloud console (instance stopped, disk full, network). On the node: "
45
+ "`systemctl status kubelet containerd` and `journalctl -u kubelet`. Replace it if it does not recover.",
46
+ commands=[f"kubectl describe node {name}", f"kubectl get pods -A -o wide --field-selector spec.nodeName={name}"],
47
+ since=parse_time(ready.get("lastTransitionTime")),
48
+ node=name,
49
+ )
50
+
51
+
52
+ @check("node-pressure", "Node is under resource pressure", Category.NODE, Severity.HIGH)
53
+ def node_pressure(ctx: Context) -> Iterator[Finding]:
54
+ """MemoryPressure / DiskPressure / PIDPressure: the kubelet is evicting pods."""
55
+ for node in ctx.snap.all("nodes"):
56
+ for ctype, advice in (
57
+ ("MemoryPressure", "Pods use more memory than the node has. Set memory requests near real usage."),
58
+ ("DiskPressure", "Node disk is almost full: images, container logs or emptyDir. Prune images, cap log size, "
59
+ "add ephemeral-storage limits."),
60
+ ("PIDPressure", "A process is forking too many processes/threads."),
61
+ ):
62
+ c = _cond(node, ctype)
63
+ if c.get("status") == "True":
64
+ name = meta(node).get("name", "")
65
+ yield ctx.finding(
66
+ "node-pressure", node, f"{ctype}: {c.get('message', '')[:200]}",
67
+ title=f"Node has {ctype}", evidence=[f"since {c.get('lastTransitionTime')}"],
68
+ remediation=advice, commands=[f"kubectl describe node {name}", "kubectl top pods -A --sort-by=memory | head"],
69
+ node=name, since=parse_time(c.get("lastTransitionTime")),
70
+ )
71
+
72
+
73
+ @check("node-cordoned", "Node is cordoned", Category.NODE, Severity.LOW, risk=True)
74
+ def node_cordoned(ctx: Context) -> Iterator[Finding]:
75
+ """spec.unschedulable is true; often left behind after maintenance."""
76
+ for node in ctx.snap.all("nodes"):
77
+ if (node.get("spec") or {}).get("unschedulable"):
78
+ name = meta(node).get("name", "")
79
+ yield ctx.finding("node-cordoned", node, "New pods cannot be scheduled on this node (cordoned).",
80
+ remediation="Uncordon it if maintenance is finished.",
81
+ commands=[f"kubectl uncordon {name}"], node=name)
82
+
83
+
84
+ @check("node-overcommitted", "Node requests are near capacity", Category.RESOURCES, Severity.MEDIUM)
85
+ def node_overcommitted(ctx: Context) -> Iterator[Finding]:
86
+ """CPU or memory requests above 95% of allocatable: new or rescheduled pods will not fit."""
87
+ nodes = ctx.snap.all("nodes")
88
+ full = []
89
+ for node in nodes:
90
+ name = meta(node).get("name", "")
91
+ alloc = (node.get("status") or {}).get("allocatable") or {}
92
+ for res in ("cpu", "memory"):
93
+ cap = parse_quantity(alloc.get(res))
94
+ if not cap:
95
+ continue
96
+ req = sum(parse_quantity(((c.get("resources") or {}).get("requests") or {}).get(res))
97
+ for p in pods_on(ctx, name) for c in containers_of(p.get("spec")))
98
+ if req / cap >= 0.95:
99
+ full.append((node, res, req / cap))
100
+ if nodes and len({id(n) for n, _, _ in full}) == len(nodes):
101
+ node, res, ratio = full[0]
102
+ yield ctx.finding(
103
+ "node-overcommitted", node,
104
+ f"Every node has more than 95% of its {res} requested; any new pod will stay Pending.",
105
+ severity=Severity.HIGH,
106
+ evidence=[f"{meta(n).get('name')}: {r} {ratio:.0%}" for n, r, ratio in full[:8]],
107
+ remediation="Add capacity (or enable the cluster autoscaler), or right-size over-requested workloads (`kubesift cost`).",
108
+ )
109
+
110
+
111
+ @check("node-version-skew", "Kubelet version too far behind the control plane", Category.UPGRADE, Severity.MEDIUM, risk=True)
112
+ def version_skew(ctx: Context) -> Iterator[Finding]:
113
+ """Kubelets may be at most 3 minor versions older than the API server."""
114
+ server = minor(ctx.snap.server_version)
115
+ if server is None:
116
+ return
117
+ for node in ctx.snap.all("nodes"):
118
+ info = (node.get("status") or {}).get("nodeInfo") or {}
119
+ kubelet = minor(info.get("kubeletVersion"))
120
+ if kubelet is None:
121
+ continue
122
+ if kubelet > server or server - kubelet > 3:
123
+ yield ctx.finding(
124
+ "node-version-skew", node,
125
+ f"Kubelet {info.get('kubeletVersion')} vs API server {ctx.snap.server_version}: outside the supported skew.",
126
+ severity=Severity.HIGH, remediation="Upgrade (or replace) the node before upgrading the control plane further.",
127
+ node=meta(node).get("name"),
128
+ )
129
+ elif server - kubelet == 3:
130
+ yield ctx.finding(
131
+ "node-version-skew", node,
132
+ f"Kubelet {info.get('kubeletVersion')} is 3 minors behind {ctx.snap.server_version}; "
133
+ "the next control-plane upgrade will break support for this node.",
134
+ remediation="Upgrade node pools before the next control-plane upgrade.", node=meta(node).get("name"),
135
+ )
136
+
137
+
138
+ @check("node-containerd-1x", "Node runs containerd 1.x", Category.UPGRADE, Severity.MEDIUM, risk=True)
139
+ def containerd_1x(ctx: Context) -> Iterator[Finding]:
140
+ """Kubernetes 1.36 dropped support for containerd 1.x; nodes must move to containerd 2.x."""
141
+ old = [n for n in ctx.snap.all("nodes")
142
+ if re.match(r"containerd://1\.", ((n.get("status") or {}).get("nodeInfo") or {}).get("containerRuntimeVersion", ""))]
143
+ if not old:
144
+ return
145
+ runtime = ((old[0].get("status") or {}).get("nodeInfo") or {}).get("containerRuntimeVersion")
146
+ yield ctx.finding(
147
+ "node-containerd-1x", old[0],
148
+ f"{len(old)} node(s) run {runtime}. Kubernetes 1.36+ requires containerd 2.x; upgrade the node image first.",
149
+ severity=Severity.HIGH if (minor(ctx.snap.server_version) or 0) >= 35 else Severity.MEDIUM,
150
+ evidence=[meta(n).get("name", "") for n in old[:10]],
151
+ remediation="Roll node pools to an OS image with containerd 2.x (check cgroup v2 at the same time).",
152
+ )
153
+
154
+
155
+ @check("gpu-idle", "GPU capacity is idle", Category.COST, Severity.MEDIUM, risk=True)
156
+ def gpu_idle(ctx: Context) -> Iterator[Finding]:
157
+ """GPU nodes with no pod requesting their GPUs. GPUs are the most expensive idle resource."""
158
+ gpu_keys = ("nvidia.com/gpu", "amd.com/gpu", "gpu.intel.com/i915")
159
+ idle = []
160
+ total = 0.0
161
+ for node in ctx.snap.all("nodes"):
162
+ alloc = (node.get("status") or {}).get("allocatable") or {}
163
+ gpus = sum(parse_quantity(alloc.get(k)) for k in gpu_keys)
164
+ if not gpus:
165
+ continue
166
+ total += gpus
167
+ name = meta(node).get("name", "")
168
+ used = sum(parse_quantity(((c.get("resources") or {}).get("limits") or {}).get(k))
169
+ for p in pods_on(ctx, name) for c in containers_of(p.get("spec")) for k in gpu_keys)
170
+ if used < gpus:
171
+ idle.append((name, gpus - used, gpus))
172
+ if not idle:
173
+ return
174
+ free = sum(i[1] for i in idle)
175
+ monthly = free * ctx.settings.cost_gpu_hour * 730
176
+ yield ctx.finding(
177
+ "gpu-idle", ctx.snap.all("nodes")[0] if len(idle) > 1 else next(n for n in ctx.snap.all("nodes") if meta(n).get("name") == idle[0][0]),
178
+ f"{free:.0f} of {total:.0f} GPUs are not requested by any pod (about ${monthly:,.0f}/month at "
179
+ f"${ctx.settings.cost_gpu_hour}/GPU-hour).",
180
+ evidence=[f"{n}: {f:.0f}/{t:.0f} GPUs free" for n, f, t in idle[:10]],
181
+ remediation="Scale GPU node pools to zero when idle (cluster autoscaler / Karpenter), share GPUs with time-slicing or MIG, "
182
+ "and move notebooks and batch jobs to on-demand GPU pools.",
183
+ )