kubesift 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift/__init__.py +3 -0
- kubesift/analysis/__init__.py +3 -0
- kubesift/analysis/changes.py +165 -0
- kubesift/analysis/correlate.py +160 -0
- kubesift/analysis/describe.py +83 -0
- kubesift/analysis/engine.py +362 -0
- kubesift/analysis/llm.py +152 -0
- kubesift/checks/__init__.py +17 -0
- kubesift/checks/base.py +150 -0
- kubesift/checks/config_refs.py +84 -0
- kubesift/checks/events.py +77 -0
- kubesift/checks/hygiene.py +264 -0
- kubesift/checks/lifecycle.py +202 -0
- kubesift/checks/network.py +325 -0
- kubesift/checks/nodes.py +183 -0
- kubesift/checks/operations.py +319 -0
- kubesift/checks/pods.py +485 -0
- kubesift/checks/storage.py +72 -0
- kubesift/checks/workloads.py +201 -0
- kubesift/cli.py +874 -0
- kubesift/cluster/__init__.py +4 -0
- kubesift/cluster/client.py +141 -0
- kubesift/cluster/snapshot.py +513 -0
- kubesift/config.py +134 -0
- kubesift/cost.py +150 -0
- kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
- kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
- kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
- kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
- kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
- kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
- kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
- kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
- kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
- kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
- kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
- kubesift/drill_scenarios/__init__.py +1 -0
- kubesift/drills.py +122 -0
- kubesift/knowledge.py +51 -0
- kubesift/leader.py +145 -0
- kubesift/lint.py +89 -0
- kubesift/mcp_server.py +193 -0
- kubesift/models.py +144 -0
- kubesift/notify.py +71 -0
- kubesift/postmortem.py +152 -0
- kubesift/preflight.py +435 -0
- kubesift/redact.py +38 -0
- kubesift/remediate.py +99 -0
- kubesift/report.py +193 -0
- kubesift/runbooks.py +58 -0
- kubesift/server/__init__.py +0 -0
- kubesift/server/app.py +480 -0
- kubesift/server/static/index.html +152 -0
- kubesift/slack.py +106 -0
- kubesift/store.py +219 -0
- kubesift/timetravel.py +156 -0
- kubesift/upgrade.py +51 -0
- kubesift/util.py +174 -0
- kubesift-2.0.0.dist-info/METADATA +192 -0
- kubesift-2.0.0.dist-info/RECORD +63 -0
- kubesift-2.0.0.dist-info/WHEEL +4 -0
- kubesift-2.0.0.dist-info/entry_points.txt +3 -0
- kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Services, endpoints, Ingress, Gateway API, DNS and NetworkPolicies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from datetime import timedelta
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from kubesift.checks.base import Context, check
|
|
10
|
+
from kubesift.models import Category, Finding, ResourceRef, Severity
|
|
11
|
+
from kubesift.util import age, containers_of, human_duration, label_selector_matches, meta, selector_matches
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _pods_for(ctx: Context, ns: str | None, selector: dict[str, str]) -> list[dict[str, Any]]:
|
|
15
|
+
return [p for p in ctx.snap.in_namespace("pods", ns) if selector_matches(selector, meta(p).get("labels"))
|
|
16
|
+
and (p.get("status") or {}).get("phase") not in ("Succeeded", "Failed")]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _ready(pod: dict[str, Any]) -> bool:
|
|
20
|
+
return any(c.get("type") == "Ready" and c.get("status") == "True"
|
|
21
|
+
for c in (pod.get("status") or {}).get("conditions") or [])
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _ready_endpoints(ctx: Context, svc: dict[str, Any]) -> int | None:
|
|
25
|
+
if not ctx.snap.can_read("endpointslices"):
|
|
26
|
+
return None
|
|
27
|
+
ns, name = meta(svc).get("namespace"), meta(svc).get("name")
|
|
28
|
+
count = 0
|
|
29
|
+
for es in ctx.snap.in_namespace("endpointslices", ns):
|
|
30
|
+
if (meta(es).get("labels") or {}).get("kubernetes.io/service-name") != name:
|
|
31
|
+
continue
|
|
32
|
+
for ep in es.get("endpoints") or []:
|
|
33
|
+
if (ep.get("conditions") or {}).get("ready") is not False:
|
|
34
|
+
count += 1
|
|
35
|
+
return count
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@check("service-no-endpoints", "Service has no ready backends", Category.NETWORK, Severity.HIGH)
|
|
39
|
+
def service_no_endpoints(ctx: Context) -> Iterator[Finding]:
|
|
40
|
+
"""Traffic to the Service fails: its selector matches no ready pods."""
|
|
41
|
+
for svc in ctx.snap.all("services"):
|
|
42
|
+
spec = svc.get("spec") or {}
|
|
43
|
+
selector = spec.get("selector")
|
|
44
|
+
if not selector or spec.get("type") == "ExternalName":
|
|
45
|
+
continue
|
|
46
|
+
if (a := age(meta(svc).get("creationTimestamp"))) is not None and a < timedelta(minutes=2):
|
|
47
|
+
continue
|
|
48
|
+
ns, name = meta(svc).get("namespace"), meta(svc).get("name", "")
|
|
49
|
+
ready_eps = _ready_endpoints(ctx, svc)
|
|
50
|
+
pods = _pods_for(ctx, ns, selector)
|
|
51
|
+
if ready_eps is None:
|
|
52
|
+
ready_eps = sum(1 for p in pods if _ready(p))
|
|
53
|
+
if ready_eps > 0:
|
|
54
|
+
continue
|
|
55
|
+
sel = ",".join(f"{k}={v}" for k, v in selector.items())
|
|
56
|
+
if not pods:
|
|
57
|
+
near = _closest_labels(ctx, ns, selector)
|
|
58
|
+
msg = f"Selector {sel} matches no pods in {ns}, so the Service has no endpoints."
|
|
59
|
+
evidence = [f"selector: {sel}"] + ([f"pods with similar labels: {near}"] if near else [])
|
|
60
|
+
fix = "Make the Service selector equal the pod template labels (a typo or a renamed label is the usual cause)."
|
|
61
|
+
sev = Severity.CRITICAL
|
|
62
|
+
else:
|
|
63
|
+
msg = f"{len(pods)} pod(s) match {sel} but none is ready, so the Service sends no traffic."
|
|
64
|
+
evidence = [f"{meta(p).get('name')}: phase={(p.get('status') or {}).get('phase')}" for p in pods[:5]]
|
|
65
|
+
fix = "Fix the pods' readiness (see the pod findings)."
|
|
66
|
+
sev = Severity.HIGH
|
|
67
|
+
yield ctx.finding(
|
|
68
|
+
"service-no-endpoints", svc, msg, severity=sev, evidence=evidence, remediation=fix,
|
|
69
|
+
commands=[f"kubectl get endpointslices -n {ns} -l kubernetes.io/service-name={name}",
|
|
70
|
+
f"kubectl get pods -n {ns} --show-labels"],
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _closest_labels(ctx: Context, ns: str | None, selector: dict[str, str]) -> str:
|
|
75
|
+
from difflib import SequenceMatcher
|
|
76
|
+
|
|
77
|
+
best, best_score = None, 0.5
|
|
78
|
+
for p in ctx.snap.in_namespace("pods", ns):
|
|
79
|
+
labels = meta(p).get("labels") or {}
|
|
80
|
+
score = sum(SequenceMatcher(None, str(v), str(labels[k])).ratio() for k, v in selector.items() if k in labels)
|
|
81
|
+
score /= max(len(selector), 1)
|
|
82
|
+
if score > best_score:
|
|
83
|
+
best, best_score = labels, score
|
|
84
|
+
if not best:
|
|
85
|
+
return ""
|
|
86
|
+
return ",".join(f"{k}={v}" for k, v in list(best.items())[:6])
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@check("service-port-mismatch", "Service targetPort does not match any container port", Category.NETWORK,
|
|
90
|
+
Severity.HIGH, live_only=False)
|
|
91
|
+
def service_port_mismatch(ctx: Context) -> Iterator[Finding]:
|
|
92
|
+
"""targetPort refers to a named or numbered port the selected pods do not expose."""
|
|
93
|
+
for svc in ctx.snap.all("services"):
|
|
94
|
+
selector = (svc.get("spec") or {}).get("selector")
|
|
95
|
+
if not selector:
|
|
96
|
+
continue
|
|
97
|
+
ns = meta(svc).get("namespace")
|
|
98
|
+
specs = [p.get("spec") or {} for p in _pods_for(ctx, ns, selector)] if ctx.snap.live else []
|
|
99
|
+
if not specs:
|
|
100
|
+
for w in ctx.snap.workloads():
|
|
101
|
+
tmpl = ((w.get("spec") or {}).get("template") or {})
|
|
102
|
+
if meta(w).get("namespace") == ns and selector_matches(selector, (tmpl.get("metadata") or {}).get("labels")):
|
|
103
|
+
specs.append(tmpl.get("spec") or {})
|
|
104
|
+
if not specs:
|
|
105
|
+
continue
|
|
106
|
+
names, numbers = set(), set()
|
|
107
|
+
for s in specs:
|
|
108
|
+
for c in containers_of(s):
|
|
109
|
+
for p in c.get("ports") or []:
|
|
110
|
+
if p.get("name"):
|
|
111
|
+
names.add(p["name"])
|
|
112
|
+
numbers.add(p.get("containerPort"))
|
|
113
|
+
if not numbers:
|
|
114
|
+
continue # ports not declared; nothing to compare
|
|
115
|
+
for port in (svc.get("spec") or {}).get("ports") or []:
|
|
116
|
+
target = port.get("targetPort", port.get("port"))
|
|
117
|
+
bad = (isinstance(target, str) and not target.isdigit() and target not in names) or (
|
|
118
|
+
(isinstance(target, int) or str(target).isdigit()) and int(target) not in numbers)
|
|
119
|
+
if bad:
|
|
120
|
+
yield ctx.finding(
|
|
121
|
+
"service-port-mismatch", svc,
|
|
122
|
+
f"Port {port.get('port')} forwards to targetPort {target}, but the pods expose "
|
|
123
|
+
f"{sorted(n for n in numbers if n)}{' / ' + ', '.join(sorted(names)) if names else ''}.",
|
|
124
|
+
remediation="Set targetPort to the port the container actually listens on.",
|
|
125
|
+
evidence=[f"service port {port.get('name') or ''} {port.get('port')} -> {target}"],
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@check("ingress-backend-missing", "Ingress points to a missing Service or port", Category.NETWORK, Severity.HIGH)
|
|
130
|
+
def ingress_backend(ctx: Context) -> Iterator[Finding]:
|
|
131
|
+
"""An Ingress rule routes to a Service (or port) that does not exist, which returns 503/404."""
|
|
132
|
+
for ing in ctx.snap.all("ingresses"):
|
|
133
|
+
ns = meta(ing).get("namespace")
|
|
134
|
+
spec = ing.get("spec") or {}
|
|
135
|
+
backends = []
|
|
136
|
+
if spec.get("defaultBackend"):
|
|
137
|
+
backends.append(("default", spec["defaultBackend"]))
|
|
138
|
+
for rule in spec.get("rules") or []:
|
|
139
|
+
for path in (rule.get("http") or {}).get("paths") or []:
|
|
140
|
+
backends.append((f"{rule.get('host', '*')}{path.get('path', '/')}", path.get("backend") or {}))
|
|
141
|
+
problems = []
|
|
142
|
+
for where, b in backends:
|
|
143
|
+
svc_ref = b.get("service")
|
|
144
|
+
if not svc_ref:
|
|
145
|
+
continue
|
|
146
|
+
svc = ctx.snap.get("services", ns, svc_ref.get("name", ""))
|
|
147
|
+
if svc is None:
|
|
148
|
+
problems.append(f"{where} -> Service '{svc_ref.get('name')}' does not exist")
|
|
149
|
+
continue
|
|
150
|
+
port = svc_ref.get("port") or {}
|
|
151
|
+
ports = (svc.get("spec") or {}).get("ports") or []
|
|
152
|
+
if port.get("number") and all(p.get("port") != port["number"] for p in ports):
|
|
153
|
+
problems.append(f"{where} -> Service '{svc_ref.get('name')}' has no port {port['number']}")
|
|
154
|
+
if port.get("name") and all(p.get("name") != port["name"] for p in ports):
|
|
155
|
+
problems.append(f"{where} -> Service '{svc_ref.get('name')}' has no port named {port['name']}")
|
|
156
|
+
for tls in spec.get("tls") or []:
|
|
157
|
+
secret = tls.get("secretName")
|
|
158
|
+
if secret and ctx.snap.can_read("secrets") and ctx.snap.get("secrets", ns, secret) is None:
|
|
159
|
+
problems.append(f"TLS secret '{secret}' for {', '.join(tls.get('hosts') or [])} does not exist")
|
|
160
|
+
if problems:
|
|
161
|
+
yield ctx.finding(
|
|
162
|
+
"ingress-backend-missing", ing, problems[0] + ".", evidence=problems[:6],
|
|
163
|
+
remediation="Point the Ingress at an existing Service and port in the same namespace "
|
|
164
|
+
"(or create the TLS Secret, e.g. via cert-manager).",
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
INGRESS_NGINX_IMAGES = ("ingress-nginx/controller", "k8s.gcr.io/ingress-nginx", "registry.k8s.io/ingress-nginx")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@check("ingress-nginx-retired", "ingress-nginx controller is retired", Category.UPGRADE, Severity.HIGH, risk=True)
|
|
172
|
+
def ingress_nginx_retired(ctx: Context) -> Iterator[Finding]:
|
|
173
|
+
"""The community ingress-nginx project was retired in March 2026: no more fixes or security patches."""
|
|
174
|
+
controllers = []
|
|
175
|
+
for pod in ctx.snap.all("pods"):
|
|
176
|
+
for c in containers_of(pod.get("spec")):
|
|
177
|
+
if any(s in c.get("image", "") for s in INGRESS_NGINX_IMAGES):
|
|
178
|
+
controllers.append((pod, c.get("image")))
|
|
179
|
+
break
|
|
180
|
+
classes = [ic for ic in ctx.snap.all("ingressclasses")
|
|
181
|
+
if (ic.get("spec") or {}).get("controller") == "k8s.io/ingress-nginx"]
|
|
182
|
+
if not controllers and not classes:
|
|
183
|
+
return
|
|
184
|
+
class_names = {meta(ic).get("name") for ic in classes} or {"nginx"}
|
|
185
|
+
using = [i for i in ctx.snap.all("ingresses")
|
|
186
|
+
if (i.get("spec") or {}).get("ingressClassName") in class_names
|
|
187
|
+
or (meta(i).get("annotations") or {}).get("kubernetes.io/ingress.class") in class_names]
|
|
188
|
+
annotated = sum(1 for i in using if any(k.startswith("nginx.ingress.kubernetes.io/")
|
|
189
|
+
for k in (meta(i).get("annotations") or {})))
|
|
190
|
+
image = controllers[0][1] if controllers else "unknown"
|
|
191
|
+
ref = ResourceRef(kind="IngressClass", name=sorted(class_names)[0]) if classes else ResourceRef(
|
|
192
|
+
kind="Pod", name=meta(controllers[0][0]).get("name", ""), namespace=meta(controllers[0][0]).get("namespace"))
|
|
193
|
+
yield ctx.finding(
|
|
194
|
+
"ingress-nginx-retired", ref,
|
|
195
|
+
f"The cluster runs the community ingress-nginx controller ({image}), which was retired in March 2026 "
|
|
196
|
+
f"and gets no security fixes. {len(using)} Ingress object(s) use it, {annotated} with nginx-specific annotations "
|
|
197
|
+
"that need translating.",
|
|
198
|
+
evidence=[f"controller image: {image}", f"ingresses using it: {len(using)}",
|
|
199
|
+
f"with nginx.ingress.kubernetes.io/* annotations: {annotated}"],
|
|
200
|
+
remediation="Plan a migration to Gateway API (or another maintained controller). Convert manifests with "
|
|
201
|
+
"ingress2gateway, run both paths side by side, then shift traffic gradually.",
|
|
202
|
+
commands=["kubectl get ingress -A -o wide"],
|
|
203
|
+
docs="https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/",
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@check("ingress-class-missing", "Ingress uses a class with no controller", Category.NETWORK, Severity.HIGH)
|
|
208
|
+
def ingress_class_missing(ctx: Context) -> Iterator[Finding]:
|
|
209
|
+
"""ingressClassName names an IngressClass that does not exist, so no controller serves it."""
|
|
210
|
+
if not ctx.snap.can_read("ingressclasses") or not ctx.snap.all("ingressclasses"):
|
|
211
|
+
return
|
|
212
|
+
names = {meta(ic).get("name") for ic in ctx.snap.all("ingressclasses")}
|
|
213
|
+
for ing in ctx.snap.all("ingresses"):
|
|
214
|
+
cls = (ing.get("spec") or {}).get("ingressClassName")
|
|
215
|
+
if cls and cls not in names:
|
|
216
|
+
yield ctx.finding(
|
|
217
|
+
"ingress-class-missing", ing,
|
|
218
|
+
f"ingressClassName '{cls}' does not exist (available: {', '.join(sorted(n for n in names if n))}). "
|
|
219
|
+
"No controller will route this Ingress.",
|
|
220
|
+
remediation="Use one of the installed IngressClasses or install the matching controller.",
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@check("loadbalancer-pending", "LoadBalancer has no external address", Category.NETWORK, Severity.HIGH)
|
|
225
|
+
def lb_pending(ctx: Context) -> Iterator[Finding]:
|
|
226
|
+
"""A type=LoadBalancer Service has waited 5+ minutes without an IP/hostname."""
|
|
227
|
+
for svc in ctx.snap.all("services"):
|
|
228
|
+
spec = svc.get("spec") or {}
|
|
229
|
+
if spec.get("type") != "LoadBalancer":
|
|
230
|
+
continue
|
|
231
|
+
if ((svc.get("status") or {}).get("loadBalancer") or {}).get("ingress"):
|
|
232
|
+
continue
|
|
233
|
+
a = age(meta(svc).get("creationTimestamp"))
|
|
234
|
+
if a is not None and a < timedelta(minutes=5):
|
|
235
|
+
continue
|
|
236
|
+
events = [e.get("message", "") for e in ctx.snap.events_for("Service", meta(svc).get("namespace"),
|
|
237
|
+
meta(svc).get("name", ""))
|
|
238
|
+
if e.get("type") == "Warning"]
|
|
239
|
+
yield ctx.finding(
|
|
240
|
+
"loadbalancer-pending", svc,
|
|
241
|
+
f"No external address after {human_duration(a)}." + (f" Cloud controller says: {events[0][:200]}" if events else
|
|
242
|
+
" No cloud load-balancer controller may be installed (common on kind/bare metal)."),
|
|
243
|
+
evidence=events[:3],
|
|
244
|
+
remediation="Check the cloud controller/load-balancer controller logs, subnet tags and quotas; on bare metal install MetalLB or use an Ingress.",
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@check("dns-unhealthy", "Cluster DNS is unhealthy", Category.PLATFORM, Severity.CRITICAL)
|
|
249
|
+
def dns_unhealthy(ctx: Context) -> Iterator[Finding]:
|
|
250
|
+
"""CoreDNS/kube-dns pods are not ready: every service-name lookup in the cluster may fail."""
|
|
251
|
+
pods = [p for p in ctx.snap.in_namespace("pods", "kube-system")
|
|
252
|
+
if (meta(p).get("labels") or {}).get("k8s-app") == "kube-dns"]
|
|
253
|
+
if not pods:
|
|
254
|
+
return
|
|
255
|
+
ready = [p for p in pods if _ready(p)]
|
|
256
|
+
if len(ready) == len(pods):
|
|
257
|
+
return
|
|
258
|
+
yield ctx.finding(
|
|
259
|
+
"dns-unhealthy", ResourceRef(kind="Deployment", name="coredns", namespace="kube-system"),
|
|
260
|
+
f"{len(ready)}/{len(pods)} DNS pods are ready." + (" DNS is down for the whole cluster." if not ready else
|
|
261
|
+
" Lookups may intermittently fail."),
|
|
262
|
+
severity=Severity.CRITICAL if not ready else Severity.HIGH,
|
|
263
|
+
evidence=[f"{meta(p).get('name')}: {(p.get('status') or {}).get('phase')}" for p in pods],
|
|
264
|
+
commands=["kubectl -n kube-system logs -l k8s-app=kube-dns --tail=50",
|
|
265
|
+
"kubectl -n kube-system get configmap coredns -o yaml"],
|
|
266
|
+
remediation="Check the CoreDNS logs (loop detection, upstream resolver errors, OOM) and the coredns ConfigMap.",
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
@check("netpol-blocks-dns", "NetworkPolicy blocks DNS", Category.NETWORK, Severity.HIGH, live_only=False)
|
|
271
|
+
def netpol_blocks_dns(ctx: Context) -> Iterator[Finding]:
|
|
272
|
+
"""An egress NetworkPolicy selects pods but does not allow port 53, so name lookups time out."""
|
|
273
|
+
for np in ctx.snap.all("networkpolicies"):
|
|
274
|
+
spec = np.get("spec") or {}
|
|
275
|
+
if "Egress" not in (spec.get("policyTypes") or []):
|
|
276
|
+
continue
|
|
277
|
+
egress = spec.get("egress") or []
|
|
278
|
+
allows_dns = False
|
|
279
|
+
for rule in egress:
|
|
280
|
+
ports, peers = rule.get("ports"), rule.get("to")
|
|
281
|
+
if ports and any(str(p.get("port")) in ("53", "dns", "dns-tcp") for p in ports) or not ports and (not peers or any(_peer_maybe_dns(peer) for peer in peers)):
|
|
282
|
+
allows_dns = True
|
|
283
|
+
if allows_dns:
|
|
284
|
+
continue
|
|
285
|
+
ns = meta(np).get("namespace")
|
|
286
|
+
selected = [p for p in ctx.snap.in_namespace("pods", ns)
|
|
287
|
+
if label_selector_matches(spec.get("podSelector") or {}, meta(p).get("labels"))]
|
|
288
|
+
if ctx.snap.live and not selected:
|
|
289
|
+
continue
|
|
290
|
+
yield ctx.finding(
|
|
291
|
+
"netpol-blocks-dns", np,
|
|
292
|
+
f"This policy restricts egress for {len(selected) if ctx.snap.live else 'its'} pod(s) but allows no traffic to port 53; "
|
|
293
|
+
"DNS lookups from those pods will fail.",
|
|
294
|
+
remediation="Add an egress rule to kube-system pods with label k8s-app=kube-dns on UDP and TCP port 53.",
|
|
295
|
+
evidence=[f"podSelector: {spec.get('podSelector')}"],
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _peer_maybe_dns(peer: dict[str, Any]) -> bool:
|
|
300
|
+
if peer.get("ipBlock"):
|
|
301
|
+
return True
|
|
302
|
+
ns_sel = (peer.get("namespaceSelector") or {}).get("matchLabels") or {}
|
|
303
|
+
return peer.get("namespaceSelector") == {} or ns_sel.get("kubernetes.io/metadata.name") == "kube-system"
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@check("gateway-not-accepted", "Gateway or HTTPRoute not accepted", Category.NETWORK, Severity.HIGH)
|
|
307
|
+
def gateway_status(ctx: Context) -> Iterator[Finding]:
|
|
308
|
+
"""Gateway API objects whose Accepted/Programmed/ResolvedRefs condition is False."""
|
|
309
|
+
for gw in ctx.snap.all("gateways"):
|
|
310
|
+
for c in (gw.get("status") or {}).get("conditions") or []:
|
|
311
|
+
if c.get("type") in ("Accepted", "Programmed") and c.get("status") == "False":
|
|
312
|
+
yield ctx.finding("gateway-not-accepted", gw,
|
|
313
|
+
f"Gateway {c.get('type')}=False ({c.get('reason')}): {c.get('message', '')[:200]}")
|
|
314
|
+
break
|
|
315
|
+
for route in ctx.snap.all("httproutes"):
|
|
316
|
+
for parent in (route.get("status") or {}).get("parents") or []:
|
|
317
|
+
bad = next((c for c in parent.get("conditions") or []
|
|
318
|
+
if c.get("type") in ("Accepted", "ResolvedRefs") and c.get("status") == "False"), None)
|
|
319
|
+
if bad:
|
|
320
|
+
yield ctx.finding("gateway-not-accepted", route,
|
|
321
|
+
f"HTTPRoute {bad.get('type')}=False on {(parent.get('parentRef') or {}).get('name')} "
|
|
322
|
+
f"({bad.get('reason')}): {bad.get('message', '')[:200]}",
|
|
323
|
+
remediation="Check the parentRef name/namespace, allowedRoutes on the Gateway listener, "
|
|
324
|
+
"and that backendRefs exist (cross-namespace refs need a ReferenceGrant).")
|
|
325
|
+
break
|
kubesift/checks/nodes.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Node health, capacity and version skew."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from kubesift.checks.base import Context, check
|
|
10
|
+
from kubesift.models import Category, Finding, Severity
|
|
11
|
+
from kubesift.util import age, containers_of, human_duration, meta, parse_quantity, parse_time
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def minor(version: str | None) -> int | None:
|
|
15
|
+
m = re.match(r"v?1\.(\d+)", version or "")
|
|
16
|
+
return int(m.group(1)) if m else None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _cond(node: dict[str, Any], ctype: str) -> dict[str, Any]:
|
|
20
|
+
return next((c for c in (node.get("status") or {}).get("conditions") or [] if c.get("type") == ctype), {})
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def pods_on(ctx: Context, node: str) -> list[dict[str, Any]]:
|
|
24
|
+
return [p for p in ctx.snap.all("pods") if (p.get("spec") or {}).get("nodeName") == node
|
|
25
|
+
and (p.get("status") or {}).get("phase") not in ("Succeeded", "Failed")]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@check("node-not-ready", "Node is not ready", Category.NODE, Severity.CRITICAL)
|
|
29
|
+
def node_not_ready(ctx: Context) -> Iterator[Finding]:
|
|
30
|
+
"""The kubelet stopped reporting or reports itself unhealthy; pods on it are being lost."""
|
|
31
|
+
for node in ctx.snap.all("nodes"):
|
|
32
|
+
ready = _cond(node, "Ready")
|
|
33
|
+
if not ready or ready.get("status") == "True":
|
|
34
|
+
continue
|
|
35
|
+
name = meta(node).get("name", "")
|
|
36
|
+
affected = pods_on(ctx, name)
|
|
37
|
+
yield ctx.finding(
|
|
38
|
+
"node-not-ready", node,
|
|
39
|
+
f"Node has been NotReady for {human_duration(age(ready.get('lastTransitionTime')))} "
|
|
40
|
+
f"({ready.get('reason')}: {ready.get('message', '')[:160]}). {len(affected)} pod(s) run on it.",
|
|
41
|
+
evidence=[f"Ready={ready.get('status')} reason={ready.get('reason')}"] + [
|
|
42
|
+
f"{c.get('type')}={c.get('status')}" for c in (node.get("status") or {}).get("conditions") or []
|
|
43
|
+
if c.get("type") != "Ready" and c.get("status") == "True"],
|
|
44
|
+
remediation="Check the node in your cloud console (instance stopped, disk full, network). On the node: "
|
|
45
|
+
"`systemctl status kubelet containerd` and `journalctl -u kubelet`. Replace it if it does not recover.",
|
|
46
|
+
commands=[f"kubectl describe node {name}", f"kubectl get pods -A -o wide --field-selector spec.nodeName={name}"],
|
|
47
|
+
since=parse_time(ready.get("lastTransitionTime")),
|
|
48
|
+
node=name,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@check("node-pressure", "Node is under resource pressure", Category.NODE, Severity.HIGH)
|
|
53
|
+
def node_pressure(ctx: Context) -> Iterator[Finding]:
|
|
54
|
+
"""MemoryPressure / DiskPressure / PIDPressure: the kubelet is evicting pods."""
|
|
55
|
+
for node in ctx.snap.all("nodes"):
|
|
56
|
+
for ctype, advice in (
|
|
57
|
+
("MemoryPressure", "Pods use more memory than the node has. Set memory requests near real usage."),
|
|
58
|
+
("DiskPressure", "Node disk is almost full: images, container logs or emptyDir. Prune images, cap log size, "
|
|
59
|
+
"add ephemeral-storage limits."),
|
|
60
|
+
("PIDPressure", "A process is forking too many processes/threads."),
|
|
61
|
+
):
|
|
62
|
+
c = _cond(node, ctype)
|
|
63
|
+
if c.get("status") == "True":
|
|
64
|
+
name = meta(node).get("name", "")
|
|
65
|
+
yield ctx.finding(
|
|
66
|
+
"node-pressure", node, f"{ctype}: {c.get('message', '')[:200]}",
|
|
67
|
+
title=f"Node has {ctype}", evidence=[f"since {c.get('lastTransitionTime')}"],
|
|
68
|
+
remediation=advice, commands=[f"kubectl describe node {name}", "kubectl top pods -A --sort-by=memory | head"],
|
|
69
|
+
node=name, since=parse_time(c.get("lastTransitionTime")),
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@check("node-cordoned", "Node is cordoned", Category.NODE, Severity.LOW, risk=True)
|
|
74
|
+
def node_cordoned(ctx: Context) -> Iterator[Finding]:
|
|
75
|
+
"""spec.unschedulable is true; often left behind after maintenance."""
|
|
76
|
+
for node in ctx.snap.all("nodes"):
|
|
77
|
+
if (node.get("spec") or {}).get("unschedulable"):
|
|
78
|
+
name = meta(node).get("name", "")
|
|
79
|
+
yield ctx.finding("node-cordoned", node, "New pods cannot be scheduled on this node (cordoned).",
|
|
80
|
+
remediation="Uncordon it if maintenance is finished.",
|
|
81
|
+
commands=[f"kubectl uncordon {name}"], node=name)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@check("node-overcommitted", "Node requests are near capacity", Category.RESOURCES, Severity.MEDIUM)
|
|
85
|
+
def node_overcommitted(ctx: Context) -> Iterator[Finding]:
|
|
86
|
+
"""CPU or memory requests above 95% of allocatable: new or rescheduled pods will not fit."""
|
|
87
|
+
nodes = ctx.snap.all("nodes")
|
|
88
|
+
full = []
|
|
89
|
+
for node in nodes:
|
|
90
|
+
name = meta(node).get("name", "")
|
|
91
|
+
alloc = (node.get("status") or {}).get("allocatable") or {}
|
|
92
|
+
for res in ("cpu", "memory"):
|
|
93
|
+
cap = parse_quantity(alloc.get(res))
|
|
94
|
+
if not cap:
|
|
95
|
+
continue
|
|
96
|
+
req = sum(parse_quantity(((c.get("resources") or {}).get("requests") or {}).get(res))
|
|
97
|
+
for p in pods_on(ctx, name) for c in containers_of(p.get("spec")))
|
|
98
|
+
if req / cap >= 0.95:
|
|
99
|
+
full.append((node, res, req / cap))
|
|
100
|
+
if nodes and len({id(n) for n, _, _ in full}) == len(nodes):
|
|
101
|
+
node, res, ratio = full[0]
|
|
102
|
+
yield ctx.finding(
|
|
103
|
+
"node-overcommitted", node,
|
|
104
|
+
f"Every node has more than 95% of its {res} requested; any new pod will stay Pending.",
|
|
105
|
+
severity=Severity.HIGH,
|
|
106
|
+
evidence=[f"{meta(n).get('name')}: {r} {ratio:.0%}" for n, r, ratio in full[:8]],
|
|
107
|
+
remediation="Add capacity (or enable the cluster autoscaler), or right-size over-requested workloads (`kubesift cost`).",
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@check("node-version-skew", "Kubelet version too far behind the control plane", Category.UPGRADE, Severity.MEDIUM, risk=True)
|
|
112
|
+
def version_skew(ctx: Context) -> Iterator[Finding]:
|
|
113
|
+
"""Kubelets may be at most 3 minor versions older than the API server."""
|
|
114
|
+
server = minor(ctx.snap.server_version)
|
|
115
|
+
if server is None:
|
|
116
|
+
return
|
|
117
|
+
for node in ctx.snap.all("nodes"):
|
|
118
|
+
info = (node.get("status") or {}).get("nodeInfo") or {}
|
|
119
|
+
kubelet = minor(info.get("kubeletVersion"))
|
|
120
|
+
if kubelet is None:
|
|
121
|
+
continue
|
|
122
|
+
if kubelet > server or server - kubelet > 3:
|
|
123
|
+
yield ctx.finding(
|
|
124
|
+
"node-version-skew", node,
|
|
125
|
+
f"Kubelet {info.get('kubeletVersion')} vs API server {ctx.snap.server_version}: outside the supported skew.",
|
|
126
|
+
severity=Severity.HIGH, remediation="Upgrade (or replace) the node before upgrading the control plane further.",
|
|
127
|
+
node=meta(node).get("name"),
|
|
128
|
+
)
|
|
129
|
+
elif server - kubelet == 3:
|
|
130
|
+
yield ctx.finding(
|
|
131
|
+
"node-version-skew", node,
|
|
132
|
+
f"Kubelet {info.get('kubeletVersion')} is 3 minors behind {ctx.snap.server_version}; "
|
|
133
|
+
"the next control-plane upgrade will break support for this node.",
|
|
134
|
+
remediation="Upgrade node pools before the next control-plane upgrade.", node=meta(node).get("name"),
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@check("node-containerd-1x", "Node runs containerd 1.x", Category.UPGRADE, Severity.MEDIUM, risk=True)
|
|
139
|
+
def containerd_1x(ctx: Context) -> Iterator[Finding]:
|
|
140
|
+
"""Kubernetes 1.36 dropped support for containerd 1.x; nodes must move to containerd 2.x."""
|
|
141
|
+
old = [n for n in ctx.snap.all("nodes")
|
|
142
|
+
if re.match(r"containerd://1\.", ((n.get("status") or {}).get("nodeInfo") or {}).get("containerRuntimeVersion", ""))]
|
|
143
|
+
if not old:
|
|
144
|
+
return
|
|
145
|
+
runtime = ((old[0].get("status") or {}).get("nodeInfo") or {}).get("containerRuntimeVersion")
|
|
146
|
+
yield ctx.finding(
|
|
147
|
+
"node-containerd-1x", old[0],
|
|
148
|
+
f"{len(old)} node(s) run {runtime}. Kubernetes 1.36+ requires containerd 2.x; upgrade the node image first.",
|
|
149
|
+
severity=Severity.HIGH if (minor(ctx.snap.server_version) or 0) >= 35 else Severity.MEDIUM,
|
|
150
|
+
evidence=[meta(n).get("name", "") for n in old[:10]],
|
|
151
|
+
remediation="Roll node pools to an OS image with containerd 2.x (check cgroup v2 at the same time).",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@check("gpu-idle", "GPU capacity is idle", Category.COST, Severity.MEDIUM, risk=True)
|
|
156
|
+
def gpu_idle(ctx: Context) -> Iterator[Finding]:
|
|
157
|
+
"""GPU nodes with no pod requesting their GPUs. GPUs are the most expensive idle resource."""
|
|
158
|
+
gpu_keys = ("nvidia.com/gpu", "amd.com/gpu", "gpu.intel.com/i915")
|
|
159
|
+
idle = []
|
|
160
|
+
total = 0.0
|
|
161
|
+
for node in ctx.snap.all("nodes"):
|
|
162
|
+
alloc = (node.get("status") or {}).get("allocatable") or {}
|
|
163
|
+
gpus = sum(parse_quantity(alloc.get(k)) for k in gpu_keys)
|
|
164
|
+
if not gpus:
|
|
165
|
+
continue
|
|
166
|
+
total += gpus
|
|
167
|
+
name = meta(node).get("name", "")
|
|
168
|
+
used = sum(parse_quantity(((c.get("resources") or {}).get("limits") or {}).get(k))
|
|
169
|
+
for p in pods_on(ctx, name) for c in containers_of(p.get("spec")) for k in gpu_keys)
|
|
170
|
+
if used < gpus:
|
|
171
|
+
idle.append((name, gpus - used, gpus))
|
|
172
|
+
if not idle:
|
|
173
|
+
return
|
|
174
|
+
free = sum(i[1] for i in idle)
|
|
175
|
+
monthly = free * ctx.settings.cost_gpu_hour * 730
|
|
176
|
+
yield ctx.finding(
|
|
177
|
+
"gpu-idle", ctx.snap.all("nodes")[0] if len(idle) > 1 else next(n for n in ctx.snap.all("nodes") if meta(n).get("name") == idle[0][0]),
|
|
178
|
+
f"{free:.0f} of {total:.0f} GPUs are not requested by any pod (about ${monthly:,.0f}/month at "
|
|
179
|
+
f"${ctx.settings.cost_gpu_hour}/GPU-hour).",
|
|
180
|
+
evidence=[f"{n}: {f:.0f}/{t:.0f} GPUs free" for n, f, t in idle[:10]],
|
|
181
|
+
remediation="Scale GPU node pools to zero when idle (cluster autoscaler / Karpenter), share GPUs with time-slicing or MIG, "
|
|
182
|
+
"and move notebooks and batch jobs to on-demand GPU pools.",
|
|
183
|
+
)
|