kubesift 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. kubesift/__init__.py +3 -0
  2. kubesift/analysis/__init__.py +3 -0
  3. kubesift/analysis/changes.py +165 -0
  4. kubesift/analysis/correlate.py +160 -0
  5. kubesift/analysis/describe.py +83 -0
  6. kubesift/analysis/engine.py +362 -0
  7. kubesift/analysis/llm.py +152 -0
  8. kubesift/checks/__init__.py +17 -0
  9. kubesift/checks/base.py +150 -0
  10. kubesift/checks/config_refs.py +84 -0
  11. kubesift/checks/events.py +77 -0
  12. kubesift/checks/hygiene.py +264 -0
  13. kubesift/checks/lifecycle.py +202 -0
  14. kubesift/checks/network.py +325 -0
  15. kubesift/checks/nodes.py +183 -0
  16. kubesift/checks/operations.py +319 -0
  17. kubesift/checks/pods.py +485 -0
  18. kubesift/checks/storage.py +72 -0
  19. kubesift/checks/workloads.py +201 -0
  20. kubesift/cli.py +874 -0
  21. kubesift/cluster/__init__.py +4 -0
  22. kubesift/cluster/client.py +141 -0
  23. kubesift/cluster/snapshot.py +513 -0
  24. kubesift/config.py +134 -0
  25. kubesift/cost.py +150 -0
  26. kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
  27. kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
  28. kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
  29. kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
  30. kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
  31. kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
  32. kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
  33. kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
  34. kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
  35. kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
  36. kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
  37. kubesift/drill_scenarios/__init__.py +1 -0
  38. kubesift/drills.py +122 -0
  39. kubesift/knowledge.py +51 -0
  40. kubesift/leader.py +145 -0
  41. kubesift/lint.py +89 -0
  42. kubesift/mcp_server.py +193 -0
  43. kubesift/models.py +144 -0
  44. kubesift/notify.py +71 -0
  45. kubesift/postmortem.py +152 -0
  46. kubesift/preflight.py +435 -0
  47. kubesift/redact.py +38 -0
  48. kubesift/remediate.py +99 -0
  49. kubesift/report.py +193 -0
  50. kubesift/runbooks.py +58 -0
  51. kubesift/server/__init__.py +0 -0
  52. kubesift/server/app.py +480 -0
  53. kubesift/server/static/index.html +152 -0
  54. kubesift/slack.py +106 -0
  55. kubesift/store.py +219 -0
  56. kubesift/timetravel.py +156 -0
  57. kubesift/upgrade.py +51 -0
  58. kubesift/util.py +174 -0
  59. kubesift-2.0.0.dist-info/METADATA +192 -0
  60. kubesift-2.0.0.dist-info/RECORD +63 -0
  61. kubesift-2.0.0.dist-info/WHEEL +4 -0
  62. kubesift-2.0.0.dist-info/entry_points.txt +3 -0
  63. kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,485 @@
1
+ """Pod runtime failures: crash loops, OOM kills, image pulls, pending pods and more."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Iterator
7
+ from datetime import timedelta
8
+ from typing import Any
9
+
10
+ from kubesift.checks.base import Context, check
11
+ from kubesift.models import Category, Finding, Severity
12
+ from kubesift.util import age, containers_of, fmt_bytes, human_duration, meta, parse_quantity, parse_time
13
+
14
+ EXIT_CODES = {
15
+ 0: "exited successfully, but the pod expects a long-running process (restartPolicy: Always)",
16
+ 1: "application error (unhandled exception or failed startup check)",
17
+ 2: "misuse of shell builtin or invalid arguments",
18
+ 126: "command found but not executable (permissions or wrong file)",
19
+ 127: "command not found (wrong command/entrypoint or missing binary in the image)",
20
+ 128: "invalid exit argument",
21
+ 134: "aborted (SIGABRT), often an assertion or fatal runtime error",
22
+ 137: "killed with SIGKILL (out of memory, or killed after a failed liveness probe)",
23
+ 139: "segmentation fault (SIGSEGV)",
24
+ 143: "terminated with SIGTERM (shutdown, eviction or failed liveness probe)",
25
+ }
26
+
27
+ LOG_HINTS: list[tuple[re.Pattern[str], str]] = [
28
+ (re.compile(r'is forbidden: User "system:serviceaccount:[^"]+"', re.I),
29
+ "The app calls the Kubernetes API and its ServiceAccount is not allowed to: add a Role/RoleBinding for the "
30
+ "verb and resource named in the log line."),
31
+ (re.compile(r"exec format error", re.I),
32
+ "The image was built for a different CPU architecture (e.g. arm64 image on amd64 nodes). "
33
+ "Rebuild with `docker buildx build --platform linux/amd64,linux/arm64`."),
34
+ (re.compile(r"KeyError: '([A-Z][A-Z0-9_]+)'|environment variable \"?([A-Z][A-Z0-9_]+)\"? (?:is )?(?:not set|missing|required|undefined)|([A-Z][A-Z0-9_]{2,}) (?:must be set|is required|is not set|not defined)", re.I),
35
+ "The application expects environment variable {0}, which is not set on the container."),
36
+ (re.compile(r"(?:connection refused|ECONNREFUSED)[^\n]*?(\b[\w.-]+:\d{2,5}\b)?", re.I),
37
+ "A dependency{0} refused the connection: it is down, not ready yet, or listening on another port."),
38
+ (re.compile(r"no such host|Name or service not known|Temporary failure in name resolution|could not resolve host|getaddrinfo ENOTFOUND", re.I),
39
+ "DNS lookup failed: the service name is wrong, in another namespace (use name.namespace.svc), or DNS/egress is blocked."),
40
+ (re.compile(r"Read-only file system", re.I),
41
+ "The app writes to a read-only filesystem; mount an emptyDir for that path or relax readOnlyRootFilesystem."),
42
+ (re.compile(r"permission denied", re.I),
43
+ "Permission denied: check runAsUser/fsGroup and file ownership in the image."),
44
+ (re.compile(r"address already in use|EADDRINUSE", re.I),
45
+ "The port is already in use: two processes or containers bind the same port."),
46
+ (re.compile(r"java\.lang\.OutOfMemoryError|JavaScript heap out of memory|MemoryError", re.I),
47
+ "The runtime ran out of heap: raise the memory limit and set the heap size relative to it (e.g. -XX:MaxRAMPercentage=75)."),
48
+ (re.compile(r"x509: certificate|certificate verify failed|SSL: CERTIFICATE_VERIFY_FAILED|certificate has expired", re.I),
49
+ "TLS verification failed: an expired certificate or a missing CA bundle in the image."),
50
+ (re.compile(r"(?:password authentication failed|Access denied for user|authentication failed|401 Unauthorized)", re.I),
51
+ "A dependency rejected the credentials: the Secret value is wrong or was rotated."),
52
+ (re.compile(r"no such file or directory", re.I),
53
+ "A file or binary the container needs does not exist: check the command, working dir and volume mounts."),
54
+ ]
55
+
56
+
57
+ def log_hints(ctx: Context, ns: str, pod: str, container: str) -> tuple[list[str], list[str]]:
58
+ entry = ctx.snap.logs.get(f"{ns}/{pod}/{container}")
59
+ if not entry:
60
+ return [], []
61
+ lines = entry.get("errors") or entry.get("tail") or []
62
+ text = "\n".join(lines)
63
+ hints = []
64
+ for pattern, template in LOG_HINTS:
65
+ m = pattern.search(text)
66
+ if not m:
67
+ continue
68
+ group = next((g for g in m.groups() if g), "")
69
+ if "{0}" not in template:
70
+ hints.append(template)
71
+ elif template.startswith("The application"):
72
+ hints.append(template.format(group or "(name not found in logs)"))
73
+ else:
74
+ hints.append(template.format(f" ({group})" if group else ""))
75
+ label = "previous run" if entry.get("previous") else "current run"
76
+ evidence = [f"log ({label}): {ln}" for ln in lines[-5:]]
77
+ return hints, evidence
78
+
79
+
80
+ def _statuses(pod: dict[str, Any]) -> Iterator[tuple[dict[str, Any], bool]]:
81
+ status = pod.get("status") or {}
82
+ for cs in status.get("initContainerStatuses") or []:
83
+ yield cs, True
84
+ for cs in status.get("containerStatuses") or []:
85
+ yield cs, False
86
+
87
+
88
+ def _container_spec(pod: dict[str, Any], name: str) -> dict[str, Any]:
89
+ for c in containers_of(pod.get("spec"), include_init=True):
90
+ if c.get("name") == name:
91
+ return c
92
+ return {}
93
+
94
+
95
+ def _pod_cmd(pod: dict[str, Any], what: str = "describe") -> str:
96
+ return f"kubectl {what} pod {meta(pod).get('name')} -n {meta(pod).get('namespace')}"
97
+
98
+
99
+ def is_crash_looping(cs: dict[str, Any]) -> bool:
100
+ """True for CrashLoopBackOff, and also between back-offs, when the container briefly shows
101
+ Error/Completed or Running again (what `kubectl get pods` prints as Error or Running with restarts)."""
102
+ state = cs.get("state") or {}
103
+ if (state.get("waiting") or {}).get("reason") == "CrashLoopBackOff":
104
+ return True
105
+ restarts = cs.get("restartCount", 0)
106
+ current = state.get("terminated") or {}
107
+ if current and current.get("exitCode", 0) != 0 and restarts >= 1:
108
+ return True
109
+ last = (cs.get("lastState") or {}).get("terminated") or {}
110
+ if restarts >= 3 and last and last.get("exitCode", 0) != 0:
111
+ a = age(last.get("finishedAt"))
112
+ return a is not None and a < timedelta(minutes=10)
113
+ return False
114
+
115
+
116
+ @check("pod-crashloop", "Container is crash looping", Category.WORKLOAD, Severity.CRITICAL)
117
+ def crashloop(ctx: Context) -> Iterator[Finding]:
118
+ """A container keeps exiting with an error and Kubernetes keeps restarting it."""
119
+ for pod in ctx.snap.all("pods"):
120
+ if meta(pod).get("deletionTimestamp"):
121
+ continue
122
+ for cs, is_init in _statuses(pod):
123
+ if not is_crash_looping(cs):
124
+ continue
125
+ last = (cs.get("lastState") or {}).get("terminated") or (cs.get("state") or {}).get("terminated") or {}
126
+ code = last.get("exitCode")
127
+ reason = last.get("reason", "")
128
+ if reason == "OOMKilled":
129
+ continue # reported by pod-oomkilled with better advice
130
+ name = cs.get("name", "")
131
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name")
132
+ hints, log_ev = log_hints(ctx, ns, pname, name)
133
+ evidence = [f"container {name}{' (init)' if is_init else ''}: restarts={cs.get('restartCount', 0)}"]
134
+ if code is not None:
135
+ evidence.append(f"last exit code {code}: {EXIT_CODES.get(code, 'application-specific error')}")
136
+ if last.get("message"):
137
+ evidence.append(f"termination message: {last['message'][:300]}")
138
+ evidence += log_ev
139
+ if code == 137 and _liveness_killed(ctx, pod):
140
+ hints.insert(0, "The liveness probe is failing and the kubelet kills the container; "
141
+ "the app may be slow to start (add a startupProbe) or the probe checks the wrong port/path.")
142
+ msg = f"Container '{name}' keeps crashing ({cs.get('restartCount', 0)} restarts"
143
+ msg += f", last exit code {code})." if code is not None else ")."
144
+ if hints:
145
+ msg += " " + hints[0]
146
+ yield ctx.finding(
147
+ "pod-crashloop", pod, msg,
148
+ evidence=evidence,
149
+ remediation=" ".join(hints[1:3]) or "Read the previous logs to see why the process exits, fix the cause, then redeploy.",
150
+ commands=[f"kubectl logs {pname} -n {ns} -c {name} --previous --tail=100", _pod_cmd(pod)],
151
+ since=parse_time(last.get("startedAt")) or parse_time((pod.get("status") or {}).get("startTime")),
152
+ node=(pod.get("spec") or {}).get("nodeName"),
153
+ )
154
+
155
+
156
+ def _liveness_killed(ctx: Context, pod: dict[str, Any]) -> bool:
157
+ for e in ctx.snap.events_for("Pod", meta(pod).get("namespace"), meta(pod).get("name", "")):
158
+ if "Liveness probe failed" in (e.get("message") or ""):
159
+ return True
160
+ return False
161
+
162
+
163
+ @check("pod-oomkilled", "Container was killed for using too much memory", Category.RESOURCES, Severity.HIGH)
164
+ def oomkilled(ctx: Context) -> Iterator[Finding]:
165
+ """The kernel killed a container because it exceeded its memory limit."""
166
+ for pod in ctx.snap.all("pods"):
167
+ for cs, _ in _statuses(pod):
168
+ last = (cs.get("lastState") or {}).get("terminated") or {}
169
+ cur = (cs.get("state") or {}).get("terminated") or {}
170
+ term = last if last.get("reason") == "OOMKilled" else cur if cur.get("reason") == "OOMKilled" else None
171
+ if not term:
172
+ continue
173
+ name = cs.get("name", "")
174
+ spec = _container_spec(pod, name)
175
+ limit = parse_quantity(((spec.get("resources") or {}).get("limits") or {}).get("memory"))
176
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name")
177
+ usage = ctx.snap.metrics.get("pods", {}).get(f"{ns}/{pname}/{name}", {}).get("memory")
178
+ evidence = [f"container {name}: restarts={cs.get('restartCount', 0)}, last killed at {term.get('finishedAt')}"]
179
+ if limit:
180
+ evidence.append(f"memory limit {fmt_bytes(limit)}")
181
+ if usage:
182
+ evidence.append(f"current memory use {fmt_bytes(usage)}")
183
+ hints, log_ev = log_hints(ctx, ns, pname, name)
184
+ suggestion = fmt_bytes(limit * 2) if limit else "a value above peak usage"
185
+ waiting = ((cs.get("state") or {}).get("waiting") or {}).get("reason") == "CrashLoopBackOff"
186
+ owner = ctx.snap.owner_of(pod)
187
+ yield ctx.finding(
188
+ "pod-oomkilled", pod,
189
+ f"Container '{name}' was OOMKilled"
190
+ + (f": it needs more than its {fmt_bytes(limit)} memory limit." if limit else ".")
191
+ + (" It is now crash looping." if waiting else ""),
192
+ severity=Severity.CRITICAL if waiting else Severity.HIGH,
193
+ evidence=evidence + log_ev,
194
+ remediation=f"Raise the memory limit (try {suggestion}) and set the request close to real usage, "
195
+ "or fix a memory leak. For JVM/Node apps also cap the heap below the limit.",
196
+ commands=[
197
+ f"kubectl set resources {owner.kind.lower()}/{owner.name} -n {ns} -c {name} --limits=memory={suggestion}"
198
+ if owner.kind in ("Deployment", "StatefulSet", "DaemonSet") and limit else _pod_cmd(pod),
199
+ f"kubectl top pod {pname} -n {ns} --containers",
200
+ ],
201
+ since=parse_time(term.get("finishedAt")),
202
+ node=(pod.get("spec") or {}).get("nodeName"),
203
+ )
204
+
205
+
206
+ _PULL_HINTS = [
207
+ (re.compile(r"toomanyrequests|rate limit", re.I),
208
+ "Docker Hub rate limit reached. Authenticate pulls with an imagePullSecret, use a pull-through cache, "
209
+ "or mirror the image to your own registry.", Severity.HIGH),
210
+ (re.compile(r"not found|manifest unknown|does not exist", re.I),
211
+ "The image or tag does not exist. Check the spelling and that the tag was pushed.", Severity.CRITICAL),
212
+ (re.compile(r"unauthorized|authentication required|denied|403 Forbidden", re.I),
213
+ "The registry rejected the pull: add or fix the imagePullSecret, or grant the node role access to the registry.",
214
+ Severity.CRITICAL),
215
+ (re.compile(r"no such host|i/o timeout|dial tcp|connection refused|TLS handshake timeout", re.I),
216
+ "Nodes cannot reach the registry: check egress firewall/NAT, proxy settings and DNS.", Severity.CRITICAL),
217
+ (re.compile(r"no match for platform", re.I),
218
+ "The image has no build for the node's CPU architecture.", Severity.CRITICAL),
219
+ ]
220
+
221
+
222
+ @check("pod-image-pull", "Image cannot be pulled", Category.WORKLOAD, Severity.CRITICAL)
223
+ def image_pull(ctx: Context) -> Iterator[Finding]:
224
+ """ErrImagePull / ImagePullBackOff / InvalidImageName."""
225
+ for pod in ctx.snap.all("pods"):
226
+ for cs, _ in _statuses(pod):
227
+ waiting = (cs.get("state") or {}).get("waiting") or {}
228
+ reason = waiting.get("reason", "")
229
+ if reason not in ("ErrImagePull", "ImagePullBackOff", "InvalidImageName", "ErrImageNeverPull"):
230
+ continue
231
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name", "")
232
+ messages = [waiting.get("message", "")] + [
233
+ e.get("message", "") for e in ctx.snap.events_for("Pod", ns, pname) if e.get("reason") in ("Failed", "BackOff")
234
+ ]
235
+ text = "\n".join(m for m in messages if m)
236
+ advice, sev = "Check the image name, tag and registry credentials.", Severity.CRITICAL
237
+ for pattern, hint, s in _PULL_HINTS:
238
+ if pattern.search(text):
239
+ advice, sev = hint, s
240
+ break
241
+ image = cs.get("image") or _container_spec(pod, cs.get("name", "")).get("image", "?")
242
+ yield ctx.finding(
243
+ "pod-image-pull", pod,
244
+ f"Image '{image}' for container '{cs.get('name')}' cannot be pulled ({reason}). {advice}",
245
+ severity=sev,
246
+ evidence=[m[:300] for m in messages if m][:3],
247
+ remediation=advice,
248
+ commands=[_pod_cmd(pod), f"kubectl get events -n {ns} --field-selector involvedObject.name={pname}"],
249
+ since=parse_time(meta(pod).get("creationTimestamp")),
250
+ node=(pod.get("spec") or {}).get("nodeName"),
251
+ )
252
+
253
+
254
+ @check("pod-config-error", "Container cannot start: configuration error", Category.CONFIG, Severity.CRITICAL)
255
+ def config_error(ctx: Context) -> Iterator[Finding]:
256
+ """CreateContainerConfigError / CreateContainerError / RunContainerError."""
257
+ for pod in ctx.snap.all("pods"):
258
+ for cs, _ in _statuses(pod):
259
+ waiting = (cs.get("state") or {}).get("waiting") or {}
260
+ reason = waiting.get("reason", "")
261
+ if reason not in ("CreateContainerConfigError", "CreateContainerError", "RunContainerError",
262
+ "ContainerCannotRun"):
263
+ continue
264
+ msg = waiting.get("message", "")
265
+ advice = "Fix the referenced configuration and the pod will start on its own."
266
+ if "secret" in msg.lower():
267
+ advice = "Create the missing Secret/key (or mark the reference optional), in the same namespace."
268
+ elif "configmap" in msg.lower():
269
+ advice = "Create the missing ConfigMap/key (or mark the reference optional), in the same namespace."
270
+ elif "executable file not found" in msg or "no such file" in msg:
271
+ advice = "The container command does not exist in the image: fix `command`/`args` or the image entrypoint."
272
+ yield ctx.finding(
273
+ "pod-config-error", pod,
274
+ f"Container '{cs.get('name')}' cannot start ({reason}): {msg[:200]}",
275
+ evidence=[msg[:400]] if msg else [],
276
+ remediation=advice,
277
+ commands=[_pod_cmd(pod)],
278
+ since=parse_time(meta(pod).get("creationTimestamp")),
279
+ )
280
+
281
+
282
+ _SCHED_HINTS = [
283
+ (re.compile(r"Insufficient (cpu|memory|ephemeral-storage|nvidia\.com/gpu|[\w./-]+)"),
284
+ "Not enough free {0} on any node. Lower the request, remove idle workloads, or add nodes (check the cluster autoscaler)."),
285
+ (re.compile(r"untolerated taint|had taint|taint\(s\)"),
286
+ "All candidate nodes have taints the pod does not tolerate. Add a toleration or schedule on untainted nodes."),
287
+ (re.compile(r"didn't match Pod's node affinity|node affinity/selector|node\(s\) didn't match node selector"),
288
+ "No node matches the pod's nodeSelector/affinity. Check node labels and the selector."),
289
+ (re.compile(r"unbound immediate PersistentVolumeClaims|persistentvolumeclaim .* not found|volume node affinity conflict"),
290
+ "A volume blocks scheduling: the PVC is unbound or the volume is in another zone than the free nodes."),
291
+ (re.compile(r"Too many pods"),
292
+ "Nodes reached their max pod count (often the CNI's IP limit). Add nodes or raise max pods."),
293
+ (re.compile(r"didn't match pod anti-affinity|didn't satisfy existing pods anti-affinity"),
294
+ "Pod anti-affinity leaves no valid node; there are more replicas than eligible nodes/zones."),
295
+ (re.compile(r"didn't match pod topology spread constraints"),
296
+ "Topology spread constraints cannot be satisfied with the current nodes/zones."),
297
+ (re.compile(r"node\(s\) were unschedulable"),
298
+ "Nodes are cordoned (unschedulable). Uncordon them or add capacity."),
299
+ ]
300
+
301
+
302
+ @check("pod-pending", "Pod cannot be scheduled", Category.SCHEDULING, Severity.HIGH)
303
+ def pending(ctx: Context) -> Iterator[Finding]:
304
+ """Pod stays Pending; explains the scheduler's reason."""
305
+ for pod in ctx.snap.all("pods"):
306
+ status = pod.get("status") or {}
307
+ if status.get("phase") != "Pending" or (pod.get("spec") or {}).get("nodeName"):
308
+ continue
309
+ a = age(meta(pod).get("creationTimestamp"))
310
+ if a is not None and a < timedelta(minutes=2):
311
+ continue
312
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name", "")
313
+ cond = next((c for c in status.get("conditions") or [] if c.get("type") == "PodScheduled"), {})
314
+ messages = [cond.get("message", "")] + [
315
+ e.get("message", "") for e in ctx.snap.events_for("Pod", ns, pname) if e.get("reason") == "FailedScheduling"
316
+ ]
317
+ text = "\n".join(m for m in messages if m)
318
+ hints = []
319
+ for pattern, template in _SCHED_HINTS:
320
+ m = pattern.search(text)
321
+ if m:
322
+ hints.append(template.format(*(m.groups() or ("",))))
323
+ nominated = status.get("nominatedNodeName")
324
+ yield ctx.finding(
325
+ "pod-pending", pod,
326
+ f"Pod has been Pending for {human_duration(a)}. " + (hints[0] if hints else "The scheduler has not placed it yet."),
327
+ evidence=[m[:400] for m in messages if m][:3] + ([f"nominated node {nominated}"] if nominated else []),
328
+ remediation=" ".join(hints[1:]) or "Run `kubectl describe pod` and read the scheduler events.",
329
+ commands=[_pod_cmd(pod), "kubectl describe nodes | grep -A6 'Allocated resources'"],
330
+ since=parse_time(meta(pod).get("creationTimestamp")),
331
+ )
332
+
333
+
334
+ @check("pod-not-ready", "Pod is running but not ready", Category.WORKLOAD, Severity.HIGH)
335
+ def not_ready(ctx: Context) -> Iterator[Finding]:
336
+ """Running for 5+ minutes but failing readiness, so it receives no traffic."""
337
+ for pod in ctx.snap.all("pods"):
338
+ status = pod.get("status") or {}
339
+ if status.get("phase") != "Running" or meta(pod).get("deletionTimestamp"):
340
+ continue
341
+ ready = next((c for c in status.get("conditions") or [] if c.get("type") == "Ready"), None)
342
+ if not ready or ready.get("status") == "True":
343
+ continue
344
+ since = age(ready.get("lastTransitionTime"))
345
+ if since is not None and since < timedelta(minutes=5):
346
+ continue
347
+ bad = [cs for cs in status.get("containerStatuses") or []
348
+ if not cs.get("ready") and (cs.get("state") or {}).get("running")]
349
+ if not bad:
350
+ continue # crash loops and waiting states are covered by other checks
351
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name", "")
352
+ probe_events = [e.get("message", "") for e in ctx.snap.events_for("Pod", ns, pname)
353
+ if "Readiness probe failed" in (e.get("message") or "")]
354
+ evidence = [f"not ready for {human_duration(since)}"] + [m[:300] for m in probe_events[:2]]
355
+ mesh = [c.get("name") for c in bad if c.get("name") in ("istio-proxy", "linkerd-proxy", "envoy", "cilium-envoy")]
356
+ if mesh:
357
+ evidence.insert(0, f"service-mesh sidecar {mesh[0]} is the unready container: check the mesh control "
358
+ "plane (istiod / linkerd-destination) and the sidecar logs, not the app")
359
+ for cs in bad:
360
+ evidence += log_hints(ctx, ns, pname, cs.get("name", ""))[1][:3]
361
+ yield ctx.finding(
362
+ "pod-not-ready", pod,
363
+ f"Containers {', '.join(c.get('name', '') for c in bad)} are running but fail their readiness probe, "
364
+ "so the pod gets no Service traffic.",
365
+ evidence=evidence,
366
+ remediation="Check that the probe path/port matches the app, that dependencies the readiness check "
367
+ "calls are reachable, and that timeoutSeconds is long enough.",
368
+ commands=[_pod_cmd(pod), f"kubectl logs {pname} -n {ns} --tail=50"],
369
+ since=parse_time(ready.get("lastTransitionTime")),
370
+ node=(pod.get("spec") or {}).get("nodeName"),
371
+ )
372
+
373
+
374
+ @check("pod-restarting", "Container restarts often", Category.WORKLOAD, Severity.MEDIUM)
375
+ def restarting(ctx: Context) -> Iterator[Finding]:
376
+ """Running now, but restarted 5+ times: an intermittent crash or liveness kill."""
377
+ for pod in ctx.snap.all("pods"):
378
+ for cs, _ in _statuses(pod):
379
+ restarts = cs.get("restartCount", 0)
380
+ state = cs.get("state") or {}
381
+ if restarts < 5 or not state.get("running") or is_crash_looping(cs):
382
+ continue
383
+ last = (cs.get("lastState") or {}).get("terminated") or {}
384
+ if last.get("reason") == "OOMKilled":
385
+ continue
386
+ finished = parse_time(last.get("finishedAt"))
387
+ if finished and age(last.get("finishedAt")) > timedelta(hours=24):
388
+ continue # old restarts, now stable
389
+ code = last.get("exitCode")
390
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name", "")
391
+ liveness = _liveness_killed(ctx, pod)
392
+ yield ctx.finding(
393
+ "pod-restarting", pod,
394
+ f"Container '{cs.get('name')}' restarted {restarts} times"
395
+ + (f", last exit code {code} ({EXIT_CODES.get(code, 'app-specific')})" if code is not None else "")
396
+ + (". The liveness probe is killing it." if liveness else "."),
397
+ evidence=[f"last restart finished at {last.get('finishedAt')}"] + log_hints(ctx, ns, pname, cs.get("name", ""))[1],
398
+ remediation="If the liveness probe kills it, make the probe cheaper or add a startupProbe; "
399
+ "otherwise read the previous logs for the crash.",
400
+ commands=[f"kubectl logs {pname} -n {ns} -c {cs.get('name')} --previous --tail=100"],
401
+ since=finished,
402
+ node=(pod.get("spec") or {}).get("nodeName"),
403
+ )
404
+
405
+
406
+ @check("pod-evicted", "Pods were evicted", Category.RESOURCES, Severity.MEDIUM)
407
+ def evicted(ctx: Context) -> Iterator[Finding]:
408
+ """The kubelet evicted pods under node pressure (memory, disk, ephemeral storage)."""
409
+ for pod in ctx.snap.all("pods"):
410
+ status = pod.get("status") or {}
411
+ if status.get("reason") != "Evicted":
412
+ continue
413
+ msg = status.get("message", "")
414
+ hint = "Set requests close to real usage so the scheduler does not overpack nodes."
415
+ if "ephemeral-storage" in msg or "ephemeral" in msg:
416
+ hint = "The pod wrote too much to local disk (logs, /tmp, emptyDir). Set an ephemeral-storage limit or use a volume."
417
+ elif "memory" in msg:
418
+ hint = "The node ran low on memory. Pods using more memory than they request are evicted first; raise requests."
419
+ yield ctx.finding(
420
+ "pod-evicted", pod, f"Pod was evicted: {msg[:200]}",
421
+ evidence=[msg[:400]],
422
+ remediation=hint + " Evicted pods stay listed until deleted.",
423
+ commands=[f"kubectl delete pods -n {meta(pod).get('namespace')} --field-selector=status.phase=Failed"],
424
+ since=parse_time(status.get("startTime")),
425
+ node=(pod.get("spec") or {}).get("nodeName"),
426
+ )
427
+
428
+
429
+ @check("pod-stuck-terminating", "Pod is stuck terminating", Category.WORKLOAD, Severity.MEDIUM)
430
+ def stuck_terminating(ctx: Context) -> Iterator[Finding]:
431
+ """Deletion requested long ago but the pod is still there (finalizers or an unreachable node)."""
432
+ for pod in ctx.snap.all("pods"):
433
+ deleted = meta(pod).get("deletionTimestamp")
434
+ if not deleted:
435
+ continue
436
+ grace = (pod.get("spec") or {}).get("terminationGracePeriodSeconds") or 30
437
+ a = age(deleted)
438
+ if a is None or a < timedelta(seconds=grace + 300):
439
+ continue
440
+ finalizers = meta(pod).get("finalizers") or []
441
+ node = (pod.get("spec") or {}).get("nodeName")
442
+ node_obj = ctx.snap.get("nodes", None, node or "")
443
+ node_ready = _node_ready(node_obj) if node_obj else None
444
+ reason = (f"finalizers {finalizers} are not being removed" if finalizers
445
+ else f"node {node} is not ready, so the kubelet cannot confirm shutdown" if node_ready is False
446
+ else "the kubelet has not confirmed the container stopped")
447
+ yield ctx.finding(
448
+ "pod-stuck-terminating", pod,
449
+ f"Pod has been terminating for {human_duration(a)}: {reason}.",
450
+ evidence=[f"deletionTimestamp {deleted}", f"finalizers: {finalizers or 'none'}"],
451
+ remediation="Find what owns the finalizer and why it is stuck. Force deletion is a last resort "
452
+ "and can leave the process running on the node.",
453
+ commands=[_pod_cmd(pod, "get") + " -o yaml"],
454
+ )
455
+
456
+
457
+ def _node_ready(node: dict[str, Any]) -> bool | None:
458
+ for c in (node.get("status") or {}).get("conditions") or []:
459
+ if c.get("type") == "Ready":
460
+ return c.get("status") == "True"
461
+ return None
462
+
463
+
464
+ @check("pod-init-blocked", "Init container has not finished", Category.WORKLOAD, Severity.HIGH)
465
+ def init_blocked(ctx: Context) -> Iterator[Finding]:
466
+ """An init container is still running after 10 minutes (often waiting on a dependency)."""
467
+ for pod in ctx.snap.all("pods"):
468
+ status = pod.get("status") or {}
469
+ if status.get("phase") != "Pending":
470
+ continue
471
+ for cs in status.get("initContainerStatuses") or []:
472
+ running = (cs.get("state") or {}).get("running")
473
+ if not running:
474
+ continue
475
+ a = age(running.get("startedAt"))
476
+ if a is None or a < timedelta(minutes=10):
477
+ continue
478
+ ns, pname = meta(pod).get("namespace"), meta(pod).get("name", "")
479
+ yield ctx.finding(
480
+ "pod-init-blocked", pod,
481
+ f"Init container '{cs.get('name')}' has been running for {human_duration(a)}; the app containers cannot start.",
482
+ evidence=log_hints(ctx, ns, pname, cs.get("name", ""))[1],
483
+ remediation="Init containers usually wait for a database, migration or service. Check that the dependency exists and is reachable.",
484
+ commands=[f"kubectl logs {pname} -n {ns} -c {cs.get('name')} --tail=50"],
485
+ )
@@ -0,0 +1,72 @@
1
+ """PersistentVolumeClaims, StorageClasses and volume mount failures."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterator
6
+ from datetime import timedelta
7
+
8
+ from kubesift.checks.base import Context, check
9
+ from kubesift.models import Category, Finding, Severity
10
+ from kubesift.util import age, human_duration, meta
11
+
12
+
13
+ @check("pvc-pending", "Volume claim is not bound", Category.STORAGE, Severity.HIGH)
14
+ def pvc_pending(ctx: Context) -> Iterator[Finding]:
15
+ """A PVC is Pending or Lost, so pods using it cannot start."""
16
+ classes = {meta(sc).get("name"): sc for sc in ctx.snap.all("storageclasses")}
17
+ default = [n for n, sc in classes.items()
18
+ if (meta(sc).get("annotations") or {}).get("storageclass.kubernetes.io/is-default-class") == "true"]
19
+ for pvc in ctx.snap.all("pvcs"):
20
+ phase = (pvc.get("status") or {}).get("phase")
21
+ if phase not in ("Pending", "Lost"):
22
+ continue
23
+ a = age(meta(pvc).get("creationTimestamp"))
24
+ if phase == "Pending" and a is not None and a < timedelta(minutes=2):
25
+ continue
26
+ spec = pvc.get("spec") or {}
27
+ sc_name = spec.get("storageClassName")
28
+ events = [e.get("message", "") for e in ctx.snap.events_for("PersistentVolumeClaim", meta(pvc).get("namespace"),
29
+ meta(pvc).get("name", ""))]
30
+ sc = classes.get(sc_name) if sc_name else None
31
+ if phase == "Lost":
32
+ msg, fix = "The bound PersistentVolume was deleted; data may be gone.", "Restore the PV from backup or recreate the claim."
33
+ elif sc_name and ctx.snap.can_read("storageclasses") and sc is None:
34
+ msg = f"StorageClass '{sc_name}' does not exist (available: {', '.join(sorted(classes)) or 'none'})."
35
+ fix = "Use an existing StorageClass or install the CSI driver that provides it."
36
+ elif not sc_name and ctx.snap.can_read("storageclasses") and not default:
37
+ msg, fix = "No storageClassName and the cluster has no default StorageClass.", "Set storageClassName or mark a default class."
38
+ elif sc and sc.get("volumeBindingMode") == "WaitForFirstConsumer" and any("waiting for first consumer" in e for e in events):
39
+ continue # normal: binds when a pod uses it
40
+ else:
41
+ msg = f"Pending for {human_duration(a)}." + (f" Provisioner says: {events[0][:200]}" if events else "")
42
+ fix = "Check the CSI driver/provisioner logs, cloud quotas and that the requested size and access mode are supported."
43
+ yield ctx.finding(
44
+ "pvc-pending", pvc, msg, severity=Severity.CRITICAL if phase == "Lost" else Severity.HIGH,
45
+ evidence=[e[:300] for e in events[:3]], remediation=fix,
46
+ commands=[f"kubectl describe pvc {meta(pvc).get('name')} -n {meta(pvc).get('namespace')}"],
47
+ )
48
+
49
+
50
+ @check("volume-mount-failed", "Volume cannot be attached or mounted", Category.STORAGE, Severity.HIGH)
51
+ def mount_failed(ctx: Context) -> Iterator[Finding]:
52
+ """FailedMount / FailedAttachVolume events on pods that are not running."""
53
+ for pod in ctx.snap.all("pods"):
54
+ if (pod.get("status") or {}).get("phase") != "Pending":
55
+ continue
56
+ ns, name = meta(pod).get("namespace"), meta(pod).get("name", "")
57
+ events = [e for e in ctx.snap.events_for("Pod", ns, name)
58
+ if e.get("reason") in ("FailedMount", "FailedAttachVolume")]
59
+ if not events:
60
+ continue
61
+ msg = events[0].get("message", "")
62
+ fix = "Check the volume and its driver."
63
+ if "Multi-Attach" in msg:
64
+ fix = ("A ReadWriteOnce volume is still attached to another node (usually the old pod during a rollout). "
65
+ "Use the Recreate strategy for single-writer volumes, or wait for the old node to release it.")
66
+ elif "not found" in msg:
67
+ fix = "The Secret/ConfigMap/PVC used as a volume does not exist."
68
+ elif "timed out" in msg:
69
+ fix = "The CSI driver did not finish in time: check the CSI node/controller pods and cloud API errors."
70
+ yield ctx.finding("volume-mount-failed", pod, f"{events[0].get('reason')}: {msg[:250]}",
71
+ evidence=[e.get("message", "")[:300] for e in events[:3]], remediation=fix,
72
+ node=(pod.get("spec") or {}).get("nodeName"))