kubesift 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift/__init__.py +3 -0
- kubesift/analysis/__init__.py +3 -0
- kubesift/analysis/changes.py +165 -0
- kubesift/analysis/correlate.py +160 -0
- kubesift/analysis/describe.py +83 -0
- kubesift/analysis/engine.py +362 -0
- kubesift/analysis/llm.py +152 -0
- kubesift/checks/__init__.py +17 -0
- kubesift/checks/base.py +150 -0
- kubesift/checks/config_refs.py +84 -0
- kubesift/checks/events.py +77 -0
- kubesift/checks/hygiene.py +264 -0
- kubesift/checks/lifecycle.py +202 -0
- kubesift/checks/network.py +325 -0
- kubesift/checks/nodes.py +183 -0
- kubesift/checks/operations.py +319 -0
- kubesift/checks/pods.py +485 -0
- kubesift/checks/storage.py +72 -0
- kubesift/checks/workloads.py +201 -0
- kubesift/cli.py +874 -0
- kubesift/cluster/__init__.py +4 -0
- kubesift/cluster/client.py +141 -0
- kubesift/cluster/snapshot.py +513 -0
- kubesift/config.py +134 -0
- kubesift/cost.py +150 -0
- kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
- kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
- kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
- kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
- kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
- kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
- kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
- kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
- kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
- kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
- kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
- kubesift/drill_scenarios/__init__.py +1 -0
- kubesift/drills.py +122 -0
- kubesift/knowledge.py +51 -0
- kubesift/leader.py +145 -0
- kubesift/lint.py +89 -0
- kubesift/mcp_server.py +193 -0
- kubesift/models.py +144 -0
- kubesift/notify.py +71 -0
- kubesift/postmortem.py +152 -0
- kubesift/preflight.py +435 -0
- kubesift/redact.py +38 -0
- kubesift/remediate.py +99 -0
- kubesift/report.py +193 -0
- kubesift/runbooks.py +58 -0
- kubesift/server/__init__.py +0 -0
- kubesift/server/app.py +480 -0
- kubesift/server/static/index.html +152 -0
- kubesift/slack.py +106 -0
- kubesift/store.py +219 -0
- kubesift/timetravel.py +156 -0
- kubesift/upgrade.py +51 -0
- kubesift/util.py +174 -0
- kubesift-2.0.0.dist-info/METADATA +192 -0
- kubesift-2.0.0.dist-info/RECORD +63 -0
- kubesift-2.0.0.dist-info/WHEEL +4 -0
- kubesift-2.0.0.dist-info/entry_points.txt +3 -0
- kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"""Day-to-day operational failures from public postmortems: rollout 502s, stale config, stuck namespaces,
|
|
2
|
+
fragile DNS, single-node replicas, autoscaler thrashing, Karpenter, and baseline security gaps."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
from collections import Counter, defaultdict
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from datetime import timedelta
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from kubesift.analysis.changes import last_user_update
|
|
13
|
+
from kubesift.checks.base import Context, check
|
|
14
|
+
from kubesift.checks.hygiene import templates
|
|
15
|
+
from kubesift.cluster.snapshot import event_time
|
|
16
|
+
from kubesift.models import Category, Finding, ResourceRef, Severity
|
|
17
|
+
from kubesift.util import (
|
|
18
|
+
age,
|
|
19
|
+
containers_of,
|
|
20
|
+
human_duration,
|
|
21
|
+
meta,
|
|
22
|
+
parse_time,
|
|
23
|
+
selector_matches,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _prestop_sleep(container: dict[str, Any]) -> int | None:
|
|
28
|
+
pre = ((container.get("lifecycle") or {}).get("preStop")) or {}
|
|
29
|
+
if not pre:
|
|
30
|
+
return None
|
|
31
|
+
if pre.get("sleep"):
|
|
32
|
+
return int(pre["sleep"].get("seconds", 0))
|
|
33
|
+
cmd = " ".join(str(x) for x in ((pre.get("exec") or {}).get("command") or []))
|
|
34
|
+
m = re.search(r"sleep\s+(\d+)", cmd)
|
|
35
|
+
return int(m.group(1)) if m else 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@check("rollout-drops-traffic", "Rollouts can drop live traffic (502s)", Category.RELIABILITY, Severity.MEDIUM,
|
|
39
|
+
live_only=False, risk=True)
|
|
40
|
+
def rollout_drops_traffic(ctx: Context) -> Iterator[Finding]:
|
|
41
|
+
"""Pods behind a Service with no preStop delay stop before load balancers stop routing to them,
|
|
42
|
+
so every deploy returns 502s / connection resets for a few seconds."""
|
|
43
|
+
for obj, spec in templates(ctx):
|
|
44
|
+
if obj.get("kind") not in ("Deployment", "StatefulSet"):
|
|
45
|
+
continue
|
|
46
|
+
labels = (((obj.get("spec") or {}).get("template") or {}).get("metadata") or {}).get("labels") or {}
|
|
47
|
+
ns = meta(obj).get("namespace")
|
|
48
|
+
served = any(selector_matches((s.get("spec") or {}).get("selector"), labels)
|
|
49
|
+
for s in ctx.snap.in_namespace("services", ns))
|
|
50
|
+
if not served:
|
|
51
|
+
continue
|
|
52
|
+
grace = spec.get("terminationGracePeriodSeconds", 30)
|
|
53
|
+
serving = [c for c in containers_of(spec) if c.get("ports")]
|
|
54
|
+
for c in serving:
|
|
55
|
+
sleep = _prestop_sleep(c)
|
|
56
|
+
if sleep is None:
|
|
57
|
+
yield ctx.finding(
|
|
58
|
+
"rollout-drops-traffic", obj,
|
|
59
|
+
f"Container '{c.get('name')}' has no preStop hook. During rollouts and scale-downs it gets SIGTERM "
|
|
60
|
+
"while load balancers and kube-proxy still send it traffic, so some requests fail.",
|
|
61
|
+
remediation="Add `lifecycle: {preStop: {sleep: {seconds: 10}}}` (or `exec: [sleep, '10']` on clusters "
|
|
62
|
+
"before 1.30), make the app drain on SIGTERM, and keep terminationGracePeriodSeconds above "
|
|
63
|
+
"the sleep plus the app's shutdown time.")
|
|
64
|
+
break
|
|
65
|
+
if sleep and grace <= sleep:
|
|
66
|
+
yield ctx.finding(
|
|
67
|
+
"rollout-drops-traffic", obj,
|
|
68
|
+
f"terminationGracePeriodSeconds ({grace}s) is not longer than the preStop sleep ({sleep}s): the pod "
|
|
69
|
+
"is killed before the app can finish in-flight requests.",
|
|
70
|
+
severity=Severity.HIGH,
|
|
71
|
+
remediation=f"Set terminationGracePeriodSeconds to at least {sleep + 20}.")
|
|
72
|
+
break
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@check("config-stale-in-pod", "Pods run with old config values", Category.CONFIG, Severity.MEDIUM)
|
|
76
|
+
def config_stale(ctx: Context) -> Iterator[Finding]:
|
|
77
|
+
"""A ConfigMap/Secret used through env vars changed after the pod started; env vars are only read at start."""
|
|
78
|
+
for pod in ctx.snap.all("pods"):
|
|
79
|
+
st = pod.get("status") or {}
|
|
80
|
+
if st.get("phase") != "Running":
|
|
81
|
+
continue
|
|
82
|
+
started = parse_time(st.get("startTime"))
|
|
83
|
+
if not started:
|
|
84
|
+
continue
|
|
85
|
+
ns = meta(pod).get("namespace")
|
|
86
|
+
stale = []
|
|
87
|
+
for c in containers_of(pod.get("spec"), include_init=True):
|
|
88
|
+
refs: set[tuple[str, str]] = set()
|
|
89
|
+
for ef in c.get("envFrom") or []:
|
|
90
|
+
if ef.get("configMapRef"):
|
|
91
|
+
refs.add(("configmaps", ef["configMapRef"].get("name", "")))
|
|
92
|
+
if ef.get("secretRef"):
|
|
93
|
+
refs.add(("secrets", ef["secretRef"].get("name", "")))
|
|
94
|
+
for env in c.get("env") or []:
|
|
95
|
+
vf = env.get("valueFrom") or {}
|
|
96
|
+
if vf.get("configMapKeyRef"):
|
|
97
|
+
refs.add(("configmaps", vf["configMapKeyRef"].get("name", "")))
|
|
98
|
+
if vf.get("secretKeyRef"):
|
|
99
|
+
refs.add(("secrets", vf["secretKeyRef"].get("name", "")))
|
|
100
|
+
for key, name in refs:
|
|
101
|
+
obj = ctx.snap.get(key, ns, name)
|
|
102
|
+
changed, who = last_user_update(obj) if obj else (None, None)
|
|
103
|
+
if changed and changed > started + timedelta(seconds=30):
|
|
104
|
+
stale.append((obj.get("kind"), name, changed, who))
|
|
105
|
+
if stale:
|
|
106
|
+
kind, name, changed, who = stale[0]
|
|
107
|
+
yield ctx.finding(
|
|
108
|
+
"config-stale-in-pod", pod,
|
|
109
|
+
f"{kind} '{name}' was changed at {changed:%Y-%m-%d %H:%M} UTC" + (f" by {who}" if who else "")
|
|
110
|
+
+ ", after this pod started. It still uses the old values (env vars are read only at start).",
|
|
111
|
+
evidence=[f"pod started {st.get('startTime')}"] + [f"{k} {n} updated {t:%H:%M}" for k, n, t, _ in stale[:4]],
|
|
112
|
+
remediation="Restart the workload to pick up the change, or add a config checksum annotation to the pod "
|
|
113
|
+
"template (or use a reloader) so changes roll out automatically.",
|
|
114
|
+
commands=[f"kubectl rollout restart {ctx.snap.owner_of(pod).kind.lower()}/{ctx.snap.owner_of(pod).name} -n {ns}"]
|
|
115
|
+
if ctx.snap.owner_of(pod).kind in ("Deployment", "StatefulSet", "DaemonSet") else [],
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@check("namespace-stuck-terminating", "Namespace stuck deleting", Category.PLATFORM, Severity.HIGH)
|
|
120
|
+
def namespace_stuck(ctx: Context) -> Iterator[Finding]:
|
|
121
|
+
"""A namespace has been Terminating for 10+ minutes, usually a finalizer or an unavailable APIService."""
|
|
122
|
+
for ns in ctx.snap.all("namespaces"):
|
|
123
|
+
if (ns.get("status") or {}).get("phase") != "Terminating":
|
|
124
|
+
continue
|
|
125
|
+
a = age(meta(ns).get("deletionTimestamp"))
|
|
126
|
+
if a is not None and a < timedelta(minutes=10):
|
|
127
|
+
continue
|
|
128
|
+
conds = [c for c in (ns.get("status") or {}).get("conditions") or [] if c.get("status") == "True"]
|
|
129
|
+
reason = conds[0].get("message", "") if conds else "resources with finalizers are still present"
|
|
130
|
+
hint = "Remove what blocks it; force-removing finalizers can leave cloud resources behind."
|
|
131
|
+
if any(c.get("type") in ("NamespaceDeletionDiscoveryFailure", "NamespaceDeletionGroupVersionParsingFailure")
|
|
132
|
+
for c in conds):
|
|
133
|
+
hint = "An aggregated API (APIService) is unavailable, so Kubernetes cannot list everything to delete. " \
|
|
134
|
+
"Fix or delete the broken APIService (see the apiservice-unavailable finding)."
|
|
135
|
+
yield ctx.finding(
|
|
136
|
+
"namespace-stuck-terminating", ns, f"Terminating for {human_duration(a)}: {reason[:250]}",
|
|
137
|
+
evidence=[f"{c.get('type')}: {c.get('message', '')[:200]}" for c in conds[:4]],
|
|
138
|
+
remediation=hint,
|
|
139
|
+
commands=[f"kubectl api-resources --verbs=list --namespaced -o name | xargs -n 1 kubectl get --show-kind "
|
|
140
|
+
f"--ignore-not-found -n {meta(ns).get('name')}"],
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@check("apiservice-unavailable", "Extension API is unavailable", Category.PLATFORM, Severity.HIGH)
|
|
145
|
+
def apiservice_unavailable(ctx: Context) -> Iterator[Finding]:
|
|
146
|
+
"""An aggregated APIService (metrics-server, custom metrics, ...) is down: it breaks HPAs, `kubectl top`,
|
|
147
|
+
API discovery and namespace deletion."""
|
|
148
|
+
for svc in ctx.snap.all("apiservices"):
|
|
149
|
+
if not (svc.get("spec") or {}).get("service"):
|
|
150
|
+
continue # built-in, served by the API server itself
|
|
151
|
+
cond = next((c for c in (svc.get("status") or {}).get("conditions") or [] if c.get("type") == "Available"), {})
|
|
152
|
+
if cond.get("status") != "False":
|
|
153
|
+
continue
|
|
154
|
+
backend = (svc.get("spec") or {}).get("service") or {}
|
|
155
|
+
name = meta(svc).get("name", "")
|
|
156
|
+
impact = "HPAs and `kubectl top` stop working" if "metrics" in name else "API discovery and namespace deletion fail"
|
|
157
|
+
yield ctx.finding(
|
|
158
|
+
"apiservice-unavailable", svc,
|
|
159
|
+
f"{name} is unavailable ({cond.get('reason')}): {impact}.",
|
|
160
|
+
evidence=[cond.get("message", "")[:300], f"backend: {backend.get('namespace')}/{backend.get('name')}"],
|
|
161
|
+
remediation="Fix the backing Service/pods, or delete the APIService if its add-on was uninstalled.",
|
|
162
|
+
commands=[f"kubectl get apiservice {name} -o yaml", f"kubectl -n {backend.get('namespace')} get pods"],
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
@check("dns-fragile", "Cluster DNS is a single point of failure", Category.PLATFORM, Severity.MEDIUM, risk=True)
|
|
167
|
+
def dns_fragile(ctx: Context) -> Iterator[Finding]:
|
|
168
|
+
"""CoreDNS with one replica, or with a CPU limit (throttling made DNS fail cluster-wide in public postmortems)."""
|
|
169
|
+
for d in ctx.snap.in_namespace("deployments", "kube-system"):
|
|
170
|
+
labels = (((d.get("spec") or {}).get("template") or {}).get("metadata") or {}).get("labels") or {}
|
|
171
|
+
if labels.get("k8s-app") != "kube-dns" and meta(d).get("name") != "coredns":
|
|
172
|
+
continue
|
|
173
|
+
replicas = (d.get("spec") or {}).get("replicas", 1)
|
|
174
|
+
if replicas < 2:
|
|
175
|
+
yield ctx.finding("dns-fragile", d, f"CoreDNS runs {replicas} replica: if that pod or its node fails, every "
|
|
176
|
+
"service-name lookup in the cluster fails.",
|
|
177
|
+
remediation="Run at least 2 replicas spread across nodes (or enable the dns-autoscaler).",
|
|
178
|
+
commands=["kubectl -n kube-system scale deployment coredns --replicas=2"])
|
|
179
|
+
for c in containers_of((d.get("spec") or {}).get("template", {}).get("spec")):
|
|
180
|
+
if ((c.get("resources") or {}).get("limits") or {}).get("cpu"):
|
|
181
|
+
yield ctx.finding("dns-fragile", d, "CoreDNS has a CPU limit. Under load it gets CPU-throttled, lookups "
|
|
182
|
+
"time out and apps retry, which makes it worse.",
|
|
183
|
+
severity=Severity.LOW,
|
|
184
|
+
remediation="Remove the CPU limit (keep a CPU request), consider NodeLocal DNSCache, and "
|
|
185
|
+
"lower ndots for apps that call external hosts.")
|
|
186
|
+
break
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@check("replicas-single-node", "All replicas on one node or zone", Category.RELIABILITY, Severity.MEDIUM, risk=True)
|
|
190
|
+
def replicas_single_node(ctx: Context) -> Iterator[Finding]:
|
|
191
|
+
"""Several replicas but every pod is on the same node (or zone): one failure takes the whole app down."""
|
|
192
|
+
zone_of = {meta(n).get("name"): (meta(n).get("labels") or {}).get("topology.kubernetes.io/zone")
|
|
193
|
+
for n in ctx.snap.all("nodes")}
|
|
194
|
+
if len(zone_of) < 2:
|
|
195
|
+
return
|
|
196
|
+
zones = {z for z in zone_of.values() if z}
|
|
197
|
+
by_owner: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
198
|
+
for p in ctx.snap.all("pods"):
|
|
199
|
+
if (p.get("status") or {}).get("phase") == "Running" and (p.get("spec") or {}).get("nodeName") \
|
|
200
|
+
and not ctx.is_system(meta(p).get("namespace")):
|
|
201
|
+
by_owner[str(ctx.snap.owner_of(p))].append(p)
|
|
202
|
+
for owner, pods in by_owner.items():
|
|
203
|
+
if len(pods) < 2 or not owner.startswith(("Deployment/", "StatefulSet/")):
|
|
204
|
+
continue
|
|
205
|
+
nodes = {(p.get("spec") or {}).get("nodeName") for p in pods}
|
|
206
|
+
pod_zones = {zone_of.get(n) for n in nodes}
|
|
207
|
+
kind, ns, name = owner.split("/", 2)
|
|
208
|
+
ref = ResourceRef(kind=kind, namespace=ns, name=name)
|
|
209
|
+
if len(nodes) == 1:
|
|
210
|
+
yield ctx.finding("replicas-single-node", ref,
|
|
211
|
+
f"All {len(pods)} replicas run on node {next(iter(nodes))}: losing that node is a full outage.",
|
|
212
|
+
severity=Severity.HIGH,
|
|
213
|
+
remediation="Add topologySpreadConstraints on kubernetes.io/hostname (whenUnsatisfiable: "
|
|
214
|
+
"ScheduleAnyway) or pod anti-affinity.")
|
|
215
|
+
elif len(zones) > 1 and len(pod_zones) == 1 and None not in pod_zones:
|
|
216
|
+
yield ctx.finding("replicas-single-node", ref,
|
|
217
|
+
f"All {len(pods)} replicas run in zone {next(iter(pod_zones))}: a zone outage takes it down.",
|
|
218
|
+
remediation="Spread across zones with topologySpreadConstraints on topology.kubernetes.io/zone.")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
@check("hpa-flapping", "Autoscaler keeps scaling up and down", Category.RESOURCES, Severity.MEDIUM)
|
|
222
|
+
def hpa_flapping(ctx: Context) -> Iterator[Finding]:
|
|
223
|
+
"""Frequent alternating rescales in the last hour: pods start and stop constantly, hurting latency and cost."""
|
|
224
|
+
rescales: dict[tuple[str | None, str], list[dict[str, Any]]] = defaultdict(list)
|
|
225
|
+
for e in ctx.snap.all("events"):
|
|
226
|
+
inv = e.get("involvedObject") or {}
|
|
227
|
+
if e.get("reason") != "SuccessfulRescale" or inv.get("kind") != "HorizontalPodAutoscaler":
|
|
228
|
+
continue
|
|
229
|
+
a = age(event_time(e))
|
|
230
|
+
if a is not None and a <= timedelta(hours=1):
|
|
231
|
+
rescales[(inv.get("namespace"), inv.get("name", ""))].append(e)
|
|
232
|
+
for (ns, name), events in rescales.items():
|
|
233
|
+
total = sum(e.get("count", 1) or 1 for e in events)
|
|
234
|
+
text = " ".join(e.get("message", "") for e in events)
|
|
235
|
+
if total >= 6 and "above target" in text and "below target" in text:
|
|
236
|
+
hpa = ctx.snap.get("hpas", ns, name)
|
|
237
|
+
yield ctx.finding(
|
|
238
|
+
"hpa-flapping", hpa or ResourceRef(kind="HorizontalPodAutoscaler", namespace=ns, name=name),
|
|
239
|
+
f"Rescaled {total} times in the last hour, both up and down.",
|
|
240
|
+
evidence=[e.get("message", "")[:200] for e in events[:3]],
|
|
241
|
+
remediation="Add a scale-down stabilization window (behavior.scaleDown.stabilizationWindowSeconds: 300), "
|
|
242
|
+
"check that requests match real usage, and avoid targets near normal load.",
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
@check("karpenter-nodeclaim-failed", "Karpenter cannot launch nodes", Category.NODE, Severity.HIGH)
|
|
247
|
+
def karpenter_failed(ctx: Context) -> Iterator[Finding]:
|
|
248
|
+
"""A Karpenter NodeClaim has not become Ready: capacity errors, launch template or IAM problems."""
|
|
249
|
+
for nc in ctx.snap.all("nodeclaims"):
|
|
250
|
+
a = age(meta(nc).get("creationTimestamp"))
|
|
251
|
+
if a is not None and a < timedelta(minutes=10):
|
|
252
|
+
continue
|
|
253
|
+
bad = next((c for c in (nc.get("status") or {}).get("conditions") or []
|
|
254
|
+
if c.get("type") in ("Launched", "Registered", "Initialized", "Ready") and c.get("status") == "False"),
|
|
255
|
+
None)
|
|
256
|
+
if not bad:
|
|
257
|
+
continue
|
|
258
|
+
msg = bad.get("message", "")
|
|
259
|
+
hint = "Check the Karpenter controller logs and the NodePool/EC2NodeClass."
|
|
260
|
+
if re.search(r"InsufficientInstanceCapacity|insufficient capacity", msg, re.I):
|
|
261
|
+
hint = "The cloud has no capacity for the allowed instance types: allow more types/zones or use on-demand."
|
|
262
|
+
elif re.search(r"UnauthorizedOperation|AccessDenied|not authorized", msg, re.I):
|
|
263
|
+
hint = "The Karpenter or node IAM role lacks a permission named in the message."
|
|
264
|
+
yield ctx.finding("karpenter-nodeclaim-failed", nc,
|
|
265
|
+
f"NodeClaim {bad.get('type')}=False for {human_duration(a)} ({bad.get('reason')}): {msg[:200]}",
|
|
266
|
+
remediation=hint, commands=["kubectl -n kube-system logs deploy/karpenter --tail=100"])
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
@check("pvc-stuck-terminating", "Volume claim stuck deleting", Category.STORAGE, Severity.MEDIUM)
|
|
270
|
+
def pvc_stuck(ctx: Context) -> Iterator[Finding]:
|
|
271
|
+
"""PVC deletion is blocked by kubernetes.io/pvc-protection because a pod still mounts it."""
|
|
272
|
+
for pvc in ctx.snap.all("pvcs"):
|
|
273
|
+
deleted = meta(pvc).get("deletionTimestamp")
|
|
274
|
+
if not deleted or (age(deleted) or timedelta()) < timedelta(minutes=10):
|
|
275
|
+
continue
|
|
276
|
+
ns, name = meta(pvc).get("namespace"), meta(pvc).get("name")
|
|
277
|
+
users = [meta(p).get("name") for p in ctx.snap.in_namespace("pods", ns)
|
|
278
|
+
if any((v.get("persistentVolumeClaim") or {}).get("claimName") == name
|
|
279
|
+
for v in (p.get("spec") or {}).get("volumes") or [])]
|
|
280
|
+
yield ctx.finding("pvc-stuck-terminating", pvc,
|
|
281
|
+
f"Deletion requested {human_duration(age(deleted))} ago but blocked"
|
|
282
|
+
+ (f": still mounted by {', '.join(users[:3])}." if users else f" by finalizers {meta(pvc).get('finalizers')}."),
|
|
283
|
+
remediation="Delete or scale down the pods using it; the claim is removed automatically afterwards.")
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _app_namespaces(ctx: Context) -> list[str]:
|
|
287
|
+
counts = Counter(meta(p).get("namespace") for p in ctx.snap.all("pods"))
|
|
288
|
+
return sorted(ns for ns in counts if ns and not ctx.is_system(ns) and ns not in ("ingress-nginx", "cert-manager"))
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
@check("security-pss-not-enforced", "Pod Security Standards not enforced", Category.SECURITY, Severity.LOW, risk=True)
|
|
292
|
+
def pss_not_enforced(ctx: Context) -> Iterator[Finding]:
|
|
293
|
+
"""Application namespaces without a pod-security.kubernetes.io/enforce label accept privileged pods."""
|
|
294
|
+
if not ctx.snap.can_read("namespaces"):
|
|
295
|
+
return
|
|
296
|
+
labels = {meta(n).get("name"): meta(n).get("labels") or {} for n in ctx.snap.all("namespaces")}
|
|
297
|
+
missing = [ns for ns in _app_namespaces(ctx) if "pod-security.kubernetes.io/enforce" not in labels.get(ns, {})]
|
|
298
|
+
if missing:
|
|
299
|
+
yield ctx.finding("security-pss-not-enforced", ResourceRef(kind="Namespace", name=missing[0]),
|
|
300
|
+
f"{len(missing)} application namespace(s) don't enforce a Pod Security level: "
|
|
301
|
+
f"{', '.join(missing[:6])}{'…' if len(missing) > 6 else ''}.",
|
|
302
|
+
remediation="Label them with pod-security.kubernetes.io/enforce=baseline (or restricted); start "
|
|
303
|
+
"with the warn label to see what would break.",
|
|
304
|
+
commands=[f"kubectl label namespace {missing[0]} pod-security.kubernetes.io/warn=restricted"])
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
@check("security-no-network-policy", "Namespaces with no NetworkPolicy", Category.SECURITY, Severity.LOW, risk=True)
|
|
308
|
+
def no_network_policy(ctx: Context) -> Iterator[Finding]:
|
|
309
|
+
"""Without any NetworkPolicy every pod can reach every other pod: one compromised pod reaches all of them."""
|
|
310
|
+
if not ctx.snap.can_read("networkpolicies"):
|
|
311
|
+
return
|
|
312
|
+
covered = {meta(n).get("namespace") for n in ctx.snap.all("networkpolicies")}
|
|
313
|
+
missing = [ns for ns in _app_namespaces(ctx) if ns not in covered]
|
|
314
|
+
if missing:
|
|
315
|
+
yield ctx.finding("security-no-network-policy", ResourceRef(kind="Namespace", name=missing[0]),
|
|
316
|
+
f"{len(missing)} application namespace(s) have no NetworkPolicy: {', '.join(missing[:6])}"
|
|
317
|
+
f"{'…' if len(missing) > 6 else ''}.",
|
|
318
|
+
remediation="Start with a default-deny ingress policy per namespace plus explicit allows "
|
|
319
|
+
"(remember to allow DNS egress if you also restrict egress).")
|