kubesift 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift/__init__.py +3 -0
- kubesift/analysis/__init__.py +3 -0
- kubesift/analysis/changes.py +165 -0
- kubesift/analysis/correlate.py +160 -0
- kubesift/analysis/describe.py +83 -0
- kubesift/analysis/engine.py +362 -0
- kubesift/analysis/llm.py +152 -0
- kubesift/checks/__init__.py +17 -0
- kubesift/checks/base.py +150 -0
- kubesift/checks/config_refs.py +84 -0
- kubesift/checks/events.py +77 -0
- kubesift/checks/hygiene.py +264 -0
- kubesift/checks/lifecycle.py +202 -0
- kubesift/checks/network.py +325 -0
- kubesift/checks/nodes.py +183 -0
- kubesift/checks/operations.py +319 -0
- kubesift/checks/pods.py +485 -0
- kubesift/checks/storage.py +72 -0
- kubesift/checks/workloads.py +201 -0
- kubesift/cli.py +874 -0
- kubesift/cluster/__init__.py +4 -0
- kubesift/cluster/client.py +141 -0
- kubesift/cluster/snapshot.py +513 -0
- kubesift/config.py +134 -0
- kubesift/cost.py +150 -0
- kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
- kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
- kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
- kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
- kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
- kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
- kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
- kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
- kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
- kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
- kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
- kubesift/drill_scenarios/__init__.py +1 -0
- kubesift/drills.py +122 -0
- kubesift/knowledge.py +51 -0
- kubesift/leader.py +145 -0
- kubesift/lint.py +89 -0
- kubesift/mcp_server.py +193 -0
- kubesift/models.py +144 -0
- kubesift/notify.py +71 -0
- kubesift/postmortem.py +152 -0
- kubesift/preflight.py +435 -0
- kubesift/redact.py +38 -0
- kubesift/remediate.py +99 -0
- kubesift/report.py +193 -0
- kubesift/runbooks.py +58 -0
- kubesift/server/__init__.py +0 -0
- kubesift/server/app.py +480 -0
- kubesift/server/static/index.html +152 -0
- kubesift/slack.py +106 -0
- kubesift/store.py +219 -0
- kubesift/timetravel.py +156 -0
- kubesift/upgrade.py +51 -0
- kubesift/util.py +174 -0
- kubesift-2.0.0.dist-info/METADATA +192 -0
- kubesift-2.0.0.dist-info/RECORD +63 -0
- kubesift-2.0.0.dist-info/WHEEL +4 -0
- kubesift-2.0.0.dist-info/entry_points.txt +3 -0
- kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Controllers: Deployments, StatefulSets, DaemonSets, Jobs, CronJobs and HPAs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from datetime import timedelta
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from kubesift.checks.base import Context, check
|
|
10
|
+
from kubesift.models import Category, Finding, Severity
|
|
11
|
+
from kubesift.util import age, human_duration, meta, parse_time
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _cond(obj: dict[str, Any], ctype: str) -> dict[str, Any]:
|
|
15
|
+
return next((c for c in (obj.get("status") or {}).get("conditions") or [] if c.get("type") == ctype), {})
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@check("deployment-unavailable", "Deployment has fewer ready replicas than desired", Category.WORKLOAD, Severity.HIGH)
|
|
19
|
+
def deployment_unavailable(ctx: Context) -> Iterator[Finding]:
|
|
20
|
+
"""Ready replicas are below spec.replicas for more than 3 minutes."""
|
|
21
|
+
for d in ctx.snap.all("deployments"):
|
|
22
|
+
spec, status = d.get("spec") or {}, d.get("status") or {}
|
|
23
|
+
want = spec.get("replicas", 1)
|
|
24
|
+
ready = status.get("readyReplicas", 0) or 0
|
|
25
|
+
if want == 0 or ready >= want or spec.get("paused"):
|
|
26
|
+
continue
|
|
27
|
+
avail = _cond(d, "Available")
|
|
28
|
+
since = age(avail.get("lastTransitionTime")) if avail.get("status") == "False" else None
|
|
29
|
+
if since is not None and since < timedelta(minutes=3):
|
|
30
|
+
continue
|
|
31
|
+
ns, name = meta(d).get("namespace"), meta(d).get("name")
|
|
32
|
+
sev = Severity.CRITICAL if ready == 0 else Severity.HIGH
|
|
33
|
+
yield ctx.finding(
|
|
34
|
+
"deployment-unavailable", d,
|
|
35
|
+
f"{ready}/{want} replicas ready" + (" — the app is down." if ready == 0 else "."),
|
|
36
|
+
severity=sev,
|
|
37
|
+
evidence=[f"updated={status.get('updatedReplicas', 0)} available={status.get('availableReplicas', 0)} "
|
|
38
|
+
f"unavailable={status.get('unavailableReplicas', 0)}",
|
|
39
|
+
*([f"Available=False since {human_duration(since)}: {avail.get('message', '')}"] if since else [])],
|
|
40
|
+
remediation="The pod-level findings for this Deployment show why its pods are not ready.",
|
|
41
|
+
commands=[f"kubectl rollout status deployment/{name} -n {ns}", f"kubectl get pods -n {ns} -l "
|
|
42
|
+
+ ",".join(f"{k}={v}" for k, v in ((spec.get('selector') or {}).get('matchLabels') or {}).items())],
|
|
43
|
+
since=parse_time(avail.get("lastTransitionTime")),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@check("deployment-rollout-stuck", "Rollout is stuck", Category.WORKLOAD, Severity.HIGH)
|
|
48
|
+
def rollout_stuck(ctx: Context) -> Iterator[Finding]:
|
|
49
|
+
"""The Deployment exceeded progressDeadlineSeconds: the new version never became ready."""
|
|
50
|
+
for d in ctx.snap.all("deployments"):
|
|
51
|
+
prog = _cond(d, "Progressing")
|
|
52
|
+
if prog.get("reason") != "ProgressDeadlineExceeded":
|
|
53
|
+
continue
|
|
54
|
+
ns, name = meta(d).get("namespace"), meta(d).get("name")
|
|
55
|
+
yield ctx.finding(
|
|
56
|
+
"deployment-rollout-stuck", d,
|
|
57
|
+
f"The latest rollout did not finish: {prog.get('message', '')}",
|
|
58
|
+
evidence=[f"Progressing=False reason=ProgressDeadlineExceeded at {prog.get('lastUpdateTime')}"],
|
|
59
|
+
remediation="The new ReplicaSet's pods are failing. Fix them, or roll back to the last working version.",
|
|
60
|
+
commands=[f"kubectl rollout undo deployment/{name} -n {ns}", f"kubectl rollout history deployment/{name} -n {ns}"],
|
|
61
|
+
since=parse_time(prog.get("lastTransitionTime")),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@check("statefulset-unready", "StatefulSet has unready replicas", Category.WORKLOAD, Severity.HIGH)
|
|
66
|
+
def statefulset_unready(ctx: Context) -> Iterator[Finding]:
|
|
67
|
+
"""Ready replicas below desired; StatefulSets roll one pod at a time, so one bad pod blocks the rest."""
|
|
68
|
+
for s in ctx.snap.all("statefulsets"):
|
|
69
|
+
want = (s.get("spec") or {}).get("replicas", 1)
|
|
70
|
+
status = s.get("status") or {}
|
|
71
|
+
ready = status.get("readyReplicas", 0) or 0
|
|
72
|
+
if want == 0 or ready >= want:
|
|
73
|
+
continue
|
|
74
|
+
rolling = status.get("updateRevision") and status.get("currentRevision") != status.get("updateRevision")
|
|
75
|
+
yield ctx.finding(
|
|
76
|
+
"statefulset-unready", s,
|
|
77
|
+
f"{ready}/{want} replicas ready" + (" during an update; the rollout waits on the unready pod." if rolling else "."),
|
|
78
|
+
severity=Severity.CRITICAL if ready == 0 else Severity.HIGH,
|
|
79
|
+
evidence=[f"currentRevision={status.get('currentRevision')} updateRevision={status.get('updateRevision')}"],
|
|
80
|
+
remediation="Fix the unready pod (often a PVC or a peer it waits for). With OrderedReady, "
|
|
81
|
+
"a broken pod after a bad update must be deleted after reverting the template.",
|
|
82
|
+
commands=[f"kubectl rollout status statefulset/{meta(s).get('name')} -n {meta(s).get('namespace')}"],
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@check("daemonset-unavailable", "DaemonSet pods missing or unready", Category.WORKLOAD, Severity.HIGH)
|
|
87
|
+
def daemonset_unavailable(ctx: Context) -> Iterator[Finding]:
|
|
88
|
+
"""Nodes that should run the DaemonSet don't have a ready pod (agents, CNI, log shippers)."""
|
|
89
|
+
for ds in ctx.snap.all("daemonsets"):
|
|
90
|
+
st = ds.get("status") or {}
|
|
91
|
+
desired, ready = st.get("desiredNumberScheduled", 0), st.get("numberReady", 0)
|
|
92
|
+
if desired == 0 or ready >= desired:
|
|
93
|
+
continue
|
|
94
|
+
system = ctx.is_system(meta(ds).get("namespace"))
|
|
95
|
+
yield ctx.finding(
|
|
96
|
+
"daemonset-unavailable", ds,
|
|
97
|
+
f"{ready}/{desired} nodes have a ready pod" + (" (system component: node features may be broken)." if system else "."),
|
|
98
|
+
severity=Severity.CRITICAL if system and ready < desired else Severity.HIGH,
|
|
99
|
+
evidence=[f"unavailable={st.get('numberUnavailable', 0)} misscheduled={st.get('numberMisscheduled', 0)}"],
|
|
100
|
+
remediation="Look at the DaemonSet's pods on the affected nodes; taints or node resources are common causes.",
|
|
101
|
+
commands=[f"kubectl get pods -n {meta(ds).get('namespace')} -o wide | grep {meta(ds).get('name')}"],
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@check("job-failed", "Job failed", Category.WORKLOAD, Severity.HIGH)
|
|
106
|
+
def job_failed(ctx: Context) -> Iterator[Finding]:
|
|
107
|
+
"""A Job hit its backoffLimit or activeDeadlineSeconds."""
|
|
108
|
+
for job in ctx.snap.all("jobs"):
|
|
109
|
+
failed = _cond(job, "Failed")
|
|
110
|
+
if failed.get("status") != "True":
|
|
111
|
+
continue
|
|
112
|
+
a = age(failed.get("lastTransitionTime"))
|
|
113
|
+
if a is not None and a > timedelta(days=2):
|
|
114
|
+
continue
|
|
115
|
+
cron = next((r.get("name") for r in meta(job).get("ownerReferences") or [] if r.get("kind") == "CronJob"), None)
|
|
116
|
+
yield ctx.finding(
|
|
117
|
+
"job-failed", job,
|
|
118
|
+
f"Job failed ({failed.get('reason', '')}): {failed.get('message', '')}"
|
|
119
|
+
+ (f" It was started by CronJob {cron}." if cron else ""),
|
|
120
|
+
evidence=[f"failed pods: {(job.get('status') or {}).get('failed', 0)}",
|
|
121
|
+
f"backoffLimit: {(job.get('spec') or {}).get('backoffLimit', 6)}"],
|
|
122
|
+
remediation="Read the logs of the job's failed pods. Set ttlSecondsAfterFinished so finished Jobs are cleaned up.",
|
|
123
|
+
commands=[f"kubectl logs job/{meta(job).get('name')} -n {meta(job).get('namespace')} --tail=100"],
|
|
124
|
+
since=parse_time(failed.get("lastTransitionTime")),
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@check("cronjob-failing", "CronJob's recent runs fail", Category.WORKLOAD, Severity.HIGH)
|
|
129
|
+
def cronjob_failing(ctx: Context) -> Iterator[Finding]:
|
|
130
|
+
"""The last scheduled run failed or there has been no success since the last schedule."""
|
|
131
|
+
jobs_by_cron: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
132
|
+
for job in ctx.snap.all("jobs"):
|
|
133
|
+
for r in meta(job).get("ownerReferences") or []:
|
|
134
|
+
if r.get("kind") == "CronJob":
|
|
135
|
+
jobs_by_cron.setdefault((meta(job).get("namespace"), r.get("name")), []).append(job)
|
|
136
|
+
for cj in ctx.snap.all("cronjobs"):
|
|
137
|
+
if (cj.get("spec") or {}).get("suspend"):
|
|
138
|
+
continue
|
|
139
|
+
key = (meta(cj).get("namespace"), meta(cj).get("name"))
|
|
140
|
+
jobs = sorted(jobs_by_cron.get(key, []), key=lambda j: meta(j).get("creationTimestamp", ""))
|
|
141
|
+
if not jobs:
|
|
142
|
+
continue
|
|
143
|
+
latest = jobs[-1]
|
|
144
|
+
if _cond(latest, "Failed").get("status") != "True":
|
|
145
|
+
continue
|
|
146
|
+
status = cj.get("status") or {}
|
|
147
|
+
yield ctx.finding(
|
|
148
|
+
"cronjob-failing", cj,
|
|
149
|
+
f"The latest run ({meta(latest).get('name')}) failed. Last success: {status.get('lastSuccessfulTime') or 'never'}.",
|
|
150
|
+
evidence=[f"lastScheduleTime={status.get('lastScheduleTime')}",
|
|
151
|
+
f"{sum(1 for j in jobs if _cond(j, 'Failed').get('status') == 'True')}/{len(jobs)} kept jobs failed"],
|
|
152
|
+
remediation="Read the failed job's logs. Missed backups and reports often go unnoticed; alert on this.",
|
|
153
|
+
commands=[f"kubectl logs job/{meta(latest).get('name')} -n {key[0]} --tail=100"],
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@check("hpa-maxed-out", "Autoscaler is at its maximum", Category.RESOURCES, Severity.MEDIUM)
|
|
158
|
+
def hpa_maxed(ctx: Context) -> Iterator[Finding]:
|
|
159
|
+
"""The HPA wants more replicas than maxReplicas allows, so load is not being absorbed."""
|
|
160
|
+
for h in ctx.snap.all("hpas"):
|
|
161
|
+
spec, st = h.get("spec") or {}, h.get("status") or {}
|
|
162
|
+
mx = spec.get("maxReplicas")
|
|
163
|
+
if not mx or st.get("currentReplicas", 0) < mx or st.get("desiredReplicas", 0) < mx:
|
|
164
|
+
continue
|
|
165
|
+
limited = _cond(h, "ScalingLimited")
|
|
166
|
+
if limited.get("status") != "True" or limited.get("reason") != "TooManyReplicas":
|
|
167
|
+
continue
|
|
168
|
+
target = spec.get("scaleTargetRef") or {}
|
|
169
|
+
yield ctx.finding(
|
|
170
|
+
"hpa-maxed-out", h,
|
|
171
|
+
f"Running at maxReplicas={mx} and still above its target metrics; {target.get('kind')}/{target.get('name')} is capped.",
|
|
172
|
+
evidence=[f"current={st.get('currentReplicas')} desired={st.get('desiredReplicas')}",
|
|
173
|
+
limited.get("message", "")],
|
|
174
|
+
remediation="Raise maxReplicas if the cluster has room, or make each replica handle more load.",
|
|
175
|
+
commands=[f"kubectl describe hpa {meta(h).get('name')} -n {meta(h).get('namespace')}"],
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@check("hpa-cannot-scale", "Autoscaler cannot read metrics", Category.RESOURCES, Severity.HIGH)
|
|
180
|
+
def hpa_broken(ctx: Context) -> Iterator[Finding]:
|
|
181
|
+
"""ScalingActive=False: missing metrics-server, missing resource requests, or a bad custom metric."""
|
|
182
|
+
for h in ctx.snap.all("hpas"):
|
|
183
|
+
active = _cond(h, "ScalingActive")
|
|
184
|
+
able = _cond(h, "AbleToScale")
|
|
185
|
+
bad = active if active.get("status") == "False" else able if able.get("status") == "False" else None
|
|
186
|
+
if not bad:
|
|
187
|
+
continue
|
|
188
|
+
msg = bad.get("message", "")
|
|
189
|
+
hint = "Install metrics-server or fix the metrics adapter."
|
|
190
|
+
if "missing request" in msg:
|
|
191
|
+
hint = "The target's containers have no CPU/memory request, so utilization cannot be computed. Add requests."
|
|
192
|
+
elif "not find the requested resource" in msg or "unable to get metrics" in msg:
|
|
193
|
+
hint = "The metrics API is not available. Install metrics-server (or the custom metrics adapter)."
|
|
194
|
+
yield ctx.finding(
|
|
195
|
+
"hpa-cannot-scale", h,
|
|
196
|
+
f"The autoscaler is not working ({bad.get('reason')}): {msg[:200]}",
|
|
197
|
+
evidence=[msg[:400]],
|
|
198
|
+
remediation=hint + " Until fixed, the workload will not scale under load.",
|
|
199
|
+
commands=[f"kubectl describe hpa {meta(h).get('name')} -n {meta(h).get('namespace')}",
|
|
200
|
+
"kubectl get apiservice v1beta1.metrics.k8s.io"],
|
|
201
|
+
)
|