kubesift 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. kubesift/__init__.py +3 -0
  2. kubesift/analysis/__init__.py +3 -0
  3. kubesift/analysis/changes.py +165 -0
  4. kubesift/analysis/correlate.py +160 -0
  5. kubesift/analysis/describe.py +83 -0
  6. kubesift/analysis/engine.py +362 -0
  7. kubesift/analysis/llm.py +152 -0
  8. kubesift/checks/__init__.py +17 -0
  9. kubesift/checks/base.py +150 -0
  10. kubesift/checks/config_refs.py +84 -0
  11. kubesift/checks/events.py +77 -0
  12. kubesift/checks/hygiene.py +264 -0
  13. kubesift/checks/lifecycle.py +202 -0
  14. kubesift/checks/network.py +325 -0
  15. kubesift/checks/nodes.py +183 -0
  16. kubesift/checks/operations.py +319 -0
  17. kubesift/checks/pods.py +485 -0
  18. kubesift/checks/storage.py +72 -0
  19. kubesift/checks/workloads.py +201 -0
  20. kubesift/cli.py +874 -0
  21. kubesift/cluster/__init__.py +4 -0
  22. kubesift/cluster/client.py +141 -0
  23. kubesift/cluster/snapshot.py +513 -0
  24. kubesift/config.py +134 -0
  25. kubesift/cost.py +150 -0
  26. kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
  27. kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
  28. kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
  29. kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
  30. kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
  31. kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
  32. kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
  33. kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
  34. kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
  35. kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
  36. kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
  37. kubesift/drill_scenarios/__init__.py +1 -0
  38. kubesift/drills.py +122 -0
  39. kubesift/knowledge.py +51 -0
  40. kubesift/leader.py +145 -0
  41. kubesift/lint.py +89 -0
  42. kubesift/mcp_server.py +193 -0
  43. kubesift/models.py +144 -0
  44. kubesift/notify.py +71 -0
  45. kubesift/postmortem.py +152 -0
  46. kubesift/preflight.py +435 -0
  47. kubesift/redact.py +38 -0
  48. kubesift/remediate.py +99 -0
  49. kubesift/report.py +193 -0
  50. kubesift/runbooks.py +58 -0
  51. kubesift/server/__init__.py +0 -0
  52. kubesift/server/app.py +480 -0
  53. kubesift/server/static/index.html +152 -0
  54. kubesift/slack.py +106 -0
  55. kubesift/store.py +219 -0
  56. kubesift/timetravel.py +156 -0
  57. kubesift/upgrade.py +51 -0
  58. kubesift/util.py +174 -0
  59. kubesift-2.0.0.dist-info/METADATA +192 -0
  60. kubesift-2.0.0.dist-info/RECORD +63 -0
  61. kubesift-2.0.0.dist-info/WHEEL +4 -0
  62. kubesift-2.0.0.dist-info/entry_points.txt +3 -0
  63. kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,201 @@
1
+ """Controllers: Deployments, StatefulSets, DaemonSets, Jobs, CronJobs and HPAs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterator
6
+ from datetime import timedelta
7
+ from typing import Any
8
+
9
+ from kubesift.checks.base import Context, check
10
+ from kubesift.models import Category, Finding, Severity
11
+ from kubesift.util import age, human_duration, meta, parse_time
12
+
13
+
14
+ def _cond(obj: dict[str, Any], ctype: str) -> dict[str, Any]:
15
+ return next((c for c in (obj.get("status") or {}).get("conditions") or [] if c.get("type") == ctype), {})
16
+
17
+
18
+ @check("deployment-unavailable", "Deployment has fewer ready replicas than desired", Category.WORKLOAD, Severity.HIGH)
19
+ def deployment_unavailable(ctx: Context) -> Iterator[Finding]:
20
+ """Ready replicas are below spec.replicas for more than 3 minutes."""
21
+ for d in ctx.snap.all("deployments"):
22
+ spec, status = d.get("spec") or {}, d.get("status") or {}
23
+ want = spec.get("replicas", 1)
24
+ ready = status.get("readyReplicas", 0) or 0
25
+ if want == 0 or ready >= want or spec.get("paused"):
26
+ continue
27
+ avail = _cond(d, "Available")
28
+ since = age(avail.get("lastTransitionTime")) if avail.get("status") == "False" else None
29
+ if since is not None and since < timedelta(minutes=3):
30
+ continue
31
+ ns, name = meta(d).get("namespace"), meta(d).get("name")
32
+ sev = Severity.CRITICAL if ready == 0 else Severity.HIGH
33
+ yield ctx.finding(
34
+ "deployment-unavailable", d,
35
+ f"{ready}/{want} replicas ready" + (" — the app is down." if ready == 0 else "."),
36
+ severity=sev,
37
+ evidence=[f"updated={status.get('updatedReplicas', 0)} available={status.get('availableReplicas', 0)} "
38
+ f"unavailable={status.get('unavailableReplicas', 0)}",
39
+ *([f"Available=False since {human_duration(since)}: {avail.get('message', '')}"] if since else [])],
40
+ remediation="The pod-level findings for this Deployment show why its pods are not ready.",
41
+ commands=[f"kubectl rollout status deployment/{name} -n {ns}", f"kubectl get pods -n {ns} -l "
42
+ + ",".join(f"{k}={v}" for k, v in ((spec.get('selector') or {}).get('matchLabels') or {}).items())],
43
+ since=parse_time(avail.get("lastTransitionTime")),
44
+ )
45
+
46
+
47
+ @check("deployment-rollout-stuck", "Rollout is stuck", Category.WORKLOAD, Severity.HIGH)
48
+ def rollout_stuck(ctx: Context) -> Iterator[Finding]:
49
+ """The Deployment exceeded progressDeadlineSeconds: the new version never became ready."""
50
+ for d in ctx.snap.all("deployments"):
51
+ prog = _cond(d, "Progressing")
52
+ if prog.get("reason") != "ProgressDeadlineExceeded":
53
+ continue
54
+ ns, name = meta(d).get("namespace"), meta(d).get("name")
55
+ yield ctx.finding(
56
+ "deployment-rollout-stuck", d,
57
+ f"The latest rollout did not finish: {prog.get('message', '')}",
58
+ evidence=[f"Progressing=False reason=ProgressDeadlineExceeded at {prog.get('lastUpdateTime')}"],
59
+ remediation="The new ReplicaSet's pods are failing. Fix them, or roll back to the last working version.",
60
+ commands=[f"kubectl rollout undo deployment/{name} -n {ns}", f"kubectl rollout history deployment/{name} -n {ns}"],
61
+ since=parse_time(prog.get("lastTransitionTime")),
62
+ )
63
+
64
+
65
+ @check("statefulset-unready", "StatefulSet has unready replicas", Category.WORKLOAD, Severity.HIGH)
66
+ def statefulset_unready(ctx: Context) -> Iterator[Finding]:
67
+ """Ready replicas below desired; StatefulSets roll one pod at a time, so one bad pod blocks the rest."""
68
+ for s in ctx.snap.all("statefulsets"):
69
+ want = (s.get("spec") or {}).get("replicas", 1)
70
+ status = s.get("status") or {}
71
+ ready = status.get("readyReplicas", 0) or 0
72
+ if want == 0 or ready >= want:
73
+ continue
74
+ rolling = status.get("updateRevision") and status.get("currentRevision") != status.get("updateRevision")
75
+ yield ctx.finding(
76
+ "statefulset-unready", s,
77
+ f"{ready}/{want} replicas ready" + (" during an update; the rollout waits on the unready pod." if rolling else "."),
78
+ severity=Severity.CRITICAL if ready == 0 else Severity.HIGH,
79
+ evidence=[f"currentRevision={status.get('currentRevision')} updateRevision={status.get('updateRevision')}"],
80
+ remediation="Fix the unready pod (often a PVC or a peer it waits for). With OrderedReady, "
81
+ "a broken pod after a bad update must be deleted after reverting the template.",
82
+ commands=[f"kubectl rollout status statefulset/{meta(s).get('name')} -n {meta(s).get('namespace')}"],
83
+ )
84
+
85
+
86
+ @check("daemonset-unavailable", "DaemonSet pods missing or unready", Category.WORKLOAD, Severity.HIGH)
87
+ def daemonset_unavailable(ctx: Context) -> Iterator[Finding]:
88
+ """Nodes that should run the DaemonSet don't have a ready pod (agents, CNI, log shippers)."""
89
+ for ds in ctx.snap.all("daemonsets"):
90
+ st = ds.get("status") or {}
91
+ desired, ready = st.get("desiredNumberScheduled", 0), st.get("numberReady", 0)
92
+ if desired == 0 or ready >= desired:
93
+ continue
94
+ system = ctx.is_system(meta(ds).get("namespace"))
95
+ yield ctx.finding(
96
+ "daemonset-unavailable", ds,
97
+ f"{ready}/{desired} nodes have a ready pod" + (" (system component: node features may be broken)." if system else "."),
98
+ severity=Severity.CRITICAL if system and ready < desired else Severity.HIGH,
99
+ evidence=[f"unavailable={st.get('numberUnavailable', 0)} misscheduled={st.get('numberMisscheduled', 0)}"],
100
+ remediation="Look at the DaemonSet's pods on the affected nodes; taints or node resources are common causes.",
101
+ commands=[f"kubectl get pods -n {meta(ds).get('namespace')} -o wide | grep {meta(ds).get('name')}"],
102
+ )
103
+
104
+
105
+ @check("job-failed", "Job failed", Category.WORKLOAD, Severity.HIGH)
106
+ def job_failed(ctx: Context) -> Iterator[Finding]:
107
+ """A Job hit its backoffLimit or activeDeadlineSeconds."""
108
+ for job in ctx.snap.all("jobs"):
109
+ failed = _cond(job, "Failed")
110
+ if failed.get("status") != "True":
111
+ continue
112
+ a = age(failed.get("lastTransitionTime"))
113
+ if a is not None and a > timedelta(days=2):
114
+ continue
115
+ cron = next((r.get("name") for r in meta(job).get("ownerReferences") or [] if r.get("kind") == "CronJob"), None)
116
+ yield ctx.finding(
117
+ "job-failed", job,
118
+ f"Job failed ({failed.get('reason', '')}): {failed.get('message', '')}"
119
+ + (f" It was started by CronJob {cron}." if cron else ""),
120
+ evidence=[f"failed pods: {(job.get('status') or {}).get('failed', 0)}",
121
+ f"backoffLimit: {(job.get('spec') or {}).get('backoffLimit', 6)}"],
122
+ remediation="Read the logs of the job's failed pods. Set ttlSecondsAfterFinished so finished Jobs are cleaned up.",
123
+ commands=[f"kubectl logs job/{meta(job).get('name')} -n {meta(job).get('namespace')} --tail=100"],
124
+ since=parse_time(failed.get("lastTransitionTime")),
125
+ )
126
+
127
+
128
+ @check("cronjob-failing", "CronJob's recent runs fail", Category.WORKLOAD, Severity.HIGH)
129
+ def cronjob_failing(ctx: Context) -> Iterator[Finding]:
130
+ """The last scheduled run failed or there has been no success since the last schedule."""
131
+ jobs_by_cron: dict[tuple[str, str], list[dict[str, Any]]] = {}
132
+ for job in ctx.snap.all("jobs"):
133
+ for r in meta(job).get("ownerReferences") or []:
134
+ if r.get("kind") == "CronJob":
135
+ jobs_by_cron.setdefault((meta(job).get("namespace"), r.get("name")), []).append(job)
136
+ for cj in ctx.snap.all("cronjobs"):
137
+ if (cj.get("spec") or {}).get("suspend"):
138
+ continue
139
+ key = (meta(cj).get("namespace"), meta(cj).get("name"))
140
+ jobs = sorted(jobs_by_cron.get(key, []), key=lambda j: meta(j).get("creationTimestamp", ""))
141
+ if not jobs:
142
+ continue
143
+ latest = jobs[-1]
144
+ if _cond(latest, "Failed").get("status") != "True":
145
+ continue
146
+ status = cj.get("status") or {}
147
+ yield ctx.finding(
148
+ "cronjob-failing", cj,
149
+ f"The latest run ({meta(latest).get('name')}) failed. Last success: {status.get('lastSuccessfulTime') or 'never'}.",
150
+ evidence=[f"lastScheduleTime={status.get('lastScheduleTime')}",
151
+ f"{sum(1 for j in jobs if _cond(j, 'Failed').get('status') == 'True')}/{len(jobs)} kept jobs failed"],
152
+ remediation="Read the failed job's logs. Missed backups and reports often go unnoticed; alert on this.",
153
+ commands=[f"kubectl logs job/{meta(latest).get('name')} -n {key[0]} --tail=100"],
154
+ )
155
+
156
+
157
+ @check("hpa-maxed-out", "Autoscaler is at its maximum", Category.RESOURCES, Severity.MEDIUM)
158
+ def hpa_maxed(ctx: Context) -> Iterator[Finding]:
159
+ """The HPA wants more replicas than maxReplicas allows, so load is not being absorbed."""
160
+ for h in ctx.snap.all("hpas"):
161
+ spec, st = h.get("spec") or {}, h.get("status") or {}
162
+ mx = spec.get("maxReplicas")
163
+ if not mx or st.get("currentReplicas", 0) < mx or st.get("desiredReplicas", 0) < mx:
164
+ continue
165
+ limited = _cond(h, "ScalingLimited")
166
+ if limited.get("status") != "True" or limited.get("reason") != "TooManyReplicas":
167
+ continue
168
+ target = spec.get("scaleTargetRef") or {}
169
+ yield ctx.finding(
170
+ "hpa-maxed-out", h,
171
+ f"Running at maxReplicas={mx} and still above its target metrics; {target.get('kind')}/{target.get('name')} is capped.",
172
+ evidence=[f"current={st.get('currentReplicas')} desired={st.get('desiredReplicas')}",
173
+ limited.get("message", "")],
174
+ remediation="Raise maxReplicas if the cluster has room, or make each replica handle more load.",
175
+ commands=[f"kubectl describe hpa {meta(h).get('name')} -n {meta(h).get('namespace')}"],
176
+ )
177
+
178
+
179
+ @check("hpa-cannot-scale", "Autoscaler cannot read metrics", Category.RESOURCES, Severity.HIGH)
180
+ def hpa_broken(ctx: Context) -> Iterator[Finding]:
181
+ """ScalingActive=False: missing metrics-server, missing resource requests, or a bad custom metric."""
182
+ for h in ctx.snap.all("hpas"):
183
+ active = _cond(h, "ScalingActive")
184
+ able = _cond(h, "AbleToScale")
185
+ bad = active if active.get("status") == "False" else able if able.get("status") == "False" else None
186
+ if not bad:
187
+ continue
188
+ msg = bad.get("message", "")
189
+ hint = "Install metrics-server or fix the metrics adapter."
190
+ if "missing request" in msg:
191
+ hint = "The target's containers have no CPU/memory request, so utilization cannot be computed. Add requests."
192
+ elif "not find the requested resource" in msg or "unable to get metrics" in msg:
193
+ hint = "The metrics API is not available. Install metrics-server (or the custom metrics adapter)."
194
+ yield ctx.finding(
195
+ "hpa-cannot-scale", h,
196
+ f"The autoscaler is not working ({bad.get('reason')}): {msg[:200]}",
197
+ evidence=[msg[:400]],
198
+ remediation=hint + " Until fixed, the workload will not scale under load.",
199
+ commands=[f"kubectl describe hpa {meta(h).get('name')} -n {meta(h).get('namespace')}",
200
+ "kubectl get apiservice v1beta1.metrics.k8s.io"],
201
+ )