kubesift 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kubesift/__init__.py +3 -0
- kubesift/analysis/__init__.py +3 -0
- kubesift/analysis/changes.py +165 -0
- kubesift/analysis/correlate.py +160 -0
- kubesift/analysis/describe.py +83 -0
- kubesift/analysis/engine.py +362 -0
- kubesift/analysis/llm.py +152 -0
- kubesift/checks/__init__.py +17 -0
- kubesift/checks/base.py +150 -0
- kubesift/checks/config_refs.py +84 -0
- kubesift/checks/events.py +77 -0
- kubesift/checks/hygiene.py +264 -0
- kubesift/checks/lifecycle.py +202 -0
- kubesift/checks/network.py +325 -0
- kubesift/checks/nodes.py +183 -0
- kubesift/checks/operations.py +319 -0
- kubesift/checks/pods.py +485 -0
- kubesift/checks/storage.py +72 -0
- kubesift/checks/workloads.py +201 -0
- kubesift/cli.py +874 -0
- kubesift/cluster/__init__.py +4 -0
- kubesift/cluster/client.py +141 -0
- kubesift/cluster/snapshot.py +513 -0
- kubesift/config.py +134 -0
- kubesift/cost.py +150 -0
- kubesift/drill_scenarios/01-crashloop-missing-env.yaml +24 -0
- kubesift/drill_scenarios/02-imagepull-wrong-tag.yaml +23 -0
- kubesift/drill_scenarios/03-oomkilled-low-memory.yaml +24 -0
- kubesift/drill_scenarios/04-service-selector-mismatch.yaml +34 -0
- kubesift/drill_scenarios/05-missing-probe-crashloop.yaml +24 -0
- kubesift/drill_scenarios/06-pending-insufficient-resources.yaml +26 -0
- kubesift/drill_scenarios/07-missing-secret.yaml +28 -0
- kubesift/drill_scenarios/08-pvc-missing-storageclass.yaml +38 -0
- kubesift/drill_scenarios/09-netpol-blocks-dns.yaml +14 -0
- kubesift/drill_scenarios/10a-rollout-good.yaml +27 -0
- kubesift/drill_scenarios/10b-rollout-bad.yaml +25 -0
- kubesift/drill_scenarios/__init__.py +1 -0
- kubesift/drills.py +122 -0
- kubesift/knowledge.py +51 -0
- kubesift/leader.py +145 -0
- kubesift/lint.py +89 -0
- kubesift/mcp_server.py +193 -0
- kubesift/models.py +144 -0
- kubesift/notify.py +71 -0
- kubesift/postmortem.py +152 -0
- kubesift/preflight.py +435 -0
- kubesift/redact.py +38 -0
- kubesift/remediate.py +99 -0
- kubesift/report.py +193 -0
- kubesift/runbooks.py +58 -0
- kubesift/server/__init__.py +0 -0
- kubesift/server/app.py +480 -0
- kubesift/server/static/index.html +152 -0
- kubesift/slack.py +106 -0
- kubesift/store.py +219 -0
- kubesift/timetravel.py +156 -0
- kubesift/upgrade.py +51 -0
- kubesift/util.py +174 -0
- kubesift-2.0.0.dist-info/METADATA +192 -0
- kubesift-2.0.0.dist-info/RECORD +63 -0
- kubesift-2.0.0.dist-info/WHEEL +4 -0
- kubesift-2.0.0.dist-info/entry_points.txt +3 -0
- kubesift-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Thin wrapper over the official Kubernetes Python client that returns plain JSON dicts.
|
|
2
|
+
|
|
3
|
+
Using raw API paths (instead of the typed models) keeps objects identical to
|
|
4
|
+
`kubectl get -o json`, so every check works the same on live data, saved
|
|
5
|
+
snapshots and manifest files.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class KubeError(Exception):
|
|
16
|
+
def __init__(self, status: int, message: str):
|
|
17
|
+
super().__init__(f"HTTP {status}: {message}")
|
|
18
|
+
self.status = status
|
|
19
|
+
self.message = message
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class KubeClient:
|
|
23
|
+
def __init__(self, kubeconfig: str | None = None, context: str | None = None, timeout: int = 20):
|
|
24
|
+
from kubernetes import client, config
|
|
25
|
+
|
|
26
|
+
self.timeout = timeout
|
|
27
|
+
if not kubeconfig and not context and os.environ.get("KUBERNETES_SERVICE_HOST"):
|
|
28
|
+
config.load_incluster_config()
|
|
29
|
+
cfg = client.Configuration.get_default_copy()
|
|
30
|
+
self.context_name = "in-cluster"
|
|
31
|
+
else:
|
|
32
|
+
cfg = client.Configuration()
|
|
33
|
+
config.load_kube_config(config_file=kubeconfig, context=context, client_configuration=cfg)
|
|
34
|
+
if context:
|
|
35
|
+
self.context_name = context
|
|
36
|
+
else:
|
|
37
|
+
_, active = config.list_kube_config_contexts(config_file=kubeconfig)
|
|
38
|
+
self.context_name = (active or {}).get("name", "default")
|
|
39
|
+
self.api = client.ApiClient(cfg)
|
|
40
|
+
|
|
41
|
+
def request(
|
|
42
|
+
self,
|
|
43
|
+
method: str,
|
|
44
|
+
path: str,
|
|
45
|
+
query: dict[str, Any] | None = None,
|
|
46
|
+
body: Any = None,
|
|
47
|
+
content_type: str | None = None,
|
|
48
|
+
raw: bool = False,
|
|
49
|
+
) -> Any:
|
|
50
|
+
from kubernetes.client.exceptions import ApiException
|
|
51
|
+
|
|
52
|
+
headers = {"Accept": "*/*" if raw else "application/json"}
|
|
53
|
+
if content_type:
|
|
54
|
+
headers["Content-Type"] = content_type
|
|
55
|
+
query_params = [(k, v) for k, v in (query or {}).items() if v is not None]
|
|
56
|
+
try:
|
|
57
|
+
if hasattr(self.api, "param_serialize"): # kubernetes >= 37 (new generator)
|
|
58
|
+
params = self.api.param_serialize(
|
|
59
|
+
method=method, resource_path=path, query_params=query_params,
|
|
60
|
+
header_params=headers, body=body, auth_settings=["BearerToken"],
|
|
61
|
+
)
|
|
62
|
+
resp = self.api.call_api(*params, _request_timeout=self.timeout)
|
|
63
|
+
data = resp.read()
|
|
64
|
+
status = resp.status
|
|
65
|
+
else: # kubernetes <= 36
|
|
66
|
+
resp = self.api.call_api(
|
|
67
|
+
path, method, query_params=query_params, header_params=headers, body=body,
|
|
68
|
+
auth_settings=["BearerToken"], _preload_content=False, _return_http_data_only=True,
|
|
69
|
+
_request_timeout=self.timeout,
|
|
70
|
+
)
|
|
71
|
+
data = resp.data
|
|
72
|
+
status = resp.status
|
|
73
|
+
except ApiException as exc:
|
|
74
|
+
raise KubeError(exc.status or 0, _message(exc.body) or str(exc.reason)) from exc
|
|
75
|
+
if status >= 400:
|
|
76
|
+
raise KubeError(status, _message(data))
|
|
77
|
+
text = data.decode("utf-8", "replace") if isinstance(data, bytes) else (data or "")
|
|
78
|
+
if raw:
|
|
79
|
+
return text
|
|
80
|
+
return json.loads(text) if text else {}
|
|
81
|
+
|
|
82
|
+
def get(self, path: str, **query: Any) -> Any:
|
|
83
|
+
return self.request("GET", path, query=query)
|
|
84
|
+
|
|
85
|
+
def list_all(self, path: str, max_items: int = 5000, page: int = 500) -> tuple[list[dict], bool]:
|
|
86
|
+
"""List a collection with pagination. Returns (items, truncated)."""
|
|
87
|
+
items: list[dict] = []
|
|
88
|
+
token: str | None = None
|
|
89
|
+
while True:
|
|
90
|
+
data = self.get(path, limit=page, **({"continue": token} if token else {}))
|
|
91
|
+
items.extend(data.get("items") or [])
|
|
92
|
+
token = (data.get("metadata") or {}).get("continue")
|
|
93
|
+
if not token:
|
|
94
|
+
return items, False
|
|
95
|
+
if len(items) >= max_items:
|
|
96
|
+
return items[:max_items], True
|
|
97
|
+
|
|
98
|
+
def pod_logs(self, namespace: str, pod: str, container: str, previous: bool, tail: int = 200) -> str:
|
|
99
|
+
return self.request(
|
|
100
|
+
"GET",
|
|
101
|
+
f"/api/v1/namespaces/{namespace}/pods/{pod}/log",
|
|
102
|
+
query={"container": container, "previous": "true" if previous else None, "tailLines": tail},
|
|
103
|
+
raw=True,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
def server_version(self) -> str | None:
|
|
107
|
+
try:
|
|
108
|
+
return self.get("/version").get("gitVersion")
|
|
109
|
+
except Exception:
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _message(body: Any) -> str:
|
|
114
|
+
if isinstance(body, bytes):
|
|
115
|
+
body = body.decode("utf-8", "replace")
|
|
116
|
+
if isinstance(body, str):
|
|
117
|
+
try:
|
|
118
|
+
return json.loads(body).get("message", body)
|
|
119
|
+
except (ValueError, AttributeError):
|
|
120
|
+
return body[:300]
|
|
121
|
+
return str(body)[:300]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def list_contexts(kubeconfig: str | None = None) -> tuple[list[dict[str, str]], str | None]:
|
|
125
|
+
"""Return (contexts, current) from the kubeconfig. Each context: name, cluster, namespace."""
|
|
126
|
+
from kubernetes import config
|
|
127
|
+
|
|
128
|
+
if not kubeconfig and os.environ.get("KUBERNETES_SERVICE_HOST") and not os.path.exists(
|
|
129
|
+
os.path.expanduser("~/.kube/config")
|
|
130
|
+
):
|
|
131
|
+
return [{"name": "in-cluster", "cluster": "in-cluster", "namespace": ""}], "in-cluster"
|
|
132
|
+
contexts, active = config.list_kube_config_contexts(config_file=kubeconfig)
|
|
133
|
+
out = [
|
|
134
|
+
{
|
|
135
|
+
"name": c["name"],
|
|
136
|
+
"cluster": (c.get("context") or {}).get("cluster", ""),
|
|
137
|
+
"namespace": (c.get("context") or {}).get("namespace", ""),
|
|
138
|
+
}
|
|
139
|
+
for c in contexts or []
|
|
140
|
+
]
|
|
141
|
+
return out, (active or {}).get("name")
|
|
@@ -0,0 +1,513 @@
|
|
|
1
|
+
"""A point-in-time copy of the cluster state that every check reads from.
|
|
2
|
+
|
|
3
|
+
A Snapshot can be collected live, saved to JSON (already redacted) and loaded
|
|
4
|
+
again, or built from manifest files for offline linting.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import contextlib
|
|
11
|
+
import gzip
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from collections import defaultdict
|
|
15
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from datetime import datetime
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from kubesift.cluster.client import KubeClient, KubeError
|
|
21
|
+
from kubesift.models import ResourceRef
|
|
22
|
+
from kubesift.redact import redact
|
|
23
|
+
from kubesift.util import meta, now, parse_time
|
|
24
|
+
|
|
25
|
+
# (key, kind, api prefix, plural, namespaced, optional)
|
|
26
|
+
KINDS: list[tuple[str, str, str, str, bool, bool]] = [
|
|
27
|
+
("pods", "Pod", "/api/v1", "pods", True, False),
|
|
28
|
+
("events", "Event", "/api/v1", "events", True, False),
|
|
29
|
+
("nodes", "Node", "/api/v1", "nodes", False, False),
|
|
30
|
+
("namespaces", "Namespace", "/api/v1", "namespaces", False, False),
|
|
31
|
+
("services", "Service", "/api/v1", "services", True, False),
|
|
32
|
+
("endpointslices", "EndpointSlice", "/apis/discovery.k8s.io/v1", "endpointslices", True, False),
|
|
33
|
+
("configmaps", "ConfigMap", "/api/v1", "configmaps", True, False),
|
|
34
|
+
("secrets", "Secret", "/api/v1", "secrets", True, True),
|
|
35
|
+
("serviceaccounts", "ServiceAccount", "/api/v1", "serviceaccounts", True, False),
|
|
36
|
+
("pvcs", "PersistentVolumeClaim", "/api/v1", "persistentvolumeclaims", True, False),
|
|
37
|
+
("pvs", "PersistentVolume", "/api/v1", "persistentvolumes", False, False),
|
|
38
|
+
("resourcequotas", "ResourceQuota", "/api/v1", "resourcequotas", True, False),
|
|
39
|
+
("limitranges", "LimitRange", "/api/v1", "limitranges", True, False),
|
|
40
|
+
("deployments", "Deployment", "/apis/apps/v1", "deployments", True, False),
|
|
41
|
+
("replicasets", "ReplicaSet", "/apis/apps/v1", "replicasets", True, False),
|
|
42
|
+
("statefulsets", "StatefulSet", "/apis/apps/v1", "statefulsets", True, False),
|
|
43
|
+
("daemonsets", "DaemonSet", "/apis/apps/v1", "daemonsets", True, False),
|
|
44
|
+
("jobs", "Job", "/apis/batch/v1", "jobs", True, False),
|
|
45
|
+
("cronjobs", "CronJob", "/apis/batch/v1", "cronjobs", True, False),
|
|
46
|
+
("hpas", "HorizontalPodAutoscaler", "/apis/autoscaling/v2", "horizontalpodautoscalers", True, False),
|
|
47
|
+
("pdbs", "PodDisruptionBudget", "/apis/policy/v1", "poddisruptionbudgets", True, False),
|
|
48
|
+
("ingresses", "Ingress", "/apis/networking.k8s.io/v1", "ingresses", True, False),
|
|
49
|
+
("ingressclasses", "IngressClass", "/apis/networking.k8s.io/v1", "ingressclasses", False, False),
|
|
50
|
+
("networkpolicies", "NetworkPolicy", "/apis/networking.k8s.io/v1", "networkpolicies", True, False),
|
|
51
|
+
("storageclasses", "StorageClass", "/apis/storage.k8s.io/v1", "storageclasses", False, False),
|
|
52
|
+
("clusterrolebindings", "ClusterRoleBinding", "/apis/rbac.authorization.k8s.io/v1", "clusterrolebindings", False, True),
|
|
53
|
+
("rolebindings", "RoleBinding", "/apis/rbac.authorization.k8s.io/v1", "rolebindings", True, True),
|
|
54
|
+
("clusterroles", "ClusterRole", "/apis/rbac.authorization.k8s.io/v1", "clusterroles", False, True),
|
|
55
|
+
("roles", "Role", "/apis/rbac.authorization.k8s.io/v1", "roles", True, True),
|
|
56
|
+
("validatingwebhooks", "ValidatingWebhookConfiguration", "/apis/admissionregistration.k8s.io/v1",
|
|
57
|
+
"validatingwebhookconfigurations", False, True),
|
|
58
|
+
("mutatingwebhooks", "MutatingWebhookConfiguration", "/apis/admissionregistration.k8s.io/v1",
|
|
59
|
+
"mutatingwebhookconfigurations", False, True),
|
|
60
|
+
# Optional add-ons: missing APIs are skipped silently.
|
|
61
|
+
("gateways", "Gateway", "/apis/gateway.networking.k8s.io/v1", "gateways", True, True),
|
|
62
|
+
("httproutes", "HTTPRoute", "/apis/gateway.networking.k8s.io/v1", "httproutes", True, True),
|
|
63
|
+
("certificates", "Certificate", "/apis/cert-manager.io/v1", "certificates", True, True),
|
|
64
|
+
("argoapps", "Application", "/apis/argoproj.io/v1alpha1", "applications", True, True),
|
|
65
|
+
("apiservices", "APIService", "/apis/apiregistration.k8s.io/v1", "apiservices", False, True),
|
|
66
|
+
("nodeclaims", "NodeClaim", "/apis/karpenter.sh/v1", "nodeclaims", False, True),
|
|
67
|
+
]
|
|
68
|
+
KIND_TO_KEY = {kind: key for key, kind, *_ in KINDS}
|
|
69
|
+
CLUSTER_SCOPED = {key for key, _, _, _, namespaced, _ in KINDS if not namespaced}
|
|
70
|
+
_LAST_APPLIED = "kubectl.kubernetes.io/last-applied-configuration"
|
|
71
|
+
APPLIED_API = "kubesift.dev/applied-api-version"
|
|
72
|
+
SOURCE = "kubesift.dev/source"
|
|
73
|
+
_ERROR_LINE = re.compile(
|
|
74
|
+
r"(?i)(error|exception|fatal|panic|traceback|refused|denied|forbidden|timeout|timed out|not found|"
|
|
75
|
+
r"no such|unable to|cannot|could not|failed|killed|exec format|oom|out of memory|unauthorized|invalid)"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
_NOT_LOGS = re.compile(r"(?i)^(unable to retrieve container logs|failed to try resolving symlinks|"
|
|
80
|
+
r"previous terminated container .* not found|container .* is waiting to start)")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class Snapshot:
|
|
85
|
+
context: str = "offline"
|
|
86
|
+
namespace: str | None = None
|
|
87
|
+
live: bool = True
|
|
88
|
+
collected_at: datetime = field(default_factory=now)
|
|
89
|
+
server_version: str | None = None
|
|
90
|
+
objects: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
|
|
91
|
+
warnings: list[str] = field(default_factory=list)
|
|
92
|
+
logs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
93
|
+
metrics: dict[str, dict[str, dict[str, float]]] = field(default_factory=dict)
|
|
94
|
+
helm_releases: list[dict[str, Any]] = field(default_factory=list)
|
|
95
|
+
readable: set[str] = field(default_factory=set)
|
|
96
|
+
|
|
97
|
+
# ----------------------------------------------------------------- access
|
|
98
|
+
def all(self, key: str) -> list[dict[str, Any]]:
|
|
99
|
+
return self.objects.get(key, [])
|
|
100
|
+
|
|
101
|
+
def can_read(self, key: str) -> bool:
|
|
102
|
+
"""False when the kind could not be listed (no permission), so absence means nothing."""
|
|
103
|
+
return not self.live or key in self.readable
|
|
104
|
+
|
|
105
|
+
def get(self, key: str, namespace: str | None, name: str) -> dict[str, Any] | None:
|
|
106
|
+
return self._index().get((key, namespace if key not in CLUSTER_SCOPED else None, name))
|
|
107
|
+
|
|
108
|
+
def _index(self) -> dict[tuple[str, str | None, str], dict[str, Any]]:
|
|
109
|
+
if not hasattr(self, "_idx"):
|
|
110
|
+
idx = {}
|
|
111
|
+
for key, items in self.objects.items():
|
|
112
|
+
for o in items:
|
|
113
|
+
m = meta(o)
|
|
114
|
+
idx[(key, None if key in CLUSTER_SCOPED else m.get("namespace"), m.get("name", ""))] = o
|
|
115
|
+
self._idx = idx # type: ignore[attr-defined]
|
|
116
|
+
return self._idx # type: ignore[attr-defined]
|
|
117
|
+
|
|
118
|
+
def in_namespace(self, key: str, namespace: str | None) -> list[dict[str, Any]]:
|
|
119
|
+
cache = self.__dict__.setdefault("_ns_idx", {})
|
|
120
|
+
if key not in cache:
|
|
121
|
+
by_ns: dict[str | None, list[dict[str, Any]]] = defaultdict(list)
|
|
122
|
+
for o in self.all(key):
|
|
123
|
+
by_ns[meta(o).get("namespace")].append(o)
|
|
124
|
+
cache[key] = by_ns
|
|
125
|
+
return cache[key].get(namespace, [])
|
|
126
|
+
|
|
127
|
+
def events_for(self, kind: str, namespace: str | None, name: str) -> list[dict[str, Any]]:
|
|
128
|
+
if "_ev_idx" not in self.__dict__:
|
|
129
|
+
idx: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
|
130
|
+
for e in self.all("events"):
|
|
131
|
+
inv = e.get("involvedObject") or e.get("regarding") or {}
|
|
132
|
+
idx[(inv.get("kind", ""), inv.get("name", ""))].append(e)
|
|
133
|
+
for items in idx.values():
|
|
134
|
+
items.sort(key=event_time, reverse=True)
|
|
135
|
+
self.__dict__["_ev_idx"] = idx
|
|
136
|
+
return [e for e in self.__dict__["_ev_idx"].get((kind, name), [])
|
|
137
|
+
if namespace is None or ((e.get("involvedObject") or e.get("regarding") or {}).get("namespace")
|
|
138
|
+
in (namespace, None, ""))]
|
|
139
|
+
|
|
140
|
+
def owner_of(self, obj: dict[str, Any]) -> ResourceRef:
|
|
141
|
+
"""Follow ownerReferences up to the top-level workload (Pod -> ReplicaSet -> Deployment)."""
|
|
142
|
+
current = obj
|
|
143
|
+
for _ in range(4):
|
|
144
|
+
refs = meta(current).get("ownerReferences") or []
|
|
145
|
+
ref = next((r for r in refs if r.get("controller")), refs[0] if refs else None)
|
|
146
|
+
if not ref:
|
|
147
|
+
break
|
|
148
|
+
key = KIND_TO_KEY.get(ref.get("kind", ""))
|
|
149
|
+
parent = self.get(key, meta(current).get("namespace"), ref.get("name", "")) if key else None
|
|
150
|
+
if parent is None:
|
|
151
|
+
return ResourceRef(kind=ref.get("kind", "?"), name=ref.get("name", "?"),
|
|
152
|
+
namespace=meta(current).get("namespace"))
|
|
153
|
+
current = parent
|
|
154
|
+
kind = current.get("kind") or "Pod"
|
|
155
|
+
return ResourceRef(kind=kind, name=meta(current).get("name", ""), namespace=meta(current).get("namespace"))
|
|
156
|
+
|
|
157
|
+
def workloads(self) -> list[dict[str, Any]]:
|
|
158
|
+
"""Objects that carry a pod template (what people actually write and deploy)."""
|
|
159
|
+
out = []
|
|
160
|
+
for key in ("deployments", "statefulsets", "daemonsets", "cronjobs"):
|
|
161
|
+
out.extend(self.all(key))
|
|
162
|
+
out.extend(j for j in self.all("jobs") if not meta(j).get("ownerReferences"))
|
|
163
|
+
if not self.live:
|
|
164
|
+
out.extend(p for p in self.all("pods") if not meta(p).get("ownerReferences"))
|
|
165
|
+
return out
|
|
166
|
+
|
|
167
|
+
# ------------------------------------------------------------ persistence
|
|
168
|
+
def to_dict(self) -> dict[str, Any]:
|
|
169
|
+
return {
|
|
170
|
+
"kubesift_snapshot": 1,
|
|
171
|
+
"context": self.context,
|
|
172
|
+
"namespace": self.namespace,
|
|
173
|
+
"collected_at": self.collected_at.isoformat(),
|
|
174
|
+
"server_version": self.server_version,
|
|
175
|
+
"objects": dict(self.objects),
|
|
176
|
+
"warnings": self.warnings,
|
|
177
|
+
"logs": self.logs,
|
|
178
|
+
"metrics": self.metrics,
|
|
179
|
+
"helm_releases": self.helm_releases,
|
|
180
|
+
"readable": sorted(self.readable),
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
@classmethod
|
|
184
|
+
def from_dict(cls, data: dict[str, Any]) -> Snapshot:
|
|
185
|
+
snap = cls(
|
|
186
|
+
context=data.get("context", "snapshot"),
|
|
187
|
+
namespace=data.get("namespace"),
|
|
188
|
+
live=True,
|
|
189
|
+
collected_at=parse_time(data.get("collected_at")) or now(),
|
|
190
|
+
server_version=data.get("server_version"),
|
|
191
|
+
warnings=list(data.get("warnings") or []),
|
|
192
|
+
logs=dict(data.get("logs") or {}),
|
|
193
|
+
metrics=dict(data.get("metrics") or {}),
|
|
194
|
+
helm_releases=list(data.get("helm_releases") or []),
|
|
195
|
+
)
|
|
196
|
+
snap.objects = defaultdict(list, {k: list(v) for k, v in (data.get("objects") or {}).items()})
|
|
197
|
+
snap.readable = set(data.get("readable") or snap.objects.keys())
|
|
198
|
+
return snap
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def from_manifests(cls, docs: list[dict[str, Any]]) -> Snapshot:
|
|
202
|
+
snap = cls(context="manifests", live=False)
|
|
203
|
+
for doc in docs:
|
|
204
|
+
kind = doc.get("kind")
|
|
205
|
+
if kind == "List":
|
|
206
|
+
for item in doc.get("items") or []:
|
|
207
|
+
item.setdefault("metadata", {}).setdefault("annotations", {})
|
|
208
|
+
src = meta(doc).get("annotations", {}).get(SOURCE)
|
|
209
|
+
if src:
|
|
210
|
+
item["metadata"]["annotations"][SOURCE] = src
|
|
211
|
+
snap._add_manifest(item)
|
|
212
|
+
else:
|
|
213
|
+
snap._add_manifest(doc)
|
|
214
|
+
return snap
|
|
215
|
+
|
|
216
|
+
def _add_manifest(self, doc: dict[str, Any]) -> None:
|
|
217
|
+
key = KIND_TO_KEY.get(doc.get("kind", ""))
|
|
218
|
+
doc.setdefault("metadata", {}).setdefault("namespace", None if key in CLUSTER_SCOPED else "default")
|
|
219
|
+
self.objects[key or f"other:{doc.get('kind')}"].append(doc)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def event_time(e: dict[str, Any]) -> str:
|
|
223
|
+
return e.get("lastTimestamp") or e.get("eventTime") or meta(e).get("creationTimestamp") or ""
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# --------------------------------------------------------------------- collect
|
|
227
|
+
def collect(
|
|
228
|
+
client: KubeClient,
|
|
229
|
+
namespace: str | None = None,
|
|
230
|
+
max_items: int = 5000,
|
|
231
|
+
with_logs: bool = True,
|
|
232
|
+
kinds: set[str] | None = None,
|
|
233
|
+
) -> Snapshot:
|
|
234
|
+
from kubesift.util import check_name
|
|
235
|
+
|
|
236
|
+
check_name(namespace, "namespace")
|
|
237
|
+
snap = Snapshot(context=client.context_name, namespace=namespace, live=True)
|
|
238
|
+
snap.server_version = client.server_version()
|
|
239
|
+
|
|
240
|
+
def fetch(entry: tuple[str, str, str, str, bool, bool]) -> None:
|
|
241
|
+
key, kind, prefix, plural, namespaced, optional = entry
|
|
242
|
+
if kinds is not None and key not in kinds:
|
|
243
|
+
return
|
|
244
|
+
if namespace and namespaced:
|
|
245
|
+
path = f"{prefix}/namespaces/{namespace}/{plural}"
|
|
246
|
+
else:
|
|
247
|
+
path = f"{prefix}/{plural}"
|
|
248
|
+
try:
|
|
249
|
+
items, truncated = client.list_all(path, max_items=max_items)
|
|
250
|
+
except KubeError as exc:
|
|
251
|
+
if exc.status == 404:
|
|
252
|
+
return
|
|
253
|
+
if exc.status == 403:
|
|
254
|
+
if not optional or key == "secrets":
|
|
255
|
+
snap.warnings.append(f"No permission to list {plural}; checks that need them were skipped.")
|
|
256
|
+
return
|
|
257
|
+
snap.warnings.append(f"Could not list {plural}: {exc.message}")
|
|
258
|
+
return
|
|
259
|
+
except Exception as exc: # network errors etc.
|
|
260
|
+
snap.warnings.append(f"Could not list {plural}: {exc}")
|
|
261
|
+
return
|
|
262
|
+
api_version = prefix.removeprefix("/apis/").removeprefix("/api/")
|
|
263
|
+
for item in items:
|
|
264
|
+
item["kind"] = kind
|
|
265
|
+
item["apiVersion"] = api_version
|
|
266
|
+
if truncated:
|
|
267
|
+
snap.warnings.append(f"Only the first {max_items} {plural} were read.")
|
|
268
|
+
if key == "secrets":
|
|
269
|
+
snap.helm_releases.extend(_helm_releases(items))
|
|
270
|
+
items = [_sanitize_secret(s) for s in items]
|
|
271
|
+
elif key == "configmaps":
|
|
272
|
+
items = [_sanitize_configmap(c) for c in items]
|
|
273
|
+
for item in items:
|
|
274
|
+
_shrink(item)
|
|
275
|
+
snap.objects[key] = items
|
|
276
|
+
snap.readable.add(key)
|
|
277
|
+
|
|
278
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
279
|
+
list(pool.map(fetch, KINDS))
|
|
280
|
+
|
|
281
|
+
if "pods" not in snap.readable and not snap.warnings:
|
|
282
|
+
snap.warnings.append("Could not read pods from the cluster.")
|
|
283
|
+
_collect_metrics(client, snap, namespace)
|
|
284
|
+
_collect_kube_proxy_mode(client, snap)
|
|
285
|
+
if with_logs:
|
|
286
|
+
collect_logs(client, snap)
|
|
287
|
+
return snap
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
CHANGE_TRACKED = {"ConfigMap", "Secret", "Deployment", "StatefulSet", "DaemonSet", "CronJob", "Service", "Ingress",
|
|
291
|
+
"NetworkPolicy", "HorizontalPodAutoscaler", "ResourceQuota"}
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _shrink(obj: dict[str, Any]) -> None:
|
|
295
|
+
m = obj.get("metadata") or {}
|
|
296
|
+
ann = m.get("annotations") or {}
|
|
297
|
+
applied = ann.pop(_LAST_APPLIED, None)
|
|
298
|
+
if applied:
|
|
299
|
+
with contextlib.suppress(ValueError):
|
|
300
|
+
ann[APPLIED_API] = json.loads(applied).get("apiVersion", "")
|
|
301
|
+
_redact_env(obj)
|
|
302
|
+
mf = m.pop("managedFields", None)
|
|
303
|
+
if mf and obj.get("kind") in CHANGE_TRACKED:
|
|
304
|
+
# keep only who changed the object and when; used by `kubesift changes`
|
|
305
|
+
m["managedFields"] = [{"manager": f.get("manager"), "operation": f.get("operation"), "time": f.get("time"),
|
|
306
|
+
"subresource": f.get("subresource")} for f in mf]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _redact_env(obj: dict[str, Any]) -> None:
|
|
310
|
+
"""Literal env values are part of pod specs, not Secrets; mask the ones that look like credentials."""
|
|
311
|
+
from kubesift.redact import SECRET_ENV_NAME
|
|
312
|
+
from kubesift.util import containers_of, pod_spec_of
|
|
313
|
+
|
|
314
|
+
spec = pod_spec_of(obj) if obj.get("kind") in ("Pod", "Deployment", "StatefulSet", "DaemonSet", "Job",
|
|
315
|
+
"CronJob", "ReplicaSet") else None
|
|
316
|
+
for c in containers_of(spec, include_init=True):
|
|
317
|
+
for env in c.get("env") or []:
|
|
318
|
+
value = env.get("value")
|
|
319
|
+
if not isinstance(value, str) or not value:
|
|
320
|
+
continue
|
|
321
|
+
if SECRET_ENV_NAME.search(env.get("name", "")):
|
|
322
|
+
env["value"] = "[REDACTED]"
|
|
323
|
+
else:
|
|
324
|
+
env["value"] = redact(value)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _sanitize_configmap(cm: dict[str, Any]) -> dict[str, Any]:
|
|
328
|
+
return {
|
|
329
|
+
"kind": cm.get("kind"),
|
|
330
|
+
"apiVersion": cm.get("apiVersion"),
|
|
331
|
+
"metadata": cm.get("metadata") or {},
|
|
332
|
+
"keys": sorted(list((cm.get("data") or {}).keys()) + list((cm.get("binaryData") or {}).keys())),
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _sanitize_secret(secret: dict[str, Any]) -> dict[str, Any]:
|
|
337
|
+
"""Keep only key names, the type and certificate expiry. Secret values never leave this function."""
|
|
338
|
+
data = secret.get("data") or {}
|
|
339
|
+
out: dict[str, Any] = {
|
|
340
|
+
"kind": "Secret",
|
|
341
|
+
"apiVersion": "v1",
|
|
342
|
+
"metadata": secret.get("metadata") or {},
|
|
343
|
+
"type": secret.get("type", "Opaque"),
|
|
344
|
+
"keys": sorted(data.keys()),
|
|
345
|
+
}
|
|
346
|
+
if out["type"] == "kubernetes.io/tls" or "tls.crt" in data:
|
|
347
|
+
out["tls"] = _cert_info(data.get("tls.crt"))
|
|
348
|
+
return out
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _cert_info(b64: str | None) -> dict[str, Any] | None:
|
|
352
|
+
if not b64:
|
|
353
|
+
return None
|
|
354
|
+
try:
|
|
355
|
+
from cryptography import x509
|
|
356
|
+
|
|
357
|
+
pem = base64.b64decode(b64)
|
|
358
|
+
cert = x509.load_pem_x509_certificate(pem)
|
|
359
|
+
try:
|
|
360
|
+
not_after = cert.not_valid_after_utc
|
|
361
|
+
except AttributeError: # cryptography < 42
|
|
362
|
+
not_after = cert.not_valid_after
|
|
363
|
+
names: list[str] = []
|
|
364
|
+
try:
|
|
365
|
+
ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName)
|
|
366
|
+
names = ext.value.get_values_for_type(x509.DNSName)
|
|
367
|
+
except Exception:
|
|
368
|
+
pass
|
|
369
|
+
return {"not_after": not_after.isoformat(), "subject": cert.subject.rfc4514_string(), "dns_names": names[:10]}
|
|
370
|
+
except Exception:
|
|
371
|
+
return {"error": "could not parse certificate"}
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _helm_releases(secrets: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
375
|
+
"""Decode the latest revision of each Helm 3 release (resource list only, no values)."""
|
|
376
|
+
latest: dict[tuple[str, str], dict[str, Any]] = {}
|
|
377
|
+
for s in secrets:
|
|
378
|
+
if s.get("type") != "helm.sh/release.v1":
|
|
379
|
+
continue
|
|
380
|
+
labels = meta(s).get("labels") or {}
|
|
381
|
+
key = (meta(s).get("namespace", ""), labels.get("name", ""))
|
|
382
|
+
try:
|
|
383
|
+
version = int(labels.get("version", "0"))
|
|
384
|
+
except ValueError:
|
|
385
|
+
version = 0
|
|
386
|
+
prev = latest.get(key)
|
|
387
|
+
if prev is None or version > prev["revision"]:
|
|
388
|
+
latest[key] = {"secret": s, "revision": version, "status": labels.get("status", "")}
|
|
389
|
+
out = []
|
|
390
|
+
for (ns, name), item in latest.items():
|
|
391
|
+
rel: dict[str, Any] = {"name": name, "namespace": ns, "revision": item["revision"], "status": item["status"],
|
|
392
|
+
"updated": meta(item["secret"]).get("creationTimestamp"), "resources": []}
|
|
393
|
+
try:
|
|
394
|
+
raw = base64.b64decode(base64.b64decode((item["secret"].get("data") or {}).get("release", "")))
|
|
395
|
+
if raw[:2] == b"\x1f\x8b":
|
|
396
|
+
raw = gzip.decompress(raw)
|
|
397
|
+
payload = json.loads(raw)
|
|
398
|
+
chart = ((payload.get("chart") or {}).get("metadata") or {})
|
|
399
|
+
rel["chart"] = f"{chart.get('name', '')}-{chart.get('version', '')}"
|
|
400
|
+
rel["app_version"] = chart.get("appVersion")
|
|
401
|
+
rel["updated"] = (payload.get("info") or {}).get("last_deployed") or rel["updated"]
|
|
402
|
+
rel["resources"] = _manifest_index(payload.get("manifest") or "")
|
|
403
|
+
except Exception:
|
|
404
|
+
pass
|
|
405
|
+
out.append(rel)
|
|
406
|
+
return out
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _manifest_index(manifest: str) -> list[dict[str, str]]:
|
|
410
|
+
import yaml
|
|
411
|
+
|
|
412
|
+
out = []
|
|
413
|
+
try:
|
|
414
|
+
for doc in yaml.safe_load_all(manifest):
|
|
415
|
+
if isinstance(doc, dict) and doc.get("kind"):
|
|
416
|
+
out.append({"apiVersion": doc.get("apiVersion", ""), "kind": doc["kind"],
|
|
417
|
+
"name": meta(doc).get("name", "")})
|
|
418
|
+
except yaml.YAMLError:
|
|
419
|
+
pass
|
|
420
|
+
return out
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _collect_metrics(client: KubeClient, snap: Snapshot, namespace: str | None) -> None:
|
|
424
|
+
from kubesift.util import parse_quantity
|
|
425
|
+
|
|
426
|
+
base = "/apis/metrics.k8s.io/v1beta1"
|
|
427
|
+
try:
|
|
428
|
+
pods = client.get(f"{base}/namespaces/{namespace}/pods" if namespace else f"{base}/pods")
|
|
429
|
+
nodes = client.get(f"{base}/nodes")
|
|
430
|
+
except Exception:
|
|
431
|
+
return
|
|
432
|
+
pm: dict[str, dict[str, float]] = {}
|
|
433
|
+
for p in pods.get("items") or []:
|
|
434
|
+
for c in p.get("containers") or []:
|
|
435
|
+
k = f"{meta(p).get('namespace')}/{meta(p).get('name')}/{c.get('name')}"
|
|
436
|
+
usage = c.get("usage") or {}
|
|
437
|
+
pm[k] = {"cpu": parse_quantity(usage.get("cpu")), "memory": parse_quantity(usage.get("memory"))}
|
|
438
|
+
nm = {}
|
|
439
|
+
for n in nodes.get("items") or []:
|
|
440
|
+
usage = n.get("usage") or {}
|
|
441
|
+
nm[meta(n).get("name", "")] = {"cpu": parse_quantity(usage.get("cpu")),
|
|
442
|
+
"memory": parse_quantity(usage.get("memory"))}
|
|
443
|
+
snap.metrics = {"pods": pm, "nodes": nm}
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _collect_kube_proxy_mode(client: KubeClient, snap: Snapshot) -> None:
|
|
447
|
+
try:
|
|
448
|
+
cm = client.get("/api/v1/namespaces/kube-system/configmaps/kube-proxy")
|
|
449
|
+
except Exception:
|
|
450
|
+
return
|
|
451
|
+
for value in (cm.get("data") or {}).values():
|
|
452
|
+
m = re.search(r"(?m)^\s*mode:\s*\"?(\w+)", value or "")
|
|
453
|
+
if m:
|
|
454
|
+
snap.metrics.setdefault("cluster", {})["kube_proxy"] = {"mode_" + m.group(1): 1.0}
|
|
455
|
+
return
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def failing_containers(pod: dict[str, Any]) -> list[tuple[str, bool]]:
|
|
459
|
+
"""Containers worth reading logs from: (name, use_previous_logs)."""
|
|
460
|
+
out = []
|
|
461
|
+
statuses = (pod.get("status") or {}).get("containerStatuses") or []
|
|
462
|
+
statuses += (pod.get("status") or {}).get("initContainerStatuses") or []
|
|
463
|
+
for cs in statuses:
|
|
464
|
+
waiting = (cs.get("state") or {}).get("waiting") or {}
|
|
465
|
+
terminated = (cs.get("state") or {}).get("terminated") or {}
|
|
466
|
+
last = (cs.get("lastState") or {}).get("terminated") or {}
|
|
467
|
+
if waiting.get("reason") == "CrashLoopBackOff" or (last and last.get("exitCode", 0) != 0):
|
|
468
|
+
out.append((cs.get("name", ""), bool(last)))
|
|
469
|
+
elif terminated and terminated.get("exitCode", 0) != 0 or cs.get("ready") is False and (cs.get("state") or {}).get("running"):
|
|
470
|
+
out.append((cs.get("name", ""), False))
|
|
471
|
+
return out
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def summarize_log(text: str, max_lines: int = 25) -> dict[str, Any]:
|
|
475
|
+
lines = [ln.rstrip() for ln in text.splitlines() if ln.strip()]
|
|
476
|
+
errors: list[str] = []
|
|
477
|
+
seen: set[str] = set()
|
|
478
|
+
for ln in lines:
|
|
479
|
+
if _ERROR_LINE.search(ln):
|
|
480
|
+
key = re.sub(r"\d+", "#", ln)[:160]
|
|
481
|
+
if key not in seen:
|
|
482
|
+
seen.add(key)
|
|
483
|
+
errors.append(redact(ln[:400]))
|
|
484
|
+
return {"errors": errors[-max_lines:], "tail": [redact(ln[:400]) for ln in lines[-12:]]}
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def collect_logs(client: KubeClient, snap: Snapshot, max_containers: int = 25) -> None:
|
|
488
|
+
targets = []
|
|
489
|
+
for pod in snap.all("pods"):
|
|
490
|
+
for container, previous in failing_containers(pod):
|
|
491
|
+
targets.append((meta(pod).get("namespace", ""), meta(pod).get("name", ""), container, previous))
|
|
492
|
+
targets = targets[:max_containers]
|
|
493
|
+
|
|
494
|
+
def read(ns: str, pod: str, container: str, previous: bool) -> str | None:
|
|
495
|
+
try:
|
|
496
|
+
text = client.pod_logs(ns, pod, container, previous=previous)
|
|
497
|
+
except Exception:
|
|
498
|
+
return None
|
|
499
|
+
# The API answers 200 with an error sentence when the runtime has already removed the old container.
|
|
500
|
+
if not text.strip() or _NOT_LOGS.match(text.lstrip()):
|
|
501
|
+
return None
|
|
502
|
+
return text
|
|
503
|
+
|
|
504
|
+
def fetch(t: tuple[str, str, str, bool]) -> None:
|
|
505
|
+
ns, pod, container, previous = t
|
|
506
|
+
for prev in ((True, False) if previous else (False,)):
|
|
507
|
+
text = read(ns, pod, container, prev)
|
|
508
|
+
if text is not None:
|
|
509
|
+
snap.logs[f"{ns}/{pod}/{container}"] = {"previous": prev, **summarize_log(text)}
|
|
510
|
+
return
|
|
511
|
+
|
|
512
|
+
with ThreadPoolExecutor(max_workers=6) as pool:
|
|
513
|
+
list(pool.map(fetch, targets))
|