pubdeploy 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubdeploy/__init__.py +532 -0
- pubdeploy/channels.py +188 -0
- pubdeploy/cli.py +120 -0
- pubdeploy/config.py +140 -0
- pubdeploy/contract.py +87 -0
- pubdeploy/github.py +61 -0
- pubdeploy/hooks.py +82 -0
- pubdeploy/implied.py +179 -0
- pubdeploy/journal.json +207 -0
- pubdeploy/k8s.py +472 -0
- pubdeploy/machine.py +363 -0
- pubdeploy/model.py +229 -0
- pubdeploy/names.py +25 -0
- pubdeploy/order.py +101 -0
- pubdeploy/providers.py +75 -0
- pubdeploy/registry.py +96 -0
- pubdeploy/release-commit +1 -0
- pubdeploy/schema.json +127 -0
- pubdeploy/schema.py +238 -0
- pubdeploy/store.py +127 -0
- pubdeploy/testing/__init__.py +94 -0
- pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
- pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
- pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
- pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
- pubdeploy/testing/plugin.py +26 -0
- pubdeploy/testing/steps.py +1124 -0
- pubdeploy/testing/world.py +569 -0
- pubdeploy/types/__init__.py +181 -0
- pubdeploy/types/auth.py +199 -0
- pubdeploy/types/github.py +110 -0
- pubdeploy/types/k8s.py +609 -0
- pubdeploy/types/net.py +237 -0
- pubdeploy/types/rollout.py +186 -0
- pubdeploy/units.py +177 -0
- pubdeploy/walk.py +325 -0
- pubdeploy-0.8.0.dist-info/METADATA +520 -0
- pubdeploy-0.8.0.dist-info/RECORD +45 -0
- pubdeploy-0.8.0.dist-info/WHEEL +5 -0
- pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
- pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
- pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/k8s.py
ADDED
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
"""`pubdeploy k8s preflight` and `pubdeploy k8s diagnose`: what a deploy to
|
|
2
|
+
Kubernetes checks before it applies anything, and what it shows when a
|
|
3
|
+
rollout fails.
|
|
4
|
+
|
|
5
|
+
pubdeploy k8s preflight --namespace NS [--kubectl CMD] [--offline] < MANIFESTS
|
|
6
|
+
pubdeploy k8s diagnose --namespace NS [--kubectl CMD] [--selector SEL] [--tail N]
|
|
7
|
+
|
|
8
|
+
`preflight` reads rendered manifests on standard input, and opens no
|
|
9
|
+
file: YAML, or JSON where PyYAML is not installed. It collects every
|
|
10
|
+
Secret their pods use: env secretKeyRef, envFrom secretRef, secret and projected secret
|
|
11
|
+
volumes (with the keys their items name), and imagePullSecrets, of every
|
|
12
|
+
container and init container of a Pod, Deployment, StatefulSet,
|
|
13
|
+
DaemonSet, ReplicaSet, Job or CronJob. A reference marked `optional: true`
|
|
14
|
+
may be missing. Each Secret is then read, once, with `kubectl get secret`,
|
|
15
|
+
in the workload's namespace or NS: it must be there, with every key a
|
|
16
|
+
reference names. A Secret the manifests themselves hold counts as there.
|
|
17
|
+
|
|
18
|
+
Some values have a shape the program reading them requires, which a
|
|
19
|
+
Secret's mere presence does not show. One so far: oauth2-proxy's cookie
|
|
20
|
+
secret (the env OAUTH2_PROXY_COOKIE_SECRET, or --cookie-secret of a
|
|
21
|
+
container whose image is oauth2-proxy's) must be 16, 24 or 32 bytes, as it
|
|
22
|
+
is or after URL-safe base64 decoding, which is oauth2-proxy's own rule;
|
|
23
|
+
standard base64, with + or /, is the likely mistake, and is named as such.
|
|
24
|
+
|
|
25
|
+
No value is ever printed: a problem names the Secret, the key and what
|
|
26
|
+
reads it, and a value's shape is described by its length alone. Any
|
|
27
|
+
problem fails the command, in the seconds kubectl takes, before anything
|
|
28
|
+
is applied. `--offline` reads no cluster: it lists what the manifests
|
|
29
|
+
need, and checks the values they hold themselves.
|
|
30
|
+
|
|
31
|
+
`diagnose` is for after a rollout fails: for every pod that is not ready,
|
|
32
|
+
or whose containers restarted, it prints the pod's phase, each container's
|
|
33
|
+
state, its last termination, its warning events, and the last lines of
|
|
34
|
+
its logs (the previous container's too, after a restart).
|
|
35
|
+
|
|
36
|
+
`kubectl` is `kubectl` unless --kubectl gives another command, such as
|
|
37
|
+
`k3s kubectl --kubeconfig FILE`. The file needs nothing of pubdeploy's, and
|
|
38
|
+
PyYAML only for YAML, so a server's deploy program can run it as it is:
|
|
39
|
+
`python3 k8s.py preflight --namespace NS --kubectl '...' < manifests`.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import argparse
|
|
45
|
+
import base64
|
|
46
|
+
import binascii
|
|
47
|
+
import json
|
|
48
|
+
import re
|
|
49
|
+
import shlex
|
|
50
|
+
import subprocess
|
|
51
|
+
import sys
|
|
52
|
+
from dataclasses import dataclass, field
|
|
53
|
+
from typing import Any, Callable
|
|
54
|
+
|
|
55
|
+
TIMEOUT = 30
|
|
56
|
+
# Where each kind keeps its pod spec.
|
|
57
|
+
POD_SPEC = {
|
|
58
|
+
"Pod": ("spec",),
|
|
59
|
+
"Deployment": ("spec", "template", "spec"),
|
|
60
|
+
"StatefulSet": ("spec", "template", "spec"),
|
|
61
|
+
"DaemonSet": ("spec", "template", "spec"),
|
|
62
|
+
"ReplicaSet": ("spec", "template", "spec"),
|
|
63
|
+
"ReplicationController": ("spec", "template", "spec"),
|
|
64
|
+
"Job": ("spec", "template", "spec"),
|
|
65
|
+
"CronJob": ("spec", "jobTemplate", "spec", "template", "spec"),
|
|
66
|
+
}
|
|
67
|
+
# Kubernetes' own names: a lower-case DNS subdomain, and a Secret's key.
|
|
68
|
+
NAME = re.compile(r"[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?")
|
|
69
|
+
KEY = re.compile(r"[-._a-zA-Z0-9]{1,253}")
|
|
70
|
+
AES = (16, 24, 32)
|
|
71
|
+
URL_SAFE = re.compile(rb"[A-Za-z0-9_-]*")
|
|
72
|
+
STANDARD = re.compile(rb"[A-Za-z0-9+/]*={0,2}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def url_decoded(value: bytes) -> bytes | None:
|
|
76
|
+
"""`value` as oauth2-proxy decodes it: '=' trimmed from the end, then
|
|
77
|
+
unpadded URL-safe base64, where line breaks are skipped (Go's
|
|
78
|
+
base64.RawURLEncoding). None when it is not that."""
|
|
79
|
+
text = value.rstrip(b"=").replace(b"\r", b"").replace(b"\n", b"")
|
|
80
|
+
if not URL_SAFE.fullmatch(text) or len(text) % 4 == 1:
|
|
81
|
+
return None
|
|
82
|
+
return base64.urlsafe_b64decode(text + b"=" * (-len(text) % 4))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def cookie_secret(value: bytes) -> str | None:
|
|
86
|
+
"""Why oauth2-proxy refuses `value` as its cookie secret, or None. It
|
|
87
|
+
takes the URL-safe base64 decoding when that is an AES key's length, and
|
|
88
|
+
the bytes as they are otherwise, which must then be one."""
|
|
89
|
+
decoded = url_decoded(value)
|
|
90
|
+
if len(value) in AES or (decoded is not None and len(decoded) in AES):
|
|
91
|
+
return None
|
|
92
|
+
shape = f"{len(value)} bytes" + (f", {len(decoded)} after URL-safe base64 decoding"
|
|
93
|
+
if decoded is not None else "")
|
|
94
|
+
rule = "; oauth2-proxy needs 16, 24 or 32 bytes, as they are or after URL-safe base64 decoding"
|
|
95
|
+
stripped = value.strip()
|
|
96
|
+
if stripped != value and cookie_secret(stripped) is None:
|
|
97
|
+
return f"{shape}, with white space around it{rule}: drop the trailing newline"
|
|
98
|
+
if STANDARD.fullmatch(stripped) and (b"+" in stripped or b"/" in stripped):
|
|
99
|
+
try:
|
|
100
|
+
length = len(base64.b64decode(stripped, validate=True))
|
|
101
|
+
except binascii.Error:
|
|
102
|
+
length = 0
|
|
103
|
+
if length in AES:
|
|
104
|
+
return (f"{shape}: it is standard base64 (with + or /) of {length} bytes{rule}; "
|
|
105
|
+
"make it URL-safe, + as - and / as _, as `openssl rand -base64 32 | tr -- '+/' '-_'` does")
|
|
106
|
+
return shape + rule
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass(frozen=True)
|
|
110
|
+
class Shape:
|
|
111
|
+
"""A value with a form the program reading it requires."""
|
|
112
|
+
what: str
|
|
113
|
+
problem: Callable[[bytes], str | None]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
COOKIE_SECRET = Shape("oauth2-proxy's cookie secret", cookie_secret)
|
|
117
|
+
# By the env variable that holds the value, in any container.
|
|
118
|
+
ENV_SHAPES = {"OAUTH2_PROXY_COOKIE_SECRET": COOKIE_SECRET}
|
|
119
|
+
# By a command-line flag, in a container whose image is the program's.
|
|
120
|
+
FLAG_SHAPES = {"oauth2-proxy": {"--cookie-secret": COOKIE_SECRET}}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@dataclass
|
|
124
|
+
class Need:
|
|
125
|
+
"""A Secret something reads: `keys` it names (none for envFrom, or a
|
|
126
|
+
volume without items), whether it may be missing, and what reads it."""
|
|
127
|
+
namespace: str
|
|
128
|
+
secret: str
|
|
129
|
+
where: str
|
|
130
|
+
keys: tuple[str, ...] = ()
|
|
131
|
+
optional: bool = False
|
|
132
|
+
shapes: dict[str, Shape] = field(default_factory=dict) # by key
|
|
133
|
+
prefix: str | None = None # envFrom: every key, as an env variable
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass
|
|
137
|
+
class Literal:
|
|
138
|
+
"""A value with a shape written into the manifests as it is."""
|
|
139
|
+
where: str
|
|
140
|
+
value: bytes
|
|
141
|
+
shape: Shape
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass
|
|
145
|
+
class Found:
|
|
146
|
+
"""What the manifests use and hold."""
|
|
147
|
+
needs: list[Need] = field(default_factory=list)
|
|
148
|
+
literals: list[Literal] = field(default_factory=list)
|
|
149
|
+
held: dict[tuple[str, str], dict[str, bytes]] = field(default_factory=dict)
|
|
150
|
+
problems: list[str] = field(default_factory=list)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def documents(text: str) -> list[dict[str, Any]]:
|
|
154
|
+
"""The objects in `text`: YAML documents, or JSON where PyYAML is not
|
|
155
|
+
installed, with each List's items in its place."""
|
|
156
|
+
try:
|
|
157
|
+
import yaml # noqa: PLC0415
|
|
158
|
+
except ImportError:
|
|
159
|
+
yaml = None
|
|
160
|
+
if yaml is not None:
|
|
161
|
+
docs = list(yaml.safe_load_all(text))
|
|
162
|
+
else:
|
|
163
|
+
try:
|
|
164
|
+
loaded = json.loads(text)
|
|
165
|
+
except ValueError:
|
|
166
|
+
raise SystemExit("k8s: the manifests are not JSON, and PyYAML, to read YAML, "
|
|
167
|
+
"is not installed") from None
|
|
168
|
+
docs = loaded if isinstance(loaded, list) else [loaded]
|
|
169
|
+
out = []
|
|
170
|
+
for doc in docs:
|
|
171
|
+
if isinstance(doc, dict) and doc.get("kind") == "List":
|
|
172
|
+
out += [d for d in doc.get("items") or [] if isinstance(d, dict)]
|
|
173
|
+
elif isinstance(doc, dict):
|
|
174
|
+
out.append(doc)
|
|
175
|
+
return out
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def dig(doc: Any, path: tuple[str, ...]) -> Any:
|
|
179
|
+
for step in path:
|
|
180
|
+
doc = doc.get(step) if isinstance(doc, dict) else None
|
|
181
|
+
return doc
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def items(value: Any) -> list[dict[str, Any]]:
|
|
185
|
+
return [v for v in value if isinstance(v, dict)] if isinstance(value, list) else []
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def secret_data(doc: dict[str, Any]) -> dict[str, bytes]:
|
|
189
|
+
"""A Secret's keys and values: data decoded, stringData as it is."""
|
|
190
|
+
out = {}
|
|
191
|
+
for key, value in (doc.get("data") or {}).items():
|
|
192
|
+
try:
|
|
193
|
+
out[key] = base64.b64decode(value or "", validate=True)
|
|
194
|
+
except (binascii.Error, TypeError):
|
|
195
|
+
out[key] = b""
|
|
196
|
+
for key, value in (doc.get("stringData") or {}).items():
|
|
197
|
+
out[key] = str(value).encode()
|
|
198
|
+
return out
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def flag_values(args: list[str], flags: dict[str, Shape]) -> list[tuple[str, str, Shape]]:
|
|
202
|
+
"""Each flag of `flags` in `args`, as --flag=VALUE or --flag VALUE."""
|
|
203
|
+
out = []
|
|
204
|
+
for i, arg in enumerate(args):
|
|
205
|
+
name, eq, value = arg.partition("=")
|
|
206
|
+
if name in flags and eq:
|
|
207
|
+
out.append((name, value, flags[name]))
|
|
208
|
+
elif name in flags and i + 1 < len(args):
|
|
209
|
+
out.append((name, args[i + 1], flags[name]))
|
|
210
|
+
return out
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def collect(docs: list[dict[str, Any]], namespace: str) -> Found:
|
|
214
|
+
"""Every Secret the pods in `docs` read, and every Secret `docs` hold."""
|
|
215
|
+
found = Found()
|
|
216
|
+
for doc in docs:
|
|
217
|
+
kind, meta = doc.get("kind"), doc.get("metadata") or {}
|
|
218
|
+
ns = meta.get("namespace") or namespace
|
|
219
|
+
if kind == "Secret" and meta.get("name"):
|
|
220
|
+
found.held[(ns, meta["name"])] = secret_data(doc)
|
|
221
|
+
spec = dig(doc, POD_SPEC[kind]) if kind in POD_SPEC else None
|
|
222
|
+
if isinstance(spec, dict):
|
|
223
|
+
pod(found, spec, ns, f"{kind}/{meta.get('name', '?')}")
|
|
224
|
+
return found
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def pod(found: Found, spec: dict[str, Any], ns: str, workload: str) -> None:
|
|
228
|
+
"""The Secrets one pod spec reads."""
|
|
229
|
+
def need(secret: Any, where: str, **kw: Any) -> None:
|
|
230
|
+
if not isinstance(secret, str) or not NAME.fullmatch(secret):
|
|
231
|
+
found.problems.append(f"{where} names no Secret, or one that is not a Secret's name: {secret!r}")
|
|
232
|
+
return
|
|
233
|
+
found.needs.append(Need(ns, secret, where, **kw))
|
|
234
|
+
|
|
235
|
+
containers = [("container", c) for c in items(spec.get("containers"))]
|
|
236
|
+
containers += [("init container", c) for c in items(spec.get("initContainers"))]
|
|
237
|
+
for label, c in containers:
|
|
238
|
+
at = f"{workload} {label} {c.get('name', '?')}"
|
|
239
|
+
env = {e.get("name"): e for e in items(c.get("env"))}
|
|
240
|
+
shapes = {name: shape for name, shape in ENV_SHAPES.items() if name in env}
|
|
241
|
+
image = str(c.get("image", "")).rsplit("/", 1)[-1].split(":")[0].split("@")[0]
|
|
242
|
+
argv = [str(a) for a in (c.get("command") or []) + (c.get("args") or [])]
|
|
243
|
+
for flag, value, shape in flag_values(argv, FLAG_SHAPES.get(image, {})):
|
|
244
|
+
var = re.fullmatch(r"\$\(([A-Za-z_][A-Za-z0-9_]*)\)", value)
|
|
245
|
+
if var and var[1] in env:
|
|
246
|
+
shapes[var[1]] = shape
|
|
247
|
+
elif not var:
|
|
248
|
+
found.literals.append(Literal(f"{flag} of {at}", value.encode(), shape))
|
|
249
|
+
for name, e in env.items():
|
|
250
|
+
ref = dig(e, ("valueFrom", "secretKeyRef"))
|
|
251
|
+
shape = shapes.get(name)
|
|
252
|
+
if isinstance(ref, dict):
|
|
253
|
+
key = ref.get("key")
|
|
254
|
+
where = f"env {name} of {at}"
|
|
255
|
+
if not isinstance(key, str) or not KEY.fullmatch(key):
|
|
256
|
+
found.problems.append(f"{where} names no key of its Secret")
|
|
257
|
+
continue
|
|
258
|
+
need(ref.get("name"), where, keys=(key,), optional=ref.get("optional") is True,
|
|
259
|
+
shapes={key: shape} if shape else {})
|
|
260
|
+
elif shape and "value" in e:
|
|
261
|
+
found.literals.append(Literal(f"env {name} of {at}", str(e["value"]).encode(), shape))
|
|
262
|
+
for e in items(c.get("envFrom")):
|
|
263
|
+
ref = e.get("secretRef")
|
|
264
|
+
if isinstance(ref, dict):
|
|
265
|
+
prefix = str(e.get("prefix") or "")
|
|
266
|
+
need(ref.get("name"), f"envFrom of {at}", optional=ref.get("optional") is True,
|
|
267
|
+
prefix=prefix)
|
|
268
|
+
for v in items(spec.get("volumes")):
|
|
269
|
+
where = f"volume {v.get('name', '?')} of {workload}"
|
|
270
|
+
sources = [v["secret"] | {"name": v["secret"].get("secretName")}] if isinstance(
|
|
271
|
+
v.get("secret"), dict) else []
|
|
272
|
+
sources += [s["secret"] for s in items(dig(v, ("projected", "sources")))
|
|
273
|
+
if isinstance(s.get("secret"), dict)]
|
|
274
|
+
for s in sources:
|
|
275
|
+
keys = tuple(str(i.get("key")) for i in items(s.get("items")))
|
|
276
|
+
need(s.get("name"), where, keys=keys, optional=s.get("optional") is True)
|
|
277
|
+
for p in items(spec.get("imagePullSecrets")):
|
|
278
|
+
need(p.get("name"), f"imagePullSecrets of {workload}")
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def run_kubectl(argv: list[str]) -> subprocess.CompletedProcess[str]:
|
|
282
|
+
"""Run one kubectl command line, as Cluster does unless given another way."""
|
|
283
|
+
return subprocess.run(argv, capture_output=True, text=True, timeout=TIMEOUT, check=False)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class Cluster:
|
|
287
|
+
"""Secrets read through kubectl, each once. `run` runs a command line and
|
|
288
|
+
returns its exit code, stdout and stderr, as subprocess.run does; another
|
|
289
|
+
program, such as a test, may answer in kubectl's place."""
|
|
290
|
+
|
|
291
|
+
def __init__(self, kubectl: list[str], run: Callable[[list[str]], Any] = run_kubectl) -> None:
|
|
292
|
+
self.kubectl = kubectl
|
|
293
|
+
self.run = run
|
|
294
|
+
self.read: dict[tuple[str, str], dict[str, bytes] | str | None] = {}
|
|
295
|
+
|
|
296
|
+
def secret(self, ns: str, name: str) -> dict[str, bytes] | str | None:
|
|
297
|
+
"""The Secret's keys and values, None when it is not there, or why
|
|
298
|
+
it could not be read."""
|
|
299
|
+
if (ns, name) not in self.read:
|
|
300
|
+
self.read[(ns, name)] = self.get(ns, name)
|
|
301
|
+
return self.read[(ns, name)]
|
|
302
|
+
|
|
303
|
+
def get(self, ns: str, name: str) -> dict[str, bytes] | str | None:
|
|
304
|
+
try:
|
|
305
|
+
done = self.run([*self.kubectl, "-n", ns, "get", "secret", name, "-o", "json"])
|
|
306
|
+
except (OSError, subprocess.TimeoutExpired) as error:
|
|
307
|
+
return f"kubectl did not run: {error}"
|
|
308
|
+
if done.returncode == 0:
|
|
309
|
+
try:
|
|
310
|
+
return secret_data(json.loads(done.stdout))
|
|
311
|
+
except ValueError:
|
|
312
|
+
return "kubectl's answer is not JSON"
|
|
313
|
+
err = done.stderr.strip().splitlines()[-1:] or [f"exit {done.returncode}"]
|
|
314
|
+
if "(NotFound)" in done.stderr or " not found" in done.stderr:
|
|
315
|
+
return None
|
|
316
|
+
if "(Forbidden)" in done.stderr or "forbidden" in done.stderr:
|
|
317
|
+
return (f"kubectl may not read Secrets in {ns}: the deploy identity needs get on "
|
|
318
|
+
f"secrets there ({err[0]})")
|
|
319
|
+
return f"kubectl failed: {err[0]}"
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def preflight(text: str, namespace: str, cluster: Cluster | None) -> int:
|
|
323
|
+
"""Check the Secrets the manifests in `text` read; 1 on any problem."""
|
|
324
|
+
found = collect(documents(text), namespace)
|
|
325
|
+
problems = list(found.problems)
|
|
326
|
+
for lit in found.literals:
|
|
327
|
+
if (why := lit.shape.problem(lit.value)) is not None:
|
|
328
|
+
problems.append(f"{lit.where}, {lit.shape.what}: {why}")
|
|
329
|
+
by_secret: dict[tuple[str, str], list[Need]] = {}
|
|
330
|
+
for n in found.needs:
|
|
331
|
+
by_secret.setdefault((n.namespace, n.secret), []).append(n)
|
|
332
|
+
unchecked = []
|
|
333
|
+
for (ns, name), needs in sorted(by_secret.items()):
|
|
334
|
+
held = found.held.get((ns, name))
|
|
335
|
+
data = held if held is not None else (cluster.secret(ns, name) if cluster else False)
|
|
336
|
+
if data is False:
|
|
337
|
+
keys = sorted({k for n in needs for k in n.keys})
|
|
338
|
+
unchecked.append(f"{name}" + (f" ({', '.join(keys)})" if keys else ""))
|
|
339
|
+
continue
|
|
340
|
+
if isinstance(data, str):
|
|
341
|
+
problems.append(f"Secret {name} in {ns}: {data}")
|
|
342
|
+
continue
|
|
343
|
+
required = [n for n in needs if not n.optional]
|
|
344
|
+
if data is None:
|
|
345
|
+
if required:
|
|
346
|
+
problems.append(f"Secret {name} is not in namespace {ns}; "
|
|
347
|
+
f"{'; '.join(n.where for n in required)} read it")
|
|
348
|
+
else:
|
|
349
|
+
print(f"Secret {name} is not in {ns}, and every reference to it is optional")
|
|
350
|
+
continue
|
|
351
|
+
bad = check_keys(name, needs, data)
|
|
352
|
+
problems += bad
|
|
353
|
+
if not bad:
|
|
354
|
+
keys = sorted({k for n in needs for k in n.keys})
|
|
355
|
+
print(f"ok: Secret {name} in {ns}" + (f", keys {', '.join(keys)}" if keys else ""))
|
|
356
|
+
if unchecked:
|
|
357
|
+
print(f"not read (offline), the manifests need the Secrets {', '.join(unchecked)}")
|
|
358
|
+
for p in problems:
|
|
359
|
+
print(f"error: {p}")
|
|
360
|
+
count = len(by_secret)
|
|
361
|
+
if problems:
|
|
362
|
+
print(f"preflight: {len(problems)} problem{'s' * (len(problems) != 1)} with the Secrets "
|
|
363
|
+
f"in {namespace}: fix them before this deploys", file=sys.stderr)
|
|
364
|
+
return 1
|
|
365
|
+
print(f"preflight: the manifests read {count} Secret{'s' * (count != 1)}, "
|
|
366
|
+
+ ("listed above" if cluster is None else "and each is as they need"))
|
|
367
|
+
return 0
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def check_keys(name: str, needs: list[Need], data: dict[str, bytes]) -> list[str]:
|
|
371
|
+
"""What is wrong with one Secret that is there, for what reads it."""
|
|
372
|
+
out = []
|
|
373
|
+
for n in needs:
|
|
374
|
+
for key in n.keys:
|
|
375
|
+
if key not in data:
|
|
376
|
+
if not n.optional:
|
|
377
|
+
out.append(f"Secret {name} has no key {key}, which {n.where} reads")
|
|
378
|
+
continue
|
|
379
|
+
shape = n.shapes.get(key)
|
|
380
|
+
if shape and (why := shape.problem(data[key])) is not None:
|
|
381
|
+
out.append(f"Secret {name} key {key} ({n.where}), {shape.what}: {why}")
|
|
382
|
+
if n.prefix is not None:
|
|
383
|
+
for key, value in sorted(data.items()):
|
|
384
|
+
shape = ENV_SHAPES.get(n.prefix + key)
|
|
385
|
+
if shape and (why := shape.problem(value)) is not None:
|
|
386
|
+
out.append(f"Secret {name} key {key} ({n.where}, as {n.prefix + key}), "
|
|
387
|
+
f"{shape.what}: {why}")
|
|
388
|
+
return out
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def kubectl_json(kubectl: list[str], ns: str, *args: str) -> dict[str, Any]:
|
|
392
|
+
done = subprocess.run([*kubectl, "-n", ns, *args, "-o", "json"], capture_output=True,
|
|
393
|
+
text=True, timeout=TIMEOUT, check=False)
|
|
394
|
+
if done.returncode != 0:
|
|
395
|
+
print(f"kubectl {' '.join(args)}: {done.stderr.strip()}")
|
|
396
|
+
return {}
|
|
397
|
+
try:
|
|
398
|
+
return json.loads(done.stdout)
|
|
399
|
+
except ValueError:
|
|
400
|
+
return {}
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def diagnose(kubectl: list[str], ns: str, selector: str, tail: int) -> int:
|
|
404
|
+
"""Print what is wrong with the pods in `ns` that are not ready or
|
|
405
|
+
restarted. Always 0: it explains a failure, and is not one."""
|
|
406
|
+
args = ["get", "pods"] + (["-l", selector] if selector else [])
|
|
407
|
+
pods = kubectl_json(kubectl, ns, *args).get("items") or []
|
|
408
|
+
events = kubectl_json(kubectl, ns, "get", "events").get("items") or []
|
|
409
|
+
shown = 0
|
|
410
|
+
for p in pods:
|
|
411
|
+
name, status = dig(p, ("metadata", "name")), p.get("status") or {}
|
|
412
|
+
statuses = items(status.get("initContainerStatuses")) + items(status.get("containerStatuses"))
|
|
413
|
+
ready = any(c.get("type") == "Ready" and c.get("status") == "True"
|
|
414
|
+
for c in items(status.get("conditions")))
|
|
415
|
+
if ready and not any(s.get("restartCount") for s in statuses):
|
|
416
|
+
continue
|
|
417
|
+
shown += 1
|
|
418
|
+
print(f"--- pod {name}: {status.get('phase', '?')}"
|
|
419
|
+
+ (f", {status['reason']}" if status.get("reason") else ""))
|
|
420
|
+
for s in statuses:
|
|
421
|
+
state = s.get("state") or {}
|
|
422
|
+
now = next(iter(state), "?")
|
|
423
|
+
detail = state.get(now) or {}
|
|
424
|
+
why = " ".join(str(detail[k]) for k in ("reason", "message") if detail.get(k))
|
|
425
|
+
last = dig(s, ("lastState", "terminated")) or {}
|
|
426
|
+
print(f" container {s.get('name')}: {now}" + (f" ({why})" if why else "")
|
|
427
|
+
+ f", {s.get('restartCount', 0)} restarts"
|
|
428
|
+
+ (f", last exit {last.get('exitCode')} {last.get('reason', '')}".rstrip()
|
|
429
|
+
if last else ""))
|
|
430
|
+
for e in [e for e in events if dig(e, ("involvedObject", "name")) == name
|
|
431
|
+
and e.get("type") == "Warning"][-10:]:
|
|
432
|
+
print(f" event {e.get('reason')}: {e.get('message', '').strip()}")
|
|
433
|
+
for s in statuses:
|
|
434
|
+
for previous in ([True] if s.get("restartCount") else []) + [False]:
|
|
435
|
+
done = subprocess.run([*kubectl, "-n", ns, "logs", name, "-c", str(s.get("name")),
|
|
436
|
+
f"--tail={tail}", *(["--previous"] if previous else [])],
|
|
437
|
+
capture_output=True, text=True, timeout=TIMEOUT, check=False)
|
|
438
|
+
lines = (done.stdout if done.returncode == 0 else done.stderr).rstrip()
|
|
439
|
+
if lines:
|
|
440
|
+
print(f" logs of {s.get('name')}{' (previous)' if previous else ''}:")
|
|
441
|
+
print("\n".join(f" {line}" for line in lines.splitlines()))
|
|
442
|
+
if not shown:
|
|
443
|
+
print(f"diagnose: every pod in {ns} is ready and none restarted")
|
|
444
|
+
return 0
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def main(argv: list[str] | None = None) -> int:
|
|
448
|
+
parser = argparse.ArgumentParser(prog="pubdeploy k8s", description="Check the Secrets rendered manifests "
|
|
449
|
+
"read before a deploy, or show why a rollout failed.")
|
|
450
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
451
|
+
p = commands.add_parser("preflight", help="every Secret the manifests on standard input read is "
|
|
452
|
+
"there, with its keys, and values of a known shape have it")
|
|
453
|
+
p.add_argument("--offline", action="store_true",
|
|
454
|
+
help="read no cluster: list what is needed, and check what the manifests hold")
|
|
455
|
+
p = commands.add_parser("diagnose", help="what is wrong with the pods that are not ready")
|
|
456
|
+
p.add_argument("--selector", default="", help="only the pods with these labels, such as app=web")
|
|
457
|
+
p.add_argument("--tail", type=int, default=30, help="the lines of each container's logs")
|
|
458
|
+
for p in commands.choices.values():
|
|
459
|
+
p.add_argument("--namespace", "-n", required=True, help="the namespace to read")
|
|
460
|
+
p.add_argument("--kubectl", default="kubectl",
|
|
461
|
+
help="the kubectl command, such as 'k3s kubectl --kubeconfig FILE'")
|
|
462
|
+
args = parser.parse_args(argv)
|
|
463
|
+
if not NAME.fullmatch(args.namespace):
|
|
464
|
+
parser.error(f"--namespace: {args.namespace!r} is not a namespace's name")
|
|
465
|
+
kubectl = shlex.split(args.kubectl)
|
|
466
|
+
if args.command == "diagnose":
|
|
467
|
+
return diagnose(kubectl, args.namespace, args.selector, args.tail)
|
|
468
|
+
return preflight(sys.stdin.read(), args.namespace, None if args.offline else Cluster(kubectl))
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
if __name__ == "__main__":
|
|
472
|
+
sys.exit(main())
|