pubdeploy 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubdeploy/__init__.py +532 -0
- pubdeploy/channels.py +188 -0
- pubdeploy/cli.py +120 -0
- pubdeploy/config.py +140 -0
- pubdeploy/contract.py +87 -0
- pubdeploy/github.py +61 -0
- pubdeploy/hooks.py +82 -0
- pubdeploy/implied.py +179 -0
- pubdeploy/journal.json +207 -0
- pubdeploy/k8s.py +472 -0
- pubdeploy/machine.py +363 -0
- pubdeploy/model.py +229 -0
- pubdeploy/names.py +25 -0
- pubdeploy/order.py +101 -0
- pubdeploy/providers.py +75 -0
- pubdeploy/registry.py +96 -0
- pubdeploy/release-commit +1 -0
- pubdeploy/schema.json +127 -0
- pubdeploy/schema.py +238 -0
- pubdeploy/store.py +127 -0
- pubdeploy/testing/__init__.py +94 -0
- pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
- pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
- pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
- pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
- pubdeploy/testing/plugin.py +26 -0
- pubdeploy/testing/steps.py +1124 -0
- pubdeploy/testing/world.py +569 -0
- pubdeploy/types/__init__.py +181 -0
- pubdeploy/types/auth.py +199 -0
- pubdeploy/types/github.py +110 -0
- pubdeploy/types/k8s.py +609 -0
- pubdeploy/types/net.py +237 -0
- pubdeploy/types/rollout.py +186 -0
- pubdeploy/units.py +177 -0
- pubdeploy/walk.py +325 -0
- pubdeploy-0.8.0.dist-info/METADATA +520 -0
- pubdeploy-0.8.0.dist-info/RECORD +45 -0
- pubdeploy-0.8.0.dist-info/WHEEL +5 -0
- pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
- pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
- pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/types/k8s.py
ADDED
|
@@ -0,0 +1,609 @@
|
|
|
1
|
+
"""k8s.namespace, k8s.secrets-declared, k8s.manifests, k8s.cronjob, k8s.job-once and k8s.jwks:
|
|
2
|
+
what the cluster must hold, read through kubectl, and what the rendered
|
|
3
|
+
manifests read, which needs no cluster. They use pubdeploy.k8s, which needs
|
|
4
|
+
nothing of this package's. k8s.secret, which reads a Secret's values and
|
|
5
|
+
makes one from Bitwarden's, is pubsec's.
|
|
6
|
+
|
|
7
|
+
A fulfil writes through the kubectl channel or runs on the server, so each
|
|
8
|
+
needs the admin channel or the plane; k8s.namespace's, which runs sustenaut-app
|
|
9
|
+
on the server, the admin channel alone (on the plane it stays a laptop step until
|
|
10
|
+
the deploy channel no longer needs the node's kubeconfig)."""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import copy
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from datetime import datetime, timedelta
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
|
|
23
|
+
from .. import k8s
|
|
24
|
+
from ..model import Context, Need, Result, Stop
|
|
25
|
+
from . import Base, failed, kubectl_json, kubectl_refused, last_success, not_found, run_job, schema
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Namespace(Base):
|
|
29
|
+
"""The namespace, with the deploy service account and its role, as the
|
|
30
|
+
server's sustenaut-app program makes them."""
|
|
31
|
+
type = "k8s.namespace"
|
|
32
|
+
grade = "auto"
|
|
33
|
+
# Four kubectl calls: over the admin channel's kept connection about 0.35 s
|
|
34
|
+
# each, as measured to the k3s server over the tailnet.
|
|
35
|
+
budget = 4.0
|
|
36
|
+
fulfil_budget = 60.0
|
|
37
|
+
singleton = True
|
|
38
|
+
needs_of_context = ("server",)
|
|
39
|
+
|
|
40
|
+
def action(self, need: Need, ctx: Context) -> str:
|
|
41
|
+
return f"make namespace {need.name} on the server: sustenaut-app {need.name}"
|
|
42
|
+
|
|
43
|
+
def fulfil(self, need: Need, ctx: Context) -> Result:
|
|
44
|
+
ran = ctx.server(["sustenaut-app", need.name], None)
|
|
45
|
+
if ran.returncode != 0:
|
|
46
|
+
return failed(f"sustenaut-app {need.name} failed on the server", ran)
|
|
47
|
+
return Result("ready", f"made namespace {need.name}, with the deploy service account and its role")
|
|
48
|
+
|
|
49
|
+
def check(self, need: Need, ctx: Context) -> Result:
|
|
50
|
+
ns = need.name
|
|
51
|
+
server = ctx.env.get("SUSTENAUT_SERVER", "<server>")
|
|
52
|
+
fix = f"ssh root@{server} sustenaut-app {ns}"
|
|
53
|
+
if ctx.channel == "deploy":
|
|
54
|
+
# The deploy identity may read neither namespaces nor roles.
|
|
55
|
+
return Result("unknown", f"namespace {ns} is judged over the admin channel only",
|
|
56
|
+
fix="run the walk with --server HOST, the admin channel, or from the plane")
|
|
57
|
+
_, ran = kubectl_json(ctx, "get", "namespace", ns)
|
|
58
|
+
if ran.returncode != 0:
|
|
59
|
+
if not_found(ran):
|
|
60
|
+
return Result("missing", f"namespace {ns} is not there", fix=fix)
|
|
61
|
+
return kubectl_refused(ran, ns, "namespaces")
|
|
62
|
+
missing = []
|
|
63
|
+
for kind in ("serviceaccount", "role", "rolebinding"):
|
|
64
|
+
_, ran = kubectl_json(ctx, "-n", ns, "get", kind, "deploy")
|
|
65
|
+
if ran.returncode != 0 and not not_found(ran):
|
|
66
|
+
return kubectl_refused(ran, ns, f"{kind}s")
|
|
67
|
+
if ran.returncode != 0:
|
|
68
|
+
missing.append(f"{kind} deploy")
|
|
69
|
+
if ctx.channel == "admin":
|
|
70
|
+
# The server's deploy command refuses an app without the deploy user's kubeconfig,
|
|
71
|
+
# which sustenaut-app writes last: the cluster's objects alone are not enough.
|
|
72
|
+
ran = ctx.server(["test", "-f", f"/home/deploy/.kube/{ns}"], None)
|
|
73
|
+
if ran.returncode != 0:
|
|
74
|
+
missing.append(f"deploy kubeconfig (/home/deploy/.kube/{ns} on the server)")
|
|
75
|
+
if missing:
|
|
76
|
+
return Result("missing", f"namespace {ns} has no {', '.join(missing)}", fix=fix)
|
|
77
|
+
return Result("ready", f"namespace {ns} is there, with the deploy service account and its role")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class SecretsDeclared(Base):
|
|
81
|
+
"""Every Secret the rendered manifests read is one a need fills, and every
|
|
82
|
+
Secret a need fills is read."""
|
|
83
|
+
type = "k8s.secrets-declared"
|
|
84
|
+
budget = 1.0
|
|
85
|
+
offline = True
|
|
86
|
+
singleton = True
|
|
87
|
+
|
|
88
|
+
def check(self, need: Need, ctx: Context) -> Result:
|
|
89
|
+
from ..implied import declarers # noqa: PLC0415
|
|
90
|
+
from ..registry import types # noqa: PLC0415
|
|
91
|
+
try:
|
|
92
|
+
found = k8s.collect(k8s.documents(ctx.rendered()), ctx.namespace)
|
|
93
|
+
except Stop as stop:
|
|
94
|
+
if stop.missing_tool:
|
|
95
|
+
return Result("unknown", "the overlay could not be rendered", fix=str(stop))
|
|
96
|
+
return Result("drift", "the overlay does not render", fix=str(stop))
|
|
97
|
+
except (ValueError, yaml.YAMLError) as e:
|
|
98
|
+
return Result("drift", "the rendered manifests are not YAML", fix=f"see what the overlay renders "
|
|
99
|
+
f"({type(e).__name__})")
|
|
100
|
+
declared = declarers(ctx.block, ctx.types or types(), ctx.namespace)
|
|
101
|
+
held = {name for ns, name in found.held if ns == ctx.namespace}
|
|
102
|
+
reads: dict[str, list[k8s.Need]] = {}
|
|
103
|
+
for n in found.needs:
|
|
104
|
+
if n.namespace == ctx.namespace:
|
|
105
|
+
reads.setdefault(n.secret, []).append(n)
|
|
106
|
+
lines = list(found.problems)
|
|
107
|
+
for name, refs in sorted(reads.items()):
|
|
108
|
+
required = [r for r in refs if not r.optional]
|
|
109
|
+
if name in held or not required:
|
|
110
|
+
continue
|
|
111
|
+
fillers = declared.get(name)
|
|
112
|
+
if not fillers:
|
|
113
|
+
lines.append(f"Secret {name} is read by {', '.join(r.where for r in required)}, and no need "
|
|
114
|
+
f"says where it comes from: add it to deploy.needs, or a need's secret")
|
|
115
|
+
continue
|
|
116
|
+
if any(keys is None for _, keys in fillers):
|
|
117
|
+
continue
|
|
118
|
+
provided = {k for _, keys in fillers for k in keys or []}
|
|
119
|
+
for r in required:
|
|
120
|
+
for key in r.keys:
|
|
121
|
+
if key not in provided:
|
|
122
|
+
lines.append(f"Secret {name} key {key} is read by {r.where}, and "
|
|
123
|
+
f"{', '.join(i for i, _ in fillers)} provides only {', '.join(sorted(provided))}")
|
|
124
|
+
for name, fillers in sorted(declared.items()):
|
|
125
|
+
if name not in reads:
|
|
126
|
+
lines.append(f"Secret {name} is declared by {', '.join(i for i, _ in fillers)} but nothing reads it")
|
|
127
|
+
count = len([n for n, refs in reads.items() if n not in held and any(not r.optional for r in refs)])
|
|
128
|
+
left = leftovers(ctx, set(reads) | set(declared) | held)
|
|
129
|
+
lines += [f"Secret {name} is in the cluster, and nothing reads or declares it: "
|
|
130
|
+
f"kubectl -n {ctx.namespace} delete secret {name}" for name in left]
|
|
131
|
+
if lines:
|
|
132
|
+
return Result("drift", f"the manifests and the needs disagree about {len(lines)} "
|
|
133
|
+
f"Secret{'s' * (len(lines) != 1)}", details=lines,
|
|
134
|
+
fix="make deploy.needs say where each Secret the manifests read comes from, "
|
|
135
|
+
"drop what nothing reads, and delete a Secret left in the cluster")
|
|
136
|
+
return Result("ready", f"the manifests read {count} Secret{'s' * (count != 1)}, "
|
|
137
|
+
f"and a need says where each comes from")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def leftovers(ctx: Context, known: set[str]) -> list[str]:
|
|
141
|
+
"""The Opaque Secrets in the namespace that nothing reads or declares: one a need
|
|
142
|
+
filled before the need went, say a key's once its service reaches Google without
|
|
143
|
+
one. None when the walk is offline or kubectl may not list them (the check stands
|
|
144
|
+
on the manifests alone then)."""
|
|
145
|
+
if ctx.offline or ctx.channel == "deploy":
|
|
146
|
+
return []
|
|
147
|
+
# Names and types alone: no Secret's data reaches the walk, nor crosses the admin channel.
|
|
148
|
+
ran = ctx.kubectl(["-n", ctx.namespace, "get", "secrets", "--no-headers",
|
|
149
|
+
"-o", "custom-columns=NAME:.metadata.name,TYPE:.type"])
|
|
150
|
+
if ran.returncode != 0:
|
|
151
|
+
return []
|
|
152
|
+
rows = [line.split() for line in ran.stdout.splitlines()]
|
|
153
|
+
return sorted(row[0] for row in rows if len(row) == 2 and row[1] == "Opaque" and row[0] not in known)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class Manifests(Base):
|
|
157
|
+
"""A manifest file applied by hand, such as an admin's: it parses, and,
|
|
158
|
+
online, each object in it is in the cluster."""
|
|
159
|
+
type = "k8s.manifests"
|
|
160
|
+
grade = "auto"
|
|
161
|
+
budget = 5.0
|
|
162
|
+
fulfil_budget = 60.0
|
|
163
|
+
offline = True
|
|
164
|
+
needs_of_context = ("kubectl-write",)
|
|
165
|
+
schema = schema({"path": {"type": "string", "pattern": r"^(?!/)(?!.*(^|/)\.\.(/|$))[A-Za-z0-9_./-]+\.ya?ml$",
|
|
166
|
+
"x-phrase": "must be a YAML file in the repository"}}, ("path",))
|
|
167
|
+
|
|
168
|
+
def default_name(self, params: dict[str, Any]) -> str | None:
|
|
169
|
+
return params.get("path")
|
|
170
|
+
|
|
171
|
+
def action(self, need: Need, ctx: Context) -> str:
|
|
172
|
+
return f"kubectl apply -f {need.params['path']}"
|
|
173
|
+
|
|
174
|
+
def fulfil(self, need: Need, ctx: Context) -> Result:
|
|
175
|
+
path = need.params["path"]
|
|
176
|
+
try:
|
|
177
|
+
text = (Path(ctx.root) / path).read_bytes()
|
|
178
|
+
except OSError as e:
|
|
179
|
+
return Result("missing", f"{path} cannot be read", fix=type(e).__name__)
|
|
180
|
+
try:
|
|
181
|
+
docs = k8s.documents(text.decode("utf-8"))
|
|
182
|
+
except (UnicodeDecodeError, ValueError, SystemExit, yaml.YAMLError) as e:
|
|
183
|
+
return Result("drift", f"{path} cannot be read as manifests", fix=type(e).__name__)
|
|
184
|
+
# An object that names its namespace is applied there, the rest in the unit's:
|
|
185
|
+
# kubectl refuses -n NS for an object of another namespace.
|
|
186
|
+
groups: dict[str, list[Any]] = {}
|
|
187
|
+
for doc in docs:
|
|
188
|
+
groups.setdefault((doc.get("metadata") or {}).get("namespace") or ctx.namespace, []).append(doc)
|
|
189
|
+
applied = 0
|
|
190
|
+
for ns, group in groups.items():
|
|
191
|
+
ran = ctx.kubectl(["-n", ns, "apply", "--server-side", "--force-conflicts", "-f", "-"],
|
|
192
|
+
input=yaml.safe_dump_all(group, sort_keys=False).encode())
|
|
193
|
+
if ran.returncode != 0:
|
|
194
|
+
return failed(f"kubectl apply -f {path} failed", ran)
|
|
195
|
+
applied += len(ran.stdout.splitlines())
|
|
196
|
+
return Result("ready", f"applied {path}: {applied} object{'s' * (applied != 1)}")
|
|
197
|
+
|
|
198
|
+
def files(self, params: dict[str, Any]) -> list[str]:
|
|
199
|
+
return [params["path"]] if isinstance(params.get("path"), str) else []
|
|
200
|
+
|
|
201
|
+
def check(self, need: Need, ctx: Context) -> Result:
|
|
202
|
+
path = need.params["path"]
|
|
203
|
+
try:
|
|
204
|
+
text = (Path(ctx.root) / path).read_text(encoding="utf-8")
|
|
205
|
+
docs = k8s.documents(text)
|
|
206
|
+
except (OSError, ValueError, SystemExit, yaml.YAMLError) as e:
|
|
207
|
+
return Result("drift", f"{path} cannot be read as manifests",
|
|
208
|
+
fix=f"make {path} YAML that kubectl applies ({type(e).__name__})")
|
|
209
|
+
if not docs:
|
|
210
|
+
return Result("drift", f"{path} holds no object", fix=f"put the objects to apply in {path}")
|
|
211
|
+
objects = [(str(d.get("kind")), str((d.get("metadata") or {}).get("name")),
|
|
212
|
+
(d.get("metadata") or {}).get("namespace") or ctx.namespace) for d in docs]
|
|
213
|
+
if ctx.offline:
|
|
214
|
+
return Result("ready", f"{path} holds {len(objects)} object{'s' * (len(objects) != 1)}")
|
|
215
|
+
missing = []
|
|
216
|
+
for kind, name, ns in objects:
|
|
217
|
+
ran = ctx.kubectl(["-n", ns, "get", f"{kind.lower()}/{name}", "-o", "name"])
|
|
218
|
+
if ran.returncode != 0 and not not_found(ran):
|
|
219
|
+
return kubectl_refused(ran, ns, f"{kind} objects")
|
|
220
|
+
if ran.returncode != 0:
|
|
221
|
+
missing.append(f"{kind}/{name}")
|
|
222
|
+
if missing:
|
|
223
|
+
return Result("missing", f"{', '.join(missing)} from {path} {'is' if len(missing) == 1 else 'are'} "
|
|
224
|
+
f"not in {ctx.namespace}", fix=f"kubectl apply -f {path}, as an admin")
|
|
225
|
+
# There, but as the file says? A server-side dry run, authorised as the apply would be.
|
|
226
|
+
groups: dict[str, list[Any]] = {}
|
|
227
|
+
for doc in docs:
|
|
228
|
+
groups.setdefault((doc.get("metadata") or {}).get("namespace") or ctx.namespace, []).append(doc)
|
|
229
|
+
for ns, group in groups.items():
|
|
230
|
+
# --force-conflicts: objects applied client-side have their fields owned by kubectl's
|
|
231
|
+
# client-side apply; a dry run takes them over in its answer and writes nothing.
|
|
232
|
+
ran = ctx.kubectl(["-n", ns, "diff", "--server-side", "--force-conflicts", "-f", "-"],
|
|
233
|
+
input=yaml.safe_dump_all(group, sort_keys=False).encode())
|
|
234
|
+
if ran.returncode == 1:
|
|
235
|
+
changed = sorted({line.split()[1].rsplit("/", 1)[0] for line in ran.stdout.splitlines()
|
|
236
|
+
if line.startswith("diff -u") and len(line.split()) > 1})
|
|
237
|
+
return Result("drift", f"{path} differs from what is in the cluster"
|
|
238
|
+
+ (f" ({len(changed)} object{'s' * (len(changed) != 1)})" if changed else ""),
|
|
239
|
+
fix=f"kubectl apply --server-side --force-conflicts -f {path}, as an admin")
|
|
240
|
+
if ran.returncode != 0:
|
|
241
|
+
said = (ran.stderr.strip().splitlines() or [f"exit {ran.returncode}"])[-1][:200]
|
|
242
|
+
return Result("unknown", f"the objects in {path} are there; whether they match it could not be "
|
|
243
|
+
f"compared: {said}", fix="run the walk over the admin channel")
|
|
244
|
+
return Result("ready", f"the {len(objects)} object{'s' * (len(objects) != 1)} in {path} are applied, "
|
|
245
|
+
"as the file says")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
DURATION = {"type": "string", "pattern": r"^[1-9][0-9]*[mhd]$", "x-phrase": "must be a duration: minutes, hours "
|
|
249
|
+
"or days, such as 90m, 36h or 2d"}
|
|
250
|
+
UNITS = {"m": timedelta(minutes=1), "h": timedelta(hours=1), "d": timedelta(days=1)}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def duration(text: str) -> timedelta:
|
|
254
|
+
return int(text[:-1]) * UNITS[text[-1]]
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def interval(schedule: str) -> timedelta | None:
|
|
258
|
+
"""How often a schedule of a common form runs, or None for another."""
|
|
259
|
+
named = {"@hourly": timedelta(hours=1), "@daily": timedelta(days=1), "@midnight": timedelta(days=1),
|
|
260
|
+
"@weekly": timedelta(days=7), "@monthly": timedelta(days=31), "@yearly": timedelta(days=366),
|
|
261
|
+
"@annually": timedelta(days=366)}
|
|
262
|
+
if schedule.strip() in named:
|
|
263
|
+
return named[schedule.strip()]
|
|
264
|
+
fields = schedule.split()
|
|
265
|
+
if len(fields) != 5:
|
|
266
|
+
return None
|
|
267
|
+
minute, hour, day, month, weekday = fields
|
|
268
|
+
|
|
269
|
+
def every(field: str) -> int | None:
|
|
270
|
+
"""1 for *, N for */N, None for anything else."""
|
|
271
|
+
if field == "*":
|
|
272
|
+
return 1
|
|
273
|
+
m = re.fullmatch(r"\*/([1-9][0-9]*)", field)
|
|
274
|
+
return int(m[1]) if m else None
|
|
275
|
+
|
|
276
|
+
def listed(field: str) -> int | None:
|
|
277
|
+
"""How many fixed values a field lists: 5, or 1,2,3."""
|
|
278
|
+
return len(field.split(",")) if re.fullmatch(r"[0-9]+(,[0-9]+)*", field) else None
|
|
279
|
+
|
|
280
|
+
if (n := every(minute)) and hour == day == month == weekday == "*":
|
|
281
|
+
return timedelta(minutes=n)
|
|
282
|
+
if listed(minute) and month == "*":
|
|
283
|
+
if (n := every(hour)) and day == weekday == "*":
|
|
284
|
+
return timedelta(hours=n) / listed(minute)
|
|
285
|
+
if (count := listed(hour)) and day == weekday == "*":
|
|
286
|
+
return timedelta(days=1) / count
|
|
287
|
+
if listed(hour) == 1 and day == "*" and (count := listed(weekday)):
|
|
288
|
+
return timedelta(days=7) / count
|
|
289
|
+
if listed(hour) == 1 and weekday == "*" and listed(day) == 1:
|
|
290
|
+
return timedelta(days=31)
|
|
291
|
+
return None
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def when(text: str) -> datetime:
|
|
295
|
+
return datetime.fromisoformat(str(text).replace("Z", "+00:00"))
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
class CronJob(Base):
|
|
299
|
+
"""A CronJob that succeeds as often as it runs: a service without a host."""
|
|
300
|
+
type = "k8s.cronjob"
|
|
301
|
+
stop = "shorten"
|
|
302
|
+
grade = "auto"
|
|
303
|
+
budget = 4.0
|
|
304
|
+
fulfil_budget = 300.0
|
|
305
|
+
needs_of_context = ("kubectl-write",)
|
|
306
|
+
schema = schema({"within": DURATION})
|
|
307
|
+
|
|
308
|
+
def action(self, need: Need, ctx: Context) -> str:
|
|
309
|
+
return f"run a Job from CronJob {need.name}, and wait for it"
|
|
310
|
+
|
|
311
|
+
def check(self, need: Need, ctx: Context) -> Result:
|
|
312
|
+
ns, name = ctx.namespace, need.name
|
|
313
|
+
found, ran = kubectl_json(ctx, "-n", ns, "get", "cronjob", name)
|
|
314
|
+
if found is None:
|
|
315
|
+
if not_found(ran):
|
|
316
|
+
return Result("missing", f"CronJob {name} is not in {ns}", fixable=False,
|
|
317
|
+
fix=f"see that the overlay {ctx.block.get('overlay', 'k8s')} holds it")
|
|
318
|
+
return kubectl_refused(ran, ns, "cronjobs")
|
|
319
|
+
spec, status = found.get("spec") or {}, found.get("status") or {}
|
|
320
|
+
given = need.params.get("within")
|
|
321
|
+
schedule = str(spec.get("schedule", ""))
|
|
322
|
+
within = duration(given) if given else (2 * every if (every := interval(schedule)) else None)
|
|
323
|
+
if within is None:
|
|
324
|
+
return Result("unknown", f"CronJob {name}'s schedule {schedule} is not one pubdeploy reads",
|
|
325
|
+
fix=f"give the need's within: how recently {name} must have succeeded, such as 36h")
|
|
326
|
+
if spec.get("suspend") is True:
|
|
327
|
+
return Result("drift", f"CronJob {name} is suspended", fixable=False,
|
|
328
|
+
fix=f"kubectl -n {ns} patch cronjob {name} -p '{{\"spec\":{{\"suspend\":false}}}}'")
|
|
329
|
+
last, scheduled = last_success(ctx, name, status), status.get("lastScheduleTime")
|
|
330
|
+
hours = f"{within.total_seconds() / 3600:g} h"
|
|
331
|
+
if last and ctx.now() - when(last) <= within:
|
|
332
|
+
return Result("ready", f"CronJob {name} succeeded at {last}, within {hours}")
|
|
333
|
+
fix = f"pubdeploy k8s diagnose --namespace {ns}"
|
|
334
|
+
if last:
|
|
335
|
+
return Result("drift", f"CronJob {name} last succeeded at {last}, over {hours} ago", fix=fix)
|
|
336
|
+
if scheduled:
|
|
337
|
+
return Result("drift", f"CronJob {name}'s last run did not succeed", fix=fix)
|
|
338
|
+
return Result("missing", f"CronJob {name} never ran",
|
|
339
|
+
fix=f"kubectl -n {ns} create job --from=cronjob/{name} {name}-now")
|
|
340
|
+
|
|
341
|
+
def fulfil(self, need: Need, ctx: Context) -> Result:
|
|
342
|
+
return run_job(ctx, need.name, self.fulfil_budget)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
# A clone of the file's repository older than this may hold a pin that has since moved.
|
|
346
|
+
STALE_HOURS = 24
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class Jwks(Base):
|
|
350
|
+
"""A file that pins the cluster's service-account signing keys is the
|
|
351
|
+
cluster's: each key `kubectl get --raw /openid/v1/jwks` gives, and no other;
|
|
352
|
+
with `keys`, the cluster publishes at least that many (two: one signs, the
|
|
353
|
+
other is the fallback a swap or a lost key falls back to).
|
|
354
|
+
What trusts the cluster's tokens without asking it (Google's Workload
|
|
355
|
+
Identity providers, storage's principals) reads the file, so a key the
|
|
356
|
+
cluster rotated fails all of them at once, and only at run time."""
|
|
357
|
+
type = "k8s.jwks"
|
|
358
|
+
grade = "human"
|
|
359
|
+
# One kubectl call.
|
|
360
|
+
budget = 4.0
|
|
361
|
+
schema = schema({"file": {"type": "string", "minLength": 1,
|
|
362
|
+
"x-phrase": "must be the pinned file's path, from the repository's root"},
|
|
363
|
+
"keys": {"type": "integer", "minimum": 1, "maximum": 10,
|
|
364
|
+
"x-phrase": "must be how many signing keys the cluster publishes, 1 to 10"},
|
|
365
|
+
"repository": {"type": "string", "pattern": r"^[A-Za-z0-9-]+/[A-Za-z0-9._-]+$",
|
|
366
|
+
"x-phrase": "must be the OWNER/NAME of the repository the file is in"}},
|
|
367
|
+
("file",))
|
|
368
|
+
|
|
369
|
+
def default_name(self, params: dict[str, Any]) -> str | None:
|
|
370
|
+
file = params.get("file")
|
|
371
|
+
return Path(file).stem if isinstance(file, str) and file else None
|
|
372
|
+
|
|
373
|
+
def check(self, need: Need, ctx: Context) -> Result:
|
|
374
|
+
import json # noqa: PLC0415
|
|
375
|
+
file, owner = need.params["file"], need.params.get("repository")
|
|
376
|
+
where = f"{owner}'s {Path(file).name}" if owner else file
|
|
377
|
+
fix = (f"put the output of `kubectl get --raw /openid/v1/jwks` in {where} as jwks, and in every other "
|
|
378
|
+
f"copy of it, then apply what reads them")
|
|
379
|
+
path = Path(ctx.root) / file
|
|
380
|
+
read = read_at(path)
|
|
381
|
+
try:
|
|
382
|
+
pinned = keys_of(json.loads(path.read_text(encoding="utf-8")).get("jwks"))
|
|
383
|
+
except OSError:
|
|
384
|
+
# A file in another repository's clone (the plane's zones repository, say): no clone, no check.
|
|
385
|
+
return Result("drift", f"{where} could not be read", fixable=False, details=read,
|
|
386
|
+
fix=f"see that {owner + ' is cloned beside this repository' if owner else file + ' is there'}")
|
|
387
|
+
except (ValueError, AttributeError):
|
|
388
|
+
return Result("drift", f"{where} is not a JSON object with the cluster's jwks", fixable=False, fix=fix,
|
|
389
|
+
details=read)
|
|
390
|
+
age = fetched_hours(path) if owner else None
|
|
391
|
+
if age is not None and age > STALE_HOURS:
|
|
392
|
+
return Result("unknown", f"{owner}'s clone was last fetched {age:.0f} h ago; the pin may have moved",
|
|
393
|
+
details=read, fix=f"refresh the clone of {owner}")
|
|
394
|
+
result = self._against_cluster(ctx, file, where, fix, pinned, need.params.get("keys"))
|
|
395
|
+
result.details[:0] = read
|
|
396
|
+
return result
|
|
397
|
+
|
|
398
|
+
def _against_cluster(self, ctx: Context, file: str, where: str, fix: str, pinned: dict, wanted: Any) -> Result:
|
|
399
|
+
if ctx.channel == "deploy":
|
|
400
|
+
# The server's read-only verb takes get with its own flags, and --raw is not one.
|
|
401
|
+
return Result("unknown", "the cluster's signing keys are read over the admin channel or from the plane",
|
|
402
|
+
fix="run the walk with --server HOST, the admin channel, or from the plane")
|
|
403
|
+
ran = ctx.kubectl(["get", "--raw", "/openid/v1/jwks"])
|
|
404
|
+
if ran.returncode != 0:
|
|
405
|
+
return kubectl_refused(ran, ctx.namespace, "the cluster's signing keys (/openid/v1/jwks)")
|
|
406
|
+
try:
|
|
407
|
+
live = keys_of(json.loads(ran.stdout))
|
|
408
|
+
except (ValueError, AttributeError):
|
|
409
|
+
return Result("unknown", "the cluster's /openid/v1/jwks is not a JSON key set",
|
|
410
|
+
fix="see what kubectl get --raw /openid/v1/jwks gives")
|
|
411
|
+
if live == pinned and wanted is not None and len(live) < wanted:
|
|
412
|
+
# One key short of the pair: a lost key would have no fallback to take over.
|
|
413
|
+
return Result("drift", f"the cluster publishes {len(live)} signing key{'s' * (len(live) != 1)}, "
|
|
414
|
+
f"not the {wanted} the need asks for", fixable=False,
|
|
415
|
+
fix="give the cluster its second signing key: shared-infra's scripts/go-live-signing-keys.sh")
|
|
416
|
+
if live == pinned:
|
|
417
|
+
return Result("ready", f"{where} pins the cluster's {len(live)} signing key{'s' * (len(live) != 1)}")
|
|
418
|
+
details = [f"the cluster signs with {kid}, which {file} does not pin" for kid in sorted(set(live) - set(pinned))]
|
|
419
|
+
details += [f"{file} pins {kid}, which the cluster no longer has" for kid in sorted(set(pinned) - set(live))]
|
|
420
|
+
details += [f"{kid} differs between {file} and the cluster" for kid in sorted(set(live) & set(pinned))
|
|
421
|
+
if live[kid] != pinned[kid]]
|
|
422
|
+
return Result("drift", f"{file} does not pin the cluster's signing keys", details=details,
|
|
423
|
+
fixable=False, fix=fix)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def read_at(path: Path) -> list[str]:
|
|
427
|
+
"""Which commit of which clone the pinned file was read at, as a detail; none outside git."""
|
|
428
|
+
import subprocess # noqa: PLC0415
|
|
429
|
+
|
|
430
|
+
def git(*args: str) -> str:
|
|
431
|
+
ran = subprocess.run(["git", "-C", str(path.parent), *args], capture_output=True, text=True,
|
|
432
|
+
timeout=10, check=False)
|
|
433
|
+
return ran.stdout.strip() if ran.returncode == 0 else ""
|
|
434
|
+
try:
|
|
435
|
+
top, head = git("rev-parse", "--show-toplevel"), git("log", "-1", "--format=%h %cI")
|
|
436
|
+
shallow = git("rev-parse", "--is-shallow-repository") == "true"
|
|
437
|
+
changed = "" if shallow else git("log", "-1", "--format=%h %cI", "--", path.name)
|
|
438
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
439
|
+
return []
|
|
440
|
+
if not top or not head:
|
|
441
|
+
return []
|
|
442
|
+
name, (commit, when) = Path(top).name, head.split()
|
|
443
|
+
# A shallow clone (the engine's are depth 1) knows its tip alone: the file's own commit is unknown.
|
|
444
|
+
said = f"read from {name}'s clone at {commit} ({when})"
|
|
445
|
+
if changed:
|
|
446
|
+
said += f", where {path.name} was last changed in {changed.split()[0]} ({changed.split()[1]})"
|
|
447
|
+
return [said]
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def fetched_hours(path: Path) -> float | None:
|
|
451
|
+
"""How long ago the clone the file sits in was last fetched or made (FETCH_HEAD's time, else
|
|
452
|
+
HEAD's), in hours; None outside git."""
|
|
453
|
+
import subprocess # noqa: PLC0415
|
|
454
|
+
import time # noqa: PLC0415
|
|
455
|
+
try:
|
|
456
|
+
ran = subprocess.run(["git", "-C", str(path.parent), "rev-parse", "--git-dir"], capture_output=True,
|
|
457
|
+
text=True, timeout=10, check=False)
|
|
458
|
+
if ran.returncode != 0:
|
|
459
|
+
return None
|
|
460
|
+
git_dir = Path(ran.stdout.strip())
|
|
461
|
+
git_dir = git_dir if git_dir.is_absolute() else path.parent / git_dir
|
|
462
|
+
# A fetch writes FETCH_HEAD; a clone writes none, but it, like a checkout, writes HEAD.
|
|
463
|
+
for name in ("FETCH_HEAD", "HEAD"):
|
|
464
|
+
if (git_dir / name).is_file():
|
|
465
|
+
return (time.time() - (git_dir / name).stat().st_mtime) / 3600
|
|
466
|
+
return None
|
|
467
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
468
|
+
return None
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def keys_of(jwks: Any) -> dict[str, dict[str, Any]]:
|
|
472
|
+
"""A JSON key set's keys by their kid; an AttributeError when it is not one."""
|
|
473
|
+
found = jwks.get("keys")
|
|
474
|
+
if not isinstance(found, list) or not all(isinstance(k, dict) for k in found):
|
|
475
|
+
raise AttributeError("keys")
|
|
476
|
+
return {str(k.get("kid", "")): k for k in found}
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
TIMEOUT = {"type": "string", "pattern": r"^[1-9][0-9]*[mhd]$",
|
|
480
|
+
"x-phrase": "must be how long the Job may run: minutes, hours or days, such as 90m or 6h"}
|
|
481
|
+
JOB_NAME = {"type": "string", "pattern": r"^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$",
|
|
482
|
+
"x-phrase": "must be a Job's or CronJob's name, in lower case"}
|
|
483
|
+
MANIFEST = {"type": "string", "pattern": r"^(?!/)(?!.*(^|/)\.\.(/|$))[A-Za-z0-9_./-]+\.ya?ml$",
|
|
484
|
+
"x-phrase": "must be a YAML file in the repository holding one Job"}
|
|
485
|
+
ANNOTATION = "pubdeploy/need"
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
class JobOnce(Base):
|
|
489
|
+
"""A one-off Job, such as a 90-day backfill, run once: made from a CronJob's
|
|
490
|
+
template (`cronjob`) or a manifest holding one Job (`manifest`), with `args`
|
|
491
|
+
added to its first container's, and `timeout` as its active deadline. The Job
|
|
492
|
+
stays as the record that it ran: its time-to-live is dropped. A failed one is
|
|
493
|
+
made again only with `rerun: if-failed`, and then only that Job is deleted.
|
|
494
|
+
The fulfil makes it and returns: the next walk sees how it went."""
|
|
495
|
+
type = "k8s.job-once"
|
|
496
|
+
stop = "continue"
|
|
497
|
+
stop_warning = "the Job keeps running in the cluster; deleting it may leave its work half done"
|
|
498
|
+
grade = "auto"
|
|
499
|
+
budget = 4.0 # one or two kubectl reads
|
|
500
|
+
fulfil_budget = 60.0
|
|
501
|
+
needs_of_context = ("kubectl-write",)
|
|
502
|
+
# One source of the two, which the check says when a need gives both or neither.
|
|
503
|
+
schema = schema({"cronjob": JOB_NAME, "manifest": MANIFEST,
|
|
504
|
+
"args": {"type": "array", "items": {"type": "string", "maxLength": 200},
|
|
505
|
+
"x-phrase": "must list the arguments added to the Job's container"},
|
|
506
|
+
"timeout": TIMEOUT, "rerun": {"enum": ["never", "if-failed"]}})
|
|
507
|
+
|
|
508
|
+
def _source(self, need: Need) -> str:
|
|
509
|
+
p = need.params
|
|
510
|
+
return f"CronJob {p['cronjob']}" if p.get("cronjob") else p["manifest"]
|
|
511
|
+
|
|
512
|
+
def action(self, need: Need, ctx: Context) -> str:
|
|
513
|
+
args = need.params.get("args") or []
|
|
514
|
+
make = (f"create Job {need.name} in namespace {ctx.namespace} from {self._source(need)}"
|
|
515
|
+
+ (f" with the arguments {' '.join(args)}" if args else ""))
|
|
516
|
+
found = ctx.results.get(need.id)
|
|
517
|
+
if found is not None and found.state == "drift":
|
|
518
|
+
return f"delete Job {need.name} in namespace {ctx.namespace} (no data is deleted), then {make}"
|
|
519
|
+
return make
|
|
520
|
+
|
|
521
|
+
def check(self, need: Need, ctx: Context) -> Result:
|
|
522
|
+
ns, name = ctx.namespace, need.name
|
|
523
|
+
if bool(need.params.get("cronjob")) == bool(need.params.get("manifest")):
|
|
524
|
+
return Result("drift", f"k8s.job-once {name} names {'both' if need.params.get('cronjob') else 'neither'} "
|
|
525
|
+
"of cronjob and manifest", fixable=False,
|
|
526
|
+
fix="give the need one source: cronjob, a CronJob's name, or manifest, a file with one Job")
|
|
527
|
+
job, ran = kubectl_json(ctx, "-n", ns, "get", "job", name)
|
|
528
|
+
if job is None and not not_found(ran):
|
|
529
|
+
return kubectl_refused(ran, ns, "jobs")
|
|
530
|
+
if job is None:
|
|
531
|
+
return self._absent(need, ctx)
|
|
532
|
+
status = job.get("status") or {}
|
|
533
|
+
conditions = {c.get("type"): c.get("status") for c in status.get("conditions") or [] if isinstance(c, dict)}
|
|
534
|
+
if (status.get("succeeded") or 0) >= 1 or conditions.get("Complete") == "True":
|
|
535
|
+
when_ = status.get("completionTime") or ""
|
|
536
|
+
return Result("ready", f"Job {name} completed{' at ' + when_ if when_ else ''}")
|
|
537
|
+
if conditions.get("Failed") == "True":
|
|
538
|
+
again = need.params.get("rerun") == "if-failed"
|
|
539
|
+
return Result("drift", f"Job {name} failed", fixable=again,
|
|
540
|
+
fix=(f"pubdeploy k8s diagnose --namespace {ns}; walk --fulfil deletes Job {name} and makes "
|
|
541
|
+
"it again" if again else
|
|
542
|
+
f"pubdeploy k8s diagnose --namespace {ns}, then kubectl -n {ns} delete job {name} and "
|
|
543
|
+
"walk --fulfil, or give the need rerun: if-failed"))
|
|
544
|
+
return Result("unknown", f"Job {name} is running", fixable=False,
|
|
545
|
+
fix=f"wait for it: kubectl -n {ns} get job {name}")
|
|
546
|
+
|
|
547
|
+
def _absent(self, need: Need, ctx: Context) -> Result:
|
|
548
|
+
ns, name, cronjob = ctx.namespace, need.name, need.params.get("cronjob")
|
|
549
|
+
if cronjob:
|
|
550
|
+
_, ran = kubectl_json(ctx, "-n", ns, "get", "cronjob", cronjob)
|
|
551
|
+
if ran.returncode != 0 and not not_found(ran):
|
|
552
|
+
return kubectl_refused(ran, ns, "cronjobs")
|
|
553
|
+
if ran.returncode != 0:
|
|
554
|
+
return Result("missing", f"Job {name} has not run, and CronJob {cronjob} is not in {ns} yet",
|
|
555
|
+
fixable=False, fix="deploy the service first: the Job is made from its CronJob")
|
|
556
|
+
return Result("missing", f"Job {name} has not run in {ns}",
|
|
557
|
+
fix=f"./pubdeploy walk --fulfil makes it from {self._source(need)}")
|
|
558
|
+
|
|
559
|
+
def _job(self, need: Need, ctx: Context) -> dict[str, Any] | Result:
|
|
560
|
+
"""The Job to make, or why it cannot be."""
|
|
561
|
+
ns, p = ctx.namespace, need.params
|
|
562
|
+
if p.get("cronjob"):
|
|
563
|
+
found, ran = kubectl_json(ctx, "-n", ns, "get", "cronjob", p["cronjob"])
|
|
564
|
+
if found is None:
|
|
565
|
+
return failed(f"CronJob {p['cronjob']} could not be read", ran)
|
|
566
|
+
spec = copy.deepcopy(((found.get("spec") or {}).get("jobTemplate") or {}).get("spec") or {})
|
|
567
|
+
else:
|
|
568
|
+
try:
|
|
569
|
+
docs = k8s.documents((Path(ctx.root) / p["manifest"]).read_text(encoding="utf-8"))
|
|
570
|
+
except (OSError, ValueError, SystemExit, yaml.YAMLError) as e:
|
|
571
|
+
return Result("drift", f"{p['manifest']} cannot be read as a Job", fix=type(e).__name__)
|
|
572
|
+
jobs = [d for d in docs if d.get("kind") == "Job"]
|
|
573
|
+
if len(jobs) != 1:
|
|
574
|
+
return Result("drift", f"{p['manifest']} holds {len(jobs)} Jobs, not one",
|
|
575
|
+
fix=f"put the one Job to run in {p['manifest']}")
|
|
576
|
+
spec = copy.deepcopy(jobs[0].get("spec") or {})
|
|
577
|
+
containers = ((spec.get("template") or {}).get("spec") or {}).get("containers") or []
|
|
578
|
+
if not containers:
|
|
579
|
+
return Result("drift", f"{self._source(need)} gives a Job with no container",
|
|
580
|
+
fix=f"see {self._source(need)}")
|
|
581
|
+
if p.get("args"):
|
|
582
|
+
containers[0]["args"] = [*(containers[0].get("args") or []), *p["args"]]
|
|
583
|
+
spec.pop("ttlSecondsAfterFinished", None) # the Job stays as the record that it ran
|
|
584
|
+
if p.get("timeout"):
|
|
585
|
+
spec["activeDeadlineSeconds"] = int(duration(p["timeout"]).total_seconds())
|
|
586
|
+
return {"apiVersion": "batch/v1", "kind": "Job",
|
|
587
|
+
"metadata": {"name": need.name, "namespace": ns, "annotations": {ANNOTATION: need.id}},
|
|
588
|
+
"spec": spec}
|
|
589
|
+
|
|
590
|
+
def fulfil(self, need: Need, ctx: Context) -> Result:
|
|
591
|
+
ns, name = ctx.namespace, need.name
|
|
592
|
+
job = self._job(need, ctx)
|
|
593
|
+
if isinstance(job, Result):
|
|
594
|
+
return job
|
|
595
|
+
found = ctx.results.get(need.id)
|
|
596
|
+
details = []
|
|
597
|
+
if found is not None and found.state == "drift":
|
|
598
|
+
if need.params.get("rerun") != "if-failed":
|
|
599
|
+
return Result("drift", f"Job {name} failed, and the need does not run it again", fixable=False,
|
|
600
|
+
fix=found.fix)
|
|
601
|
+
ran = ctx.kubectl(["-n", ns, "delete", "job", name, "--wait=true"])
|
|
602
|
+
if ran.returncode != 0 and not not_found(ran):
|
|
603
|
+
return failed(f"Job {name} could not be deleted", ran)
|
|
604
|
+
details.append(f"deleted Job {name}, which had failed")
|
|
605
|
+
ran = ctx.kubectl(["-n", ns, "create", "-f", "-"], input=json.dumps(job).encode())
|
|
606
|
+
if ran.returncode != 0:
|
|
607
|
+
return failed(f"Job {name} could not be made", ran)
|
|
608
|
+
return Result("ready", f"made Job {name} from {self._source(need)}: the next walk sees how it went",
|
|
609
|
+
details=details)
|