pubdeploy 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubdeploy/__init__.py +532 -0
- pubdeploy/channels.py +188 -0
- pubdeploy/cli.py +120 -0
- pubdeploy/config.py +140 -0
- pubdeploy/contract.py +87 -0
- pubdeploy/github.py +61 -0
- pubdeploy/hooks.py +82 -0
- pubdeploy/implied.py +179 -0
- pubdeploy/journal.json +207 -0
- pubdeploy/k8s.py +472 -0
- pubdeploy/machine.py +363 -0
- pubdeploy/model.py +229 -0
- pubdeploy/names.py +25 -0
- pubdeploy/order.py +101 -0
- pubdeploy/providers.py +75 -0
- pubdeploy/registry.py +96 -0
- pubdeploy/release-commit +1 -0
- pubdeploy/schema.json +127 -0
- pubdeploy/schema.py +238 -0
- pubdeploy/store.py +127 -0
- pubdeploy/testing/__init__.py +94 -0
- pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
- pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
- pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
- pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
- pubdeploy/testing/plugin.py +26 -0
- pubdeploy/testing/steps.py +1124 -0
- pubdeploy/testing/world.py +569 -0
- pubdeploy/types/__init__.py +181 -0
- pubdeploy/types/auth.py +199 -0
- pubdeploy/types/github.py +110 -0
- pubdeploy/types/k8s.py +609 -0
- pubdeploy/types/net.py +237 -0
- pubdeploy/types/rollout.py +186 -0
- pubdeploy/units.py +177 -0
- pubdeploy/walk.py +325 -0
- pubdeploy-0.8.0.dist-info/METADATA +520 -0
- pubdeploy-0.8.0.dist-info/RECORD +45 -0
- pubdeploy-0.8.0.dist-info/WHEEL +5 -0
- pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
- pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
- pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/machine.py
ADDED
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""A unit's needs as a state machine, which the convergence engine runs live.
|
|
2
|
+
|
|
3
|
+
`pubdeploy machine --unit U` prints it: a documented subset of XState v5 config,
|
|
4
|
+
as JSON, so Stately's editor and inspector draw it and XState runs it as well as
|
|
5
|
+
`load()` here (python-statemachine) does:
|
|
6
|
+
|
|
7
|
+
id, initial, states, type ("parallel", "final"), on (event -> {target, guard?}
|
|
8
|
+
or a list of them, tried in order), entry, exit, meta
|
|
9
|
+
|
|
10
|
+
Guards and actions are names only; the runner provides them. The root holds one
|
|
11
|
+
parallel state, `running`, with a region per need. A region's states:
|
|
12
|
+
|
|
13
|
+
waiting --start.K--> checking
|
|
14
|
+
checking --checked.K [is_ready]--> ready (final)
|
|
15
|
+
--checked.K [is_found]--> found (missing or drift)
|
|
16
|
+
--checked.K [is_unknown]--> unknown (final: what the walk could not judge)
|
|
17
|
+
--checked.K--> blocked (final)
|
|
18
|
+
--error.K--> failed (final)
|
|
19
|
+
found final in mode check, and for a need of grade propose, human or none;
|
|
20
|
+
for an auto need in mode fulfil --approved.K--> fulfilling
|
|
21
|
+
fulfilling --fulfilled.K [is_ok]--> checking (a fulfil ends with a re-check)
|
|
22
|
+
--fulfilled.K--> failed
|
|
23
|
+
--error.K--> failed
|
|
24
|
+
waiting --predecessor_failed.K--> blocked
|
|
25
|
+
any state not final --stop--> stopped (final)
|
|
26
|
+
|
|
27
|
+
K is the region's key: the need's id with each character a JavaScript or Python
|
|
28
|
+
name cannot hold as `_`, made unique. The need itself is in the region's meta:
|
|
29
|
+
its id, type, grade, mode, the needs it is after (keys), its seconds budget, and
|
|
30
|
+
how a Stop treats it (`stop`: signal, shorten or continue, and `stop_warning`).
|
|
31
|
+
Events carry what the guards read: `checked` a `state`, `fulfilled` an `ok`.
|
|
32
|
+
|
|
33
|
+
A runner blocks the needs after one that ends unknown, blocked, failed or stopped
|
|
34
|
+
(`predecessor_failed`), and fails a run only on a need failed or blocked: a run whose only
|
|
35
|
+
needs not ready are unknown ends done, with a note.
|
|
36
|
+
|
|
37
|
+
What runs, and in which order, is the deploy block's `run` section's, or the
|
|
38
|
+
default generated from the needs (`compile`): every need in `default_mode`, the
|
|
39
|
+
`after` edges the walk's order already has, the needs of a serial kind chained,
|
|
40
|
+
needs touching one resource (a Secret, a Terraform state, a database) chained,
|
|
41
|
+
and 4 at a time. A `run` section may only narrow that: change a need's mode, add
|
|
42
|
+
after-edges, lower `parallel`.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import json
|
|
48
|
+
import re
|
|
49
|
+
from collections.abc import Callable, Mapping
|
|
50
|
+
from typing import Any
|
|
51
|
+
|
|
52
|
+
SCHEMA = 1
|
|
53
|
+
MODES = ("check", "fulfil")
|
|
54
|
+
STOPS = ("signal", "shorten", "continue")
|
|
55
|
+
PARALLEL = 4
|
|
56
|
+
SECTION_KEYS = ("schema", "default_mode", "parallel", "needs")
|
|
57
|
+
NEED_KEYS = ("mode", "after")
|
|
58
|
+
FINAL = ("ready", "found", "unknown", "blocked", "failed", "stopped")
|
|
59
|
+
# A transformer per schema change: TRANSFORMERS[n] turns a version n section into n + 1.
|
|
60
|
+
TRANSFORMERS: dict[int, Callable[[dict[str, Any]], dict[str, Any]]] = {}
|
|
61
|
+
UNSAFE = re.compile(r"[^A-Za-z0-9_]")
|
|
62
|
+
# What two needs touch alike, by their params: they are chained, one after the other.
|
|
63
|
+
RESOURCES = (("secret", "name"), ("state", None), ("database", None))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def key_of(need_id: str, taken: set[str]) -> str:
|
|
67
|
+
"""A region's key: the need's id, as a JavaScript and a Python name may hold it, unique."""
|
|
68
|
+
key = UNSAFE.sub("_", need_id)
|
|
69
|
+
out, n = key, 2
|
|
70
|
+
while out in taken:
|
|
71
|
+
out, n = f"{key}_{n}", n + 1
|
|
72
|
+
taken.add(out)
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def fulfils_need(meta: Mapping[str, Any]) -> bool:
|
|
77
|
+
"""Whether a need found missing or drifted goes on to fulfilling: in mode fulfil, and of
|
|
78
|
+
grade auto, the one grade a walk acts on. A propose need's fulfil only gives its proposal
|
|
79
|
+
as the fix and changes nothing; a human or none need is a person's to act on. Those end at
|
|
80
|
+
found, as every need does in mode check."""
|
|
81
|
+
return meta.get("mode") == "fulfil" and meta.get("grade") == "auto"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def region(key: str, meta: dict[str, Any]) -> dict[str, Any]:
|
|
85
|
+
"""One need's states, as the module says."""
|
|
86
|
+
fulfils = fulfils_need(meta)
|
|
87
|
+
stop = {"stop": {"target": "stopped"}}
|
|
88
|
+
found: dict[str, Any] = {"type": "final"}
|
|
89
|
+
if fulfils:
|
|
90
|
+
found = {"on": {f"approved.{key}": {"target": "fulfilling"}, **stop}}
|
|
91
|
+
return {
|
|
92
|
+
"initial": "waiting",
|
|
93
|
+
"meta": meta,
|
|
94
|
+
"states": {
|
|
95
|
+
"waiting": {"on": {f"start.{key}": {"target": "checking"},
|
|
96
|
+
f"predecessor_failed.{key}": {"target": "blocked"}, **stop}},
|
|
97
|
+
"checking": {"on": {f"checked.{key}": [{"target": "ready", "guard": "is_ready"},
|
|
98
|
+
{"target": "found", "guard": "is_found"},
|
|
99
|
+
{"target": "unknown", "guard": "is_unknown"},
|
|
100
|
+
{"target": "blocked"}],
|
|
101
|
+
f"error.{key}": {"target": "failed"}, **stop}},
|
|
102
|
+
"found": found,
|
|
103
|
+
"fulfilling": {"on": {f"fulfilled.{key}": [{"target": "checking", "guard": "is_ok"},
|
|
104
|
+
{"target": "failed"}],
|
|
105
|
+
f"error.{key}": {"target": "failed"}, **stop}},
|
|
106
|
+
"ready": {"type": "final"},
|
|
107
|
+
"unknown": {"type": "final"},
|
|
108
|
+
"blocked": {"type": "final"},
|
|
109
|
+
"failed": {"type": "final"},
|
|
110
|
+
"stopped": {"type": "final"},
|
|
111
|
+
},
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def default_section(needs: list[dict[str, Any]], default_mode: str = "fulfil") -> dict[str, Any]:
|
|
116
|
+
"""The run section a block without one has: every need in default_mode, with its
|
|
117
|
+
after-edges, 4 at a time."""
|
|
118
|
+
return {"schema": SCHEMA, "default_mode": default_mode, "parallel": PARALLEL,
|
|
119
|
+
"needs": {n["id"]: {"mode": default_mode, "after": sorted(n["after"])} for n in needs}}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def generated_after(needs: list[dict[str, Any]]) -> dict[str, set[str]]:
|
|
123
|
+
"""Each need's after-edges: the walk's order, then a serial kind's needs chained, then
|
|
124
|
+
needs touching one resource chained, each chain in id order."""
|
|
125
|
+
after = {n["id"]: set(n["after"]) for n in needs}
|
|
126
|
+
by_id = list(needs) # in the order given: the walk's
|
|
127
|
+
|
|
128
|
+
def chain(group: list[dict[str, Any]]) -> None:
|
|
129
|
+
for earlier, later in zip(group, group[1:]):
|
|
130
|
+
if earlier["id"] not in after[later["id"]] and not reaches(after, earlier["id"], later["id"]):
|
|
131
|
+
after[later["id"]].add(earlier["id"])
|
|
132
|
+
for kind in sorted({n["type"] for n in needs if n.get("serial")}):
|
|
133
|
+
chain([n for n in by_id if n["type"] == kind])
|
|
134
|
+
for param, field in RESOURCES:
|
|
135
|
+
groups: dict[str, list[dict[str, Any]]] = {}
|
|
136
|
+
for n in by_id:
|
|
137
|
+
value = (n.get("params") or {}).get(param)
|
|
138
|
+
value = value.get(field) if field and isinstance(value, dict) else (None if field else value)
|
|
139
|
+
if isinstance(value, str) and value:
|
|
140
|
+
groups.setdefault(value, []).append(n)
|
|
141
|
+
for group in groups.values():
|
|
142
|
+
chain(group)
|
|
143
|
+
return after
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def reaches(after: Mapping[str, set[str]], start: str, goal: str) -> bool:
|
|
147
|
+
"""Whether `start` waits, by after-edges, for `goal`: a chain must not close a cycle."""
|
|
148
|
+
seen, todo = set(), [start]
|
|
149
|
+
while todo:
|
|
150
|
+
i = todo.pop()
|
|
151
|
+
if i == goal:
|
|
152
|
+
return True
|
|
153
|
+
if i not in seen:
|
|
154
|
+
seen.add(i)
|
|
155
|
+
todo += list(after.get(i, ()))
|
|
156
|
+
return False
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def upgrade(section: dict[str, Any]) -> dict[str, Any] | str:
|
|
160
|
+
"""A run section at this schema version, or why it is not one pubdeploy reads."""
|
|
161
|
+
version = section.get("schema")
|
|
162
|
+
if not isinstance(version, int) or isinstance(version, bool) or version < 1:
|
|
163
|
+
return f'deploy.run must give "schema": {SCHEMA}'
|
|
164
|
+
if version > SCHEMA:
|
|
165
|
+
return f"deploy.run's schema {version} is newer than this pubdeploy's ({SCHEMA}): move to a newer pubdeploy"
|
|
166
|
+
while version < SCHEMA:
|
|
167
|
+
section = TRANSFORMERS[version](section)
|
|
168
|
+
version += 1
|
|
169
|
+
return section
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def section_problems(section: Any, needs: list[dict[str, Any]]) -> list[str]:
|
|
173
|
+
"""What is wrong with a run section, against the needs it runs: it may only narrow the
|
|
174
|
+
generated default."""
|
|
175
|
+
if not isinstance(section, dict):
|
|
176
|
+
return ["deploy.run must be an object, such as {\"schema\": 1, \"default_mode\": \"fulfil\"}"]
|
|
177
|
+
out = [f"deploy.run has no setting {k}; it takes {', '.join(SECTION_KEYS)}" for k in sorted(set(section) - set(SECTION_KEYS))]
|
|
178
|
+
upgraded = upgrade(section)
|
|
179
|
+
if isinstance(upgraded, str):
|
|
180
|
+
return [*out, upgraded]
|
|
181
|
+
if upgraded.get("default_mode", "fulfil") not in MODES:
|
|
182
|
+
out.append("deploy.run.default_mode must be check or fulfil")
|
|
183
|
+
parallel = upgraded.get("parallel", PARALLEL)
|
|
184
|
+
if not isinstance(parallel, int) or isinstance(parallel, bool) or not 1 <= parallel <= PARALLEL:
|
|
185
|
+
out.append(f"deploy.run.parallel may only lower the default: 1 to {PARALLEL}, not {parallel!r}")
|
|
186
|
+
known = {n["id"] for n in needs}
|
|
187
|
+
given = upgraded.get("needs", {})
|
|
188
|
+
if not isinstance(given, dict):
|
|
189
|
+
return [*out, "deploy.run.needs must be an object, by need id"]
|
|
190
|
+
for name, entry in sorted(given.items()):
|
|
191
|
+
if name not in known:
|
|
192
|
+
out.append(f"deploy.run.needs names {name}, which is no need of this unit")
|
|
193
|
+
continue
|
|
194
|
+
if not isinstance(entry, dict):
|
|
195
|
+
out.append(f"deploy.run.needs.{name} must be an object with mode and after")
|
|
196
|
+
continue
|
|
197
|
+
out += [f"deploy.run.needs.{name} has no setting {k}; it takes {', '.join(NEED_KEYS)}"
|
|
198
|
+
for k in sorted(set(entry) - set(NEED_KEYS))]
|
|
199
|
+
if "mode" in entry and entry["mode"] not in MODES:
|
|
200
|
+
out.append(f"deploy.run.needs.{name}.mode must be check or fulfil")
|
|
201
|
+
after = entry.get("after", [])
|
|
202
|
+
if not isinstance(after, list) or not all(isinstance(a, str) for a in after):
|
|
203
|
+
out.append(f"deploy.run.needs.{name}.after must list need ids")
|
|
204
|
+
continue
|
|
205
|
+
out += [f"deploy.run.needs.{name} is after {a}, which is no need of this unit" for a in after if a not in known]
|
|
206
|
+
if not out:
|
|
207
|
+
edges = generated_after(needs)
|
|
208
|
+
for name, entry in given.items():
|
|
209
|
+
edges[name] |= set(entry.get("after", []))
|
|
210
|
+
if any(reaches(edges, a, name) for name, entry in given.items() for a in entry.get("after", [])):
|
|
211
|
+
out.append("deploy.run.needs' after-edges make needs wait for each other")
|
|
212
|
+
return out
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def compile(plan: list[dict[str, Any]], run_section: dict[str, Any] | None = None,
|
|
216
|
+
modes: Mapping[str, str] | None = None) -> dict[str, Any]:
|
|
217
|
+
"""The unit's machine: `plan` is each need (id, type, grade, after, seconds, stop,
|
|
218
|
+
stop_warning, serial, params), `run_section` the deploy block's run section or None,
|
|
219
|
+
`modes` a run's own choice of mode per need id, over the section's. ValueError when
|
|
220
|
+
the section or the modes are not ones to run."""
|
|
221
|
+
if bad := section_problems(run_section if run_section is not None else default_section(plan), plan):
|
|
222
|
+
raise ValueError("; ".join(bad))
|
|
223
|
+
section = upgrade(run_section) if run_section is not None else default_section(plan)
|
|
224
|
+
assert isinstance(section, dict) # noqa: S101 - section_problems said so
|
|
225
|
+
default_mode = section.get("default_mode", "fulfil")
|
|
226
|
+
after = generated_after(plan)
|
|
227
|
+
for name, entry in (section.get("needs") or {}).items():
|
|
228
|
+
after[name] |= set(entry.get("after", []))
|
|
229
|
+
unknown = sorted(set(modes or {}) - {n["id"] for n in plan})
|
|
230
|
+
if unknown or any(m not in MODES for m in (modes or {}).values()):
|
|
231
|
+
raise ValueError(f"the modes name {', '.join(unknown) or 'a mode'} that is not check or fulfil of a need here")
|
|
232
|
+
taken: set[str] = set()
|
|
233
|
+
keys = {n["id"]: key_of(n["id"], taken) for n in sorted(plan, key=lambda n: n["id"])}
|
|
234
|
+
regions = {}
|
|
235
|
+
for n in sorted(plan, key=lambda n: n["id"]):
|
|
236
|
+
mode = (modes or {}).get(n["id"]) or (section.get("needs") or {}).get(n["id"], {}).get("mode") or default_mode
|
|
237
|
+
meta = {"need": n["id"], "type": n["type"], "grade": n["grade"], "mode": mode,
|
|
238
|
+
"after": sorted(keys[a] for a in after[n["id"]] if a in keys), "seconds": n.get("seconds", 60.0),
|
|
239
|
+
"stop": n.get("stop", "signal")}
|
|
240
|
+
if n.get("stop_warning"):
|
|
241
|
+
meta["stop_warning"] = n["stop_warning"]
|
|
242
|
+
regions[keys[n["id"]]] = region(keys[n["id"]], meta)
|
|
243
|
+
return {"id": "service-needs", "initial": "running",
|
|
244
|
+
"meta": {"schema": SCHEMA, "parallel": section.get("parallel", PARALLEL)},
|
|
245
|
+
"states": {"running": {"type": "parallel", "states": regions}}}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def targets(on: Any) -> list[str]:
|
|
249
|
+
listed = on if isinstance(on, list) else [on]
|
|
250
|
+
return [t.get("target") for t in listed if isinstance(t, dict)]
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def validate(machine: Any) -> list[str]:
|
|
254
|
+
"""What keeps `machine` from being one this module makes and runs."""
|
|
255
|
+
if not isinstance(machine, dict) or machine.get("initial") != "running":
|
|
256
|
+
return ['the machine must be an object whose initial state is "running"']
|
|
257
|
+
running = (machine.get("states") or {}).get("running") or {}
|
|
258
|
+
if running.get("type") != "parallel" or not isinstance(running.get("states"), dict):
|
|
259
|
+
return ["the machine's running state must be parallel, a region per need"]
|
|
260
|
+
out = []
|
|
261
|
+
keys = set(running["states"])
|
|
262
|
+
for key, part in running["states"].items():
|
|
263
|
+
meta = part.get("meta") or {}
|
|
264
|
+
states = part.get("states") or {}
|
|
265
|
+
if part.get("initial") not in states:
|
|
266
|
+
out.append(f"{key}: its initial state {part.get('initial')!r} is not one of its states")
|
|
267
|
+
if meta.get("mode") not in MODES:
|
|
268
|
+
out.append(f"{key}: meta.mode must be check or fulfil")
|
|
269
|
+
if meta.get("stop", "signal") not in STOPS:
|
|
270
|
+
out.append(f"{key}: meta.stop must be one of {', '.join(STOPS)}")
|
|
271
|
+
if meta.get("stop") == "continue" and not meta.get("stop_warning"):
|
|
272
|
+
out.append(f"{key}: meta.stop continue needs a stop_warning")
|
|
273
|
+
out += [f"{key}: it is after {a}, which is no region" for a in meta.get("after", []) if a not in keys]
|
|
274
|
+
for name, state in states.items():
|
|
275
|
+
for event, on in (state.get("on") or {}).items():
|
|
276
|
+
out += [f"{key}.{name}: {event} goes to {t}, which is no state of {key}" for t in targets(on)
|
|
277
|
+
if t not in states]
|
|
278
|
+
if not fulfils_need(meta) and "fulfilling" in {t for s in states.values()
|
|
279
|
+
for on in (s.get("on") or {}).values()
|
|
280
|
+
for t in targets(on)}:
|
|
281
|
+
out.append(f"{key}: only an auto need in mode fulfil enters fulfilling")
|
|
282
|
+
after = {k: set((p.get("meta") or {}).get("after", [])) for k, p in running["states"].items()}
|
|
283
|
+
if not out and any(reaches(after, a, k) for k, v in after.items() for a in v):
|
|
284
|
+
out.append("the regions' after-edges make needs wait for each other")
|
|
285
|
+
return out
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def problems(deploy: dict[str, Any], needs: list[dict[str, Any]]) -> list[str]:
|
|
289
|
+
"""The deploy block's run section, as `pubrules sync --check` checks it: none when it
|
|
290
|
+
has none."""
|
|
291
|
+
if "run" not in deploy:
|
|
292
|
+
return []
|
|
293
|
+
return section_problems(deploy["run"], needs)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class Guards:
|
|
297
|
+
"""What the machine's guards read of an event, for `load()`. python-statemachine passes
|
|
298
|
+
its own `state` to a guard, so `send()` hands the event's state over as `need_state`."""
|
|
299
|
+
|
|
300
|
+
def is_ready(self, need_state: str | None = None, **_: Any) -> bool:
|
|
301
|
+
return need_state == "ready"
|
|
302
|
+
|
|
303
|
+
def is_found(self, need_state: str | None = None, **_: Any) -> bool:
|
|
304
|
+
return need_state in ("missing", "drift")
|
|
305
|
+
|
|
306
|
+
def is_unknown(self, need_state: str | None = None, **_: Any) -> bool:
|
|
307
|
+
return need_state == "unknown"
|
|
308
|
+
|
|
309
|
+
def is_ok(self, ok: bool | None = None, **_: Any) -> bool:
|
|
310
|
+
return ok is True
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def send(chart: Any, event: Mapping[str, Any]) -> None:
|
|
314
|
+
"""Send an event, as XState takes it ({"type": "checked.KEY", "state": "missing"}), to a
|
|
315
|
+
chart `load()` made."""
|
|
316
|
+
data = {("need_state" if k == "state" else k): v for k, v in event.items() if k != "type"}
|
|
317
|
+
chart.send(event["type"], **data)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def native(machine: dict[str, Any]) -> dict[str, Any]:
|
|
321
|
+
"""The machine in python-statemachine's own JSON form: each state's id made unique by
|
|
322
|
+
its region's key, each guard a condition on the bound model."""
|
|
323
|
+
running = machine["states"]["running"]
|
|
324
|
+
regions = {}
|
|
325
|
+
for key, part in running["states"].items():
|
|
326
|
+
states = {}
|
|
327
|
+
for name, state in part["states"].items():
|
|
328
|
+
out: dict[str, Any] = {}
|
|
329
|
+
if name == part["initial"]:
|
|
330
|
+
out["initial"] = True
|
|
331
|
+
if state.get("type") == "final":
|
|
332
|
+
out["final"] = True
|
|
333
|
+
transitions = []
|
|
334
|
+
for event, on in (state.get("on") or {}).items():
|
|
335
|
+
for t in on if isinstance(on, list) else [on]:
|
|
336
|
+
move = {"event": event, "target": f"{key}__{t['target']}"}
|
|
337
|
+
if t.get("guard"):
|
|
338
|
+
move["cond"] = t["guard"]
|
|
339
|
+
transitions.append(move)
|
|
340
|
+
if transitions:
|
|
341
|
+
out["transitions"] = transitions
|
|
342
|
+
states[f"{key}__{name}"] = out
|
|
343
|
+
regions[key] = {"states": states}
|
|
344
|
+
return {"name": "ServiceNeeds", "states": {"running": {"initial": True, "parallel": True, "states": regions}}}
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def load(machine: dict[str, Any]) -> Any:
|
|
348
|
+
"""The machine as a python-statemachine StateChart, bound to the guards: give it events
|
|
349
|
+
with `send(chart, {"type": "checked.KEY", "state": "missing"})`, and read `states(chart)`."""
|
|
350
|
+
from statemachine.io import loader # noqa: PLC0415
|
|
351
|
+
if bad := validate(machine):
|
|
352
|
+
raise ValueError("; ".join(bad))
|
|
353
|
+
return loader.load(json.dumps(native(machine)), format="json")(model=Guards())
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def states(chart: Any) -> dict[str, str]:
|
|
357
|
+
"""Each region's state, by key, as the chart holds them."""
|
|
358
|
+
out = {}
|
|
359
|
+
for state in chart.configuration:
|
|
360
|
+
key, sep, name = str(state.id).rpartition("__")
|
|
361
|
+
if sep:
|
|
362
|
+
out[key] = name
|
|
363
|
+
return dict(sorted(out.items()))
|
pubdeploy/model.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""What a deploy needs, what a check finds, and what a check may touch.
|
|
2
|
+
|
|
3
|
+
A Need is one thing a service needs before it runs: a namespace, a Secret,
|
|
4
|
+
a GitHub variable, an upstream that answers. A NeedType knows how to check
|
|
5
|
+
one kind; its check returns a Result, in one of five states, and its fulfil
|
|
6
|
+
makes what is missing. A Context holds every way a check or a fulfil reaches
|
|
7
|
+
the world (kubectl, a command on the server, the GitHub API, gh's setters,
|
|
8
|
+
gcloud, Bitwarden's writer, HTTP, the environment, the clock, the rendered
|
|
9
|
+
manifests), so a test can give it fakes and no type reaches anything else.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import time
|
|
16
|
+
from collections.abc import Callable, Mapping
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Literal, Protocol, runtime_checkable
|
|
21
|
+
|
|
22
|
+
State = Literal["ready", "missing", "drift", "blocked", "unknown"]
|
|
23
|
+
Grade = Literal["auto", "propose", "human", "none"]
|
|
24
|
+
STATES: tuple[State, ...] = ("ready", "missing", "drift", "blocked", "unknown")
|
|
25
|
+
GRADES: tuple[Grade, ...] = ("auto", "propose", "human", "none")
|
|
26
|
+
SUMMARY = 120 # the longest summary, in characters
|
|
27
|
+
FULFIL_BUDGET = 60.0 # seconds, for a type that declares none
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Stop(Exception):
|
|
31
|
+
"""A step a check depends on could not be taken, such as rendering the
|
|
32
|
+
overlay: its message says why, and never holds a value. `missing_tool`
|
|
33
|
+
when a program is not installed, rather than failing."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, message: str, missing_tool: bool = False) -> None:
|
|
36
|
+
super().__init__(message)
|
|
37
|
+
self.missing_tool = missing_tool
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Need:
|
|
42
|
+
type: str
|
|
43
|
+
name: str
|
|
44
|
+
params: dict[str, Any] # the need as written, validated; never mutated
|
|
45
|
+
implied_by: str | None = None # "namespace", "host", "secrets", "manifests", "secret of <id>"
|
|
46
|
+
after: tuple[str, ...] = () # explicit ids
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def id(self) -> str:
|
|
50
|
+
return f"{self.type}:{self.name}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class Result:
|
|
55
|
+
state: State
|
|
56
|
+
summary: str # one line, no value
|
|
57
|
+
fix: str = "" # what a person does when it is not ready: a command or a place, never a value
|
|
58
|
+
details: list[str] = field(default_factory=list)
|
|
59
|
+
checked_at: str = "" # ISO 8601 UTC, set by the walk
|
|
60
|
+
seconds: float = 0.0 # set by the walk
|
|
61
|
+
fulfilled: bool = False # set by the walk: this is the check after a fulfil
|
|
62
|
+
# Neither is journaled. False when the type's fulfil cannot make what the
|
|
63
|
+
# check found, such as an account only Terraform makes.
|
|
64
|
+
fixable: bool = True
|
|
65
|
+
step: str = "" # which of its steps the type's fulfil takes, for a type with several
|
|
66
|
+
|
|
67
|
+
def __post_init__(self) -> None:
|
|
68
|
+
line = " ".join(self.summary.split())
|
|
69
|
+
self.summary = line if len(line) <= SUMMARY else line[:SUMMARY - 1] + "…"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class Ran:
|
|
74
|
+
"""A command's exit code and output, as subprocess.run gives them."""
|
|
75
|
+
returncode: int
|
|
76
|
+
stdout: str = ""
|
|
77
|
+
stderr: str = ""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class Kubectl(Protocol):
|
|
81
|
+
"""kubectl with `args`, `input` on its standard input: a manifest, never in argv."""
|
|
82
|
+
|
|
83
|
+
def __call__(self, args: list[str], input: bytes | None = None) -> Ran: ... # noqa: A002
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
Server = Callable[[list[str], "bytes | None"], Ran] # a command on the server, and its standard input
|
|
87
|
+
Api = Callable[[str, str, Any], Any]
|
|
88
|
+
HttpGet = Callable[[str, float], tuple[int, bytes]]
|
|
89
|
+
Resolve = Callable[[str, str], "list[str] | None"]
|
|
90
|
+
Send = Callable[[str, str, str, str, "str | None"], "str | None"] # app_secrets.send
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class Writer(Protocol):
|
|
94
|
+
"""Bitwarden, with a write token the writer keeps: the token is never the type's."""
|
|
95
|
+
|
|
96
|
+
def write(self, name: str, value: str, note: str) -> None:
|
|
97
|
+
"""Create or update the secret `name` in the walk's project, or raise; the error holds no value."""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def no_kubectl(_args: list[str], input: bytes | None = None) -> Ran: # noqa: A002
|
|
101
|
+
return Ran(127, "", "kubectl: not given to this check")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def no_server(_args: list[str], _input: bytes | None = None) -> Ran:
|
|
105
|
+
return Ran(127, "", "no command runs on the server: not the admin channel")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def no_send(kind: str, name: str, _repo: str, _value: str, _environment: str | None = None) -> str | None:
|
|
109
|
+
return f"gh {kind} set {name}: not given to this walk"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def no_gh(method: str, path: str, _body: Any = None) -> Any:
|
|
113
|
+
from .github import ApiError # noqa: PLC0415
|
|
114
|
+
raise ApiError(f"gh api -X {method} {path}: not given to this check")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def no_http(url: str, _timeout: float) -> tuple[int, bytes]:
|
|
118
|
+
raise ConnectionRefusedError(f"no HTTP in this check: {url}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def no_resolve(_host: str, _rtype: str) -> list[str] | None:
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def no_render() -> str:
|
|
126
|
+
raise Stop("the overlay was not rendered", missing_tool=True)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass
|
|
130
|
+
class Context:
|
|
131
|
+
"""Everything a check touches."""
|
|
132
|
+
root: Path
|
|
133
|
+
config: dict[str, Any]
|
|
134
|
+
block: dict[str, Any]
|
|
135
|
+
namespace: str
|
|
136
|
+
kubectl: Kubectl = no_kubectl
|
|
137
|
+
server: Server = no_server
|
|
138
|
+
# How kubectl reaches the cluster: admin (over the server's root SSH),
|
|
139
|
+
# deploy (the server's read-only verb), plain (a kubeconfig) or plane.
|
|
140
|
+
channel: str = "plain"
|
|
141
|
+
gh: Api = no_gh
|
|
142
|
+
send: Send = no_send
|
|
143
|
+
bitwarden_writer: Writer | None = None # None without a write token
|
|
144
|
+
http_get: HttpGet = no_http
|
|
145
|
+
env: Mapping[str, str] = field(default_factory=dict)
|
|
146
|
+
now: Callable[[], datetime] = lambda: datetime.now(UTC)
|
|
147
|
+
rendered: Callable[[], str] = no_render
|
|
148
|
+
offline: bool = False
|
|
149
|
+
slug: str = "" # OWNER/NAME of the repository on GitHub, or "" when origin is not one
|
|
150
|
+
zones: dict[str, Any] | None = None # the hosting block's zones, from config or --zones
|
|
151
|
+
resolve: Resolve = no_resolve # a host's records of a type; None without dig
|
|
152
|
+
unit: str = "" # the unit walked, by name
|
|
153
|
+
sleep: Callable[[float], None] = time.sleep # between polls, in a fulfil that waits
|
|
154
|
+
# What a fulfil made in this walk, by Bitwarden name: never journaled, printed or kept.
|
|
155
|
+
values: dict[str, bytes] = field(default_factory=dict)
|
|
156
|
+
dispatched: dict[str, datetime] = field(default_factory=dict) # by workflow, when this walk ran it
|
|
157
|
+
# The most a check or a fulfil may take, below its type's budget: what is left of a
|
|
158
|
+
# run's time (walk --budget-seconds, a Stop that shortens), or None for the type's.
|
|
159
|
+
budget_cap: float | None = None
|
|
160
|
+
# Filled in by the walk: the types, every need in the plan, and the results so far.
|
|
161
|
+
types: dict[str, Any] = field(default_factory=dict)
|
|
162
|
+
needs: dict[str, Need] = field(default_factory=dict)
|
|
163
|
+
results: dict[str, Result] = field(default_factory=dict)
|
|
164
|
+
|
|
165
|
+
@classmethod
|
|
166
|
+
def for_tests(cls, **overrides: Any) -> Context:
|
|
167
|
+
"""A context that reaches nothing: each tool refuses until a test
|
|
168
|
+
gives a fake for it."""
|
|
169
|
+
block = overrides.pop("block", {"namespace": "example"})
|
|
170
|
+
given = {"root": Path("."), "config": {"kind": "service", "deploy": block}, "block": block,
|
|
171
|
+
"namespace": block.get("namespace", "example")}
|
|
172
|
+
return cls(**(given | overrides))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@runtime_checkable
|
|
176
|
+
class NeedType(Protocol):
|
|
177
|
+
type: str # "k8s.secret"
|
|
178
|
+
schema: dict[str, Any] # JSON schema of the need's params, additionalProperties false
|
|
179
|
+
grade: Grade # what fulfil would be; "none" when only a person can act
|
|
180
|
+
budget: float # seconds a check may take; the walk enforces it
|
|
181
|
+
offline: bool # True when check() runs under --offline (touching no network then)
|
|
182
|
+
# And optionally: fulfil_budget (seconds a fulfil may take, at most 900,
|
|
183
|
+
# default 60); grade_for(need), a need's own grade; requires(need), what
|
|
184
|
+
# its fulfil needs of the context ("server", "kubectl-write",
|
|
185
|
+
# "write-token"); action(need, ctx), the plan's line; value(need, ctx),
|
|
186
|
+
# the bytes a need with a `secret` puts in it; secret_keys(params), the
|
|
187
|
+
# keys it fills when its secret names none.
|
|
188
|
+
|
|
189
|
+
def default_name(self, params: dict[str, Any]) -> str | None: ...
|
|
190
|
+
|
|
191
|
+
def depends_on(self, need: Need, needs: Mapping[str, Need]) -> list[str]: ...
|
|
192
|
+
|
|
193
|
+
def check(self, need: Need, ctx: Context) -> Result: ...
|
|
194
|
+
|
|
195
|
+
def fulfil(self, need: Need, ctx: Context) -> Result:
|
|
196
|
+
"""Make what check found missing or drifted, and say what was done in
|
|
197
|
+
`summary`, state ready. It raises nothing: a failure is a Result with
|
|
198
|
+
state missing or drift and the cause in fix, or blocked when what stops it
|
|
199
|
+
is someone else's to change (a grant, say), which the walk then records
|
|
200
|
+
as the need's state. A propose or human type's changes nothing, and gives
|
|
201
|
+
the change to make as fix."""
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
SCHEMA = "deploy-journal/2"
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@dataclass
|
|
208
|
+
class Journal:
|
|
209
|
+
"""One walk: who ran it, where, and each need's result, in walk order."""
|
|
210
|
+
service: dict[str, str]
|
|
211
|
+
walk: dict[str, Any]
|
|
212
|
+
entries: list[tuple[Need, Result]] = field(default_factory=list)
|
|
213
|
+
grades: dict[str, str] = field(default_factory=dict) # by need id
|
|
214
|
+
|
|
215
|
+
def counts(self) -> dict[str, int]:
|
|
216
|
+
return {s: sum(r.state == s for _, r in self.entries) for s in STATES}
|
|
217
|
+
|
|
218
|
+
def document(self) -> dict[str, Any]:
|
|
219
|
+
"""The deploy-journal/2 document: no need's params, so no value."""
|
|
220
|
+
return {"schema": SCHEMA, "service": self.service, "walk": self.walk,
|
|
221
|
+
"needs": [{"id": n.id, "type": n.type, "name": n.name, "implied_by": n.implied_by,
|
|
222
|
+
"after": list(n.after), "grade": self.grades.get(n.id, "none"), "state": r.state,
|
|
223
|
+
"fulfilled": r.fulfilled, "summary": r.summary, "fix": r.fix, "details": r.details,
|
|
224
|
+
"seconds": round(r.seconds, 3), "checked_at": r.checked_at}
|
|
225
|
+
for n, r in self.entries],
|
|
226
|
+
"summary": self.counts()}
|
|
227
|
+
|
|
228
|
+
def json(self) -> str:
|
|
229
|
+
return json.dumps(self.document(), indent=2)
|
pubdeploy/names.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""The DNS names and record types the dns.site check may ask a resolver
|
|
2
|
+
about: absolute, lower case, labels of letters, digits, hyphens and
|
|
3
|
+
underscores; types in capitals. Anything else stops the check. After
|
|
4
|
+
pubrules's dns.NAME, checked and checked_type."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
NAME = re.compile(r"(?:[a-z0-9_][a-z0-9_-]{0,62}\.)+")
|
|
12
|
+
TYPE = re.compile(r"[A-Z][A-Z0-9]{0,9}")
|
|
13
|
+
LONGEST = 254 # an absolute name's length, its final dot included
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def checked(name: str) -> str:
|
|
17
|
+
if not NAME.fullmatch(name) or len(name) > LONGEST:
|
|
18
|
+
sys.exit(f"pubdeploy: {name!r} is not a DNS name the deploy walk asks about")
|
|
19
|
+
return name
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def checked_type(rtype: str) -> str:
|
|
23
|
+
if not TYPE.fullmatch(rtype):
|
|
24
|
+
sys.exit(f"pubdeploy: {rtype!r} is not a DNS record type, such as A or MX")
|
|
25
|
+
return rtype
|