pubdeploy 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubdeploy/__init__.py +532 -0
- pubdeploy/channels.py +188 -0
- pubdeploy/cli.py +120 -0
- pubdeploy/config.py +140 -0
- pubdeploy/contract.py +87 -0
- pubdeploy/github.py +61 -0
- pubdeploy/hooks.py +82 -0
- pubdeploy/implied.py +179 -0
- pubdeploy/journal.json +207 -0
- pubdeploy/k8s.py +472 -0
- pubdeploy/machine.py +363 -0
- pubdeploy/model.py +229 -0
- pubdeploy/names.py +25 -0
- pubdeploy/order.py +101 -0
- pubdeploy/providers.py +75 -0
- pubdeploy/registry.py +96 -0
- pubdeploy/release-commit +1 -0
- pubdeploy/schema.json +127 -0
- pubdeploy/schema.py +238 -0
- pubdeploy/store.py +127 -0
- pubdeploy/testing/__init__.py +94 -0
- pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
- pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
- pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
- pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
- pubdeploy/testing/plugin.py +26 -0
- pubdeploy/testing/steps.py +1124 -0
- pubdeploy/testing/world.py +569 -0
- pubdeploy/types/__init__.py +181 -0
- pubdeploy/types/auth.py +199 -0
- pubdeploy/types/github.py +110 -0
- pubdeploy/types/k8s.py +609 -0
- pubdeploy/types/net.py +237 -0
- pubdeploy/types/rollout.py +186 -0
- pubdeploy/units.py +177 -0
- pubdeploy/walk.py +325 -0
- pubdeploy-0.8.0.dist-info/METADATA +520 -0
- pubdeploy-0.8.0.dist-info/RECORD +45 -0
- pubdeploy-0.8.0.dist-info/WHEEL +5 -0
- pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
- pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
- pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/__init__.py
ADDED
|
@@ -0,0 +1,532 @@
|
|
|
1
|
+
"""`pubdeploy walk`, `pubdeploy check` and `pubdeploy types` (cli.py):
|
|
2
|
+
what a service needs before it deploys, whether it has it, and making it.
|
|
3
|
+
|
|
4
|
+
pubdeploy walk [--fulfil] [--yes] [--unit NAME] [--kubectl CMD] [--server HOST]
|
|
5
|
+
[--zones PATH]... [--journal URL] [--json] [--offline] [--namespace NS]
|
|
6
|
+
pubdeploy check ... walk without --fulfil
|
|
7
|
+
pubdeploy types
|
|
8
|
+
|
|
9
|
+
pubrules.json's `deploy` block names the namespace, and maybe the host, the
|
|
10
|
+
deploy workflow, the gate variable, the overlay and needs nothing implies.
|
|
11
|
+
The block, the secrets, github_app and variables blocks, and the Secrets the
|
|
12
|
+
rendered overlay reads imply the rest (implied.py). `check` walks every need
|
|
13
|
+
in order (order.py, walk.py) and prints its state: ready, missing, drift,
|
|
14
|
+
blocked or unknown. It reads, and changes nothing. `--offline` checks only
|
|
15
|
+
what needs no network: the rendered manifests against the needs, the
|
|
16
|
+
manifest files, and the host against the hosting block. `pubrules sync
|
|
17
|
+
--check` runs that too, so a block that disagrees with the manifests fails CI.
|
|
18
|
+
|
|
19
|
+
`walk --fulfil` then acts (walk.py): it prints the plan, a line per need
|
|
20
|
+
it will act on, waits for a typed `yes` (--yes skips that), asks for a
|
|
21
|
+
Bitwarden write token when a need wants one (the sustenaut.walk hook,
|
|
22
|
+
pubsec's, asks for it at a hidden prompt), and visits
|
|
23
|
+
the needs again, making each auto one that is missing. A walk from GitHub
|
|
24
|
+
Actions, or with --yes and no terminal, asks for no token, so the needs that
|
|
25
|
+
want one are unknown, unless PUBSEC_WRITE_TOKEN_FILE names a file the hook
|
|
26
|
+
reads the token from (the convergence engine's Fulfil). The channels kubectl and the server are reached by are
|
|
27
|
+
channels.py's; a block's units, units.py's: each is walked, and journaled,
|
|
28
|
+
on its own.
|
|
29
|
+
|
|
30
|
+
Exit 0 when every need checked is ready, 1 otherwise, 2 for a bad block;
|
|
31
|
+
with --tolerate-unknown, 0 unless one is missing, drift or blocked, for a
|
|
32
|
+
scheduled walk whose token may not read everything (the unknowns stay in
|
|
33
|
+
the journal).
|
|
34
|
+
`--json` prints the walk as a deploy-journal/2 document instead (journal.json),
|
|
35
|
+
a list of them for more than one unit; `--journal URL` also writes it to
|
|
36
|
+
storage (store.py). No value is printed: a check names a secret, its key,
|
|
37
|
+
its length and its kind, never what it holds.
|
|
38
|
+
|
|
39
|
+
load(root) -> Plan the needs of a unit, in walk order
|
|
40
|
+
check(plan, ctx) -> Journal the walk
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
import json
|
|
46
|
+
import os
|
|
47
|
+
import shutil
|
|
48
|
+
import signal
|
|
49
|
+
import subprocess
|
|
50
|
+
import sys
|
|
51
|
+
import threading
|
|
52
|
+
import urllib.error
|
|
53
|
+
import urllib.request
|
|
54
|
+
from dataclasses import dataclass
|
|
55
|
+
from datetime import UTC, datetime
|
|
56
|
+
from functools import cache
|
|
57
|
+
from pathlib import Path
|
|
58
|
+
from typing import Any
|
|
59
|
+
|
|
60
|
+
import yaml
|
|
61
|
+
|
|
62
|
+
from .model import Context, Journal, Need, NeedType, Result, Stop
|
|
63
|
+
from .schema import BLOCK, DEFAULT_OVERLAY
|
|
64
|
+
|
|
65
|
+
__all__ = ["Context", "Journal", "Need", "NeedType", "Plan", "Result", "__version__", "check", "load"]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _version() -> str:
|
|
69
|
+
from importlib.metadata import PackageNotFoundError, version # noqa: PLC0415
|
|
70
|
+
try:
|
|
71
|
+
return version("pubdeploy")
|
|
72
|
+
except PackageNotFoundError:
|
|
73
|
+
return "0+source"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
__version__ = _version()
|
|
77
|
+
RENDER_TIMEOUT = 60
|
|
78
|
+
TOOL_TIMEOUT = 30
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class Plan:
|
|
83
|
+
root: Path
|
|
84
|
+
config: dict[str, Any]
|
|
85
|
+
namespace: str
|
|
86
|
+
needs: dict[str, Need]
|
|
87
|
+
order: list[str]
|
|
88
|
+
problems: list[str]
|
|
89
|
+
unit: str = ""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def render(root: Path, overlay: str) -> str:
|
|
93
|
+
"""The overlay as kustomize builds it: kustomize, or kubectl's own."""
|
|
94
|
+
where = root / overlay
|
|
95
|
+
if shutil.which("kustomize"):
|
|
96
|
+
argv = ["kustomize", "build", str(where)]
|
|
97
|
+
elif shutil.which("kubectl"):
|
|
98
|
+
argv = ["kubectl", "kustomize", str(where)]
|
|
99
|
+
else:
|
|
100
|
+
raise Stop(f"neither kustomize nor kubectl is installed, to render {overlay}", missing_tool=True)
|
|
101
|
+
try:
|
|
102
|
+
done = subprocess.run(argv, capture_output=True, text=True, timeout=RENDER_TIMEOUT, check=False)
|
|
103
|
+
except (OSError, subprocess.TimeoutExpired) as e:
|
|
104
|
+
raise Stop(f"{argv[0]} did not render {overlay} ({type(e).__name__})", missing_tool=True) from None
|
|
105
|
+
if done.returncode != 0:
|
|
106
|
+
last = (done.stderr.strip().splitlines() or [f"exit {done.returncode}"])[-1]
|
|
107
|
+
raise Stop(f"{' '.join(argv[:2])} {overlay} failed: {last[:300]}")
|
|
108
|
+
return done.stdout
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def renderer(root: Path, overlay: str) -> Any:
|
|
112
|
+
"""render, once."""
|
|
113
|
+
return cache(lambda: render(root, overlay))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def kubectl(command: str) -> Any:
|
|
117
|
+
from .channels import plain # noqa: PLC0415
|
|
118
|
+
return plain(command)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def http_get(url: str, timeout: float) -> tuple[int, bytes]:
|
|
122
|
+
request = urllib.request.Request(url, headers={"User-Agent": "pubdeploy"})
|
|
123
|
+
try:
|
|
124
|
+
with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 - an https:// need
|
|
125
|
+
return response.status, response.read(1 << 16)
|
|
126
|
+
except urllib.error.HTTPError as e:
|
|
127
|
+
return e.code, b""
|
|
128
|
+
except urllib.error.URLError as e:
|
|
129
|
+
if isinstance(e.reason, TimeoutError):
|
|
130
|
+
raise TimeoutError from None
|
|
131
|
+
raise ConnectionError(type(e.reason).__name__) from None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def resolve(host: str, rtype: str) -> list[str] | None:
|
|
135
|
+
from .names import NAME, checked, checked_type # noqa: PLC0415
|
|
136
|
+
if not shutil.which("dig"):
|
|
137
|
+
return None
|
|
138
|
+
done = subprocess.run(["dig", "+short", "-q", checked(f"{host}."), "-t", checked_type(rtype)],
|
|
139
|
+
capture_output=True, text=True, timeout=TOOL_TIMEOUT, check=False)
|
|
140
|
+
return [line for line in done.stdout.split() if line and not NAME.fullmatch(line)]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def git(root: Path, *args: str) -> str:
|
|
144
|
+
try:
|
|
145
|
+
done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, check=False)
|
|
146
|
+
except OSError:
|
|
147
|
+
return ""
|
|
148
|
+
return done.stdout.strip() if done.returncode == 0 else ""
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def slug(root: Path) -> str:
|
|
152
|
+
"""OWNER/NAME of the GitHub repository origin points at, or ""."""
|
|
153
|
+
import re # noqa: PLC0415
|
|
154
|
+
m = re.search(r"github\.com[:/]([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?)(?:\.git)?/?$", git(root, "remote", "get-url", "origin"))
|
|
155
|
+
return m.group(1) if m else ""
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def zones_of(config: dict[str, Any], zones_file: Path | list[Path] | None) -> dict[str, Any] | None:
|
|
159
|
+
"""The hosting block's zones: this repository's, and each --zones's (one per owner
|
|
160
|
+
whose zones the units' hosts are in)."""
|
|
161
|
+
out: dict[str, Any] = {}
|
|
162
|
+
found = False
|
|
163
|
+
files = [] if zones_file is None else [zones_file] if isinstance(zones_file, Path) else list(zones_file)
|
|
164
|
+
for given in [config] + [json.loads(f.read_text(encoding="utf-8")) for f in files]:
|
|
165
|
+
hosting = given.get("hosting") if isinstance(given, dict) else None
|
|
166
|
+
if isinstance(hosting, dict) and isinstance(hosting.get("zones"), dict):
|
|
167
|
+
out |= hosting["zones"]
|
|
168
|
+
found = True
|
|
169
|
+
return out if found else None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def load(root: Path, config: dict[str, Any] | None = None, ctx: Context | None = None,
|
|
173
|
+
namespace: str | None = None, types: dict[str, NeedType] | None = None, unit: str | None = None) -> Plan:
|
|
174
|
+
"""The needs of a unit of `root`'s deploy block (the top level's by
|
|
175
|
+
default), with those the rendered manifests imply when they render, in
|
|
176
|
+
walk order; `problems` when the block is not sound."""
|
|
177
|
+
from . import k8s # noqa: PLC0415
|
|
178
|
+
from .implied import build # noqa: PLC0415
|
|
179
|
+
from .order import cycle_problems, topological # noqa: PLC0415
|
|
180
|
+
from .registry import types as registered # noqa: PLC0415
|
|
181
|
+
from .schema import problems # noqa: PLC0415
|
|
182
|
+
if config is None:
|
|
183
|
+
from .config import read_config # noqa: PLC0415
|
|
184
|
+
config = read_config(root)
|
|
185
|
+
types = types or registered()
|
|
186
|
+
if BLOCK not in config:
|
|
187
|
+
return Plan(root, config, "", {}, [], [f"{BLOCK}: pubrules.json has no deploy block"])
|
|
188
|
+
from .units import unit_block # noqa: PLC0415
|
|
189
|
+
if bad := problems(config, root, types):
|
|
190
|
+
return Plan(root, config, "", {}, [], bad)
|
|
191
|
+
unit = unit or config[BLOCK]["namespace"]
|
|
192
|
+
block = unit_block(config[BLOCK], unit)
|
|
193
|
+
ns = namespace or block["namespace"]
|
|
194
|
+
rendered = ctx.rendered if ctx else renderer(root, block.get("overlay", DEFAULT_OVERLAY))
|
|
195
|
+
try:
|
|
196
|
+
found = k8s.collect(k8s.documents(rendered()), ns)
|
|
197
|
+
except (Stop, ValueError, yaml.YAMLError):
|
|
198
|
+
found = None # k8s.secrets-declared says why
|
|
199
|
+
needs, bad = build(config, types, root=root, namespace=ns, found=found, unit=unit)
|
|
200
|
+
bad = bad or cycle_problems(needs, types)
|
|
201
|
+
return Plan(root, config, ns, needs, [] if bad else topological(needs, types), bad, unit)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def check(plan: Plan, ctx: Context, types: dict[str, NeedType] | None = None) -> Journal:
|
|
205
|
+
"""Walk the plan's needs."""
|
|
206
|
+
from .registry import types as registered # noqa: PLC0415
|
|
207
|
+
from .walk import walk # noqa: PLC0415
|
|
208
|
+
from . import __version__ # noqa: PLC0415
|
|
209
|
+
service = {"repository": ctx.slug, "namespace": plan.namespace, "unit": plan.unit or plan.namespace,
|
|
210
|
+
"commit": git(plan.root, "rev-parse", "HEAD"), "pubdeploy": __version__}
|
|
211
|
+
# PUBRULES_WALK_BY: who a host that walks for others (the convergence engine) says asked.
|
|
212
|
+
by = ctx.env.get("PUBRULES_WALK_BY") or ctx.env.get("GITHUB_ACTOR") or git(plan.root, "config", "user.email")
|
|
213
|
+
origin = "actions" if ctx.env.get("GITHUB_ACTIONS") == "true" else (
|
|
214
|
+
"plane" if ctx.env.get("PUBRULES_PLANE") else "laptop")
|
|
215
|
+
return walk(plan.needs, types or registered(), ctx, service, by=by, origin=origin)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def context(root: Path, config: dict[str, Any], offline: bool, kubectl_command: str | None = None,
|
|
219
|
+
zones_file: Path | list[Path] | None = None, namespace: str | None = None, server: str | None = None,
|
|
220
|
+
unit: str | None = None) -> Context:
|
|
221
|
+
"""The context `pubdeploy walk` runs a unit in: the real tools."""
|
|
222
|
+
from .github import gh, gh_set as send # noqa: PLC0415
|
|
223
|
+
from .channels import channels # noqa: PLC0415
|
|
224
|
+
from .units import unit_block # noqa: PLC0415
|
|
225
|
+
unit = unit or config[BLOCK]["namespace"]
|
|
226
|
+
block = unit_block(config[BLOCK], unit)
|
|
227
|
+
ns = namespace or block["namespace"]
|
|
228
|
+
reach = channels(kubectl_command, server, os.environ, ns)
|
|
229
|
+
if reach.connection is not None and not offline:
|
|
230
|
+
reach.connection.open() # a failure shows in the first check that uses it
|
|
231
|
+
return Context(root=root, config=config, block=block, namespace=ns, kubectl=reach.kubectl,
|
|
232
|
+
server=reach.server, channel=reach.name, gh=gh, send=send, http_get=http_get, env=os.environ,
|
|
233
|
+
now=lambda: datetime.now(UTC), rendered=renderer(root, block.get("overlay", DEFAULT_OVERLAY)),
|
|
234
|
+
offline=offline, slug=slug(root), zones=zones_of(config, zones_file), resolve=resolve, unit=unit)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def offline_lines(root: Path, config: dict[str, Any], ctx: Context | None = None) -> tuple[list[str], bool]:
|
|
238
|
+
"""For `pubrules sync`: what is wrong with the deploy block, or the offline
|
|
239
|
+
walk's needs that are not ready, and whether that fails --check. An
|
|
240
|
+
offline need that could not be checked (no kustomize, no hosting block)
|
|
241
|
+
is said, and fails nothing."""
|
|
242
|
+
from .registry import load as registry # noqa: PLC0415
|
|
243
|
+
loaded = registry()
|
|
244
|
+
lines = [f"pubdeploy: warning: {w}" for w in loaded.warnings]
|
|
245
|
+
from .units import names, problems as unit_problems # noqa: PLC0415
|
|
246
|
+
plan = load(root, config, ctx or context(root, config, offline=True), types=loaded.types)
|
|
247
|
+
if plan.problems:
|
|
248
|
+
return lines + [f"pubdeploy: {p}" for p in plan.problems], True
|
|
249
|
+
bad = unit_problems(root, config[BLOCK])
|
|
250
|
+
lines += [f"pubdeploy: {p}" for p in bad]
|
|
251
|
+
failed = bool(bad)
|
|
252
|
+
entries = []
|
|
253
|
+
for unit in names(config[BLOCK]):
|
|
254
|
+
given = ctx or context(root, config, offline=True, unit=unit)
|
|
255
|
+
plan = load(root, config, given, types=loaded.types, unit=unit)
|
|
256
|
+
if plan.problems:
|
|
257
|
+
return lines + [f"pubdeploy: {p}" for p in plan.problems], True
|
|
258
|
+
from . import machine # noqa: PLC0415
|
|
259
|
+
run = machine.problems(unit_block(config, unit), machine_needs(plan, loaded.types))
|
|
260
|
+
lines += [f"pubdeploy: {p}" for p in run]
|
|
261
|
+
failed = failed or bool(run)
|
|
262
|
+
entries += check(plan, given, loaded.types).entries
|
|
263
|
+
for need, result in entries:
|
|
264
|
+
if not loaded.types[need.type].offline or result.state == "ready":
|
|
265
|
+
continue
|
|
266
|
+
failed = failed or result.state != "unknown"
|
|
267
|
+
said = "not checked" if result.state == "unknown" else result.state
|
|
268
|
+
lines.append(f"pubdeploy: deploy: {said}, {need.id}: {result.summary}")
|
|
269
|
+
lines += [f"pubdeploy: {d}" for d in result.details]
|
|
270
|
+
if result.fix:
|
|
271
|
+
lines.append(f"pubdeploy: {result.fix}")
|
|
272
|
+
return lines, failed
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def ask(prompt: str) -> str:
|
|
276
|
+
"""A typed answer, the prompt on standard error, so --json's output is the document alone."""
|
|
277
|
+
print(prompt, end="", file=sys.stderr, flush=True)
|
|
278
|
+
return sys.stdin.readline().strip()
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# The entry-point group of the hooks a walk calls before it fulfils, such as
|
|
282
|
+
# pubsec's, which asks for the Bitwarden write token and gives the walk its
|
|
283
|
+
# writer: pubrules holds no write token itself.
|
|
284
|
+
WALK_HOOKS = "sustenaut.walk" # contract.WALK
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def prepare(ctx: Context, plan: list[tuple[str, str]], terminal: bool, points: Any = None) -> None:
|
|
288
|
+
"""Call each hook's prepare(ctx, plan, terminal), printing the line it
|
|
289
|
+
returns. A hook that fails is said, by its name and the error's type
|
|
290
|
+
alone, and the walk goes on without what it would have given."""
|
|
291
|
+
from .registry import installed # noqa: PLC0415
|
|
292
|
+
for point in sorted(installed(WALK_HOOKS) if points is None else points, key=lambda p: p.name):
|
|
293
|
+
try:
|
|
294
|
+
said = point.load()(ctx, plan, terminal)
|
|
295
|
+
except Exception as e: # noqa: BLE001 - a hook's failure is said, not fatal, and holds no value
|
|
296
|
+
print(f"pubdeploy: deploy: the {WALK_HOOKS} hook {point.name} failed ({type(e).__name__})", file=sys.stderr)
|
|
297
|
+
continue
|
|
298
|
+
if isinstance(said, str) and said:
|
|
299
|
+
print(said, file=sys.stderr)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def machine_needs(plan: Plan, types: dict[str, NeedType]) -> list[dict[str, Any]]:
|
|
303
|
+
"""The plan's needs as machine.compile takes them: id, type, grade, the needs each waits
|
|
304
|
+
for, its fulfil budget, how a Stop treats it, and its params (for shared resources)."""
|
|
305
|
+
from .order import edges # noqa: PLC0415
|
|
306
|
+
from .walk import fulfil_budget, grade_of # noqa: PLC0415
|
|
307
|
+
waits = edges(plan.needs, types)
|
|
308
|
+
out = []
|
|
309
|
+
for i, need in plan.needs.items():
|
|
310
|
+
kind = types[need.type]
|
|
311
|
+
out.append({"id": i, "type": need.type, "grade": grade_of(kind, need), "after": sorted(waits[i]),
|
|
312
|
+
"seconds": fulfil_budget(kind), "serial": bool(getattr(kind, "serial", False)),
|
|
313
|
+
"stop": getattr(kind, "stop", "signal"), "stop_warning": getattr(kind, "stop_warning", ""),
|
|
314
|
+
"params": need.params})
|
|
315
|
+
return out
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def narrow(plan: Plan, types: dict[str, NeedType], config: dict[str, Any], unit: str | None,
|
|
319
|
+
only: list[str]) -> tuple[Plan, set[str]] | str:
|
|
320
|
+
"""The plan walk --need checks: the named needs and every need they wait for, by the
|
|
321
|
+
walk's order and the run's after-edges (checked, never fulfilled), and the needs they
|
|
322
|
+
wait for directly, which must be ready. Or why it cannot be walked."""
|
|
323
|
+
from dataclasses import replace # noqa: PLC0415
|
|
324
|
+
from . import machine # noqa: PLC0415
|
|
325
|
+
unknown = sorted(set(only) - set(plan.needs))
|
|
326
|
+
if unknown:
|
|
327
|
+
return f"--need {', '.join(unknown)} is no need of this unit; it has {', '.join(sorted(plan.needs))}"
|
|
328
|
+
needs = machine_needs(plan, types)
|
|
329
|
+
after = machine.generated_after(needs)
|
|
330
|
+
section = unit_block(config, unit).get("run") or {}
|
|
331
|
+
for name, entry in (section.get("needs") or {}).items() if isinstance(section, dict) else ():
|
|
332
|
+
if name in after and isinstance(entry, dict):
|
|
333
|
+
after[name] |= {a for a in entry.get("after", []) if a in after}
|
|
334
|
+
direct = set().union(*(after[i] for i in only)) - set(only)
|
|
335
|
+
keep, todo = set(only), list(only)
|
|
336
|
+
while todo:
|
|
337
|
+
for a in after[todo.pop()]:
|
|
338
|
+
if a not in keep:
|
|
339
|
+
keep.add(a)
|
|
340
|
+
todo.append(a)
|
|
341
|
+
sub = {i: n for i, n in plan.needs.items() if i in keep}
|
|
342
|
+
return replace(plan, needs=sub, order=[i for i in plan.order if i in keep]), direct
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def unit_block(config: dict[str, Any], unit: str | None) -> dict[str, Any]:
|
|
346
|
+
"""The deploy block as the unit sees it (its run section included)."""
|
|
347
|
+
from .units import unit_block as block_of # noqa: PLC0415
|
|
348
|
+
block = config[BLOCK]
|
|
349
|
+
return block_of(block, unit) if unit and isinstance(block, dict) and "namespace" in block else block
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def run_machine(root: Path, unit: str | None, mode: str | None, modes: dict[str, str]) -> int:
|
|
353
|
+
"""`pubdeploy machine`: the unit's machine, as JSON, from pubrules.json alone."""
|
|
354
|
+
from . import machine # noqa: PLC0415
|
|
355
|
+
from .config import read_config # noqa: PLC0415
|
|
356
|
+
from .registry import load as registry # noqa: PLC0415
|
|
357
|
+
from .units import names # noqa: PLC0415
|
|
358
|
+
config = read_config(root)
|
|
359
|
+
if BLOCK not in config:
|
|
360
|
+
print(f"pubdeploy: {BLOCK}: pubrules.json has no deploy block", file=sys.stderr)
|
|
361
|
+
return 2
|
|
362
|
+
units = names(config[BLOCK]) if isinstance(config[BLOCK], dict) and "namespace" in config[BLOCK] else [None]
|
|
363
|
+
if unit is None and len(units) > 1:
|
|
364
|
+
print(f"pubdeploy: deploy: give --unit, one of {', '.join(map(str, units))}", file=sys.stderr)
|
|
365
|
+
return 2
|
|
366
|
+
unit = unit if unit is not None else units[0]
|
|
367
|
+
if unit not in units:
|
|
368
|
+
print(f"pubdeploy: deploy: --unit {unit} is no unit of the deploy block; it has {', '.join(map(str, units))}",
|
|
369
|
+
file=sys.stderr)
|
|
370
|
+
return 2
|
|
371
|
+
loaded = registry()
|
|
372
|
+
ctx = context(root, config, True, unit=unit)
|
|
373
|
+
plan = load(root, config, ctx, None, loaded.types, unit=unit)
|
|
374
|
+
if plan.problems:
|
|
375
|
+
for problem in plan.problems:
|
|
376
|
+
print(f"pubdeploy: {problem}", file=sys.stderr)
|
|
377
|
+
return 2
|
|
378
|
+
needs = machine_needs(plan, loaded.types)
|
|
379
|
+
section = unit_block(config, unit).get("run")
|
|
380
|
+
# --mode check makes every need a check; --mode fulfil is only the default, so a need the run
|
|
381
|
+
# section keeps to a check stays one. --modes, a need's own, is over both.
|
|
382
|
+
if mode == "check":
|
|
383
|
+
section = {**(section or machine.default_section(needs, mode)), "default_mode": mode}
|
|
384
|
+
section["needs"] = {k: {**v, "mode": mode} for k, v in (section.get("needs") or {}).items()}
|
|
385
|
+
elif mode == "fulfil":
|
|
386
|
+
section = {**section, "default_mode": mode} if section else machine.default_section(needs, mode)
|
|
387
|
+
try:
|
|
388
|
+
compiled = machine.compile(needs, section, modes)
|
|
389
|
+
except ValueError as e:
|
|
390
|
+
print(f"pubdeploy: deploy: {e}", file=sys.stderr)
|
|
391
|
+
return 2
|
|
392
|
+
print(json.dumps(compiled, indent=2))
|
|
393
|
+
return 0
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def run_walk(root: Path, offline: bool, as_json: bool, kubectl_command: str | None, zones: Path | list[Path] | None,
|
|
397
|
+
namespace: str | None, fulfil: bool = False, unit: str | None = None, server: str | None = None,
|
|
398
|
+
yes: bool = False, journal_url: str | None = None, tolerate_unknown: bool = False,
|
|
399
|
+
only: list[str] | None = None, budget_seconds: float | None = None) -> int:
|
|
400
|
+
from .config import read_config # noqa: PLC0415
|
|
401
|
+
from .registry import load as registry # noqa: PLC0415
|
|
402
|
+
from .units import names # noqa: PLC0415
|
|
403
|
+
from .walk import counted, fulfil as act, lines, outcomes, planned, unable, wants_token # noqa: PLC0415
|
|
404
|
+
config = read_config(root)
|
|
405
|
+
loaded = registry()
|
|
406
|
+
for warning in loaded.warnings:
|
|
407
|
+
print(f"pubdeploy: warning: {warning}", file=sys.stderr)
|
|
408
|
+
if BLOCK not in config:
|
|
409
|
+
print(f"pubdeploy: {BLOCK}: pubrules.json has no deploy block", file=sys.stderr)
|
|
410
|
+
return 2
|
|
411
|
+
if fulfil and offline:
|
|
412
|
+
print("pubdeploy: deploy: --fulfil does not run --offline", file=sys.stderr)
|
|
413
|
+
return 2
|
|
414
|
+
units = names(config[BLOCK]) if isinstance(config[BLOCK], dict) and "namespace" in config[BLOCK] else [None]
|
|
415
|
+
if unit is not None and unit not in units:
|
|
416
|
+
print(f"pubdeploy: deploy: --unit {unit} is no unit of the deploy block; it has {', '.join(map(str, units))}",
|
|
417
|
+
file=sys.stderr)
|
|
418
|
+
return 2
|
|
419
|
+
walks = []
|
|
420
|
+
from .walk import SHORTENED, shorten # noqa: PLC0415
|
|
421
|
+
SHORTENED.clear()
|
|
422
|
+
if hasattr(signal, "SIGUSR1") and threading.current_thread() is threading.main_thread():
|
|
423
|
+
signal.signal(signal.SIGUSR1, shorten) # a live run's Stop for a need that shortens
|
|
424
|
+
for name in [unit] if unit else units:
|
|
425
|
+
try:
|
|
426
|
+
ctx = context(root, config, offline, kubectl_command, zones, namespace, server=server, unit=name)
|
|
427
|
+
except (OSError, ValueError) as e:
|
|
428
|
+
print(f"pubdeploy: deploy: --zones {zones}: {type(e).__name__}", file=sys.stderr)
|
|
429
|
+
return 2
|
|
430
|
+
plan = load(root, config, ctx, namespace, loaded.types, unit=name)
|
|
431
|
+
if plan.problems:
|
|
432
|
+
for problem in plan.problems:
|
|
433
|
+
print(f"pubdeploy: {problem}", file=sys.stderr)
|
|
434
|
+
return 2
|
|
435
|
+
ctx.budget_cap = budget_seconds
|
|
436
|
+
if only:
|
|
437
|
+
narrowed = narrow(plan, loaded.types, config, name, only)
|
|
438
|
+
if isinstance(narrowed, str):
|
|
439
|
+
print(f"pubdeploy: deploy: {narrowed}", file=sys.stderr)
|
|
440
|
+
return 2
|
|
441
|
+
plan, waited = narrowed
|
|
442
|
+
journal = check(plan, ctx, loaded.types)
|
|
443
|
+
late = sorted(i for i, r in ((n.id, r) for n, r in journal.entries) if i in waited and r.state != "ready")
|
|
444
|
+
if late:
|
|
445
|
+
print(f"pubdeploy: deploy: --need {', '.join(only)} waits for {', '.join(late)}, which "
|
|
446
|
+
f"{'is' if len(late) == 1 else 'are'} not ready: walk those first", file=sys.stderr)
|
|
447
|
+
return 2
|
|
448
|
+
walks.append((ctx, journal))
|
|
449
|
+
continue
|
|
450
|
+
walks.append((ctx, check(plan, ctx, loaded.types)))
|
|
451
|
+
if fulfil:
|
|
452
|
+
env = walks[0][0].env
|
|
453
|
+
terminal = sys.stdin.isatty() and env.get("GITHUB_ACTIONS") != "true"
|
|
454
|
+
# Without a terminal, a token file the hook reads (pubsec's, for the engine's Fulfil).
|
|
455
|
+
token_coming = terminal or bool(env.get("PUBSEC_WRITE_TOKEN_FILE"))
|
|
456
|
+
# With --need, only the named needs are ever fulfilled: the needs they wait for are checked.
|
|
457
|
+
acts = [(ctx, [(i, line) for i, line in planned(journal, loaded.types, ctx, token_coming=token_coming)
|
|
458
|
+
if not only or i in only]) for ctx, journal in walks]
|
|
459
|
+
approved: set[tuple[str, str]] = set()
|
|
460
|
+
declined = False
|
|
461
|
+
if any(plan for _, plan in acts):
|
|
462
|
+
print("deploy: the walk will:", file=sys.stderr)
|
|
463
|
+
for ctx, plan in acts:
|
|
464
|
+
for _, line in plan:
|
|
465
|
+
print(f" {ctx.unit}: {line}" if len(walks) > 1 else f" {line}", file=sys.stderr)
|
|
466
|
+
if yes or ask('Type "yes" to do it: ') == "yes":
|
|
467
|
+
approved = {(ctx.unit, i) for ctx, plan in acts for i, _ in plan}
|
|
468
|
+
else:
|
|
469
|
+
declined = True
|
|
470
|
+
print("deploy: nothing was fulfilled", file=sys.stderr)
|
|
471
|
+
if approved and token_coming:
|
|
472
|
+
for ctx, plan in acts:
|
|
473
|
+
if wants_token(plan, loaded.types, ctx):
|
|
474
|
+
prepare(ctx, plan, terminal)
|
|
475
|
+
for ctx, journal in walks:
|
|
476
|
+
unable(journal, loaded.types, ctx, declined)
|
|
477
|
+
act(journal, loaded.types, ctx, {i for u, i in approved if u == ctx.unit})
|
|
478
|
+
if approved:
|
|
479
|
+
# What it did, on stderr too: with --json (the engine's Fulfil) the journal is stdout's.
|
|
480
|
+
print("deploy: the walk did:", file=sys.stderr)
|
|
481
|
+
for ctx, journal in walks:
|
|
482
|
+
said = outcomes(journal, {i for u, i in approved if u == ctx.unit})
|
|
483
|
+
if said and len(walks) > 1:
|
|
484
|
+
print(f" unit {ctx.unit}:", file=sys.stderr)
|
|
485
|
+
for line in said:
|
|
486
|
+
print(line, file=sys.stderr)
|
|
487
|
+
if only: # only the named needs are said and journaled
|
|
488
|
+
for _, journal in walks:
|
|
489
|
+
journal.entries = [(n, r) for n, r in journal.entries if n.id in only]
|
|
490
|
+
documents = [journal.document() for _, journal in walks]
|
|
491
|
+
if as_json:
|
|
492
|
+
print(json.dumps(documents[0] if len(documents) == 1 else documents, indent=2))
|
|
493
|
+
else:
|
|
494
|
+
for k, (ctx, journal) in enumerate(walks):
|
|
495
|
+
if len(walks) > 1:
|
|
496
|
+
print(("\n" if k else "") + f"unit {ctx.unit}:")
|
|
497
|
+
print("\n".join(lines(journal)))
|
|
498
|
+
url = journal_url or walks[0][0].env.get("PUBRULES_JOURNAL_URL")
|
|
499
|
+
if url:
|
|
500
|
+
from .store import write # noqa: PLC0415
|
|
501
|
+
for document in documents:
|
|
502
|
+
print(write(url, document, walks[0][0].env), file=sys.stderr)
|
|
503
|
+
passing = ("ready", "unknown") if tolerate_unknown else ("ready",)
|
|
504
|
+
ready = all(r.state in passing for _, journal in walks for _, r in counted(journal, loaded.types, offline))
|
|
505
|
+
return 0 if ready else 1
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def run_check(root: Path, offline: bool, as_json: bool, kubectl_command: str | None, zones: Path | list[Path] | None,
|
|
509
|
+
namespace: str | None, **given: Any) -> int:
|
|
510
|
+
return run_walk(root, offline, as_json, kubectl_command, zones, namespace, **given)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def run_types() -> int:
|
|
514
|
+
from .registry import load as registry # noqa: PLC0415
|
|
515
|
+
from .schema import COMMON, listed # noqa: PLC0415
|
|
516
|
+
loaded = registry()
|
|
517
|
+
for warning in loaded.warnings:
|
|
518
|
+
print(f"pubdeploy: warning: {warning}", file=sys.stderr)
|
|
519
|
+
from .walk import fulfil_budget # noqa: PLC0415
|
|
520
|
+
rows = [("type", "grade", "budget", "offline", "from", "fulfil", "keys")]
|
|
521
|
+
for name in sorted(loaded.types):
|
|
522
|
+
kind = loaded.types[name]
|
|
523
|
+
required = set(kind.schema.get("required", []))
|
|
524
|
+
keys = [f"{k}{'' if k in required else '?'}" for k in kind.schema.get("properties", {})]
|
|
525
|
+
rows.append((name, kind.grade, f"{kind.budget:g} s", "yes" if kind.offline else "no",
|
|
526
|
+
loaded.origin[name], f"{fulfil_budget(kind):g} s", ", ".join(keys) or "-"))
|
|
527
|
+
widths = [max(len(r[i]) for r in rows) for i in range(len(rows[0]) - 1)]
|
|
528
|
+
for row in rows:
|
|
529
|
+
print(" ".join(cell.ljust(w) for cell, w in zip(row, widths)) + " " + row[-1])
|
|
530
|
+
print(f"{len(loaded.types)} types; every need also takes {listed(list(COMMON))}, and one with a secret "
|
|
531
|
+
f"fills that Kubernetes Secret, a k8s.secret the walk adds")
|
|
532
|
+
return 0
|