pubdeploy 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pubdeploy/__init__.py +532 -0
- pubdeploy/channels.py +188 -0
- pubdeploy/cli.py +120 -0
- pubdeploy/config.py +140 -0
- pubdeploy/contract.py +87 -0
- pubdeploy/github.py +61 -0
- pubdeploy/hooks.py +82 -0
- pubdeploy/implied.py +179 -0
- pubdeploy/journal.json +207 -0
- pubdeploy/k8s.py +472 -0
- pubdeploy/machine.py +363 -0
- pubdeploy/model.py +229 -0
- pubdeploy/names.py +25 -0
- pubdeploy/order.py +101 -0
- pubdeploy/providers.py +75 -0
- pubdeploy/registry.py +96 -0
- pubdeploy/release-commit +1 -0
- pubdeploy/schema.json +127 -0
- pubdeploy/schema.py +238 -0
- pubdeploy/store.py +127 -0
- pubdeploy/testing/__init__.py +94 -0
- pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
- pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
- pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
- pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
- pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
- pubdeploy/testing/plugin.py +26 -0
- pubdeploy/testing/steps.py +1124 -0
- pubdeploy/testing/world.py +569 -0
- pubdeploy/types/__init__.py +181 -0
- pubdeploy/types/auth.py +199 -0
- pubdeploy/types/github.py +110 -0
- pubdeploy/types/k8s.py +609 -0
- pubdeploy/types/net.py +237 -0
- pubdeploy/types/rollout.py +186 -0
- pubdeploy/units.py +177 -0
- pubdeploy/walk.py +325 -0
- pubdeploy-0.8.0.dist-info/METADATA +520 -0
- pubdeploy-0.8.0.dist-info/RECORD +45 -0
- pubdeploy-0.8.0.dist-info/WHEEL +5 -0
- pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
- pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
- pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/order.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""The order needs are walked in: each after the needs it waits for.
|
|
2
|
+
|
|
3
|
+
The built-in rules: everything waits for k8s.namespace but the needs outside
|
|
4
|
+
the cluster (github.*, bitwarden.*, gcp.*, storage.*, dns.site, upstream) and
|
|
5
|
+
k8s.secrets-declared, which reads only the manifests; a k8s.secret waits for
|
|
6
|
+
the need whose secret fills it; github.gate waits for everything but
|
|
7
|
+
deploy.rollout and service.health; deploy.rollout waits for the gate, or,
|
|
8
|
+
without one, for everything; service.health waits for deploy.rollout. A
|
|
9
|
+
type may add its own, and a need its `after`.
|
|
10
|
+
|
|
11
|
+
The order is Kahn's algorithm with ties broken by id, so it is the same on
|
|
12
|
+
every run.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import heapq
|
|
18
|
+
from collections.abc import Mapping
|
|
19
|
+
|
|
20
|
+
from .model import Need, NeedType
|
|
21
|
+
|
|
22
|
+
OUTSIDE = ("github.", "bitwarden.", "gcp.", "storage.")
|
|
23
|
+
OUTSIDE_TYPES = {"dns.site", "upstream", "k8s.secrets-declared", "k8s.namespace"}
|
|
24
|
+
LAST = {"deploy.rollout", "service.health"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def builtin(need: Need, needs: Mapping[str, Need]) -> set[str]:
|
|
28
|
+
"""The ids `need` waits for by the built-in rules."""
|
|
29
|
+
out: set[str] = set()
|
|
30
|
+
if not need.type.startswith(OUTSIDE) and need.type not in OUTSIDE_TYPES:
|
|
31
|
+
out |= {i for i, n in needs.items() if n.type == "k8s.namespace"}
|
|
32
|
+
if need.type == "k8s.secret":
|
|
33
|
+
out |= {i for i, n in needs.items()
|
|
34
|
+
if isinstance(n.params.get("secret"), dict) and n.params["secret"].get("name") == need.name}
|
|
35
|
+
gates = {i for i, n in needs.items() if n.type == "github.gate"}
|
|
36
|
+
if need.type == "github.gate":
|
|
37
|
+
out |= {i for i, n in needs.items() if n.type not in LAST and n.type != "github.gate"}
|
|
38
|
+
elif need.type == "deploy.rollout":
|
|
39
|
+
out |= gates or {i for i, n in needs.items() if n.type not in LAST}
|
|
40
|
+
elif need.type == "service.health":
|
|
41
|
+
out |= {i for i, n in needs.items() if n.type == "deploy.rollout"}
|
|
42
|
+
out.discard(need.id)
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def edges(needs: Mapping[str, Need], types: Mapping[str, NeedType]) -> dict[str, set[str]]:
|
|
47
|
+
"""By id, the ids each need waits for: the built-in rules, its type's, and its after."""
|
|
48
|
+
return {i: (builtin(n, needs) | set(types[n.type].depends_on(n, needs)) | set(n.after)) & set(needs) - {i}
|
|
49
|
+
for i, n in needs.items()}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def unknown_after(needs: Mapping[str, Need]) -> list[str]:
|
|
53
|
+
return [f"deploy: {i} is after {a}, which is no need of this deploy"
|
|
54
|
+
for i, n in needs.items() for a in n.after if a not in needs]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def topological(needs: Mapping[str, Need], types: Mapping[str, NeedType]) -> list[str]:
|
|
58
|
+
"""Every id, each after those it waits for. ValueError naming a cycle."""
|
|
59
|
+
waits = edges(needs, types)
|
|
60
|
+
blocking: dict[str, set[str]] = {i: set() for i in needs}
|
|
61
|
+
for i, deps in waits.items():
|
|
62
|
+
for d in deps:
|
|
63
|
+
blocking[d].add(i)
|
|
64
|
+
left = {i: len(deps) for i, deps in waits.items()}
|
|
65
|
+
ready = [i for i, n in left.items() if n == 0]
|
|
66
|
+
heapq.heapify(ready)
|
|
67
|
+
out = []
|
|
68
|
+
while ready:
|
|
69
|
+
i = heapq.heappop(ready)
|
|
70
|
+
out.append(i)
|
|
71
|
+
for j in blocking[i]:
|
|
72
|
+
left[j] -= 1
|
|
73
|
+
if left[j] == 0:
|
|
74
|
+
heapq.heappush(ready, j)
|
|
75
|
+
if len(out) < len(needs):
|
|
76
|
+
raise ValueError(cycle({i: waits[i] for i in needs if i not in out}))
|
|
77
|
+
return out
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def cycle(waits: Mapping[str, set[str]]) -> str:
|
|
81
|
+
"""One cycle among needs that wait for each other: the first by id, around."""
|
|
82
|
+
start = min(waits)
|
|
83
|
+
path, seen = [start], {start}
|
|
84
|
+
while True:
|
|
85
|
+
nxt = min(d for d in waits[path[-1]] if d in waits)
|
|
86
|
+
if nxt in seen:
|
|
87
|
+
loop = path[path.index(nxt):] + [nxt]
|
|
88
|
+
return f"deploy: these needs wait for each other: {' -> '.join(loop)}"
|
|
89
|
+
path.append(nxt)
|
|
90
|
+
seen.add(nxt)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def cycle_problems(needs: Mapping[str, Need], types: Mapping[str, NeedType]) -> list[str]:
|
|
94
|
+
"""An after naming no need, or needs waiting in a cycle."""
|
|
95
|
+
if bad := unknown_after(needs):
|
|
96
|
+
return bad
|
|
97
|
+
try:
|
|
98
|
+
topological(needs, types)
|
|
99
|
+
except ValueError as e:
|
|
100
|
+
return [str(e)]
|
|
101
|
+
return []
|
pubdeploy/providers.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Moving deploy.providers to its packages' latest releases, for
|
|
2
|
+
`pubdeploy bump` (the sustenaut.bump hook). Taken from pubrules's
|
|
3
|
+
deploy/providers.py, whose floor and launcher half stays in pubrules.
|
|
4
|
+
|
|
5
|
+
A provider's wheel is a GitHub release's asset, so its latest release is
|
|
6
|
+
read through `gh`, and the asset of the same distribution in it is the new
|
|
7
|
+
wheel, with the sha256 GitHub records for it (or, for an older release
|
|
8
|
+
without one, the sha256 of the file). pubrules.json is changed in place, the
|
|
9
|
+
two strings and nothing else, so its layout stays the repository's.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import urllib.request
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
WHEEL = re.compile(r"https://github\.com/(?P<repo>[A-Za-z0-9-]+/[A-Za-z0-9_.-]+)/releases/download/[^/\s]+/"
|
|
22
|
+
r"(?P<name>[A-Za-z0-9_.]+)-(?P<version>[^-/\s]+)-py3-none-any\.whl")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def sha256(url: str) -> str:
|
|
26
|
+
digest = hashlib.sha256()
|
|
27
|
+
with urllib.request.urlopen(url, timeout=120) as response: # noqa: S310 - a release asset's address
|
|
28
|
+
for chunk in iter(lambda: response.read(1 << 16), b""):
|
|
29
|
+
digest.update(chunk)
|
|
30
|
+
return digest.hexdigest()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def latest(wheel: str, gh: Any = None) -> tuple[str, str] | None:
|
|
34
|
+
"""The newest release's wheel of the same distribution, and its sha256; None when there is none."""
|
|
35
|
+
from .github import ApiError, gh as api # noqa: PLC0415
|
|
36
|
+
m = WHEEL.fullmatch(wheel)
|
|
37
|
+
if not m:
|
|
38
|
+
return None
|
|
39
|
+
try:
|
|
40
|
+
release = (gh or api)("GET", f"repos/{m['repo']}/releases/latest", None) or {}
|
|
41
|
+
except ApiError:
|
|
42
|
+
return None
|
|
43
|
+
for asset in release.get("assets") or []:
|
|
44
|
+
name = str(asset.get("name", ""))
|
|
45
|
+
if name.startswith(f"{m['name']}-") and name.endswith("-py3-none-any.whl"):
|
|
46
|
+
url = str(asset.get("browser_download_url", ""))
|
|
47
|
+
digest = str(asset.get("digest") or "")
|
|
48
|
+
return url, digest.removeprefix("sha256:") if digest.startswith("sha256:") else sha256(url)
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def bump(root: Path, gh: Any = None, find: Any = None) -> list[str]:
|
|
53
|
+
"""Move each provider of pubrules.json to its latest release; a line per one moved."""
|
|
54
|
+
path = root / "pubrules.json"
|
|
55
|
+
text = path.read_text(encoding="utf-8")
|
|
56
|
+
deploy = json.loads(text).get("deploy")
|
|
57
|
+
out = []
|
|
58
|
+
for given in (deploy or {}).get("providers") or [] if isinstance(deploy, dict) else []:
|
|
59
|
+
if not isinstance(given, dict) or not isinstance(given.get("wheel"), str):
|
|
60
|
+
continue
|
|
61
|
+
if not isinstance(given.get("sha256"), str) or not given["sha256"]:
|
|
62
|
+
out.append(f"left deploy.providers {given.get('distribution')}: it has no sha256 to move; "
|
|
63
|
+
f"give it the wheel's sha256")
|
|
64
|
+
continue
|
|
65
|
+
found = (find or latest)(given["wheel"], gh)
|
|
66
|
+
if found is None or found[0] == given["wheel"]:
|
|
67
|
+
continue
|
|
68
|
+
text = text.replace(given["wheel"], found[0]).replace(given["sha256"], found[1])
|
|
69
|
+
out.append(f"updated pubrules.json (deploy.providers {given.get('distribution')}: "
|
|
70
|
+
f"{WHEEL.fullmatch(found[0])['version'] if WHEEL.fullmatch(found[0]) else found[0]})")
|
|
71
|
+
if any(line.startswith("updated") for line in out):
|
|
72
|
+
path.write_text(text, encoding="utf-8")
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
pubdeploy/registry.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""The need types a walk knows: pubdeploy's own, and any an installed package
|
|
2
|
+
offers under the entry-point group `sustenaut.needs`, each `module:object`
|
|
3
|
+
naming an object with what model.NeedType has: an instance, whose check and
|
|
4
|
+
fulfil the walk calls, not a class. A provider need import nothing of
|
|
5
|
+
pubdeploy's: any result with state, summary, fix and details serves
|
|
6
|
+
(contract.py holds the contract, and the checks below).
|
|
7
|
+
|
|
8
|
+
pubsec offers the types that hold a secret's value (PUBSEC_TYPES),
|
|
9
|
+
which pubrules had built in until pubsec took them.
|
|
10
|
+
|
|
11
|
+
A type is checked as it loads: its name is dotted lower-case words, its
|
|
12
|
+
schema an object schema that takes no key it does not name, its budget more
|
|
13
|
+
than 0 and at most 10 seconds, its grade one of auto, propose, human and
|
|
14
|
+
none, and its fulfil_budget, when it gives one, more than 0 and at most 900
|
|
15
|
+
seconds. One that fails, or does not load, is left out with a warning, and
|
|
16
|
+
never stops `pubrules sync --check`. A type a second package offers again is
|
|
17
|
+
left out, naming both.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from importlib.metadata import EntryPoint, entry_points
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from .contract import NEEDS, problems
|
|
27
|
+
from .model import NeedType
|
|
28
|
+
from .types.auth import SignInClient
|
|
29
|
+
from .types.github import VariableEntry
|
|
30
|
+
from .types.k8s import CronJob, JobOnce, Jwks, Manifests, Namespace, SecretsDeclared
|
|
31
|
+
from .types.net import Health, Site, Tenant, Upstream
|
|
32
|
+
from .types.rollout import Gate, Rollout
|
|
33
|
+
|
|
34
|
+
GROUP = NEEDS
|
|
35
|
+
BUILTIN: tuple[NeedType, ...] = (Namespace(), SecretsDeclared(), Manifests(), CronJob(), JobOnce(), Jwks(), VariableEntry(), Gate(),
|
|
36
|
+
Upstream(), Site(), Tenant(), Rollout(), Health(), SignInClient())
|
|
37
|
+
PUBDEPLOY = "pubdeploy"
|
|
38
|
+
# The types that hold a secret's value, which pubsec offers: the walk implies
|
|
39
|
+
# several (a secret's k8s.secret, the secrets block's github.*), so a plan
|
|
40
|
+
# with one needs pubsec installed, as deploy.providers lists it.
|
|
41
|
+
PUBSEC = "pubsec"
|
|
42
|
+
PUBSEC_TYPES = ("bitwarden.adopt", "bitwarden.generated", "bitwarden.secret", "gcp.service-account-key", "github.deploy-settings",
|
|
43
|
+
"github.secret", "github.variable", "k8s.secret")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class Registry:
|
|
48
|
+
types: dict[str, NeedType] = field(default_factory=dict)
|
|
49
|
+
origin: dict[str, str] = field(default_factory=dict) # by type, the distribution it came from
|
|
50
|
+
warnings: list[str] = field(default_factory=list)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def installed(group: str) -> list[EntryPoint]:
|
|
54
|
+
"""The entry points of `group` the installed packages offer: what the walk
|
|
55
|
+
loads, and what pubdeploy.testing's world puts its providers in place of."""
|
|
56
|
+
return list(entry_points(group=group))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def load(points: list[EntryPoint] | None = None) -> Registry:
|
|
60
|
+
"""pubdeploy's types, and those the entry points offer."""
|
|
61
|
+
out = Registry()
|
|
62
|
+
for kind in BUILTIN:
|
|
63
|
+
out.types[kind.type] = kind
|
|
64
|
+
out.origin[kind.type] = PUBDEPLOY
|
|
65
|
+
found = sorted(installed(GROUP) if points is None else points, key=lambda p: (p.name, p.value))
|
|
66
|
+
for point in found:
|
|
67
|
+
dist = getattr(getattr(point, "dist", None), "name", None) or "an unknown package"
|
|
68
|
+
label = f"{GROUP} entry point {point.name} ({point.value}, from {dist})"
|
|
69
|
+
try:
|
|
70
|
+
kind = point.load()
|
|
71
|
+
except Exception as e: # noqa: BLE001 - a bad provider is reported, not fatal
|
|
72
|
+
out.warnings.append(f"{label} does not load ({type(e).__name__}), and is left out")
|
|
73
|
+
continue
|
|
74
|
+
if bad := problems(kind):
|
|
75
|
+
out.warnings.append(f"{label} is left out: {'; '.join(bad)}")
|
|
76
|
+
continue
|
|
77
|
+
if kind.type in out.types:
|
|
78
|
+
out.warnings.append(f"{label} offers {kind.type}, which {out.origin[kind.type]} offers too; "
|
|
79
|
+
f"{dist}'s is left out")
|
|
80
|
+
continue
|
|
81
|
+
out.types[kind.type] = kind
|
|
82
|
+
out.origin[kind.type] = dist
|
|
83
|
+
return out
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def lacking(needs: Any, types: dict[str, NeedType]) -> list[str]:
|
|
87
|
+
"""For each type of `needs` that is pubsec's and not installed, what to do: one phrase."""
|
|
88
|
+
missing = sorted({n.type for n in needs if n.type not in types and n.type in PUBSEC_TYPES})
|
|
89
|
+
if not missing:
|
|
90
|
+
return []
|
|
91
|
+
return [f"deploy: {', '.join(missing)} {'is' if len(missing) == 1 else 'are'} {PUBSEC}'s, which is not installed: "
|
|
92
|
+
f"list {PUBSEC} in deploy.providers, as `pubrules sync` does, and run ./pubdeploy, which runs it"]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def types() -> dict[str, NeedType]:
|
|
96
|
+
return load().types
|
pubdeploy/release-commit
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
89ad7f788ed76e8958db39cd55795c2a8adc622e
|
pubdeploy/schema.json
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/Sustenaut/pubdeploy/blob/main/src/pubdeploy/schema.json",
|
|
4
|
+
"title": "pubrules.json's deploy block",
|
|
5
|
+
"description": "What a service needs before it deploys. Each need's own keys are its type's schema (pubdeploy types).",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"x-phrase": "must be an object, with namespace and maybe host, workflow, gate, overlay, units, providers and needs",
|
|
8
|
+
"required": [
|
|
9
|
+
"namespace"
|
|
10
|
+
],
|
|
11
|
+
"additionalProperties": false,
|
|
12
|
+
"properties": {
|
|
13
|
+
"namespace": {
|
|
14
|
+
"description": "The k3s namespace the service runs in.",
|
|
15
|
+
"type": "string",
|
|
16
|
+
"pattern": "^[a-z][a-z0-9-]{0,30}$",
|
|
17
|
+
"x-phrase": "must be a lower-case name of at most 31 characters"
|
|
18
|
+
},
|
|
19
|
+
"host": {
|
|
20
|
+
"description": "The public host; it implies dns.site, and service.health asks it.",
|
|
21
|
+
"type": "string",
|
|
22
|
+
"pattern": "^(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z][a-z0-9-]{0,61}[a-z0-9]$",
|
|
23
|
+
"x-phrase": "must be a host name in lower case, such as example.example.eu"
|
|
24
|
+
},
|
|
25
|
+
"workflow": {
|
|
26
|
+
"description": "The deploy workflow's file in .github/workflows (default deploy.yml).",
|
|
27
|
+
"type": "string",
|
|
28
|
+
"pattern": "^[A-Za-z0-9_.-]+\\.ya?ml$",
|
|
29
|
+
"x-phrase": "must be a workflow's file name, such as deploy.yml"
|
|
30
|
+
},
|
|
31
|
+
"gate": {
|
|
32
|
+
"description": "The repository variable that must be on for the deploy; it implies github.gate.",
|
|
33
|
+
"type": "string",
|
|
34
|
+
"pattern": "^(?!GITHUB_)[A-Z_][A-Z0-9_]*$",
|
|
35
|
+
"x-phrase": "must be a variable's name: upper case letters, digits and _, not starting GITHUB_"
|
|
36
|
+
},
|
|
37
|
+
"overlay": {
|
|
38
|
+
"description": "The kustomize overlay the manifest checks render (default k8s).",
|
|
39
|
+
"type": "string",
|
|
40
|
+
"pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[A-Za-z0-9_./-]+$",
|
|
41
|
+
"x-phrase": "must be a directory in the repository, such as k8s"
|
|
42
|
+
},
|
|
43
|
+
"units": {
|
|
44
|
+
"description": "More units, by name, each its own namespace, overlay, gate and host; a need's unit scopes it to one.",
|
|
45
|
+
"type": "object",
|
|
46
|
+
"x-phrase": "must be an object of units, by name",
|
|
47
|
+
"additionalProperties": {
|
|
48
|
+
"type": "object",
|
|
49
|
+
"required": [
|
|
50
|
+
"namespace"
|
|
51
|
+
],
|
|
52
|
+
"additionalProperties": false,
|
|
53
|
+
"x-name": "a unit",
|
|
54
|
+
"x-phrase": "must be an object, with namespace and maybe overlay, gate and host",
|
|
55
|
+
"properties": {
|
|
56
|
+
"namespace": {
|
|
57
|
+
"$ref": "#/properties/namespace"
|
|
58
|
+
},
|
|
59
|
+
"overlay": {
|
|
60
|
+
"$ref": "#/properties/overlay"
|
|
61
|
+
},
|
|
62
|
+
"gate": {
|
|
63
|
+
"$ref": "#/properties/gate"
|
|
64
|
+
},
|
|
65
|
+
"host": {
|
|
66
|
+
"$ref": "#/properties/host"
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
"providers": {
|
|
72
|
+
"description": "The packages whose sustenaut.needs types the needs use: ./pubdeploy runs with each wheel, checked against its sha256.",
|
|
73
|
+
"type": "array",
|
|
74
|
+
"x-phrase": "must be a list of packages, each with distribution, wheel and sha256",
|
|
75
|
+
"items": {
|
|
76
|
+
"type": "object",
|
|
77
|
+
"required": [
|
|
78
|
+
"distribution",
|
|
79
|
+
"wheel",
|
|
80
|
+
"sha256"
|
|
81
|
+
],
|
|
82
|
+
"additionalProperties": false,
|
|
83
|
+
"x-name": "a provider",
|
|
84
|
+
"x-phrase": "must be an object with distribution, wheel and sha256",
|
|
85
|
+
"properties": {
|
|
86
|
+
"distribution": {
|
|
87
|
+
"type": "string",
|
|
88
|
+
"pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$",
|
|
89
|
+
"x-phrase": "must be a package's name"
|
|
90
|
+
},
|
|
91
|
+
"wheel": {
|
|
92
|
+
"type": "string",
|
|
93
|
+
"pattern": "^https://github\\.com/[A-Za-z0-9-]+/[A-Za-z0-9_.-]+/releases/download/[^/\\s]+/[A-Za-z0-9_.]+-[^-/\\s]+-py3-none-any\\.whl$",
|
|
94
|
+
"x-phrase": "must be the address of a GitHub release's wheel, NAME-VERSION-py3-none-any.whl"
|
|
95
|
+
},
|
|
96
|
+
"sha256": {
|
|
97
|
+
"type": "string",
|
|
98
|
+
"pattern": "^[0-9a-f]{64}$",
|
|
99
|
+
"x-phrase": "must be the wheel's sha256, 64 hex digits"
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
"needs": {
|
|
105
|
+
"description": "The needs nothing implies, each an object with type and that type's keys.",
|
|
106
|
+
"type": "array",
|
|
107
|
+
"x-phrase": "must be a list of needs, each an object with a type",
|
|
108
|
+
"items": {
|
|
109
|
+
"type": "object",
|
|
110
|
+
"required": [
|
|
111
|
+
"type"
|
|
112
|
+
],
|
|
113
|
+
"x-phrase": "must be an object with a type",
|
|
114
|
+
"properties": {
|
|
115
|
+
"type": {
|
|
116
|
+
"type": "string"
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
},
|
|
121
|
+
"run": {
|
|
122
|
+
"description": "How a live run (the convergence engine's) runs the needs: schema, default_mode, parallel, and per need its mode and more after-edges; it may only narrow what pubdeploy generates (pubdeploy machine).",
|
|
123
|
+
"type": "object",
|
|
124
|
+
"x-phrase": "must be an object, such as {\"schema\": 1, \"default_mode\": \"fulfil\"}"
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
pubdeploy/schema.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""The deploy block's schema, and what is wrong with a block: phrases, one
|
|
2
|
+
per problem, in the voice `pubrules sync` prints the other blocks' in.
|
|
3
|
+
|
|
4
|
+
The block's own keys are in schema.json, shipped with pubdeploy; each need's
|
|
5
|
+
keys are its type's schema (`pubdeploy types` lists them), with the keys
|
|
6
|
+
every need may carry: type, name and after. pubdeploy has no JSON schema
|
|
7
|
+
library, so `validate` reads the part of JSON schema these schemas use:
|
|
8
|
+
type, properties, required, additionalProperties, items, enum, const,
|
|
9
|
+
pattern, minLength, maxLength, minItems, minimum, maximum and $ref within the
|
|
10
|
+
document. A schema's `x-phrase` says, in pubdeploy's words, what a value that
|
|
11
|
+
fails it must be, and `x-name` what an object is, for a key it does not take.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
from functools import cache
|
|
19
|
+
from importlib import resources
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from .model import NeedType
|
|
24
|
+
|
|
25
|
+
BLOCK = "deploy"
|
|
26
|
+
WORKFLOWS = Path(".github/workflows")
|
|
27
|
+
DEFAULT_WORKFLOW = "deploy.yml"
|
|
28
|
+
DEFAULT_OVERLAY = "k8s"
|
|
29
|
+
KUSTOMIZATIONS = ("kustomization.yaml", "kustomization.yml", "Kustomization")
|
|
30
|
+
# The keys every need may carry, beside its type's own.
|
|
31
|
+
NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/@-]{0,252}")
|
|
32
|
+
ID = re.compile(r"[a-z][a-z0-9-]*(\.[a-z][a-z0-9-]*)*:[A-Za-z0-9][A-Za-z0-9._/@-]{0,252}")
|
|
33
|
+
COMMON: dict[str, Any] = {
|
|
34
|
+
"type": {"type": "string"},
|
|
35
|
+
"name": {"type": "string", "pattern": f"^{NAME.pattern}$",
|
|
36
|
+
"x-phrase": "must be a name: letters, digits and . _ / @ -"},
|
|
37
|
+
"after": {"type": "array", "items": {"type": "string", "pattern": f"^{ID.pattern}$",
|
|
38
|
+
"x-phrase": "must be a need's id, TYPE:NAME"},
|
|
39
|
+
"x-phrase": "must be a list of needs' ids, each TYPE:NAME"},
|
|
40
|
+
"unit": {"type": "string", "x-phrase": "must be a unit's name"},
|
|
41
|
+
}
|
|
42
|
+
# Where a need's type says it fills a Kubernetes Secret: its name, the key, and
|
|
43
|
+
# whether the value is base64 that goes into the Secret decoded.
|
|
44
|
+
SECRET: dict[str, Any] = {
|
|
45
|
+
"type": "object", "required": ["name"], "additionalProperties": False,
|
|
46
|
+
"x-phrase": "must be an object with the Secret's name, and maybe key and decode",
|
|
47
|
+
"properties": {
|
|
48
|
+
"name": {"type": "string", "pattern": r"^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$",
|
|
49
|
+
"x-phrase": "must be a Secret's name, in lower case"},
|
|
50
|
+
"key": {"type": "string", "pattern": r"^[-._a-zA-Z0-9]{1,253}$",
|
|
51
|
+
"x-phrase": "must be a key of a Secret: letters, digits, - . _"},
|
|
52
|
+
"decode": {"enum": ["base64"]},
|
|
53
|
+
},
|
|
54
|
+
}
|
|
55
|
+
TYPES = {"string": "a string", "array": "a list", "object": "an object", "boolean": "true or false",
|
|
56
|
+
"integer": "a whole number", "number": "a number", "null": "null"}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@cache
|
|
60
|
+
def block_schema() -> dict[str, Any]:
|
|
61
|
+
return json.loads(resources.files("pubdeploy").joinpath("schema.json").read_text(encoding="utf-8"))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@cache
|
|
65
|
+
def journal_schema() -> dict[str, Any]:
|
|
66
|
+
return json.loads(resources.files("pubdeploy").joinpath("journal.json").read_text(encoding="utf-8"))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def listed(words: list[str] | tuple[str, ...], last: str = "and") -> str:
|
|
70
|
+
"""a, b and c."""
|
|
71
|
+
words = list(words)
|
|
72
|
+
return words[0] if len(words) == 1 else f"{', '.join(words[:-1])} {last} {words[-1]}"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def where(path: tuple[str | int, ...]) -> str:
|
|
76
|
+
"""needs[2].secret.name"""
|
|
77
|
+
out = ""
|
|
78
|
+
for step in path:
|
|
79
|
+
out += f"[{step}]" if isinstance(step, int) else (f".{step}" if out else str(step))
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def is_type(value: Any, name: str) -> bool:
|
|
84
|
+
if name == "integer":
|
|
85
|
+
return isinstance(value, int) and not isinstance(value, bool)
|
|
86
|
+
if name == "number":
|
|
87
|
+
return isinstance(value, int | float) and not isinstance(value, bool)
|
|
88
|
+
return isinstance(value, {"string": str, "array": list, "object": dict, "boolean": bool,
|
|
89
|
+
"null": type(None)}[name])
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
Found = list[tuple[tuple[str | int, ...], str, bool]]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def validate(value: Any, schema: dict[str, Any], path: tuple[str | int, ...] = (),
|
|
96
|
+
root: dict[str, Any] | None = None) -> Found:
|
|
97
|
+
"""What is wrong with `value` by `schema`: where, the phrase, and whether
|
|
98
|
+
it is about a key of the object there rather than the value."""
|
|
99
|
+
root = root or schema
|
|
100
|
+
if "$ref" in schema:
|
|
101
|
+
target: Any = root
|
|
102
|
+
for step in schema["$ref"].removeprefix("#/").split("/"):
|
|
103
|
+
target = target[step]
|
|
104
|
+
schema = {**target, **{k: v for k, v in schema.items() if k != "$ref"}}
|
|
105
|
+
own: list[str] = []
|
|
106
|
+
deeper: Found = []
|
|
107
|
+
wanted = schema.get("type")
|
|
108
|
+
names = [wanted] if isinstance(wanted, str) else wanted or []
|
|
109
|
+
if names and not any(is_type(value, n) for n in names):
|
|
110
|
+
own.append(f"must be {listed([TYPES[n] for n in names], 'or')}")
|
|
111
|
+
else:
|
|
112
|
+
if "enum" in schema and value not in schema["enum"]:
|
|
113
|
+
own.append(f"must be {listed([json.dumps(v) for v in schema['enum']], 'or')}")
|
|
114
|
+
if "const" in schema and value != schema["const"]:
|
|
115
|
+
own.append(f"must be {json.dumps(schema['const'])}")
|
|
116
|
+
if isinstance(value, str):
|
|
117
|
+
if "pattern" in schema and not re.search(schema["pattern"], value):
|
|
118
|
+
own.append(f"must match {schema['pattern']}")
|
|
119
|
+
if len(value) < schema.get("minLength", 0):
|
|
120
|
+
own.append(f"must be at least {schema['minLength']} characters")
|
|
121
|
+
if "maxLength" in schema and len(value) > schema["maxLength"]:
|
|
122
|
+
own.append(f"must be at most {schema['maxLength']} characters")
|
|
123
|
+
if is_type(value, "number"):
|
|
124
|
+
if "minimum" in schema and value < schema["minimum"]:
|
|
125
|
+
own.append(f"must be at least {schema['minimum']}")
|
|
126
|
+
if "maximum" in schema and value > schema["maximum"]:
|
|
127
|
+
own.append(f"must be at most {schema['maximum']}")
|
|
128
|
+
if isinstance(value, list):
|
|
129
|
+
if len(value) < schema.get("minItems", 0):
|
|
130
|
+
own.append(f"must list at least {schema['minItems']}")
|
|
131
|
+
if "items" in schema:
|
|
132
|
+
for i, item in enumerate(value):
|
|
133
|
+
deeper += validate(item, schema["items"], (*path, i), root)
|
|
134
|
+
if isinstance(value, dict):
|
|
135
|
+
properties = schema.get("properties", {})
|
|
136
|
+
for key in schema.get("required", []):
|
|
137
|
+
if key not in value:
|
|
138
|
+
about = properties.get(key, {}).get("description", "")
|
|
139
|
+
deeper.append((path, f"{key} is missing" + (f": {about[0].lower()}{about[1:].rstrip('.')}"
|
|
140
|
+
if about else ""), True))
|
|
141
|
+
extra = schema.get("additionalProperties", True)
|
|
142
|
+
for key in value:
|
|
143
|
+
if key in properties:
|
|
144
|
+
deeper += validate(value[key], properties[key], (*path, key), root)
|
|
145
|
+
elif extra is False:
|
|
146
|
+
of = f" of {schema['x-name']}" if "x-name" in schema else ""
|
|
147
|
+
deeper.append((path, f"{key} is not a setting{of}; it takes {listed(list(properties))}", True))
|
|
148
|
+
elif isinstance(extra, dict):
|
|
149
|
+
deeper += validate(value[key], extra, (*path, key), root)
|
|
150
|
+
if own and "x-phrase" in schema:
|
|
151
|
+
own = [schema["x-phrase"]]
|
|
152
|
+
return [(path, phrase, False) for phrase in own] + deeper
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def phrase(prefix: str, path: tuple[str | int, ...], text: str, key: bool) -> str:
|
|
156
|
+
"""`deploy: namespace must be ...`, `deploy: needs[0]: type is missing`,
|
|
157
|
+
or `deploy must be ...` for the block itself."""
|
|
158
|
+
at = where(path)
|
|
159
|
+
if key:
|
|
160
|
+
return f"{prefix}: {at}: {text}" if at else f"{prefix}: {text}"
|
|
161
|
+
return f"{prefix}: {at} {text}" if at else f"{prefix} {text}"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def need_schema(kind: NeedType) -> dict[str, Any]:
|
|
165
|
+
"""A type's schema with the keys every need carries."""
|
|
166
|
+
schema = dict(kind.schema)
|
|
167
|
+
schema["x-name"] = kind.type
|
|
168
|
+
schema["properties"] = {**COMMON, **schema.get("properties", {})}
|
|
169
|
+
schema["required"] = ["type", *[k for k in schema.get("required", []) if k != "type"]]
|
|
170
|
+
return schema
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def need_problems(entry: Any, i: int, types: dict[str, NeedType], root: Path,
|
|
174
|
+
providers: list[str] | None = None) -> list[str]:
|
|
175
|
+
"""What is wrong with one need as written."""
|
|
176
|
+
kind = types.get(entry["type"])
|
|
177
|
+
if kind is None:
|
|
178
|
+
from .registry import PUBSEC, PUBSEC_TYPES # noqa: PLC0415
|
|
179
|
+
if entry["type"] in PUBSEC_TYPES and PUBSEC not in (providers or []):
|
|
180
|
+
return [f"deploy: needs[{i}]: {entry['type']} is {PUBSEC}'s: list {PUBSEC} in deploy.providers, "
|
|
181
|
+
f"as `pubrules sync` does, and run ./pubdeploy, which runs it"]
|
|
182
|
+
if providers:
|
|
183
|
+
return [f"deploy: needs[{i}]: {entry['type']} is no type of need pubdeploy or an installed package "
|
|
184
|
+
f"offers; it would be {listed(providers, 'or')}'s, of deploy.providers, which ./pubdeploy runs "
|
|
185
|
+
f"with: run ./pubdeploy, or install {listed(providers, 'or')}"]
|
|
186
|
+
return [f"deploy: needs[{i}]: {entry['type']} is no type of need pubdeploy knows; "
|
|
187
|
+
f"it knows {listed(sorted(types))}; a package's types are named in deploy.providers"]
|
|
188
|
+
name = entry.get("name") if isinstance(entry.get("name"), str) else None
|
|
189
|
+
label = f"deploy: needs[{i}] ({entry['type']}:{name})" if name else f"deploy: needs[{i}] ({entry['type']})"
|
|
190
|
+
out = [phrase(label, *found) for found in validate(entry, need_schema(kind))]
|
|
191
|
+
if out:
|
|
192
|
+
return out
|
|
193
|
+
if name is None and kind.default_name(entry) is None and not getattr(kind, "singleton", False):
|
|
194
|
+
out.append(f"{label}: it needs a name")
|
|
195
|
+
for path in getattr(kind, "files", lambda _params: [])(entry):
|
|
196
|
+
if not (root / path).is_file():
|
|
197
|
+
out.append(f"{label}: {path} is not a file in the repository")
|
|
198
|
+
return out
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def problems(config: dict[str, Any], root: Path, types: dict[str, NeedType] | None = None) -> list[str]:
|
|
202
|
+
"""What is wrong with pubrules.json's deploy block: phrases, none when it is sound.
|
|
203
|
+
It opens no file but to see that the workflow, the overlay and each
|
|
204
|
+
k8s.manifests path are there."""
|
|
205
|
+
if types is None:
|
|
206
|
+
from .registry import types as registered # noqa: PLC0415
|
|
207
|
+
types = registered()
|
|
208
|
+
block = config.get(BLOCK)
|
|
209
|
+
out = [phrase(BLOCK, *found) for found in validate(block, block_schema())]
|
|
210
|
+
if out or not isinstance(block, dict):
|
|
211
|
+
return out
|
|
212
|
+
from .units import names, unit_block # noqa: PLC0415
|
|
213
|
+
workflow = block.get("workflow", DEFAULT_WORKFLOW)
|
|
214
|
+
if not (root / WORKFLOWS / workflow).is_file():
|
|
215
|
+
out.append(f"deploy: workflow {workflow} is not in {WORKFLOWS}")
|
|
216
|
+
units = names(block)
|
|
217
|
+
if block["namespace"] in (block.get("units") or {}):
|
|
218
|
+
out.append(f"deploy: units: {block['namespace']} is the top level's unit, named by its namespace")
|
|
219
|
+
for unit in units:
|
|
220
|
+
overlay = unit_block(block, unit).get("overlay", DEFAULT_OVERLAY)
|
|
221
|
+
if not any((root / overlay / k).is_file() for k in KUSTOMIZATIONS):
|
|
222
|
+
of = "" if unit == block["namespace"] else f" (unit {unit})"
|
|
223
|
+
out.append(f"deploy: overlay {overlay}{of} has no kustomization.yaml")
|
|
224
|
+
offered = [p["distribution"] for p in block.get("providers", [])]
|
|
225
|
+
for i, entry in enumerate(block.get("needs", [])):
|
|
226
|
+
out += need_problems(entry, i, types, root, offered)
|
|
227
|
+
if "unit" in entry and entry["unit"] not in units:
|
|
228
|
+
out.append(f"deploy: needs[{i}]: unit {entry['unit']} is no unit of the block; it has {listed(units)}")
|
|
229
|
+
if out:
|
|
230
|
+
return out
|
|
231
|
+
from .implied import build # noqa: PLC0415
|
|
232
|
+
from .order import cycle_problems # noqa: PLC0415
|
|
233
|
+
for unit in units:
|
|
234
|
+
plan, bad = build(config, types, root=root, unit=unit)
|
|
235
|
+
bad = bad or cycle_problems(plan, types)
|
|
236
|
+
if bad:
|
|
237
|
+
return bad
|
|
238
|
+
return []
|