pubdeploy 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. pubdeploy/__init__.py +532 -0
  2. pubdeploy/channels.py +188 -0
  3. pubdeploy/cli.py +120 -0
  4. pubdeploy/config.py +140 -0
  5. pubdeploy/contract.py +87 -0
  6. pubdeploy/github.py +61 -0
  7. pubdeploy/hooks.py +82 -0
  8. pubdeploy/implied.py +179 -0
  9. pubdeploy/journal.json +207 -0
  10. pubdeploy/k8s.py +472 -0
  11. pubdeploy/machine.py +363 -0
  12. pubdeploy/model.py +229 -0
  13. pubdeploy/names.py +25 -0
  14. pubdeploy/order.py +101 -0
  15. pubdeploy/providers.py +75 -0
  16. pubdeploy/registry.py +96 -0
  17. pubdeploy/release-commit +1 -0
  18. pubdeploy/schema.json +127 -0
  19. pubdeploy/schema.py +238 -0
  20. pubdeploy/store.py +127 -0
  21. pubdeploy/testing/__init__.py +94 -0
  22. pubdeploy/testing/fixtures/deploy-app/.github/workflows/deploy.yml +11 -0
  23. pubdeploy/testing/fixtures/deploy-app/k8s/admin/network-policy.yaml +11 -0
  24. pubdeploy/testing/fixtures/deploy-app/k8s/app.yaml +63 -0
  25. pubdeploy/testing/fixtures/deploy-app/k8s/kustomization.yaml +6 -0
  26. pubdeploy/testing/fixtures/deploy-app/pubrules.json +27 -0
  27. pubdeploy/testing/fixtures/deploy-app/rendered.yaml +86 -0
  28. pubdeploy/testing/fixtures/deploy-app/zones.json +8 -0
  29. pubdeploy/testing/plugin.py +26 -0
  30. pubdeploy/testing/steps.py +1124 -0
  31. pubdeploy/testing/world.py +569 -0
  32. pubdeploy/types/__init__.py +181 -0
  33. pubdeploy/types/auth.py +199 -0
  34. pubdeploy/types/github.py +110 -0
  35. pubdeploy/types/k8s.py +609 -0
  36. pubdeploy/types/net.py +237 -0
  37. pubdeploy/types/rollout.py +186 -0
  38. pubdeploy/units.py +177 -0
  39. pubdeploy/walk.py +325 -0
  40. pubdeploy-0.8.0.dist-info/METADATA +520 -0
  41. pubdeploy-0.8.0.dist-info/RECORD +45 -0
  42. pubdeploy-0.8.0.dist-info/WHEEL +5 -0
  43. pubdeploy-0.8.0.dist-info/entry_points.txt +11 -0
  44. pubdeploy-0.8.0.dist-info/licenses/LICENSE +202 -0
  45. pubdeploy-0.8.0.dist-info/top_level.txt +1 -0
pubdeploy/hooks.py ADDED
@@ -0,0 +1,82 @@
1
+ """What pubrules calls of pubdeploy, through two entry-point groups, so pubrules
2
+ imports nothing of the walk:
3
+
4
+ - `sustenaut.sync`, `check(root, config, check) -> (lines, failed)`: the deploy
5
+ block's offline check (the block and its needs, the manifests against
6
+ them, the units against the deploy workflow). It seeds no scheduled walk:
7
+ the convergence engine walks every deploy service, and a repository's own
8
+ deploy-check.yml caller, where one is left, is its own to remove. `pubrules sync` calls it after its own work, and
9
+ `--check` fails when it says so. `pubdeploy sync-check` runs it alone.
10
+ - `sustenaut.bump`, `bump(root) -> lines`: each deploy.providers entry moved
11
+ to its latest release, then this repository's uses of pubdeploy's
12
+ reusable workflows moved to the pubdeploy deploy.providers names, a caller
13
+ of k8s-deploy.yml still started by `push` rewritten to `workflow_run`.
14
+ `pubrules bump` calls it after moving pubrules's own pins; `pubdeploy bump`
15
+ runs it alone.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ from pathlib import Path
22
+ from typing import Any
23
+
24
+ from .schema import BLOCK
25
+
26
+
27
+ def check(root: Path, config: dict[str, Any], check: bool) -> tuple[list[str], bool]:
28
+ if BLOCK not in config:
29
+ return [], False
30
+ from . import offline_lines # noqa: PLC0415
31
+ lines, failed = offline_lines(root, config)
32
+ # Writing, a disagreement is said and fails nothing, as pubrules sync goes on.
33
+ return lines, check and failed
34
+
35
+
36
+ # A use of pubdeploy's reusable workflows, by commit with the version in a
37
+ # comment, or by tag.
38
+ PIN = re.compile(r"(?P<owner>[A-Za-z0-9][A-Za-z0-9-]*)/(?P<path>pubdeploy/\.github/workflows/[\w.-]+\.ya?ml@)"
39
+ r"(?:v|[0-9a-f]{40}[ \t]+#[ \t]*v)(?P<version>[0-9][^\s'\"]*)")
40
+
41
+
42
+ def tag_commit(repo: str, version: str, gh: Any = None) -> str | None:
43
+ """The commit the tag v`version` of `repo` names, or None when it cannot be read."""
44
+ from .github import ApiError, gh as api # noqa: PLC0415
45
+ call = gh or api
46
+ try:
47
+ found = call("GET", f"repos/{repo}/git/ref/tags/v{version}", None) or {}
48
+ target = found.get("object") or {}
49
+ if target.get("type") == "tag": # annotated: the tag object names the commit
50
+ target = (call("GET", f"repos/{repo}/git/tags/{target.get('sha')}", None) or {}).get("object") or {}
51
+ except ApiError:
52
+ return None
53
+ sha = target.get("sha")
54
+ return sha if isinstance(sha, str) and re.fullmatch(r"[0-9a-f]{40}", sha) else None
55
+
56
+
57
+ def bump(root: Path, gh: Any = None, find: Any = None) -> list[str]:
58
+ import json # noqa: PLC0415
59
+ from .config import ref # noqa: PLC0415
60
+ from .providers import WHEEL, bump as providers # noqa: PLC0415
61
+ from .units import K8S_DEPLOY, retrigger, trigger_problems # noqa: PLC0415
62
+ path = root / "pubrules.json"
63
+ if not path.is_file():
64
+ return []
65
+ out = providers(root, gh, find)
66
+ deploy = json.loads(path.read_text(encoding="utf-8")).get(BLOCK)
67
+ listed = deploy.get("providers") if isinstance(deploy, dict) else None
68
+ ours = next((p for p in listed or [] if isinstance(p, dict) and p.get("distribution") == "pubdeploy"), None)
69
+ m = WHEEL.fullmatch(str((ours or {}).get("wheel", "")))
70
+ if not m:
71
+ return out
72
+ version = m["version"]
73
+ commit = tag_commit(m["repo"], version, gh)
74
+ for workflow in sorted((root / ".github" / "workflows").glob("*.y*ml")):
75
+ text = workflow.read_text(encoding="utf-8")
76
+ new = PIN.sub(lambda p: f"{p['owner']}/{p['path']}{ref(version, commit)}", text)
77
+ if K8S_DEPLOY.search(new) and trigger_problems(new, workflow.name):
78
+ new = retrigger(new)
79
+ if new != text:
80
+ workflow.write_text(new, encoding="utf-8")
81
+ out.append(f"updated {workflow.relative_to(root)} (pubdeploy@{ref(version, commit)})")
82
+ return out
pubdeploy/implied.py ADDED
@@ -0,0 +1,179 @@
1
+ """The needs a deploy block implies, beside those it lists.
2
+
3
+ Derived in this order, each with `implied_by` saying where it came from:
4
+
5
+ namespace k8s.namespace, deploy.rollout, k8s.secrets-declared, and
6
+ service.health when there is a host
7
+ host dns.site
8
+ gate github.gate
9
+ secrets a github.secret or github.variable per entry, and
10
+ github.deploy-settings when one is the server's
11
+ github_app a github.secret or github.variable per slot
12
+ variables a github.variable per name, which must be set
13
+ a need's secret the k8s.secret it fills
14
+ the manifests a k8s.secret per Secret they read
15
+
16
+ A need the block lists wins over an implied one with the same id. Two
17
+ implied k8s.secret needs of one name are one, with the keys of both.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ from typing import Any
24
+
25
+ from . import k8s
26
+ from .model import Need, NeedType
27
+
28
+ # The names the server's deploy job reads (k8s-deploy.yml): one of them in the
29
+ # secrets block means the repository deploys to the server, so it needs all.
30
+ # pubsec's github.deploy-settings keeps the same table (pubsec/needs/github.py),
31
+ # which checks and sets them: a change to one is made in both.
32
+ DEPLOY_SETTINGS = {"SUSTENAUT_SERVER": "variable", "SUSTENAUT_KNOWN_HOSTS": "variable",
33
+ "SUSTENAUT_DEPLOY_SSH_KEY": "secret", "TS_GH_OAUTH_CLIENT_ID": "secret",
34
+ "TS_GH_OAUTH_SECRET": "secret"}
35
+ SERVER_NAMES = ("SUSTENAUT_SERVER", "SUSTENAUT_KNOWN_HOSTS", "SUSTENAUT_DEPLOY_SSH_KEY")
36
+ COMMON = ("type", "name", "after", "unit")
37
+ # The shapes the manifests ask of a value (pubdeploy.k8s), by the name a
38
+ # k8s.secret need gives the shape, which pubsec's type checks.
39
+ SHAPE_NAMES = {k8s.COOKIE_SECRET.what: "oauth2-proxy-cookie"}
40
+
41
+
42
+ def github_name(entry: dict[str, Any]) -> str:
43
+ """A GitHub secret's or variable's need name: environment/NAME in an environment."""
44
+ return f"{entry['environment']}/{entry['to']}" if entry.get("environment") else entry["to"]
45
+
46
+
47
+ def manifest_secrets(found: k8s.Found, namespace: str) -> dict[str, dict[str, Any]]:
48
+ """Each Secret the manifests read in `namespace` and do not hold, with
49
+ the keys and shapes its required references name. A Secret every
50
+ reference to which is optional is left out."""
51
+ out: dict[str, dict[str, Any]] = {}
52
+ for n in found.needs:
53
+ if n.optional or n.namespace != namespace or (n.namespace, n.secret) in found.held:
54
+ continue
55
+ entry = out.setdefault(n.secret, {"keys": [], "shapes": {}})
56
+ entry["keys"] = sorted(set(entry["keys"]) | set(n.keys))
57
+ entry["shapes"] |= {key: name for key, shape in n.shapes.items() if (name := SHAPE_NAMES.get(shape.what))}
58
+ return out
59
+
60
+
61
+ class Builder:
62
+ """Needs by id, in the order they are added, merged as the module says."""
63
+
64
+ def __init__(self) -> None:
65
+ self.needs: dict[str, Need] = {}
66
+
67
+ def add(self, kind: str, name: str, params: dict[str, Any], implied_by: str) -> None:
68
+ need = Need(kind, name, params, implied_by)
69
+ had = self.needs.get(need.id)
70
+ if had is None:
71
+ self.needs[need.id] = need
72
+ return
73
+ by = had.implied_by if had.implied_by is None or implied_by in had.implied_by.split(", ") \
74
+ else f"{had.implied_by}, {implied_by}"
75
+ merged = had.params
76
+ if kind == "k8s.secret":
77
+ keys = sorted(set(had.params.get("keys", [])) | set(params.get("keys", [])))
78
+ shapes = {**params.get("shapes", {}), **had.params.get("shapes", {})}
79
+ merged = {**had.params, "keys": keys} | ({"shapes": shapes} if shapes else {})
80
+ self.needs[need.id] = Need(kind, had.name, merged, by, had.after)
81
+
82
+
83
+ def build(config: dict[str, Any], types: dict[str, NeedType], root: Any = None, namespace: str | None = None,
84
+ found: k8s.Found | None = None, unit: str | None = None) -> tuple[dict[str, Need], list[str]]:
85
+ """Every need of a sound block's unit (the top level's, by default), and
86
+ what is wrong with them together: a need declared twice. `found` is what
87
+ the rendered manifests read. A unit's needs are those whose `unit` names
88
+ it, and those that name none."""
89
+ from .units import unit_block # noqa: PLC0415
90
+ unit = unit or config["deploy"]["namespace"]
91
+ block = unit_block(config["deploy"], unit)
92
+ ns = namespace or block["namespace"]
93
+ problems = []
94
+ out = Builder()
95
+ written: dict[str, str] = {}
96
+ for i, entry in enumerate(block.get("needs", [])):
97
+ kind = types[entry["type"]]
98
+ params = {k: v for k, v in entry.items() if k not in COMMON}
99
+ name = entry.get("name") or kind.default_name(params) or ns
100
+ need = Need(entry["type"], name, params, None, tuple(entry.get("after", [])))
101
+ if entry.get("unit", unit) != unit:
102
+ continue
103
+ text = json.dumps(entry, sort_keys=True)
104
+ if need.id in written:
105
+ if written[need.id] != text:
106
+ problems.append(f"deploy: needs[{i}]: {need.id} is declared twice")
107
+ continue
108
+ written[need.id] = text
109
+ out.needs[need.id] = need
110
+ explicit = list(out.needs.values())
111
+
112
+ out.add("k8s.namespace", ns, {}, "namespace")
113
+ out.add("deploy.rollout", ns, {}, "namespace")
114
+ if block.get("host"):
115
+ out.add("service.health", ns, {}, "namespace")
116
+ out.add("k8s.secrets-declared", ns, {}, "namespace")
117
+ if block.get("host"):
118
+ out.add("dns.site", block["host"], {"host": block["host"]}, "host")
119
+ if block.get("gate"):
120
+ out.add("github.gate", ns, {"variable": block["gate"]}, "gate")
121
+
122
+ from .config import APP, OTHERS, plan, secrets_sound # noqa: PLC0415
123
+ if (APP in config or OTHERS in config) and secrets_sound(config):
124
+ for item in plan(config):
125
+ params = {k: item[k] for k in ("from", "to", "environment") if k in item}
126
+ out.add(f"github.{item['kind']}", github_name(item), params,
127
+ "github_app" if item["check"] else "secrets")
128
+ if any(e["to"] in SERVER_NAMES for e in config.get(OTHERS, [])):
129
+ out.add("github.deploy-settings", ns, {}, "secrets")
130
+ variables = config.get("variables")
131
+ if isinstance(variables, dict) and isinstance(variables.get("names"), list):
132
+ for name in variables["names"]:
133
+ if isinstance(name, str):
134
+ out.add("github.variable", name, {"declared": True}, "variables")
135
+
136
+ for need in explicit:
137
+ secret = need.params.get("secret")
138
+ if isinstance(secret, dict):
139
+ shape = getattr(types[need.type], "secret_shape", lambda _p: None)(need.params)
140
+ key = secret.get("key")
141
+ params: dict[str, Any] = {"keys": [key] if key else keys_of(types[need.type], need.params)}
142
+ if shape and key:
143
+ params["shapes"] = {key: shape}
144
+ out.add("k8s.secret", secret["name"], params, f"secret of {need.id}")
145
+ if found is not None:
146
+ for name, params in manifest_secrets(found, ns).items():
147
+ out.add("k8s.secret", name, params, "manifests")
148
+ from .registry import lacking # noqa: PLC0415
149
+ problems += lacking(out.needs.values(), types)
150
+ return out.needs, problems
151
+
152
+
153
+ def keys_of(kind: NeedType, params: dict[str, Any]) -> list[str]:
154
+ """The keys a need fills when its secret names none: its type's, if it says."""
155
+ said = getattr(kind, "secret_keys", None)
156
+ return list(said(params) or []) if callable(said) else []
157
+
158
+
159
+ def declarers(block: dict[str, Any], types: dict[str, NeedType], namespace: str
160
+ ) -> dict[str, list[tuple[str, list[str] | None]]]:
161
+ """By Secret name, each need the block lists that fills it, with the keys
162
+ it provides: None for any. A need's secret provides its key, or any
163
+ without one; a k8s.secret need its keys, or any when it lists none.
164
+ From the needs as written, so no merge widens what one provides."""
165
+ out: dict[str, list[tuple[str, list[str] | None]]] = {}
166
+ for entry in block.get("needs", []):
167
+ kind = types.get(entry.get("type"))
168
+ if kind is None or entry.get("unit", block.get("unit", block.get("namespace"))) != block.get(
169
+ "unit", block.get("namespace")):
170
+ continue
171
+ params = {k: v for k, v in entry.items() if k not in COMMON}
172
+ need_id = f"{entry['type']}:{entry.get('name') or kind.default_name(params) or namespace}"
173
+ if isinstance(entry.get("secret"), dict):
174
+ key = entry["secret"].get("key")
175
+ keys = [key] if key else keys_of(kind, params) or None
176
+ out.setdefault(entry["secret"]["name"], []).append((need_id, keys))
177
+ if entry["type"] == "k8s.secret":
178
+ out.setdefault(need_id.split(":", 1)[1], []).append((need_id, list(entry.get("keys", [])) or None))
179
+ return out
pubdeploy/journal.json ADDED
@@ -0,0 +1,207 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/Sustenaut/pubdeploy/blob/main/src/pubdeploy/journal.json",
4
+ "title": "deploy-journal/2",
5
+ "description": "One walk of pubdeploy, of one unit: who ran it, where, whether it fulfilled, and what each need was found to be. It holds no need's params, and no value.",
6
+ "type": "object",
7
+ "required": [
8
+ "schema",
9
+ "service",
10
+ "walk",
11
+ "needs",
12
+ "summary"
13
+ ],
14
+ "additionalProperties": false,
15
+ "properties": {
16
+ "schema": {
17
+ "const": "deploy-journal/2"
18
+ },
19
+ "service": {
20
+ "type": "object",
21
+ "required": [
22
+ "repository",
23
+ "namespace",
24
+ "unit",
25
+ "commit",
26
+ "pubdeploy"
27
+ ],
28
+ "additionalProperties": false,
29
+ "properties": {
30
+ "repository": {
31
+ "type": "string"
32
+ },
33
+ "namespace": {
34
+ "type": "string"
35
+ },
36
+ "unit": {
37
+ "type": "string"
38
+ },
39
+ "commit": {
40
+ "type": "string"
41
+ },
42
+ "pubdeploy": {
43
+ "type": "string",
44
+ "description": "The pubdeploy that walked."
45
+ }
46
+ }
47
+ },
48
+ "walk": {
49
+ "type": "object",
50
+ "required": [
51
+ "mode",
52
+ "offline",
53
+ "started_at",
54
+ "seconds",
55
+ "by",
56
+ "from"
57
+ ],
58
+ "additionalProperties": false,
59
+ "properties": {
60
+ "mode": {
61
+ "enum": [
62
+ "check",
63
+ "fulfil"
64
+ ]
65
+ },
66
+ "offline": {
67
+ "type": "boolean"
68
+ },
69
+ "started_at": {
70
+ "type": "string"
71
+ },
72
+ "seconds": {
73
+ "type": "number",
74
+ "minimum": 0
75
+ },
76
+ "by": {
77
+ "type": "string"
78
+ },
79
+ "from": {
80
+ "enum": [
81
+ "laptop",
82
+ "actions",
83
+ "plane"
84
+ ]
85
+ }
86
+ }
87
+ },
88
+ "needs": {
89
+ "type": "array",
90
+ "items": {
91
+ "type": "object",
92
+ "required": [
93
+ "id",
94
+ "type",
95
+ "name",
96
+ "implied_by",
97
+ "after",
98
+ "grade",
99
+ "state",
100
+ "fulfilled",
101
+ "summary",
102
+ "fix",
103
+ "details",
104
+ "seconds",
105
+ "checked_at"
106
+ ],
107
+ "additionalProperties": false,
108
+ "properties": {
109
+ "id": {
110
+ "type": "string"
111
+ },
112
+ "type": {
113
+ "type": "string"
114
+ },
115
+ "name": {
116
+ "type": "string"
117
+ },
118
+ "implied_by": {
119
+ "type": [
120
+ "string",
121
+ "null"
122
+ ]
123
+ },
124
+ "after": {
125
+ "type": "array",
126
+ "items": {
127
+ "type": "string"
128
+ }
129
+ },
130
+ "grade": {
131
+ "enum": [
132
+ "auto",
133
+ "propose",
134
+ "human",
135
+ "none"
136
+ ]
137
+ },
138
+ "state": {
139
+ "enum": [
140
+ "ready",
141
+ "missing",
142
+ "drift",
143
+ "blocked",
144
+ "unknown"
145
+ ]
146
+ },
147
+ "fulfilled": {
148
+ "type": "boolean"
149
+ },
150
+ "summary": {
151
+ "type": "string",
152
+ "maxLength": 120
153
+ },
154
+ "fix": {
155
+ "type": "string"
156
+ },
157
+ "details": {
158
+ "type": "array",
159
+ "items": {
160
+ "type": "string"
161
+ }
162
+ },
163
+ "seconds": {
164
+ "type": "number",
165
+ "minimum": 0
166
+ },
167
+ "checked_at": {
168
+ "type": "string"
169
+ }
170
+ }
171
+ }
172
+ },
173
+ "summary": {
174
+ "type": "object",
175
+ "required": [
176
+ "ready",
177
+ "missing",
178
+ "drift",
179
+ "blocked",
180
+ "unknown"
181
+ ],
182
+ "additionalProperties": false,
183
+ "properties": {
184
+ "ready": {
185
+ "type": "integer",
186
+ "minimum": 0
187
+ },
188
+ "missing": {
189
+ "type": "integer",
190
+ "minimum": 0
191
+ },
192
+ "drift": {
193
+ "type": "integer",
194
+ "minimum": 0
195
+ },
196
+ "blocked": {
197
+ "type": "integer",
198
+ "minimum": 0
199
+ },
200
+ "unknown": {
201
+ "type": "integer",
202
+ "minimum": 0
203
+ }
204
+ }
205
+ }
206
+ }
207
+ }