sdtk-ops-kit 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +2 -1
  2. package/assets/manifest/toolkit-bundle.manifest.json +144 -19
  3. package/assets/manifest/toolkit-bundle.sha256.txt +35 -10
  4. package/assets/toolkit/toolkit/AGENTS.md +2 -1
  5. package/assets/toolkit/toolkit/SDTKOPS_TOOLKIT.md +4 -2
  6. package/assets/toolkit/toolkit/scripts/install-claude-skills.ps1 +4 -4
  7. package/assets/toolkit/toolkit/scripts/install-codex-skills.ps1 +4 -4
  8. package/assets/toolkit/toolkit/scripts/uninstall-claude-skills.ps1 +2 -2
  9. package/assets/toolkit/toolkit/scripts/uninstall-codex-skills.ps1 +2 -2
  10. package/assets/toolkit/toolkit/skills/ops-discover/SKILL.md +2 -0
  11. package/assets/toolkit/toolkit/skills/ops-framer-deploy/SKILL.md +15 -1
  12. package/assets/toolkit/toolkit/skills/ops-framer-deploy/package.json +10 -0
  13. package/assets/toolkit/toolkit/skills/ops-framer-deploy/scripts/framer-install.mjs +5 -1
  14. package/assets/toolkit/toolkit/skills/ops-framer-deploy/scripts/lib/config.mjs +6 -0
  15. package/assets/toolkit/toolkit/skills/ops-local-env/SKILL.md +172 -0
  16. package/assets/toolkit/toolkit/skills/ops-local-env/references/CLUSTER_CA.md +83 -0
  17. package/assets/toolkit/toolkit/skills/ops-local-env/references/GOTCHAS.md +17 -0
  18. package/assets/toolkit/toolkit/skills/ops-local-env/references/K3D_CILIUM_BOOTSTRAP.md +39 -0
  19. package/assets/toolkit/toolkit/skills/ops-local-env/references/PROFILE.md +44 -0
  20. package/assets/toolkit/toolkit/skills/ops-local-env/references/bootstrap/cilium-values.yaml +37 -0
  21. package/assets/toolkit/toolkit/skills/ops-local-env/references/bootstrap/k3d-cluster.yaml +37 -0
  22. package/assets/toolkit/toolkit/skills/ops-local-env/references/profile.example.json +80 -0
  23. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/admin_creds.py +212 -0
  24. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/cilium_heal.py +136 -0
  25. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/ensure_repos.py +69 -0
  26. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/envprofile.py +139 -0
  27. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/explain_config.py +116 -0
  28. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/health.py +109 -0
  29. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/hosts_entry.py +158 -0
  30. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/init_profile.py +217 -0
  31. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/k3d_images.py +70 -0
  32. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/preflight.py +96 -0
  33. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/registry_secret.py +64 -0
  34. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/shared_config.py +178 -0
  35. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/stack_plan.py +94 -0
  36. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/teardown.py +86 -0
  37. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/test_local_env.py +307 -0
  38. package/assets/toolkit/toolkit/skills/ops-local-env/scripts/trust_ca.py +153 -0
  39. package/package.json +1 -1
  40. package/src/lib/scope.js +1 -0
@@ -0,0 +1,212 @@
1
+ #!/usr/bin/env python3
2
+ """admin_creds.py - one admin user and one admin password for every component of the stack that has
3
+ a login (the profile's `admin.users` / `admin.passwords`), and the Kubernetes Secrets made from them
4
+ (`admin.k8s_secrets`). Never prints a password, except --reveal.
5
+
6
+ admin_creds.py --show # users per key; passwords only as set / EMPTY
7
+ admin_creds.py --user NAME # dry run: diff of the user lines
8
+ admin_creds.py --generate # dry run: a new random password, masked
9
+ admin_creds.py --user NAME --generate --apply # write, then read back
10
+ admin_creds.py --password-stdin [--apply] <<'EOF' # the person's own password, from stdin
11
+ <password>
12
+ EOF
13
+ admin_creds.py --k8s-secrets NAMESPACE # create / update admin.k8s_secrets (current context)
14
+ admin_creds.py --reveal # print the shared user and password (final summary only)
15
+
16
+ Users stay in the tracked config files; passwords go to gitignored secrets files (created with mode
17
+ 600 when missing). The password is kept, generated here (secrets.token_urlsafe), or read from stdin
18
+ -- never from the command line -- and written straight into the files. --show warns when git
19
+ tracks a password file: a `git diff` there prints the password.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import argparse
25
+ import base64
26
+ import difflib
27
+ import json
28
+ import re
29
+ import secrets
30
+ import subprocess
31
+ import sys
32
+ from pathlib import Path
33
+
34
+ from envprofile import Stack, add_common_args, fail, load_flat, set_line
35
+
36
+ USER_RX = r"^[a-z][a-z0-9_-]{2,31}$"
37
+ # goes into YAML and shell-rendered files: no space, quote, backtick, backslash or dollar
38
+ PASSWORD_RX = r'^[^\s"\'`\\$]{12,64}$'
39
+
40
+
41
+ class Admin:
42
+ def __init__(self, stack: Stack):
43
+ self.stack = stack
44
+ conf = stack.get("admin") or {}
45
+ self.users = [(u["file"], u["key"]) for u in conf.get("users") or []]
46
+ self.passwords = [(p["file"], p["key"]) for p in conf.get("passwords") or []]
47
+ self.k8s_secrets = conf.get("k8s_secrets") or []
48
+ self.user_rx = conf.get("user_rx") or USER_RX
49
+ self.password_rx = conf.get("password_rx") or PASSWORD_RX
50
+ if not self.users and not self.passwords:
51
+ raise SystemExit("error: the profile has no admin.users / admin.passwords")
52
+
53
+ def value(self, rel: str, key: str):
54
+ return load_flat(self.stack.path(rel)).get(key)
55
+
56
+ def shared(self) -> tuple[str, str] | None:
57
+ users = {self.value(rel, key) for rel, key in self.users}
58
+ passwords = {self.value(rel, key) for rel, key in self.passwords}
59
+ if len(users) != 1 or len(passwords) != 1 or None in users | passwords:
60
+ return None
61
+ return users.pop(), passwords.pop()
62
+
63
+ def tracked(self) -> list[str]:
64
+ out = []
65
+ for rel in sorted({rel for rel, _ in self.passwords}):
66
+ path = self.stack.path(rel)
67
+ res = subprocess.run(["git", "-C", str(path.parent), "ls-files", "--error-unmatch", path.name],
68
+ capture_output=True, check=False)
69
+ if res.returncode == 0:
70
+ out.append(rel)
71
+ return out
72
+
73
+ def warn_tracked(self) -> None:
74
+ files = self.tracked()
75
+ if files:
76
+ print(f"\ntracked by git, password in plain text: {', '.join(files)}\n"
77
+ " never git diff / git show / git log -p them, never stage or commit them")
78
+
79
+
80
+ def show(ad: Admin) -> int:
81
+ print(f"{'file':40} {'key':28} value")
82
+ for rel, key in ad.users:
83
+ path = ad.stack.path(rel)
84
+ value = load_flat(path).get(key, "(missing)") if path.is_file() else "(no file)"
85
+ print(f"{rel:40} {key:28} {value}")
86
+ values = []
87
+ for rel, key in ad.passwords:
88
+ path = ad.stack.path(rel)
89
+ val = load_flat(path).get(key) if path.is_file() else None
90
+ values.append(val)
91
+ print(f"{rel:40} {key:28} {'(no file)' if not path.is_file() else ('set' if val else 'EMPTY')}")
92
+ users = {str(ad.value(rel, key)) for rel, key in ad.users}
93
+ same_pw = len(set(values)) == 1 and values and values[0]
94
+ print(f"\nusers: {'all the same' if len(users) == 1 else 'DIFFER'}; "
95
+ f"passwords: {'all the same' if same_pw else 'DIFFER or EMPTY'}")
96
+ ad.warn_tracked()
97
+ return 0
98
+
99
+
100
+ def mask(lines: list[str], keys: set[str]) -> list[str]:
101
+ rx = re.compile(rf'^([-+ ]?(?:{"|".join(map(re.escape, keys))}):[ \t]*)("(?:[^"\\\n]|\\.)*"|\'[^\'\n]*\'|[^\n]*?)([ \t]*(?:(?<=[ \t])#.*)?)$')
102
+ return [rx.sub(r'\1"********"\3', line.rstrip("\n")) + "\n" for line in lines]
103
+
104
+
105
+ def change(ad: Admin, user: str | None, generate: bool, apply: bool, given: str | None) -> int:
106
+ if user is not None and not re.match(ad.user_rx, user):
107
+ return fail(f"error: user {user!r} does not match {ad.user_rx}")
108
+ if given is not None and not re.match(ad.password_rx, given):
109
+ return fail("error: the password must be 12-64 characters, with no space and none of \" ' ` \\ $")
110
+ password = secrets.token_urlsafe(18) if generate else given
111
+ edits: dict[str, list[tuple[str, str]]] = {}
112
+ if user is not None:
113
+ for rel, key in ad.users:
114
+ edits.setdefault(rel, []).append((key, user))
115
+ if password is not None:
116
+ for rel, key in ad.passwords:
117
+ edits.setdefault(rel, []).append((key, password))
118
+ if not edits:
119
+ return fail("error: pass --user NAME and/or --generate / --password-stdin")
120
+ pw_keys = {key for _, key in ad.passwords}
121
+ pw_files = {rel for rel, _ in ad.passwords}
122
+ changed = False
123
+ for rel, pairs in edits.items():
124
+ path = ad.stack.path(rel)
125
+ if not path.is_file() and rel not in pw_files:
126
+ return fail(f"error: {path} not found")
127
+ old = path.read_text(encoding="utf-8") if path.is_file() else ""
128
+ new = old
129
+ for key, value in pairs:
130
+ new = set_line(new, key, value, append=rel in pw_files)
131
+ if new == old:
132
+ continue
133
+ changed = True
134
+ sys.stdout.writelines(mask(list(difflib.unified_diff(old.splitlines(True), new.splitlines(True), rel, rel, n=0)), pw_keys))
135
+ if apply:
136
+ path.parent.mkdir(parents=True, exist_ok=True)
137
+ path.write_text(new, encoding="utf-8")
138
+ if rel in pw_files:
139
+ path.chmod(0o600)
140
+ back = load_flat(path)
141
+ for key, value in pairs:
142
+ if str(back.get(key)) != value:
143
+ return fail(f"error: {rel}: {key} does not read back as written")
144
+ if not changed:
145
+ print("nothing to change")
146
+ elif not apply:
147
+ print("\n(dry run; a generated password is discarded -- add --apply to write a new one)")
148
+ else:
149
+ print("\nwritten; passwords are in the files above, never printed")
150
+ ad.warn_tracked()
151
+ return 0
152
+
153
+
154
+ def k8s_secrets(ad: Admin, namespace: str) -> int:
155
+ """Create or update every admin.k8s_secrets entry in NAMESPACE (current kube context), from the
156
+ user and password the files share. Piped to kubectl, never shown."""
157
+ if not ad.k8s_secrets:
158
+ return fail("error: the profile lists no admin.k8s_secrets")
159
+ pair = ad.shared()
160
+ if not pair:
161
+ return fail("error: users or passwords differ or are empty; run --show, then set them first")
162
+ b64 = lambda s: base64.b64encode(s.encode()).decode()
163
+ rc = 0
164
+ for spec in ad.k8s_secrets:
165
+ secret = {"apiVersion": "v1", "kind": "Secret", "type": "Opaque",
166
+ "metadata": {"name": spec["name"], "namespace": namespace},
167
+ "data": {spec.get("user_field", "username"): b64(pair[0]),
168
+ spec.get("password_field", "password"): b64(pair[1])}}
169
+ run = subprocess.run(["kubectl", "apply", "--server-side", "--force-conflicts", "-f", "-"],
170
+ input=json.dumps(secret), text=True, capture_output=True, check=False)
171
+ if run.returncode:
172
+ rc = 1
173
+ sys.stderr.write(run.stderr)
174
+ else:
175
+ print(f"{spec['name']} in {namespace}: the shared admin user and password")
176
+ return rc
177
+
178
+
179
+ def reveal(ad: Admin) -> int:
180
+ pair = ad.shared()
181
+ if not pair:
182
+ return fail("error: users or passwords differ or are empty; run --show")
183
+ print(f"admin user: {pair[0]}\nadmin password: {pair[1]}")
184
+ return 0
185
+
186
+
187
+ def main() -> int:
188
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
189
+ add_common_args(parser)
190
+ parser.add_argument("--show", action="store_true")
191
+ parser.add_argument("--reveal", action="store_true", help="print the shared user and password (final summary only)")
192
+ parser.add_argument("--user", metavar="NAME")
193
+ pw = parser.add_mutually_exclusive_group()
194
+ pw.add_argument("--generate", action="store_true", help="one new random password for every key")
195
+ pw.add_argument("--password-stdin", action="store_true", help="read the password from stdin (first line)")
196
+ parser.add_argument("--apply", action="store_true")
197
+ parser.add_argument("--k8s-secrets", metavar="NAMESPACE", help="create / update admin.k8s_secrets in NAMESPACE")
198
+ args = parser.parse_args()
199
+ ad = Admin(Stack(args.profile, args.root))
200
+ print(f"workspace: {ad.stack.root}")
201
+ if args.reveal:
202
+ return reveal(ad)
203
+ if args.show:
204
+ return show(ad)
205
+ if args.k8s_secrets:
206
+ return k8s_secrets(ad, args.k8s_secrets)
207
+ given = sys.stdin.readline().rstrip("\r\n") if args.password_stdin else None
208
+ return change(ad, args.user, args.generate, args.apply, given)
209
+
210
+
211
+ if __name__ == "__main__":
212
+ sys.exit(main())
@@ -0,0 +1,136 @@
1
+ #!/usr/bin/env python3
2
+ """cilium_heal.py - check and repair a local k3d + Cilium cluster after a reboot.
3
+
4
+ After a reboot Docker can give the k3d node containers new IPs while Cilium keeps the old ones in its
5
+ CiliumNode objects: every pod looks Running, Services and Ingresses are dead. The bpf filesystem
6
+ Cilium needs is also gone from the node containers.
7
+
8
+ cilium_heal.py --cluster NAME --check # read-only table per node (exit 0 when every node is ok)
9
+ cilium_heal.py --cluster NAME # repair, then check again
10
+
11
+ Per node: container state and IP, the Kubernetes Node's InternalIP, the CiliumNode's InternalIP, bpffs.
12
+ State: ok | STOPPED | NO BPFFS | STALE (CiliumNode / Node IP is not the container's IP) | UNKNOWN.
13
+
14
+ Repair, in this order, each only when needed: `k3d cluster start NAME` (stopped nodes); mount bpffs
15
+ and the cgroup2 path in every node; `rollout restart ds/cilium` and wait; if a CiliumNode is still
16
+ stale, delete that CiliumNode so its agent recreates it. Every kubectl call names --context k3d-NAME;
17
+ nothing else in the cluster is touched. Exit 0 only when every node ends ok.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import argparse
23
+ import json
24
+ import re
25
+ import subprocess
26
+ import sys
27
+
28
+ MOUNTS = ("mountpoint -q /sys/fs/bpf || mount bpffs /sys/fs/bpf -t bpf; mount --make-shared /sys/fs/bpf; "
29
+ "mkdir -p /run/cilium/cgroupv2; mountpoint -q /run/cilium/cgroupv2 || "
30
+ "mount -t cgroup2 none /run/cilium/cgroupv2; mount --make-shared /run/cilium/cgroupv2")
31
+
32
+
33
+ def sh(*cmd: str, timeout: int = 60) -> subprocess.CompletedProcess:
34
+ try:
35
+ return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=False)
36
+ except (subprocess.TimeoutExpired, FileNotFoundError) as exc:
37
+ return subprocess.CompletedProcess(cmd, 1, "", str(exc))
38
+
39
+
40
+ def kube(ctx: str, *args: str, timeout: int = 60) -> subprocess.CompletedProcess:
41
+ return sh("kubectl", "--context", ctx, *args, timeout=timeout)
42
+
43
+
44
+ def node_containers(cluster: str) -> dict[str, dict]:
45
+ out = sh("docker", "ps", "-a", "--format", "{{.Names}}\t{{.State}}").stdout
46
+ nodes = {}
47
+ for line in out.splitlines():
48
+ name, _, state = line.partition("\t")
49
+ if re.fullmatch(rf"k3d-{re.escape(cluster)}-(server|agent)-\d+", name):
50
+ ip = sh("docker", "inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", name).stdout.strip()
51
+ nodes[name] = {"running": state.strip() == "running", "ip": ip}
52
+ return nodes
53
+
54
+
55
+ def internal_ips(ctx: str, kind: str) -> dict[str, str]:
56
+ res = kube(ctx, "get", kind, "-o", "json", timeout=30)
57
+ if res.returncode:
58
+ return {}
59
+ out = {}
60
+ for item in json.loads(res.stdout or "{}").get("items", []):
61
+ addrs = item.get("status", {}).get("addresses") if kind == "nodes" else item.get("spec", {}).get("addresses")
62
+ for a in addrs or []:
63
+ if a.get("type") == "InternalIP":
64
+ out[item["metadata"]["name"]] = a.get("ip") or a.get("address")
65
+ return out
66
+
67
+
68
+ def bpffs(node: str) -> bool:
69
+ return sh("docker", "exec", node, "sh", "-c", "mountpoint -q /sys/fs/bpf").returncode == 0
70
+
71
+
72
+ def table(cluster: str) -> dict[str, str]:
73
+ ctx = f"k3d-{cluster}"
74
+ containers = node_containers(cluster)
75
+ if not containers:
76
+ raise SystemExit(f"error: no node container of k3d cluster {cluster}")
77
+ nodes, cnodes = internal_ips(ctx, "nodes"), internal_ips(ctx, "ciliumnodes")
78
+ states = {}
79
+ print(f"{'node':28} {'container':10} {'container IP':16} {'Node IP':16} {'CiliumNode IP':16} {'bpffs':6} state")
80
+ for name, c in sorted(containers.items()):
81
+ nip, cip = nodes.get(name, "-"), cnodes.get(name, "-")
82
+ mounted = c["running"] and bpffs(name)
83
+ if not c["running"]:
84
+ state = "STOPPED"
85
+ elif not mounted:
86
+ state = "NO BPFFS"
87
+ elif nip == "-" or cip == "-":
88
+ state = "UNKNOWN"
89
+ elif c["ip"] not in (nip, cip) or nip != cip:
90
+ state = "STALE"
91
+ else:
92
+ state = "ok"
93
+ states[name] = state
94
+ print(f"{name:28} {'running' if c['running'] else 'stopped':10} {c['ip'] or '-':16} {nip:16} {cip:16} "
95
+ f"{'yes' if mounted else 'no':6} {state}")
96
+ return states
97
+
98
+
99
+ def main() -> int:
100
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
101
+ parser.add_argument("--cluster", required=True)
102
+ parser.add_argument("--check", action="store_true")
103
+ args = parser.parse_args()
104
+ ctx = f"k3d-{args.cluster}"
105
+ states = table(args.cluster)
106
+ if args.check or all(s == "ok" for s in states.values()):
107
+ return 0 if all(s == "ok" for s in states.values()) else 1
108
+
109
+ if any(s == "STOPPED" for s in states.values()):
110
+ print(f"\nstarting k3d cluster {args.cluster}")
111
+ if sh("k3d", "cluster", "start", args.cluster, timeout=300).returncode:
112
+ print("error: k3d cluster start failed", file=sys.stderr)
113
+ return 1
114
+ for node in node_containers(args.cluster):
115
+ sh("docker", "exec", node, "sh", "-c", MOUNTS)
116
+ states = table(args.cluster)
117
+ if any(s in ("STALE", "UNKNOWN", "NO BPFFS") for s in states.values()):
118
+ print("\nrestarting Cilium")
119
+ kube(ctx, "-n", "kube-system", "rollout", "restart", "ds/cilium")
120
+ kube(ctx, "-n", "kube-system", "rollout", "status", "ds/cilium", "--timeout=180s", timeout=200)
121
+ states = table(args.cluster)
122
+ stale = [n for n, s in states.items() if s == "STALE"]
123
+ for node in stale:
124
+ print(f"\ndeleting stale CiliumNode {node} (its agent recreates it)")
125
+ kube(ctx, "delete", "ciliumnode", node)
126
+ if stale:
127
+ kube(ctx, "-n", "kube-system", "rollout", "restart", "ds/cilium")
128
+ kube(ctx, "-n", "kube-system", "rollout", "status", "ds/cilium", "--timeout=180s", timeout=200)
129
+ states = table(args.cluster)
130
+ ok = all(s == "ok" for s in states.values())
131
+ print("\nall nodes ok" if ok else "\nstill not healthy: see references/GOTCHAS.md")
132
+ return 0 if ok else 1
133
+
134
+
135
+ if __name__ == "__main__":
136
+ sys.exit(main())
@@ -0,0 +1,69 @@
1
+ #!/usr/bin/env python3
2
+ """ensure_repos.py - every repository of the profile's `repos`, cloned side by side in the workspace,
3
+ then one line each: state, branch, local changes, position against its upstream.
4
+
5
+ Clones what is missing (SSH URL by default, `https` with --https); on an existing clone it only
6
+ fetches -- never pulls, resets or switches branches. A path that exists but is not a git clone is
7
+ BLOCKED and left alone. Git never prompts (GIT_TERMINAL_PROMPT=0): a missing credential fails fast.
8
+
9
+ ensure_repos.py [--https] [--profile FILE] [--root DIR]
10
+
11
+ Exit 0 when every repository is present, 1 when a clone failed or a path is blocked.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import argparse
17
+ import os
18
+ import subprocess
19
+ import sys
20
+
21
+ from envprofile import Stack, add_common_args
22
+
23
+ ENV = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
24
+
25
+
26
+ def git(*args: str, timeout: int = 120) -> subprocess.CompletedProcess:
27
+ return subprocess.run(["git", *args], capture_output=True, text=True, env=ENV, timeout=timeout, check=False)
28
+
29
+
30
+ def main() -> int:
31
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
32
+ add_common_args(parser)
33
+ parser.add_argument("--https", action="store_true", help="clone over HTTPS instead of SSH")
34
+ args = parser.parse_args()
35
+ stack = Stack(args.profile, args.root)
36
+ print(f"workspace: {stack.root}")
37
+ failed = 0
38
+ print(f"{'REPO':26} {'STATE':8} {'BRANCH':12} {'CHANGES':7} ORIGIN")
39
+ for repo in stack.get("repos") or []:
40
+ name, path = repo["dir"], stack.root / repo["dir"]
41
+ state = "present"
42
+ if not (path / ".git").exists():
43
+ if path.exists():
44
+ print(f"{name:26} {'BLOCKED':8} {path} exists but is not a git clone; not touching it")
45
+ failed = 1
46
+ continue
47
+ url = repo.get("https") if args.https else repo.get("url")
48
+ url = url or repo.get("url") or repo.get("https")
49
+ res = git("clone", "-q", url, str(path), timeout=600)
50
+ if res.returncode:
51
+ print(f"{name:26} {'FAILED':8} {(res.stderr.strip().splitlines() or ['clone failed'])[-1]}")
52
+ failed = 1
53
+ continue
54
+ state = "cloned"
55
+ branch = git("-C", str(path), "branch", "--show-current").stdout.strip() or "detached"
56
+ changes = len([l for l in git("-C", str(path), "status", "--porcelain").stdout.splitlines() if l])
57
+ if git("-C", str(path), "fetch", "-q", "origin").returncode == 0 and \
58
+ git("-C", str(path), "rev-parse", "-q", "--verify", "@{u}").returncode == 0:
59
+ behind = git("-C", str(path), "rev-list", "--count", "HEAD..@{u}").stdout.strip()
60
+ ahead = git("-C", str(path), "rev-list", "--count", "@{u}..HEAD").stdout.strip()
61
+ origin = f"behind {behind}, ahead {ahead}"
62
+ else:
63
+ origin = "(no upstream / fetch failed)"
64
+ print(f"{name:26} {state:8} {branch:12} {changes:<7} {origin}")
65
+ return failed
66
+
67
+
68
+ if __name__ == "__main__":
69
+ sys.exit(main())
@@ -0,0 +1,139 @@
1
+ """envprofile - the stack profile and the workspace, shared by every ops-local-env script.
2
+
3
+ A stack is a set of git repositories cloned side by side in one directory, the workspace. One of
4
+ them, the anchor repository, holds the profile `ops-env.profile.json` at its root (see
5
+ references/PROFILE.md). The profile names the repositories, the config files, the keys they must
6
+ agree on, the admin login, the host names, the local CA and the install order.
7
+
8
+ Profile lookup: --profile FILE, else $SDTK_ENV_PROFILE, else the nearest directory at or above the
9
+ current one that holds the profile itself or holds a repository (one level down) that does.
10
+ Workspace: --root DIR, else $SDTK_ENV_ROOT, else the parent of the anchor repository.
11
+
12
+ Standard library only: the scripts run before any Python package is installed, and under sudo /
13
+ an elevated prompt. Config files are read and edited as flat `key: value` lines (top-level
14
+ scalars), never through a YAML dumper, so comments and alignment survive every edit.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import os
21
+ import re
22
+ import sys
23
+ from pathlib import Path
24
+
25
+ PROFILE_NAME = "ops-env.profile.json"
26
+ NAMESPACE_RX = r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
27
+ FQDN_RX = r"^([a-z0-9]([-a-z0-9]*[a-z0-9])?\.)+[a-z]{2,}$"
28
+
29
+
30
+ class ProfileError(SystemExit):
31
+ pass
32
+
33
+
34
+ def find_profile(explicit: str | None = None) -> Path:
35
+ given = explicit or os.environ.get("SDTK_ENV_PROFILE")
36
+ if given:
37
+ path = Path(given).expanduser().resolve()
38
+ if not path.is_file():
39
+ raise ProfileError(f"error: profile {path} not found")
40
+ return path
41
+ for d in [Path.cwd(), *Path.cwd().parents]:
42
+ if (d / PROFILE_NAME).is_file():
43
+ return d / PROFILE_NAME
44
+ hits = sorted(d.glob(f"*/{PROFILE_NAME}"))
45
+ if len(hits) == 1:
46
+ return hits[0]
47
+ if len(hits) > 1:
48
+ raise ProfileError(f"error: several profiles under {d}: {', '.join(map(str, hits))}; pass --profile")
49
+ raise ProfileError(f"error: no {PROFILE_NAME} at or above {Path.cwd()}; pass --profile FILE (references/PROFILE.md)")
50
+
51
+
52
+ class Stack:
53
+ """The loaded profile plus the workspace it applies to."""
54
+
55
+ def __init__(self, profile: str | None = None, root: str | None = None):
56
+ self.profile_path = find_profile(profile)
57
+ try:
58
+ self.data = json.loads(self.profile_path.read_text(encoding="utf-8"))
59
+ except json.JSONDecodeError as exc:
60
+ raise ProfileError(f"error: {self.profile_path}: {exc}")
61
+ given = root or os.environ.get("SDTK_ENV_ROOT")
62
+ self.root = Path(given).expanduser().resolve() if given else self.profile_path.parent.parent
63
+ if not self.root.is_dir():
64
+ raise ProfileError(f"error: workspace {self.root} is not a directory")
65
+ self.name = self.data.get("name") or "stack"
66
+ self.configs = {k: self.root / v for k, v in (self.data.get("configs") or {}).items()}
67
+ self.primary = self.data.get("primary_config") or next(iter(self.configs), None)
68
+
69
+ def get(self, key: str, default=None):
70
+ return self.data.get(key, default)
71
+
72
+ def path(self, rel: str) -> Path:
73
+ return self.root / rel
74
+
75
+ def primary_value(self, key: str) -> str | None:
76
+ if not self.primary:
77
+ return None
78
+ return load_flat(self.configs[self.primary]).get(key)
79
+
80
+ def expand(self, template: str) -> str:
81
+ """`{key}` placeholders from the primary config (e.g. certs/{base_domain})."""
82
+ return re.sub(r"\{([a-z0-9_]+)\}", lambda m: self.primary_value(m.group(1)) or m.group(0), template)
83
+
84
+
85
+ def add_common_args(parser) -> None:
86
+ parser.add_argument("--profile", metavar="FILE", help=f"the stack profile ({PROFILE_NAME})")
87
+ parser.add_argument("--root", metavar="DIR", help="workspace holding the stack's repositories")
88
+
89
+
90
+ TOP_LINE = re.compile(r"^([A-Za-z0-9_]+):[ \t]*(.*)$", re.M)
91
+
92
+
93
+ def scalar(raw: str) -> str | None:
94
+ """The value of a flat YAML scalar: "double" (escapes), 'single', or plain up to ` #`."""
95
+ if raw.startswith('"'):
96
+ m = re.match(r'"((?:[^"\\]|\\.)*)"', raw)
97
+ return re.sub(r"\\(.)", r"\1", m.group(1)) if m else raw
98
+ if raw.startswith("'"):
99
+ m = re.match(r"'((?:[^']|'')*)'", raw)
100
+ return m.group(1).replace("''", "'") if m else raw
101
+ plain = re.split(r"[ \t]#", raw, maxsplit=1)[0].strip()
102
+ return plain or None
103
+
104
+
105
+ def load_flat(path: Path) -> dict[str, str | None]:
106
+ """Top-level scalar keys of a config file ({} when it does not exist)."""
107
+ if not path.is_file():
108
+ return {}
109
+ return {k: scalar(v.strip()) for k, v in TOP_LINE.findall(path.read_text(encoding="utf-8"))}
110
+
111
+
112
+ def set_line(text: str, key: str, value: str, append: bool = False) -> str:
113
+ """Replace the value on the `key: "value" # comment` line, keeping the comment and its column.
114
+ A missing key is appended only when `append` (secrets files); otherwise the text is unchanged."""
115
+ pattern = re.compile(
116
+ rf'^({re.escape(key)}:[ \t]*)("(?:[^"\\\n]|\\.)*"|\'[^\'\n]*\'|[^\n]*?)([ \t]*(?:(?<=[ \t])#.*)?)$', re.M)
117
+
118
+ def repl(m: re.Match) -> str:
119
+ width = len(m.group(2)) + len(m.group(3)) - len(m.group(3).lstrip())
120
+ body = f'"{value}"'
121
+ pad = " " * max(1, width - len(body)) if m.group(3).strip() else ""
122
+ return f"{m.group(1)}{body}{pad}{m.group(3).lstrip()}"
123
+
124
+ new, n = pattern.subn(repl, text, count=1)
125
+ if n:
126
+ return new
127
+ if append:
128
+ return text + ("" if not text or text.endswith("\n") else "\n") + f'{key}: "{value}"\n'
129
+ return text
130
+
131
+
132
+ def under(value: str, base: str) -> bool:
133
+ """`value` is exactly one label under `base` (the certificate covers base and *.base only)."""
134
+ return bool(base) and value.endswith("." + base) and "." not in value[: -len(base) - 1]
135
+
136
+
137
+ def fail(msg: str) -> int:
138
+ sys.stderr.write(msg.rstrip("\n") + "\n")
139
+ return 1
@@ -0,0 +1,116 @@
1
+ #!/usr/bin/env python3
2
+ """explain_config.py - print a config file (commented YAML) as a Markdown table: section, key, value,
3
+ and the explanation taken from the file's own comments.
4
+
5
+ explain_config.py FILE [FILE ...]
6
+
7
+ Top-level keys only; a nested list or mapping is summarised ("3 items: relay, api, web"). Values of
8
+ keys that look secret (password, secret, token, *_key) are masked. A `# ...` line on its own starts a
9
+ new section; a trailing `# ...` is the explanation. Uses PyYAML when present, otherwise reads the
10
+ file as flat lines (standard library only).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import re
16
+ import sys
17
+ from pathlib import Path
18
+
19
+ from envprofile import load_flat
20
+
21
+ try:
22
+ import yaml
23
+ except ImportError: # the flat reader is enough for a table
24
+ yaml = None
25
+
26
+ SECRET = re.compile(r"password|passwd|secret|token|_key$", re.I)
27
+ TOP = re.compile(r"^([A-Za-z0-9_.-]+):(.*)$")
28
+
29
+
30
+ def comment_of(rest: str) -> str:
31
+ inside = False
32
+ for i, ch in enumerate(rest):
33
+ if ch == '"':
34
+ inside = not inside
35
+ elif ch == "#" and not inside:
36
+ return rest[i + 1:].strip()
37
+ return ""
38
+
39
+
40
+ def summarise(value) -> str:
41
+ if isinstance(value, list):
42
+ names = [str(v.get("name", "")) if isinstance(v, dict) else str(v) for v in value]
43
+ names = [n for n in names if n]
44
+ return f"{len(value)} items" + (f": {', '.join(names)}" if names else "")
45
+ if isinstance(value, dict):
46
+ return f"{len(value)} keys: {', '.join(map(str, value))}"
47
+ return "" if value is None else str(value)
48
+
49
+
50
+ def nested_summary(lines: list[str], start: int) -> str:
51
+ """Flat reader: a key with no value followed by indented lines."""
52
+ items = []
53
+ for line in lines[start + 1:]:
54
+ if line and not line[0].isspace() and not line.startswith("#"):
55
+ break
56
+ m = re.match(r"^\s*-\s*(?:name:\s*)?\"?([^\"#:]+?)\"?\s*(?:#.*)?$", line)
57
+ if m:
58
+ items.append(m.group(1).strip())
59
+ return f"{len(items)} items: {', '.join(items)}" if items else "(nested)"
60
+
61
+
62
+ def explain(path: Path) -> None:
63
+ text = path.read_text(encoding="utf-8")
64
+ data = (yaml.safe_load(text) or {}) if yaml else None
65
+ flat = None if yaml else load_flat(path)
66
+ lines = text.splitlines()
67
+ print(f"\n**{path}**\n")
68
+ print("| section | key | value | explanation |\n|---|---|---|---|")
69
+ section, block = "", []
70
+ cell = lambda s: s.replace("|", "\\|")
71
+ for i, line in enumerate(lines):
72
+ if line.startswith("#"):
73
+ block.append(line.lstrip("#").strip())
74
+ continue
75
+ if not line.strip():
76
+ if block:
77
+ section = " ".join(block)
78
+ block = []
79
+ continue
80
+ m = TOP.match(line)
81
+ if not m:
82
+ block = []
83
+ continue
84
+ key = m.group(1)
85
+ if data is not None:
86
+ raw = data.get(key)
87
+ value = summarise(raw)
88
+ else:
89
+ raw = flat.get(key)
90
+ value = raw if raw is not None else nested_summary(lines, i)
91
+ if SECRET.search(key) and raw:
92
+ value = "********"
93
+ value = value if len(value) <= 60 else value[:57] + "..."
94
+ note = comment_of(m.group(2))
95
+ if block and not note and len(block) > 1:
96
+ note = " ".join(block)
97
+ elif block:
98
+ section = " ".join(block)
99
+ block = []
100
+ print(f"| {cell(section)} | `{key}` | {cell(value)} | {cell(note)} |")
101
+
102
+
103
+ def main() -> int:
104
+ if len(sys.argv) < 2:
105
+ sys.exit(__doc__)
106
+ for arg in sys.argv[1:]:
107
+ path = Path(arg)
108
+ if not path.is_file():
109
+ sys.stderr.write(f"error: {path} not found\n")
110
+ return 1
111
+ explain(path)
112
+ return 0
113
+
114
+
115
+ if __name__ == "__main__":
116
+ sys.exit(main())