sdtk-ops-kit 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/assets/manifest/toolkit-bundle.manifest.json +144 -19
- package/assets/manifest/toolkit-bundle.sha256.txt +35 -10
- package/assets/toolkit/toolkit/AGENTS.md +2 -1
- package/assets/toolkit/toolkit/SDTKOPS_TOOLKIT.md +4 -2
- package/assets/toolkit/toolkit/scripts/install-claude-skills.ps1 +4 -4
- package/assets/toolkit/toolkit/scripts/install-codex-skills.ps1 +4 -4
- package/assets/toolkit/toolkit/scripts/uninstall-claude-skills.ps1 +2 -2
- package/assets/toolkit/toolkit/scripts/uninstall-codex-skills.ps1 +2 -2
- package/assets/toolkit/toolkit/skills/ops-discover/SKILL.md +2 -0
- package/assets/toolkit/toolkit/skills/ops-framer-deploy/SKILL.md +15 -1
- package/assets/toolkit/toolkit/skills/ops-framer-deploy/package.json +10 -0
- package/assets/toolkit/toolkit/skills/ops-framer-deploy/scripts/framer-install.mjs +5 -1
- package/assets/toolkit/toolkit/skills/ops-framer-deploy/scripts/lib/config.mjs +6 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/SKILL.md +172 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/CLUSTER_CA.md +83 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/GOTCHAS.md +17 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/K3D_CILIUM_BOOTSTRAP.md +39 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/PROFILE.md +44 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/bootstrap/cilium-values.yaml +37 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/bootstrap/k3d-cluster.yaml +37 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/references/profile.example.json +80 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/admin_creds.py +212 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/cilium_heal.py +136 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/ensure_repos.py +69 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/envprofile.py +139 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/explain_config.py +116 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/health.py +109 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/hosts_entry.py +158 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/init_profile.py +217 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/k3d_images.py +70 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/preflight.py +96 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/registry_secret.py +64 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/shared_config.py +178 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/stack_plan.py +94 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/teardown.py +86 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/test_local_env.py +307 -0
- package/assets/toolkit/toolkit/skills/ops-local-env/scripts/trust_ca.py +153 -0
- package/package.json +1 -1
- package/src/lib/scope.js +1 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: ops-local-env
|
|
3
|
+
description: Local test environment for a multi-repository stack on Kubernetes (k3d). Use when setting up, re-running or repairing a project's local cluster end to end -- confirming tools, registry token, repositories, shared config values and one admin login, then bootstrapping the cluster and its local CA and installing every component in order through each repository's own installer, hosts file and CA trust included -- or when changing a shared domain / namespace across all repositories.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Ops Local Env
|
|
7
|
+
|
|
8
|
+
A test environment is a stack: several repositories cloned side by side in one workspace, each with
|
|
9
|
+
its own config file and installer, one local cluster, one local CA. Everything particular to a stack —
|
|
10
|
+
repositories, config files, shared keys, admin login, host names, CA, install order, health checks —
|
|
11
|
+
lives in its profile, `ops-env.profile.json` in the anchor repository (`references/PROFILE.md`,
|
|
12
|
+
`references/profile.example.json`). The skill and its scripts hold no stack names. Paths below
|
|
13
|
+
(`scripts/`, `references/`) are relative to this skill's folder; run the scripts with `python3`.
|
|
14
|
+
|
|
15
|
+
## Critical Constraints
|
|
16
|
+
|
|
17
|
+
1. **Nothing in a config file or in the cluster changes before step 0 is confirmed** (the Apply
|
|
18
|
+
summary gets a yes). Before that, only installing the missing Python packages / tools the person
|
|
19
|
+
agrees to, and cloning missing repositories after a yes.
|
|
20
|
+
2. **Never tear down** — no cluster delete, namespace / PVC deletion, `--reset`, destroy script —
|
|
21
|
+
except the reset of a **local k3d cluster** the person picked (step 0.5, or a teardown they asked
|
|
22
|
+
for), always through `scripts/teardown.py --apply --confirm <name>` after one more confirmation naming
|
|
23
|
+
the cluster. Never offer a reset of a non-k3d context.
|
|
24
|
+
3. **Confirm the kube context** (`kubectl config current-context`) at the start and before every
|
|
25
|
+
component that changes the cluster.
|
|
26
|
+
4. **Secrets stay out of sight.** Never ask for a registry token in chat, echo it or put it on a
|
|
27
|
+
command line — it lives in the environment variable the profile names. Never print a password or
|
|
28
|
+
a private key; the shared admin password is printed once, in the final summary, by
|
|
29
|
+
`admin_creds.py --reveal`. A typed password goes in through `--password-stdin` from a quoted
|
|
30
|
+
heredoc, never as an argument. Passwords live only in gitignored secrets files: never stage them
|
|
31
|
+
and never `git diff` / `git show` / `git log -p` a password file the script reports as tracked.
|
|
32
|
+
5. **Each component's own installer and its rules win.** Run the repository's own skill or command;
|
|
33
|
+
keep its gates (config confirmation, deploy go-ahead). Settled values (context, shared values,
|
|
34
|
+
admin login) count as already stated, so its interview covers only its own keys.
|
|
35
|
+
6. **Elevated commands go to the person.** Hosts file and trust store need root / Administrator: the
|
|
36
|
+
scripts print the exact command; hand it over, wait, re-check. Never run `sudo` yourself.
|
|
37
|
+
7. **Config files are edited line by line by the scripts** (value only, comment kept, read back),
|
|
38
|
+
never rewritten with a YAML dumper, never by hand when a script covers the key.
|
|
39
|
+
8. **Do not commit or push** anything unless the person asks.
|
|
40
|
+
|
|
41
|
+
## Process
|
|
42
|
+
|
|
43
|
+
The person's request picks the mode: **install / re-run** (steps 0–4), **health** ("is it still up?":
|
|
44
|
+
`scripts/health.py`, read-only, nothing to confirm), **teardown** (section below), or **new profile**.
|
|
45
|
+
|
|
46
|
+
**No profile yet:** `scripts/init_profile.py --root <workspace>` drafts one from the cloned repositories
|
|
47
|
+
(read-only). Go through its `_todo` list with the person, fix the draft, `--write` it into the anchor
|
|
48
|
+
repository, remove `_todo`, and `scripts/stack_plan.py --check` must pass before step 0.
|
|
49
|
+
|
|
50
|
+
0. **Confirm before anything** (below): admin login → tools / Python / token → repositories → core
|
|
51
|
+
values → existing clusters → extra changes → Apply summary.
|
|
52
|
+
1. **Cluster and CA** — the profile's first component (bootstrap), then its health checks.
|
|
53
|
+
2. **Components** in profile order, each through its own installer, health-checked; the CA loaded as
|
|
54
|
+
ClusterIssuer right after the component that installs cert-manager; the registry pull secret
|
|
55
|
+
before the first component that needs it; admin Secrets once their namespace exists.
|
|
56
|
+
3. **Ingresses, hosts file, CA trust**, then names resolve and answer.
|
|
57
|
+
4. **Final summary** — one table, plus the admin login printed once.
|
|
58
|
+
|
|
59
|
+
Once the Apply summary is confirmed the run goes **end to end without asking between components**.
|
|
60
|
+
It stops only at a component's own gate and on a failure (then: retry / skip this component / stop).
|
|
61
|
+
|
|
62
|
+
## 0. Confirm before anything
|
|
63
|
+
|
|
64
|
+
Ask one decision at a time or a few at once (at most 4 per round), with the current value first as
|
|
65
|
+
`Keep <value> (current)`. Use the runtime's question tool when it has one.
|
|
66
|
+
|
|
67
|
+
- **0.0 Admin login first.** `scripts/admin_creds.py --show` (standard library, runs before any
|
|
68
|
+
install): users per file, passwords as `set` / `EMPTY`. Ask: user (keep / new, `^[a-z][a-z0-9_-]{2,31}$`)
|
|
69
|
+
and password (keep — only when all are set and equal / generate / type my own). "Type my own": say it
|
|
70
|
+
will stay in the conversation transcript, ask for it in a plain message, check 12–64 characters,
|
|
71
|
+
no space and none of `"` `'` `` ` `` `\` `$`; never repeat it. A password once in a tracked file
|
|
72
|
+
(`tracked by git`) should not be kept: recommend generate.
|
|
73
|
+
- **0.1 Tools and Python.** `scripts/preflight.py`. `tools MISSING`: name them and stop. `python3
|
|
74
|
+
MISSING`: ask, then `python3 -m pip install --user <packages>` (`--break-system-packages` only after
|
|
75
|
+
asking again when pip refuses with `externally-managed-environment`). Re-run until OK.
|
|
76
|
+
- **0.2 Registry token.** `<TOKEN> MISSING / WRONG`: ask the person to export it in their own shell
|
|
77
|
+
profile and restart the agent session from that shell (for GitHub: a **classic** PAT with the
|
|
78
|
+
profile's scope; fine-grained PATs get 403). Re-run preflight; show the `OK` line (owner, scopes).
|
|
79
|
+
- **0.3 Repositories.** `scripts/ensure_repos.py [--https]`: clones what is missing, never pulls.
|
|
80
|
+
`FAILED` / `BLOCKED`: stop. Behind with no local changes: ask whether to `git pull --ff-only`. Local
|
|
81
|
+
changes or not on the default branch: report, do not touch, ask whether to go on as is. Then
|
|
82
|
+
`scripts/stack_plan.py --check` (profile problems stop the run) and `scripts/shared_config.py --show`.
|
|
83
|
+
- **0.4 Core values** (`core_keys` of the profile). `scripts/explain_config.py <primary config>`
|
|
84
|
+
first, then ask them in one round. Do not ask for other domains: every `*_domain` is one label
|
|
85
|
+
under the base domain and follows it.
|
|
86
|
+
- **0.5 Existing clusters.** `kubectl config get-contexts`, `k3d cluster list`: one table (context,
|
|
87
|
+
cluster, current, local k3d or not). For each local k3d cluster ask `Keep` / `Reset` — say a reset
|
|
88
|
+
deletes everything in it; record, delete nothing yet. For each kept one run the profile's
|
|
89
|
+
`cluster.heal` script with `--check` (none: `scripts/cilium_heal.py --cluster <name> --check`) and
|
|
90
|
+
report what step 1 will heal. Other contexts: list only.
|
|
91
|
+
Ask the scenario: everything, or which components (the bootstrap, the CA issuer, hosts and trust run
|
|
92
|
+
anyway; a CA re-issue forces every component).
|
|
93
|
+
- **0.6 Anything else?** Ingresses to enable, a domain's first label, TLS subject — only what the
|
|
94
|
+
person names.
|
|
95
|
+
- **Apply.** One summary table: every shared value old → new, every `*_domain` that follows a new
|
|
96
|
+
base domain, admin user old → new and password keep / generate, kube context, clusters to **reset**
|
|
97
|
+
(bold), components to install, Ingresses. Say that a yes starts the whole run and that a change of a
|
|
98
|
+
value in the certificate's SAN list (base domain, namespace, database name) means a new CA
|
|
99
|
+
(`references/CLUSTER_CA.md`). On yes:
|
|
100
|
+
`shared_config.py --set K=V ...` (dry run, show diff) then `--apply`;
|
|
101
|
+
`admin_creds.py [--user N] [--generate | --password-stdin]` (dry run, masked) then `--apply`;
|
|
102
|
+
re-run both `--show`: no `MISMATCH`, users and passwords `all the same`.
|
|
103
|
+
|
|
104
|
+
## 1–2. Cluster, components, CA
|
|
105
|
+
|
|
106
|
+
For each component of `scripts/stack_plan.py` in order:
|
|
107
|
+
|
|
108
|
+
1. Work from its repository. Show `scripts/explain_config.py` of its configs (secrets masked), no question.
|
|
109
|
+
Its `notes` in the profile are rules of this stack: apply them to its run (they win over its defaults).
|
|
110
|
+
2. Reset (only the one confirmed in 0.5): ask once more naming the cluster, `k3d cluster delete
|
|
111
|
+
<name>`, check it is gone. Kept cluster: heal it first (profile `cluster.heal`, else `scripts/cilium_heal.py --cluster <name>`;
|
|
112
|
+
it must end healthy). A stack without a bootstrap yet: `references/K3D_CILIUM_BOOTSTRAP.md`.
|
|
113
|
+
3. Run it: its skill (invoke it if listed; else read `<repo>/.claude/skills/<skill>/SKILL.md` or
|
|
114
|
+
`<repo>/.codex/skills/...` in full and follow it), or its `run` command. A `long` one runs in the
|
|
115
|
+
background with a log file and is checked when it exits.
|
|
116
|
+
4. Health: its `health` commands (`{key}` filled from the primary config). Healthy: one line, go on.
|
|
117
|
+
Not healthy: diagnose with that component's troubleshooting and `references/GOTCHAS.md`, then ask
|
|
118
|
+
retry / skip / stop.
|
|
119
|
+
|
|
120
|
+
At the right moments, without a question (part of the confirmed run):
|
|
121
|
+
- after the component named in `ca.issuer_after`: load the CA as `ca.cluster_issuer`
|
|
122
|
+
(`references/CLUSTER_CA.md`; a **different** root already in the cluster: ask before rotating);
|
|
123
|
+
- before the first component with `needs_registry_secret`: `scripts/registry_secret.py <namespace>`;
|
|
124
|
+
- once the apps namespace exists: `scripts/admin_creds.py --k8s-secrets <namespace>` when the profile
|
|
125
|
+
lists `admin.k8s_secrets`;
|
|
126
|
+
- locally built images: `scripts/k3d_images.py --cluster <name> <image> ...` before the component
|
|
127
|
+
that runs them.
|
|
128
|
+
|
|
129
|
+
## 3. Ingresses, hosts, trust
|
|
130
|
+
|
|
131
|
+
1. Render (`ingress.render`), check every host / namespace / `secretName` against the agreed values
|
|
132
|
+
(wrong: fix the config, never the rendered file), `kubectl diff -f <dir>`, apply, diff again = 0.
|
|
133
|
+
One TLS Secret per host; no two Ingresses on the same host and path; Envoy rejected count 0.
|
|
134
|
+
2. `scripts/hosts_entry.py --show`; not up to date: hand over the command it prints, wait, re-check.
|
|
135
|
+
3. `scripts/trust_ca.py --show`; not trusted: hand over the command; `name constraints: NONE`: back to
|
|
136
|
+
the re-issue. A re-issued CA leaves the old one in the store: hand over its removal.
|
|
137
|
+
4. Every name resolves to the profile's IP and answers over HTTPS (`curl -s -o /dev/null -w
|
|
138
|
+
'%{http_code}'`); an Ingress not applied is reported as such, not as a failure.
|
|
139
|
+
|
|
140
|
+
## 4. Final summary
|
|
141
|
+
|
|
142
|
+
One table: component, namespace, status, URL, anything left for the person. Then the admin login,
|
|
143
|
+
once: `scripts/admin_creds.py --reveal` (refuses when they differ — show `--show` instead); say which
|
|
144
|
+
UIs use it and that it is now in the transcript. Services with no Ingress: give the `kubectl
|
|
145
|
+
port-forward` command.
|
|
146
|
+
|
|
147
|
+
## Health, any time
|
|
148
|
+
|
|
149
|
+
`scripts/health.py [--only <id> ...]` runs every component's `health` commands (filled from the
|
|
150
|
+
primary config) and prints one OK / FAIL table. It runs only read-only commands and reports a changing
|
|
151
|
+
one as REFUSED. After a reboot, a FAIL on every network check with Running pods: heal the cluster
|
|
152
|
+
(`cluster.heal`, `references/GOTCHAS.md`), then run it again.
|
|
153
|
+
|
|
154
|
+
## Teardown (only when the person asks for it)
|
|
155
|
+
|
|
156
|
+
1. `scripts/teardown.py` — plan only: the k3d cluster the profile names, its nodes, and what stays
|
|
157
|
+
(repositories, config and secrets, the local CA, other clusters and contexts).
|
|
158
|
+
2. Show it and ask, naming the cluster — everything in it is lost (databases, buckets, queues).
|
|
159
|
+
3. `scripts/teardown.py --apply --confirm <name>`; it checks the cluster is gone.
|
|
160
|
+
4. Hand over the hosts-block removal it prints (`hosts_entry.py --remove`, elevated). Keep the CA
|
|
161
|
+
trusted unless the person also wants that gone.
|
|
162
|
+
|
|
163
|
+
## Common mistakes
|
|
164
|
+
|
|
165
|
+
| Mistake | Instead |
|
|
166
|
+
|---|---|
|
|
167
|
+
| Asking "continue?" after every healthy component | Go straight on; stop only at gates and failures |
|
|
168
|
+
| Re-asking the admin password inside a component's installer | Hand over the settled login; it counts as stated |
|
|
169
|
+
| Editing a rendered file to fix a host | Fix the config value, re-render |
|
|
170
|
+
| Deleting a cluster or volume to "start clean" | Only the reset confirmed in step 0.5 |
|
|
171
|
+
| Running `sudo` from the agent shell | Hand the printed command to the person |
|
|
172
|
+
| Recreating a Secret that holds state (vault / encryption key) | Never; missing with data present: stop and ask |
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# One local CA for the whole stack
|
|
2
|
+
|
|
3
|
+
The bootstrap step creates `ca.crt` / `ca.key` (the profile's `ca.dir`) once per machine. That CA is
|
|
4
|
+
the only root: it signs the Ingress leaf (`tls.crt`, one `<base_domain>-tls` Secret per namespace) and,
|
|
5
|
+
loaded as a cert-manager ClusterIssuer (`ca.cluster_issuer`), every certificate issued inside the
|
|
6
|
+
cluster (database, broker, internal service planes). Trusting it on this machine trusts all of them.
|
|
7
|
+
|
|
8
|
+
## Name constraints
|
|
9
|
+
|
|
10
|
+
The CA must be name-constrained to the names it is meant to sign: `<base_domain>`, `*.svc`,
|
|
11
|
+
`*.cluster.local`, `<apps_namespace>` (for `<svc>.<ns>`) and any bare in-cluster names cert-manager
|
|
12
|
+
issues. OpenSSL, Go and browsers reject the **whole** certificate when one SAN is outside the
|
|
13
|
+
constraints, and an unconstrained local CA trusted by the OS could sign any site. `trust_ca.py
|
|
14
|
+
--apply` refuses one.
|
|
15
|
+
|
|
16
|
+
Constraints are fixed when the CA is made: a new namespace, database name or bare name means a new CA.
|
|
17
|
+
|
|
18
|
+
## Re-issue (CA and leaf)
|
|
19
|
+
|
|
20
|
+
When the CA has no constraints, lacks a needed name (`openssl x509 -in ca.crt -noout -ext
|
|
21
|
+
nameConstraints`), or a value in the SAN list changed:
|
|
22
|
+
|
|
23
|
+
1. Ask first. Move the folder aside (`mv <ca.dir> <ca.dir>.old-<yyyymmdd>`; leaf only: move `tls.*`).
|
|
24
|
+
Never delete it; it is gitignored and per machine.
|
|
25
|
+
2. Re-run the profile's render / bootstrap command; check `Name Constraints` and the new leaf's SANs.
|
|
26
|
+
3. Keep the old CA's SHA-1 (`openssl x509 -noout -fingerprint -sha1`) to remove it from the trust store.
|
|
27
|
+
4. Every component re-deploys so it copies the new files; existing `<base_domain>-tls` Secrets are
|
|
28
|
+
overwritten; the ClusterIssuer is rotated (below); the new CA is trusted and the old one removed.
|
|
29
|
+
|
|
30
|
+
## Load the CA as the ClusterIssuer (after cert-manager exists)
|
|
31
|
+
|
|
32
|
+
`ca.issuer_after` names the component that installs cert-manager. Right after it deploys — before its
|
|
33
|
+
health check when its own pods wait for a certificate from this issuer:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
kubectl get crd certificates.cert-manager.io # missing: that component has not run
|
|
37
|
+
openssl x509 -in $CA/ca.crt -noout -fingerprint -sha256
|
|
38
|
+
kubectl -n cert-manager get secret <issuer> -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -noout -fingerprint -sha256
|
|
39
|
+
kubectl get clusterissuer <issuer> -o jsonpath='{.spec.ca.secretName} {.status.conditions[?(@.type=="Ready")].status}'
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Same fingerprint and issuer Ready: nothing to do. No Secret: create it and the issuer, no question.
|
|
43
|
+
**A different root** on a cluster that already has certificates: ask — rotating replaces every
|
|
44
|
+
certificate it signed at once.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
kubectl -n cert-manager create secret generic <issuer> --type=kubernetes.io/tls \
|
|
48
|
+
--from-file=ca.crt=$CA/ca.crt --from-file=ca.key=$CA/ca.key \
|
|
49
|
+
--from-file=tls.crt=$CA/ca.crt --from-file=tls.key=$CA/ca.key \
|
|
50
|
+
--dry-run=client -o yaml | kubectl apply -f - >/dev/null # piped, never printed: it holds the key
|
|
51
|
+
kubectl apply -f - <<'EOF'
|
|
52
|
+
apiVersion: cert-manager.io/v1
|
|
53
|
+
kind: ClusterIssuer
|
|
54
|
+
metadata:
|
|
55
|
+
name: <issuer>
|
|
56
|
+
spec:
|
|
57
|
+
ca:
|
|
58
|
+
secretName: <issuer>
|
|
59
|
+
EOF
|
|
60
|
+
kubectl wait --for=condition=Ready clusterissuer/<issuer> --timeout=60s
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
A `Certificate` named like the issuer in `cert-manager` (a self-signed layout) owns the Secret and
|
|
64
|
+
would rewrite it: delete that Certificate, its self-signed issuer and the Secret first.
|
|
65
|
+
|
|
66
|
+
### Rotation
|
|
67
|
+
|
|
68
|
+
Re-issue every Certificate of the issuer by deleting its Secret, then restart what mounts them:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
kubectl get certificate -A -o jsonpath='{range .items[?(@.spec.issuerRef.name=="<issuer>")]}{.metadata.namespace} {.spec.secretName}{"\n"}{end}' \
|
|
72
|
+
| while read ns sec; do kubectl -n "$ns" delete secret "$sec"; done
|
|
73
|
+
kubectl wait --for=condition=Ready certificate -A --all --timeout=120s
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Then `kubectl rollout restart` every Deployment / StatefulSet mounting a re-issued Secret (operators
|
|
77
|
+
such as PGO have their own restart annotation), re-run any step that copied the CA into a ConfigMap,
|
|
78
|
+
and check each re-issued Secret's `ca.crt` equals the new root (`openssl sha256`).
|
|
79
|
+
|
|
80
|
+
## Never
|
|
81
|
+
|
|
82
|
+
Print, copy, commit or share `ca.key`; copy the issuer Secret to another namespace; trust a CA without
|
|
83
|
+
name constraints.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Gotchas of local Kubernetes test environments
|
|
2
|
+
|
|
3
|
+
| Symptom | Cause | Do |
|
|
4
|
+
|---|---|---|
|
|
5
|
+
| After a reboot every pod is Running but Services / Ingresses are dead | Docker gave the k3d node containers new IPs; Cilium keeps the old ones in its CiliumNode objects | Run the profile's `cluster.heal` script, else `scripts/cilium_heal.py --cluster <name>` (`--check` first): it mounts bpffs, restarts Cilium, then deletes a CiliumNode still carrying an old IP so its agent recreates it |
|
|
6
|
+
| `helm upgrade --wait` never returns, clustermesh-apiserver stuck in `Init` | ClusterMesh enabled without `clustermesh.config.enabled: true` (the chart does not render `clustermesh-remote-users`) | Stop the upgrade, fix the values, upgrade again; check `helm get values` has nothing the values file lacks before upgrading |
|
|
7
|
+
| New Ingress routes silently never appear | Two Ingresses on one host with different `secretName`s: Envoy rejects the whole config and keeps the last good one | One TLS Secret per host name (`<base_domain>-tls`); `kubectl -n kube-system logs ds/cilium-envoy --since=2m \| grep -c rejected` must be 0 |
|
|
8
|
+
| One of two routes unreachable | Two Ingresses with the same host and path: one silently wins (`--dry-run=server` does not catch it) | `kubectl get ingress -A` and look for duplicates of host + path |
|
|
9
|
+
| A certificate is rejected although cert-manager issued it | One SAN outside the CA's name constraints | `openssl verify -x509_strict -CAfile ca.crt <cert>`; add the name and re-issue the CA (CLUSTER_CA.md) |
|
|
10
|
+
| `000` from curl although the name resolves elsewhere | Hosts file not updated or DNS cache | `hosts_entry.py --show`; `curl --resolve <host>:443:127.0.0.1` to tell them apart |
|
|
11
|
+
| TLS error with `curl -s` but not `-k` | CA not in the store this client reads | `trust_ca.py --show`; Firefox, Java (`cacerts`) and Node (`NODE_EXTRA_CA_CERTS`) have their own stores; Homebrew curl has its own bundle; WSL has its own hosts and store |
|
|
12
|
+
| Changing the namespace "renamed" nothing | The k3d cluster is named after it: a new cluster is created next to the old one | Say so before Apply; the old one stays until the person removes it |
|
|
13
|
+
| A UI still takes the old admin password | Some apps keep the first admin password in their database or read the Secret only on first boot | After a password change, log in to each UI and report which did not take |
|
|
14
|
+
| A stateful component cannot read its own data after a rebuild | A key created once (vault / encryption key) was created again | Treat such Secrets as state: never recreate them; missing while the data exists: stop and ask for the old key |
|
|
15
|
+
| Services refuse every token after the identity provider is configured | They read its discovery document / key set at start-up, before it was activated | Start or restart token-verifying services only after the identity step has passed |
|
|
16
|
+
| Pods `ErrImagePull` "no match for platform" for a locally imported image | Image built for another architecture | `k3d_images.py --cluster <name> <image>`; rebuild or re-import for the nodes' platform |
|
|
17
|
+
| macOS refuses a TLS server certificate | Valid more than 825 days, or the leaf outlives its CA | Keep both at 365 days or less |
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Bootstrapping a local k3d + Cilium cluster
|
|
2
|
+
|
|
3
|
+
For a stack whose anchor repository has no bootstrap yet. The templates are in `bootstrap/`; copy them
|
|
4
|
+
into the anchor repository, fill the placeholders, and point the profile's first component at the
|
|
5
|
+
result. These are starting points, not tested against every k3s / Cilium version: pin both versions and
|
|
6
|
+
keep what works in the anchor repository.
|
|
7
|
+
|
|
8
|
+
## Steps (the bootstrap component's `run`)
|
|
9
|
+
|
|
10
|
+
1. `k3d cluster list` — the cluster exists: heal it (`scripts/cilium_heal.py --cluster <name>`) and stop
|
|
11
|
+
here if it ends healthy. It does not: create it.
|
|
12
|
+
2. `k3d cluster create --config k3d-cluster.yaml` (flannel, kube-proxy, network policy, Traefik and
|
|
13
|
+
ServiceLB off; host 80/443 → NodePorts 30080/30443).
|
|
14
|
+
3. Mount what Cilium needs in every node container — they are lost at every restart:
|
|
15
|
+
`scripts/cilium_heal.py --cluster <name>` does it (bpffs, cgroup2), or by hand per node
|
|
16
|
+
`docker exec <node> sh -c 'mount bpffs /sys/fs/bpf -t bpf && mount --make-shared /sys/fs/bpf'`.
|
|
17
|
+
4. `helm install cilium cilium/cilium --version <pinned> -n kube-system -f cilium-values.yaml
|
|
18
|
+
--kube-context k3d-<name> --wait --timeout 10m`.
|
|
19
|
+
5. Nodes become Ready only once Cilium runs: `kubectl get nodes` all Ready;
|
|
20
|
+
`kubectl -n kube-system get pods -l k8s-app=cilium` Running;
|
|
21
|
+
`kubectl -n kube-system rollout status deploy/clustermesh-apiserver --timeout=180s`.
|
|
22
|
+
6. TLS Secrets for the Ingresses: `<base_domain>-tls` from the local CA's leaf in every namespace that
|
|
23
|
+
serves an Ingress (`kubectl create secret tls ... --dry-run=client -o yaml | kubectl apply -f -`;
|
|
24
|
+
an existing one is replaced only after asking).
|
|
25
|
+
|
|
26
|
+
## Profile entries
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
"cluster": {"kind": "k3d", "name_key": "apps_namespace"},
|
|
30
|
+
"components": [{"id": "cluster", "repo": "<anchor>", "run": "cd <anchor>/k8s && ./bootstrap.sh",
|
|
31
|
+
"long": true, "health": ["kubectl get nodes", "kubectl -n kube-system get pods -l k8s-app=cilium"]}]
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Without `cluster.heal`, the skill uses `scripts/cilium_heal.py --cluster <name>` after a reboot.
|
|
35
|
+
|
|
36
|
+
## Health after any reboot
|
|
37
|
+
|
|
38
|
+
`scripts/cilium_heal.py --cluster <name> --check` (read-only) — per node container IP, Node IP,
|
|
39
|
+
CiliumNode IP, bpffs. `STOPPED`, `NO BPFFS` or `STALE`: run it without `--check`.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Stack profile — `ops-env.profile.json`
|
|
2
|
+
|
|
3
|
+
One JSON file at the root of the stack's anchor repository (the one that bootstraps the cluster).
|
|
4
|
+
Every script of this skill reads it; nothing about a particular stack is hard-coded in the skill.
|
|
5
|
+
Start from `profile.example.json` and check it with `scripts/stack_plan.py --check`.
|
|
6
|
+
|
|
7
|
+
Lookup: `--profile FILE`, else `$SDTK_ENV_PROFILE`, else the nearest directory at or above the current
|
|
8
|
+
one that holds `ops-env.profile.json` or holds a repository that does. Workspace (the directory with
|
|
9
|
+
all the repositories side by side): `--root DIR`, else `$SDTK_ENV_ROOT`, else the anchor repository's
|
|
10
|
+
parent.
|
|
11
|
+
|
|
12
|
+
## Keys
|
|
13
|
+
|
|
14
|
+
| Key | Used by | Meaning |
|
|
15
|
+
|---|---|---|
|
|
16
|
+
| `name` | all | Short stack name; hosts-file block, CA file name in the trust store |
|
|
17
|
+
| `tools` | preflight | CLI tools that must be on PATH |
|
|
18
|
+
| `python_modules` | preflight | `{"module": "pip package"}` the stack's own scripts need (the skill's scripts need none) |
|
|
19
|
+
| `registry` | preflight, registry_secret | `host`, `token_env` (name of the env variable, never the token), `kind` (`github-classic-pat` checks owner and scope through the GitHub API), `scope`, `pull_secret`, optional `username` |
|
|
20
|
+
| `repos[]` | ensure_repos, stack_plan | `dir` (folder in the workspace), `url` (SSH), `https` |
|
|
21
|
+
| `configs` | shared_config, hosts, plan | `{"name": "repo/path/config.yaml"}` — the files whose top-level `key: value` lines carry the shared values |
|
|
22
|
+
| `primary_config` | all | The config that owns the cluster-wide values (base domain, namespaces); `{key}` placeholders are filled from it |
|
|
23
|
+
| `base_domain_key` | shared_config, hosts | Default `base_domain`. Every other `*_domain` must be one label under it |
|
|
24
|
+
| `core_keys` | skill step 0.4 | The values asked first, in one question (e.g. namespace, base domain, database name) |
|
|
25
|
+
| `shared_keys` | shared_config | Keys every config that has them must agree on |
|
|
26
|
+
| `aliases[]` | shared_config | `{config, key, file_key}`: in that config the shared `key` is stored as `file_key` (`null`: not stored there, the name means something else) |
|
|
27
|
+
| `indented_followers[]` | shared_config | `{config, line_key, follows}`: indented `line_key:` lines in that config that hold the value of `follows` and move with it |
|
|
28
|
+
| `validators` | shared_config | `{key: regex}` beyond the defaults (`*_namespace`, `*_name`: DNS label; `*_domain`: FQDN; `ssl_country`: 2 letters) |
|
|
29
|
+
| `admin.users[]` / `admin.passwords[]` | admin_creds | `{file, key}`. Users in tracked configs; passwords in gitignored secrets files |
|
|
30
|
+
| `admin.k8s_secrets[]` | admin_creds | `{name, user_field, password_field}` Secrets made from the shared login (`--k8s-secrets NAMESPACE`) |
|
|
31
|
+
| `admin.user_rx` / `password_rx` | admin_creds | Override the default rules |
|
|
32
|
+
| `hosts` | hosts_entry | `configs` to collect `*_domain` names from, `ip` (default 127.0.0.1), `extra` names |
|
|
33
|
+
| `ca` | trust_ca, skill | `dir` (with `{key}` placeholders), `ca`, `leaf`, `site_key`; optional `cluster_issuer` (cert-manager ClusterIssuer loaded from this CA) and `issuer_after` (component that installs cert-manager) |
|
|
34
|
+
| `cluster` | skill | `kind` (`k3d`), `name_key` (config key naming the cluster), optional `heal` (script that repairs node networking after a reboot) |
|
|
35
|
+
| `components[]` | stack_plan, skill | In install order: `id`, `repo`, one of `skill` (the repository's own install skill) / `run` (command from the workspace) / `steps` (free text), `configs`, `after`, `gate`, `long` (run in the background with a log), `needs_registry_secret`, `health[]` (commands, `{key}` placeholders), `notes[]` (stack rules handed to that component's run, e.g. a value that must stay off) |
|
|
36
|
+
| `ingress` | skill | `dir` of the rendered Ingresses, `render` command |
|
|
37
|
+
|
|
38
|
+
## Conventions the scripts rely on
|
|
39
|
+
|
|
40
|
+
- Config files are flat at the top: `key: "value" # explanation`. The scripts change only the value on
|
|
41
|
+
that line and read it back; they never rewrite a file with a YAML dumper (comments survive).
|
|
42
|
+
- Passwords never sit in a tracked file. Each component keeps them in a gitignored `secrets.yaml`
|
|
43
|
+
that its own launcher merges over `config.yaml`.
|
|
44
|
+
- A comment line on its own above a group of keys is a section title (`explain_config.py` shows it).
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Cilium values for a local k3d cluster (template). Replace <cluster> and <cluster-id>.
|
|
2
|
+
# helm repo add cilium https://helm.cilium.io && helm repo update
|
|
3
|
+
# helm install cilium cilium/cilium --version <version> -n kube-system -f cilium-values.yaml \
|
|
4
|
+
# --kube-context k3d-<cluster> --wait --timeout 10m
|
|
5
|
+
# Pin the chart version in your bootstrap and keep this file the whole user-supplied config: before a
|
|
6
|
+
# `helm upgrade`, `helm -n kube-system get values cilium` must hold nothing this file lacks.
|
|
7
|
+
cluster:
|
|
8
|
+
name: <cluster>
|
|
9
|
+
id: <cluster-id> # 1-255, unique among clusters you may mesh; peers also need another pod CIDR
|
|
10
|
+
k8sServiceHost: k3d-<cluster>-server-0
|
|
11
|
+
k8sServicePort: 6443
|
|
12
|
+
kubeProxyReplacement: true
|
|
13
|
+
ipam:
|
|
14
|
+
mode: kubernetes
|
|
15
|
+
operator:
|
|
16
|
+
replicas: 1
|
|
17
|
+
ingressController:
|
|
18
|
+
enabled: true
|
|
19
|
+
default: true
|
|
20
|
+
loadbalancerMode: shared
|
|
21
|
+
service:
|
|
22
|
+
type: NodePort
|
|
23
|
+
insecureNodePort: 30080
|
|
24
|
+
secureNodePort: 30443
|
|
25
|
+
hubble:
|
|
26
|
+
relay:
|
|
27
|
+
enabled: true
|
|
28
|
+
ui:
|
|
29
|
+
enabled: true
|
|
30
|
+
clustermesh:
|
|
31
|
+
useAPIServer: true
|
|
32
|
+
config:
|
|
33
|
+
enabled: true # without it the apiserver waits in Init for clustermesh-remote-users
|
|
34
|
+
apiserver:
|
|
35
|
+
service:
|
|
36
|
+
type: NodePort
|
|
37
|
+
nodePort: 32379
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# k3d cluster for Cilium (template). Replace <cluster>; pin the k3s image you test with.
|
|
2
|
+
# k3d cluster create --config k3d-cluster.yaml
|
|
3
|
+
# Cilium replaces flannel, kube-proxy and the network policy controller; Traefik and ServiceLB are off
|
|
4
|
+
# because Cilium's ingress controller serves the Ingresses on NodePorts 30080 / 30443, mapped to the
|
|
5
|
+
# host's 80 / 443 below.
|
|
6
|
+
apiVersion: k3d.io/v1alpha5
|
|
7
|
+
kind: Simple
|
|
8
|
+
metadata:
|
|
9
|
+
name: <cluster>
|
|
10
|
+
servers: 1
|
|
11
|
+
agents: 2
|
|
12
|
+
image: rancher/k3s:v1.31.4-k3s1
|
|
13
|
+
ports:
|
|
14
|
+
- port: 80:30080
|
|
15
|
+
nodeFilters: [server:0]
|
|
16
|
+
- port: 443:30443
|
|
17
|
+
nodeFilters: [server:0]
|
|
18
|
+
options:
|
|
19
|
+
k3d:
|
|
20
|
+
wait: true
|
|
21
|
+
k3s:
|
|
22
|
+
extraArgs:
|
|
23
|
+
- arg: --flannel-backend=none
|
|
24
|
+
nodeFilters: [server:*]
|
|
25
|
+
- arg: --disable-network-policy
|
|
26
|
+
nodeFilters: [server:*]
|
|
27
|
+
- arg: --disable-kube-proxy
|
|
28
|
+
nodeFilters: [server:*]
|
|
29
|
+
- arg: --disable=traefik
|
|
30
|
+
nodeFilters: [server:*]
|
|
31
|
+
- arg: --disable=servicelb
|
|
32
|
+
nodeFilters: [server:*]
|
|
33
|
+
- arg: --cluster-cidr=10.42.0.0/16
|
|
34
|
+
nodeFilters: [server:*]
|
|
35
|
+
kubeconfig:
|
|
36
|
+
updateDefaultKubeconfig: true
|
|
37
|
+
switchCurrentContext: true
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "demo",
|
|
3
|
+
"description": "Fictional stack: k3d + Cilium cluster, object storage, PostgreSQL, a message broker, monitoring and the application charts.",
|
|
4
|
+
"tools": ["docker", "k3d", "kubectl", "helm", "openssl", "git", "curl"],
|
|
5
|
+
"python_modules": {"yaml": "pyyaml", "jinja2": "jinja2"},
|
|
6
|
+
"registry": {
|
|
7
|
+
"host": "ghcr.io",
|
|
8
|
+
"token_env": "REGISTRY_TOKEN",
|
|
9
|
+
"kind": "github-classic-pat",
|
|
10
|
+
"scope": "read:packages",
|
|
11
|
+
"pull_secret": "registry-pull"
|
|
12
|
+
},
|
|
13
|
+
"repos": [
|
|
14
|
+
{"dir": "demo-infra", "url": "git@github.com:example-org/demo-infra.git", "https": "https://github.com/example-org/demo-infra.git"},
|
|
15
|
+
{"dir": "demo-storage", "url": "git@github.com:example-org/demo-storage.git", "https": "https://github.com/example-org/demo-storage.git"},
|
|
16
|
+
{"dir": "demo-db", "url": "git@github.com:example-org/demo-db.git", "https": "https://github.com/example-org/demo-db.git"},
|
|
17
|
+
{"dir": "demo-mq", "url": "git@github.com:example-org/demo-mq.git", "https": "https://github.com/example-org/demo-mq.git"},
|
|
18
|
+
{"dir": "demo-monitoring", "url": "git@github.com:example-org/demo-monitoring.git", "https": "https://github.com/example-org/demo-monitoring.git"},
|
|
19
|
+
{"dir": "demo-charts", "url": "git@github.com:example-org/demo-charts.git", "https": "https://github.com/example-org/demo-charts.git"}
|
|
20
|
+
],
|
|
21
|
+
"configs": {
|
|
22
|
+
"infra": "demo-infra/k8s/config.yaml",
|
|
23
|
+
"storage": "demo-storage/config.yaml",
|
|
24
|
+
"db": "demo-db/config.yaml",
|
|
25
|
+
"mq": "demo-mq/config.yaml",
|
|
26
|
+
"monitoring": "demo-monitoring/config.yaml"
|
|
27
|
+
},
|
|
28
|
+
"primary_config": "infra",
|
|
29
|
+
"base_domain_key": "base_domain",
|
|
30
|
+
"core_keys": ["apps_namespace", "base_domain", "db_name"],
|
|
31
|
+
"shared_keys": ["apps_namespace", "base_domain", "site_domain", "ingress_class", "db_name",
|
|
32
|
+
"ssl_country", "ssl_state", "ssl_city", "ssl_organization", "ssl_organization_unit"],
|
|
33
|
+
"aliases": [
|
|
34
|
+
{"config": "monitoring", "key": "apps_namespace", "file_key": null},
|
|
35
|
+
{"config": "monitoring", "key": "monitoring_namespace", "file_key": "apps_namespace"}
|
|
36
|
+
],
|
|
37
|
+
"indented_followers": [
|
|
38
|
+
{"config": "monitoring", "line_key": "namespace", "follows": "apps_namespace"}
|
|
39
|
+
],
|
|
40
|
+
"admin": {
|
|
41
|
+
"users": [
|
|
42
|
+
{"file": "demo-storage/config.yaml", "key": "storage_admin_user"},
|
|
43
|
+
{"file": "demo-db/config.yaml", "key": "db_admin_user"},
|
|
44
|
+
{"file": "demo-mq/config.yaml", "key": "mq_default_user"},
|
|
45
|
+
{"file": "demo-monitoring/config.yaml", "key": "grafana_admin_user"}
|
|
46
|
+
],
|
|
47
|
+
"passwords": [
|
|
48
|
+
{"file": "demo-storage/secrets.yaml", "key": "storage_admin_password"},
|
|
49
|
+
{"file": "demo-db/secrets.yaml", "key": "db_admin_password"},
|
|
50
|
+
{"file": "demo-mq/secrets.yaml", "key": "mq_default_password"},
|
|
51
|
+
{"file": "demo-monitoring/secrets.yaml", "key": "grafana_admin_password"}
|
|
52
|
+
],
|
|
53
|
+
"k8s_secrets": [
|
|
54
|
+
{"name": "demo-app-admin", "user_field": "username", "password_field": "password"}
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
"hosts": {"configs": ["infra", "storage", "monitoring"], "ip": "127.0.0.1"},
|
|
58
|
+
"ca": {"dir": "demo-infra/k8s/certs/{base_domain}", "ca": "ca.crt", "leaf": "tls.crt", "site_key": "site_domain",
|
|
59
|
+
"cluster_issuer": "demo-ca", "issuer_after": "storage"},
|
|
60
|
+
"cluster": {"kind": "k3d", "name_key": "apps_namespace", "heal": "demo-infra/k8s/cilium-heal.sh"},
|
|
61
|
+
"components": [
|
|
62
|
+
{"id": "cluster", "repo": "demo-infra", "run": "cd demo-infra/k8s && ./launch.py -d", "configs": ["infra"],
|
|
63
|
+
"gate": "the confirmed Apply summary", "long": true,
|
|
64
|
+
"health": ["kubectl get nodes", "kubectl -n kube-system get pods -l k8s-app=cilium"]},
|
|
65
|
+
{"id": "storage", "repo": "demo-storage", "skill": "install-demo-storage", "configs": ["storage"], "after": ["cluster"],
|
|
66
|
+
"gate": "its own deploy go-ahead", "long": true,
|
|
67
|
+
"health": ["kubectl -n {apps_namespace} get pods -l app=object-store"]},
|
|
68
|
+
{"id": "db", "repo": "demo-db", "skill": "install-demo-db", "configs": ["db"], "after": ["storage"],
|
|
69
|
+
"gate": "its own deploy go-ahead", "long": true,
|
|
70
|
+
"health": ["kubectl -n {apps_namespace} get postgrescluster {db_name}"]},
|
|
71
|
+
{"id": "mq", "repo": "demo-mq", "skill": "install-demo-mq", "configs": ["mq"], "after": ["storage"],
|
|
72
|
+
"health": ["kubectl -n {apps_namespace} get rabbitmqcluster"]},
|
|
73
|
+
{"id": "monitoring", "repo": "demo-monitoring", "skill": "install-demo-monitoring", "configs": ["monitoring"], "after": ["cluster"],
|
|
74
|
+
"health": ["kubectl -n {monitoring_namespace} get pods"]},
|
|
75
|
+
{"id": "apps", "repo": "demo-charts", "run": "cd demo-charts && ./deploy-charts.sh", "after": ["db", "mq"],
|
|
76
|
+
"needs_registry_secret": true,
|
|
77
|
+
"health": ["helm -n {apps_namespace} list", "kubectl -n {apps_namespace} get pods"]}
|
|
78
|
+
],
|
|
79
|
+
"ingress": {"dir": "demo-infra/k8s/ingress/{base_domain}", "render": "cd demo-infra/k8s && ./launch.py -ro"}
|
|
80
|
+
}
|