vibed-infra 0.7.0 → 0.9.0

package/README.md

@@ -41,10 +41,17 @@ Multi-app: further products’ `install-gateway.sh` only add `apps/{other}/sites
 | Path | Purpose |
 |------|---------|
 | `~/services/gateway` | Host nginx (`GATEWAY_HOME`) |
-| `~/services/vibed-infra/update-agent` | Serial pull queue + optional GHCR webhook |
+| `~/services/vibed-infra/update-agent` | Serial pull queue; GHCR notify via GitHub Actions OIDC |
 | `~/services/vibed-infra/persist-logs` | Per-app WALs + optional R2/S3 ship |
 
-Auto-update cron **enqueues** work; the agent processes one job at a time. After a GHCR push, the reusable image workflow mints a GitHub Actions OIDC JWT and notifies `https://{domain}/_vibed/hooks/ghcr` (fallback `http://{publicIp}/…`) so the pull does not wait for cron. Updates prune dangling images (`DOCKER_AUTO_PRUNE=1`).
+## Image updates (docker pull)
+
+Install registers each role in the machine **update-agent**. Two paths enqueue work; the agent runs **one pull at a time** (`update-*.sh` is digest-gated — no restart if the image digest is unchanged):
+
+1. **Immediate** — push `:main` with the reusable GHCR workflow (`id-token: write`). CI mints an OIDC JWT and POSTs `https://{site host}/_vibed/hooks/ghcr` (fallback `http://{publicIp}/…`). No GitHub webhook UI or `VIBED_WEBHOOK_SECRET`.
+2. **Cron** — if `*_AUTO_UPDATE=1`, periodic enqueue (backup if notify is missed).
+
+See [`skills/infra-update-agent/SKILL.md`](skills/infra-update-agent/SKILL.md). After a successful update, dangling images are pruned (`DOCKER_AUTO_PRUNE=1`).
 
 ## Layout
 
@@ -55,7 +62,7 @@ Auto-update cron **enqueues** work; the agent processes one job at a time. After
 | `templates/update-agent/` | Queue agent + webhook |
 | `templates/persist-logs/` | Shipper install |
 | `lib/persistlog/` | Python append / seal / replay |
-| `skills/` | system-gateway, infra-update-agent, persist-logs, dns-configure |
+| `skills/` | system-gateway, infra-update-agent, infra-cicd, infra-packager, persist-logs, dns-configure |
 
 ## Environment
 
@@ -63,8 +70,9 @@ Auto-update cron **enqueues** work; the agent processes one job at a time. After
 |----------|---------|
 | `GATEWAY_HOME` | Host gateway dir (default `~/services/gateway`) |
 | `VIBED_HOME` | Machine vibed root (default `~/services/vibed-infra`) |
+| `VIBED_UPDATE_AGENT` | Override update-agent dir (default `$VIBED_HOME/update-agent`) |
 | `GATEWAY_PUBLIC_IP` | VPS IPv4 (from `gateway.publicIp`) |
-| `TLS_EMAIL` / `TLS_MODE` | Let’s Encrypt email; `lab` or `letsencrypt` |
+| `TLS_EMAIL` / `TLS_MODE` | Let’s Encrypt email; `lab` or `letsencrypt` (docker certbot → `./certs` PEMs; optional `CERTBOT_IMAGE` / `LETSENCRYPT_HOME`) |
 | `PERSIST_LOG_DIR` | Per-service event log dir |
 | `DOCKER_AUTO_PRUNE` | Prune dangling images after update (default on) |
 
@@ -72,6 +80,7 @@ Auto-update cron **enqueues** work; the agent processes one job at a time. After
 
 ```bash
 npm test
+npm run test:oidc       # real GitHub OIDC mint; skips locally unless ACTIONS_ID_TOKEN_* set
 npm run test:dist
 npm run test:e2e-multi   # two apps, localhost wget|bash, shared host gateway
 ```

package/lib/host_gateway.sh

@@ -52,9 +52,11 @@ vibed_bootstrap_host_gateway() {
     fi
   }
 
-  # Always refresh setup-tls (idempotent re-install / multi-app)
+  # Always refresh gateway scripts (idempotent re-install / multi-app)
   _hg_fetch "templates/host-gateway/setup-tls.sh" "$home/setup-tls.sh"
-  chmod +x "$home/setup-tls.sh"
+  _hg_fetch "templates/host-gateway/start-gateway.sh" "$home/start-gateway.sh"
+  _hg_fetch "templates/host-gateway/reload-gateway.sh" "$home/reload-gateway.sh"
+  chmod +x "$home/setup-tls.sh" "$home/start-gateway.sh" "$home/reload-gateway.sh"
 
   if [[ -f "${home}/.vibed-host-gateway" ]]; then
     echo "host gateway already present: $home"
@@ -66,11 +68,9 @@ vibed_bootstrap_host_gateway() {
   _hg_fetch "templates/host-gateway/gateway/nginx.conf" "$home/gateway/nginx.conf"
   _hg_fetch "templates/host-gateway/gateway/conf.d/00-default.conf" "$home/gateway/conf.d/00-default.conf"
   _hg_fetch "templates/host-gateway/.env.example" "$home/.env.example"
-  _hg_fetch "templates/host-gateway/start-gateway.sh" "$home/start-gateway.sh"
-  _hg_fetch "templates/host-gateway/reload-gateway.sh" "$home/reload-gateway.sh"
   _hg_fetch "templates/host-gateway/update-gateway.sh" "$home/update-gateway.sh"
   _hg_fetch "lib/env.sh" "$home/lib-env.sh"
-  chmod +x "$home/start-gateway.sh" "$home/reload-gateway.sh" "$home/update-gateway.sh"
+  chmod +x "$home/update-gateway.sh"
 
   if [[ ! -f "$home/.env" ]]; then
     cp "$home/.env.example" "$home/.env"

package/lib/package.py

@@ -100,8 +100,13 @@ DOCKER_PRUNE_UNTIL=72h
 def _env_gateway(meta: dict, conf: dict, sites: list) -> str:
     au = conf["profiles"]["gateway"]["autoUpdate"]
     gw = conf["profiles"]["gateway"]
-    host = sites[0]["host"] if sites else "example.com"
-    cert_dir = sites[0].get("tlsCertDir", f"/etc/letsencrypt/live/{host}") if sites else "/etc/letsencrypt/live/example.com"
+    # Host gateway stores PEMs under ./certs (setup-tls.sh). tlsCertDir is legacy/docs only
+    # unless it already points at a relative ./certs tree.
+    cert_dir = "./certs"
+    if sites:
+        raw = sites[0].get("tlsCertDir") or ""
+        if raw.startswith("./certs") or raw.rstrip("/") == "certs":
+            cert_dir = raw.rstrip("/")
     flags = au.get("flags") or ["GATEWAY_AUTO_UPDATE"]
     flag_lines = "\n".join(f"{f}=0" for f in flags)
     public_ip = gw.get("publicIp") or meta.get("publicIp") or ""
@@ -351,6 +356,10 @@ wget -qO- https://raw.githubusercontent.com/ORG/REPO/main/dist/install-gateway.s
 Network: `{meta['network']}` (shared host edge). API container: `{meta['apiContainer']}`.
 Gateway install bootstraps `~/services/gateway` once, then drops `apps/{{product}}/sites.conf`.
 
+## Image updates
+
+wget install registers this product with the machine update-agent. A GHCR `:main` push (reusable workflow + `id-token: write`) POSTs `/_vibed/hooks/ghcr` so the VPS pulls immediately. Cron `*_AUTO_UPDATE` is a backup. See the packager’s infra-update-agent skill.
+
 ## DNS (AU agent)
 
 Copy [`DNS-SKILL.md`](DNS-SKILL.md) into an AU agent browser extension. Domains and `gateway.publicIp` are baked in when configured; confirm the IP with the operator.

package/lib/tls.sh

@@ -9,22 +9,16 @@ infra_tls_check() {
 
 infra_tls_suggest_certbot() {
   local domains=("$@")
-  local joined=""
-  local d
-  for d in "${domains[@]}"; do
-    joined+=" -d $d"
-  done
   cat <<EOF
 
-TLS certificates not found. Issue with certbot (port 80 must be free):
+TLS certificates not found. From the host gateway directory, issue with:
 
-  sudo certbot certonly --standalone${joined}
+  cd ~/services/gateway && ./setup-tls.sh --force
 
-Or after gateway HTTP is up (webroot):
+(setup-tls.sh uses docker certbot when available — no host certbot/sudo required —
+or host certbot when installed. PEMs land in ./certs/.)
 
-  sudo certbot certonly --webroot -w /var/www/certbot${joined}
-
-Then set TLS_FULLCHAIN and TLS_PRIVKEY in .env and run ./start.sh
+Or set TLS_MODE=lab for self-signed lab certs, then ./start.sh
 
 EOF
 }
@@ -41,19 +35,16 @@ infra_tls_offer_interactive() {
   if [[ ! -t 0 ]]; then
     return 0
   fi
-  read -r -p "Run certbot --standalone now? [y/N] " ans
+  local gw="${GATEWAY_HOME:-${HOME:-}/services/gateway}"
+  read -r -p "Run setup-tls.sh in ${gw}? [y/N] " ans
   case "$ans" in
     y|Y|yes|YES)
-      if ! command -v certbot >/dev/null 2>&1; then
-        echo "certbot not found — install certbot first" >&2
+      if [[ -x "${gw}/setup-tls.sh" ]]; then
+        (cd "$gw" && ./setup-tls.sh --force)
+      else
+        echo "setup-tls.sh not found at ${gw} — bootstrap the host gateway first" >&2
         return 1
       fi
-      local args=()
-      local d
-      for d in "${domains[@]}"; do
-        args+=(-d "$d")
-      done
-      sudo certbot certonly --standalone "${args[@]}"
       ;;
   esac
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vibed-infra",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "description": "Product-agnostic VPS packager: wget install, Docker Compose, TLS/nginx, auto-update",
   "license": "MIT",
   "repository": {

package/schema/packageconfig.md

@@ -27,7 +27,7 @@ gateway:
       aliases: [www.app.example.com]
       healthPath: /api/health
       createPath: /api/items
-      tlsCertDir: /etc/letsencrypt/live/app.example.com
+      tlsCertDir: ./certs  # host gateway PEMs; legacy /etc/letsencrypt/... paths ignored in .env.gateway.example
 ```
 
 Defaults: containers `{name}-api` / `{name}-ui` / `{name}-worker`; host gateway container `vps-gateway`.
@@ -39,7 +39,7 @@ Compiled `packageconfig.yaml` also includes `webhook.url` / `fallbackUrl` (from
 | Path | Purpose |
 |------|---------|
 | `install-*.sh` | wget entrypoints |
-| `notify-vps-pull.py` | Called by GHCR workflow after push; uses compiled `webhook` |
+| `notify-vps-pull.py` | GHCR workflow: mint OIDC JWT and POST webhook URLs from `webhook:` |
 | `DNS-SKILL.md` | Paste into AU browser agent; includes domains + `publicIp` when set |
 | `packageconfig.yaml` | compiled config |
 | `start-*.sh` / `update-*.sh` | lifecycle |
@@ -47,11 +47,15 @@ Compiled `packageconfig.yaml` also includes `webhook.url` / `fallbackUrl` (from
 
 Gateway install writes `$GATEWAY_HOME/apps/{name}/sites.conf` (host-extension mode).
 
-## Auto-update
+## Auto-update and docker pull
 
-When `*_AUTO_UPDATE=1`, cron **enqueues** into the machine update-agent (serial pulls). Fallback: direct `update-*.sh` if agent missing. After update: dangling image prune (`DOCKER_AUTO_PRUNE`).
+Install registers each role with the machine **update-agent**. Pulls are serial and digest-gated (`update-*.sh`).
 
-Immediate pulls: install registers the app. The reusable docker-build workflow mints a GitHub Actions OIDC JWT and POSTs to `https://{host}/_vibed/hooks/ghcr` (fallback `http://{publicIp}/_vibed/hooks/ghcr`) after pushing `:main`. The VPS verifies GitHub’s signature (`id-token: write` on the caller job). Local curl may use the compiled `webhook.token`.
+- **Immediate:** reusable GHCR workflow on the default branch mints a GitHub Actions OIDC JWT and POSTs `https://{host}/_vibed/hooks/ghcr` (fallback `http://{publicIp}/_vibed/hooks/ghcr`). Caller needs `id-token: write`. See [`skills/infra-update-agent/SKILL.md`](../skills/infra-update-agent/SKILL.md).
+- **Cron:** when `*_AUTO_UPDATE=1`, cron enqueues on an interval (backup if notify is missed). Fallback: direct `update-*.sh` if the agent is missing.
+- After update: dangling image prune (`DOCKER_AUTO_PRUNE`).
+
+`webhook.token` in compiled packageconfig is only for local curl; CI does not need `VIBED_WEBHOOK_SECRET`.
 
 ## Persist logs
 
@@ -63,6 +67,7 @@ Immediate pulls: install registers the app. The reusable docker-build workflow m
 |----------|---------|
 | `GATEWAY_HOME` | Host gateway (default `~/services/gateway`) |
 | `VIBED_HOME` | `~/services/vibed-infra` |
+| `VIBED_UPDATE_AGENT` | Override update-agent dir (default `$VIBED_HOME/update-agent`) |
 | `GATEWAY_PUBLIC_IP` / `TLS_EMAIL` / `TLS_MODE` | Host TLS (`setup-tls.sh`) |
 | `PACKAGER_RAW` / `PRODUCT_RAW` / `PACKAGECONFIG_URL` | as before |
 | `INFRA_PROFILE` | `api`, `ui`, `nodes`, `gateway` |

package/skills/infra-cicd/SKILL.md

@@ -2,8 +2,7 @@
 name: infra-cicd
 description: >-
   Set up GitHub Actions CI/CD for infra-packaged products: GHCR image build,
-  package validation, and dist e2e (api/ui/nodes). Use when adding deploy
-  pipelines for Backend/UI/worker images.
+  OIDC notify for immediate VPS pulls, package validation, and dist e2e.
 ---
 
 # Infra packager CI/CD
@@ -39,13 +38,14 @@ On the default branch, the reusable workflow mints a GitHub Actions OIDC JWT (au
 ## Product repo CI
 
 1. Run `./package.sh` and fail if `dist/` drifted from templates (optional `git diff --exit-code dist`).
-2. Build/push images to GHCR (reusable workflow notifies the VPS automatically).
+2. Build/push images to GHCR (reusable workflow notifies the VPS; requires `id-token: write`).
 3. Optional job: `test-dist.sh --profile api` on a Docker-enabled runner.
 
 ## Checklist
 
 - [ ] Four YAML templates under `templates/` (`gateway.publicIp` / `tlsEmail` / `sites[]` set for production)
 - [ ] Committed `dist/` matches `./package.sh` output (including `DNS-SKILL.md`)
-- [ ] Image names in configs match GHCR
+- [ ] Image names in configs match GHCR (`ghcr.io/{owner}/…` so OIDC `repository_owner` can match)
+- [ ] GHCR reusable job has `permissions.id-token: write`
 - [ ] `npm test` / install dry-run passes
 - [ ] Secrets not in templates — only `.env.*.example` placeholders (publicIp/domain are OK to commit)

package/skills/infra-packager/SKILL.md

@@ -2,8 +2,8 @@
 name: infra-packager
 description: >-
   Wire a new product to the infra VPS packager: four YAML configs, package.sh →
-  committed dist/, Docker images, and CI. Use when adding Backend/UI/worker
-  deploy to a repo or migrating from ad-hoc deploy/install scripts.
+  committed dist/, Docker images, GHCR OIDC pulls, and CI. Use when adding
+  Backend/UI/worker deploy to a repo or migrating from ad-hoc install scripts.
 ---
 
 # Infra packager — onboard a product
@@ -17,7 +17,7 @@ description: >-
 
 1. **Depend on** [`vibed-infra`](https://www.npmjs.com/package/vibed-infra) or clone this repo for `package.sh`.
 
-2. **Create** `templates/` with four files — see [`schema/packageconfig.md`](../../schema/packageconfig.md) and [`examples/vps-hello/templates/`](../../examples/vps-hello/templates/). Set `gateway.publicIp`, `gateway.tlsEmail`, and `gateway.sites[].host` for DNS-SKILL + HTTPS.
+2. **Create** `templates/` with four files — see [`schema/packageconfig.md`](../../schema/packageconfig.md) and [`examples/vps-hello/templates/`](../../examples/vps-hello/templates/). Set `gateway.publicIp`, `gateway.tlsEmail`, and `gateway.sites[].host` for DNS-SKILL + HTTPS (those values are also the GHCR notify URLs).
 
 3. **Add** a tiny product `package.sh`:
 
@@ -27,7 +27,7 @@ PACKAGER="$(node -e "console.log(require('path').dirname(require.resolve('vibed-
 exec bash "$PACKAGER/package.sh" --product "$ROOT" --out "$ROOT/dist"
 ```
 
-4. **Build images** — Dockerfiles in `app/`; tag `:local` for dev or push to GHCR for prod; reference image names in `*-config.yaml`. The reusable GHCR workflow notifies the VPS after push via GitHub Actions OIDC (`id-token: write`).
+4. **Images** — Dockerfiles in `app/`; tag `:local` for dev; GHCR `:main` for prod (names in `*-config.yaml`). Immediate VPS pull uses the reusable GHCR workflow with `id-token: write` — see infra-cicd and infra-update-agent. No extra GitHub secret.
 
 5. **Package and commit** `dist/` (includes `DNS-SKILL.md` with domains + public IP):
 
@@ -36,9 +36,9 @@ exec bash "$PACKAGER/package.sh" --product "$ROOT" --out "$ROOT/dist"
 git add dist && git commit && git push
 ```
 
-6. **CI** — `npm test` (packager validate + install dry-run); optional `test-dist.sh --profile api|ui|nodes` and `npm run test:e2e-multi`. See [`examples/vps-hello/test-dist.sh`](../../examples/vps-hello/test-dist.sh).
+6. **CI** — product repo: package drift check + GHCR build (notifies VPS). Packager repo: `npm test`, `oidc-webhook-e2e`, `test-dist.sh`, `npm run test:e2e-multi`.
 
-7. **VPS** — DNS first (paste `dist/DNS-SKILL.md` into AU agent), then wget:
+7. **VPS** — DNS first (paste `dist/DNS-SKILL.md` into AU agent), then wget. Install registers the app with the update-agent so the next GHCR push can pull immediately:
 
 ```bash
 wget -qO- .../dist/install-api.sh | bash
@@ -56,3 +56,4 @@ wget -qO- .../dist/install-gateway.sh | bash   # bootstraps host + setup-tls.sh
 - Gateway container names in `gateway.sites[]` must match running API/UI container names on `network.edge`.
 - UI is a separate profile; gateway is nginx-only and does not start the UI.
 - Prefer Let's Encrypt via `gateway.tlsEmail`; use `TLS_MODE=lab` only for CI/lab.
+- Product GHCR workflow must set `id-token: write` or the VPS will not get an immediate pull.

package/skills/infra-update-agent/SKILL.md

@@ -3,44 +3,50 @@ name: infra-update-agent
 description: >-
   Machine-level serial Docker update queue for vibed-infra apps. Use when
   multiple products share one VPS and must not pull images in parallel; includes
-  GHCR webhook enqueue via GitHub Actions OIDC (no extra GitHub secrets).
+  immediate GHCR pulls via GitHub Actions OIDC (no extra GitHub secrets).
 ---
 
 # Serial update agent
 
 ## Why
 
-Five apps with auto-update cron would each `docker pull` at once and overload the box. The **update-agent** is installed once; apps only **enqueue** jobs.
+Five apps with auto-update cron would each `docker pull` at once and overload the box. The **update-agent** is installed once; apps only **enqueue** jobs. The agent runs `update-*.sh`, which is **digest-gated**: pull if the registry digest changed, restart only if the running container is stale.
 
 ## Layout
 
 ```
 ~/services/vibed-infra/update-agent/
   queue/ processing/ done/ failed/
-  registry/{app}/{role}.json
-  tokens/{app}     # optional local curl; CI uses OIDC instead
+  registry/{app}/{role}.json   # image + installDir (written at wget install)
+  tokens/{app}                # optional local curl; CI uses OIDC
   agent.sh
   enqueue.sh
-  webhook_server.py # 0.0.0.0:19200 (firewall: do not expose this port)
+  webhook_server.py            # 0.0.0.0:19200 — do not open this port publicly
   lib/github_oidc.py
-  .env              # WEBHOOK_PORT; optional WEBHOOK_SECRET
+  .env                         # WEBHOOK_PORT=19200
 ```
 
-## Flow
+Override the directory with `VIBED_UPDATE_AGENT` (wins over `VIBED_HOME`).
 
-1. Install registers the app (image + install dir).
-2. Cron enqueues; agent cron (`*/5`) processes **serially**.
-3. Image push: reusable GHCR workflow mints a GitHub Actions OIDC JWT (audience = webhook URL) and POSTs `{package,tag}` to `/_vibed/hooks/ghcr`.
-4. Webhook verifies the JWT against GitHub’s JWKS (`iss`, `aud`, `exp`, RS256), then enqueues only registry images whose name contains `repository_owner`.
-5. Agent runs `update-*.sh` immediately.
+## Pull paths
 
-Gateway exposes `/_vibed/hooks/` on HTTP (port 80) and on each HTTPS site host.
+```
+git push main
+  → reusable GHCR workflow builds/pushes :main
+  → notify-vps-pull.py mints OIDC JWT (aud = webhook URL)
+  → POST https://{host}/_vibed/hooks/ghcr   (fallback http://{publicIp}/…)
+  → nginx → webhook_server (verify GitHub JWKS)
+  → enqueue matching registry entries (owner must appear in image name)
+  → agent.sh → update-{role}.sh → docker pull + restart if digest changed
+```
 
-## Zero extra GitHub setup
+- **Immediate:** default-branch image push. Caller job needs `id-token: write`. No GitHub Packages webhook and no `VIBED_WEBHOOK_SECRET`.
+- **Cron backup:** `*_AUTO_UPDATE=1` enqueues on an interval; agent cron `*/5` drains the queue if notify was missed.
+- **Manual:** `bash ~/services/vibed-infra/update-agent/enqueue.sh /path/to/api-install`
 
-`gateway.publicIp` + `gateway.sites[].host` already in product config are the URLs. Auth is GitHub’s signature — no `VIBED_WEBHOOK_SECRET`.
+Gateway HTTP (`00-default.conf`) and each HTTPS site proxy `/_vibed/hooks/` to `host.docker.internal:19200`. Keep **19200** off the public firewall; only 80/443.
 
-Product workflow must grant OIDC to the reusable job:
+## Product CI
 
 ```yaml
 jobs:
@@ -55,10 +61,11 @@ jobs:
       image: ghcr.io/${{ github.repository_owner }}/my-api
 ```
 
-Local curl can still use `X-Vibed-Secret` from `tokens/{app}` (compiled hash). Optional `WEBHOOK_SECRET` override remains.
+URLs come from `gateway.publicIp` + `gateway.sites[].host` already in config (`dist/packageconfig.yaml` `webhook.url` / `fallbackUrl`). Disable notify with `notify: false`.
 
-## Register / enqueue manually
+Local curl can send `X-Vibed-Secret` from `tokens/{app}` (compiled hash). Optional `WEBHOOK_SECRET` still works as an override.
 
-```bash
-bash ~/services/vibed-infra/update-agent/enqueue.sh /path/to/api-install
-```
+## Tests
+
+- `npm test` — openssl-signed JWT (same verifier) + enqueue under `VIBED_UPDATE_AGENT`
+- `npm run test:oidc` / CI job `oidc-webhook-e2e` — real GitHub OIDC mint against a local webhook

package/skills/system-gateway/SKILL.md

@@ -55,12 +55,12 @@ cd ~/services/gateway
 ./setup-tls.sh --force  # re-issue after host or IP change
 ```
 
-- **Production:** certbot (webroot if gateway is up, else standalone). On failure, fix DNS then re-run `setup-tls.sh`.
+- **Production:** `setup-tls.sh` issues Let’s Encrypt via docker `certbot/certbot` by default (config under `$GATEWAY_HOME/letsencrypt`, PEMs copied to `$GATEWAY_HOME/certs/`). No sudo required when Docker is available. Host certbot + sudo still preferred when present (same `--config-dir` under gateway home). Webroot if gateway is up, else standalone. On failure, fix DNS then re-run `setup-tls.sh`.
 - **Lab/CI:** `TLS_MODE=lab` → multi-SAN self-signed under `./certs/`.
 
-## Webhook path
+## Webhook path (immediate docker pull)
 
-Host `00-default.conf` (HTTP) and each app `sites.conf` (HTTPS) proxy `/_vibed/hooks/` to the update-agent on `host.docker.internal:19200`. Image builds notify this path automatically — see infra-update-agent skill.
+Host `00-default.conf` (HTTP) and each app `sites.conf` (HTTPS) proxy `/_vibed/hooks/` to the update-agent on `host.docker.internal:19200`. After a GHCR `:main` push, CI POSTs a GitHub Actions OIDC JWT to `/_vibed/hooks/ghcr` — see infra-update-agent. Do not expose port 19200 on the public firewall.
 
 ## Pitfalls
 

package/templates/host-gateway/.env.example

@@ -9,6 +9,9 @@ TLS_MODE=
 TLS_EMAIL=
 GATEWAY_PUBLIC_IP=
 CERTBOT_WWW=./certbot-www
+# Optional Let's Encrypt (docker certbot when host certbot/sudo unavailable)
+# CERTBOT_IMAGE=certbot/certbot
+# LETSENCRYPT_HOME=./letsencrypt
 HTTP_PORT=80
 HTTPS_PORT=443
 GATEWAY_MEMORY_LIMIT=64m

package/templates/host-gateway/setup-tls.sh

@@ -12,6 +12,8 @@ for arg in "$@"; do
     -h|--help)
       echo "usage: $0 [--force]"
       echo "  Issues Let's Encrypt (TLS_EMAIL / TLS_MODE=letsencrypt) or lab self-signed certs."
+      echo "  LE prefers host certbot when available (root or passwordless sudo);"
+      echo "  otherwise uses docker run certbot/certbot (no sudo). PEMs land in ./certs/."
       echo "  Re-runs when certs missing, domains/IP changed vs .vibed-tls-state, or --force."
       exit 0
       ;;
@@ -137,27 +139,104 @@ issue_letsencrypt() {
     echo "TLS_EMAIL (or gateway.tlsEmail) required for Let's Encrypt" >&2
     return 1
   fi
-  if ! command -v certbot >/dev/null 2>&1; then
-    echo "certbot not found — install certbot, or set TLS_MODE=lab for self-signed" >&2
-    return 1
+
+  local LE_HOME="${LETSENCRYPT_HOME:-${SCRIPT_DIR}/letsencrypt}"
+  local CERTBOT_IMAGE="${CERTBOT_IMAGE:-certbot/certbot}"
+  mkdir -p "$LE_HOME" "$LE_HOME/work" "$LE_HOME/logs" "$CERT_DIR" "$CERTBOT_WWW"
+
+  # Absolute webroot for docker bind mounts
+  local CERTBOT_WWW_ABS="$CERTBOT_WWW"
+  [[ "$CERTBOT_WWW_ABS" != /* ]] && CERTBOT_WWW_ABS="${SCRIPT_DIR}/${CERTBOT_WWW_ABS#./}"
+  mkdir -p "$CERTBOT_WWW_ABS"
+
+  local use_webroot=0
+  if gateway_running; then
+    use_webroot=1
   fi
-  local args=(certonly --non-interactive --agree-tos --email "$email" --cert-name "$primary")
+
+  local domain_args=()
   local d
   for d in "${all[@]}"; do
-    args+=(-d "$d")
+    domain_args+=(-d "$d")
   done
-  if gateway_running; then
-    echo "Gateway running — certbot webroot ($CERTBOT_WWW)"
-    args+=(--webroot -w "$CERTBOT_WWW")
-  else
-    echo "Gateway not running — certbot standalone (port 80 must be free)"
-    args+=(--standalone)
-  fi
+
+  local base_args=(
+    certonly
+    --agree-tos
+    --non-interactive
+    --email "$email"
+    --cert-name "$primary"
+    "${domain_args[@]}"
+  )
   if [[ "${CERTBOT_DRY_RUN:-0}" == "1" ]]; then
-    args+=(--dry-run)
+    base_args+=(--dry-run)
+  fi
+
+  local ran=0
+  local host_can_run=0
+  if command -v certbot >/dev/null 2>&1; then
+    if [[ "$(id -u)" -eq 0 ]]; then
+      host_can_run=1
+    elif sudo -n true >/dev/null 2>&1; then
+      host_can_run=1
+    fi
+  fi
+
+  if [[ "$host_can_run" == "1" ]]; then
+    local host_args=(
+      "${base_args[@]}"
+      --config-dir "$LE_HOME"
+      --work-dir "$LE_HOME/work"
+      --logs-dir "$LE_HOME/logs"
+    )
+    if [[ "$use_webroot" == "1" ]]; then
+      echo "Gateway running — host certbot webroot ($CERTBOT_WWW_ABS)"
+      host_args+=(--webroot -w "$CERTBOT_WWW_ABS")
+    else
+      echo "Gateway not running — host certbot standalone (port 80 must be free)"
+      host_args+=(--standalone)
+    fi
+    local run_certbot=(certbot)
+    if [[ "$(id -u)" -ne 0 ]]; then
+      run_certbot=(sudo certbot)
+    fi
+    if "${run_certbot[@]}" "${host_args[@]}"; then
+      ran=1
+      # Best-effort: reclaim root-owned LE tree after sudo certbot
+      if [[ "$(id -u)" -ne 0 ]]; then
+        sudo -n chown -R "$(id -u):$(id -g)" "$LE_HOME" 2>/dev/null || true
+      fi
+    fi
+  fi
+
+  if [[ "$ran" != "1" && "$host_can_run" != "1" ]] && command -v docker >/dev/null 2>&1; then
+    local docker_args=(
+      run --rm
+      -v "${LE_HOME}:/etc/letsencrypt"
+    )
+    local container_args=("${base_args[@]}")
+    if [[ "$use_webroot" == "1" ]]; then
+      echo "Gateway running — docker certbot webroot ($CERTBOT_WWW_ABS)"
+      docker_args+=(-v "${CERTBOT_WWW_ABS}:/var/www/certbot")
+      container_args+=(--webroot -w /var/www/certbot)
+    else
+      echo "Gateway not running — docker certbot standalone (-p 80:80)"
+      docker_args+=(-p 80:80)
+      container_args+=(--standalone)
+    fi
+    if docker "${docker_args[@]}" "$CERTBOT_IMAGE" "${container_args[@]}"; then
+      ran=1
+    fi
   fi
-  if ! sudo certbot "${args[@]}"; then
-    cat <<EOF >&2
+
+  if [[ "$ran" != "1" ]]; then
+    if [[ "$host_can_run" != "1" ]] && ! command -v docker >/dev/null 2>&1; then
+      cat <<EOF >&2
+Neither usable host certbot nor docker found.
+Install Docker (preferred for non-root), or certbot with sudo, or set TLS_MODE=lab.
+EOF
+    else
+      cat <<EOF >&2
 
 Let's Encrypt failed (often DNS not pointing here yet).
 1. Paste dist/DNS-SKILL.md into your AU DNS agent (or create the A records yourself).
@@ -165,15 +244,27 @@ Let's Encrypt failed (often DNS not pointing here yet).
 3. Re-run: cd ${SCRIPT_DIR} && ./setup-tls.sh --force
 
 EOF
+    fi
     return 1
   fi
-  local live="/etc/letsencrypt/live/${primary}"
+
+  # Dry-run does not write live PEMs
+  if [[ "${CERTBOT_DRY_RUN:-0}" == "1" ]]; then
+    echo "certbot dry-run succeeded (no PEMs written)"
+    return 0
+  fi
+
+  local live="${LE_HOME}/live/${primary}"
   if [[ ! -f "${live}/fullchain.pem" || ! -f "${live}/privkey.pem" ]]; then
     echo "certbot succeeded but ${live} PEMs missing" >&2
     return 1
   fi
-  patch_env_key TLS_FULLCHAIN "${live}/fullchain.pem"
-  patch_env_key TLS_PRIVKEY "${live}/privkey.pem"
+  cp -L "${live}/fullchain.pem" "${CERT_DIR}/fullchain.pem"
+  cp -L "${live}/privkey.pem" "${CERT_DIR}/privkey.pem"
+  chmod 644 "${CERT_DIR}/fullchain.pem"
+  chmod 600 "${CERT_DIR}/privkey.pem"
+  patch_env_key TLS_FULLCHAIN "${CERT_DIR}/fullchain.pem"
+  patch_env_key TLS_PRIVKEY "${CERT_DIR}/privkey.pem"
   patch_env_key TLS_MODE letsencrypt
   patch_env_key TLS_EMAIL "$email"
 }

package/templates/host-gateway/start-gateway.sh

@@ -33,7 +33,7 @@ if [[ ! -f "$TLS_FULLCHAIN" || ! -f "$TLS_PRIVKEY" ]]; then
   echo "TLS certs not found:" >&2
   echo "  $TLS_FULLCHAIN" >&2
   echo "  $TLS_PRIVKEY" >&2
-  echo "Run gen-dev-certs.sh into ./certs for lab, or certbot for production." >&2
+  echo "Run ./setup-tls.sh (lab or letsencrypt) to create ./certs PEMs." >&2
   exit 1
 fi
 
@@ -44,6 +44,10 @@ fi
 
 docker network create "$NETWORK" >/dev/null 2>&1 || true
 mkdir -p "$CERTBOT_WWW" "$APPS_DIR"
+# Absolute path for live ACME webroot bind mount
+CERTBOT_ABS="$CERTBOT_WWW"
+[[ "$CERTBOT_ABS" != /* ]] && CERTBOT_ABS="${SCRIPT_DIR}/${CERTBOT_ABS#./}"
+mkdir -p "$CERTBOT_ABS"
 
 if docker inspect "$GATEWAY_NAME" >/dev/null 2>&1; then
   echo "Removing existing container $GATEWAY_NAME ..."
@@ -58,7 +62,7 @@ STAGE="$(mktemp -d)"
 cleanup_stage() { rm -rf "$STAGE"; }
 trap cleanup_stage EXIT
 
-mkdir -p "$STAGE/conf.d" "$STAGE/certs" "$STAGE/certbot" "$STAGE/apps"
+mkdir -p "$STAGE/conf.d" "$STAGE/certs" "$STAGE/apps"
 cp "$NGINX_CONF" "$STAGE/nginx.conf"
 cp -r "$CONF_D/." "$STAGE/conf.d/"
 cp "$TLS_FULLCHAIN" "$STAGE/certs/fullchain.pem"
@@ -75,10 +79,6 @@ if [[ -d "$APPS_DIR" ]]; then
     fi
   done
 fi
-if [[ -d "$CERTBOT_WWW" ]]; then
-  cp -r "$CERTBOT_WWW/." "$STAGE/certbot/" 2>/dev/null || true
-fi
-
 tar -C "$STAGE" -cf - . | docker run --rm -i -v "${CONF_VOL}:/cfg" alpine tar -xf - -C /cfg
 
 echo "Starting $GATEWAY_NAME (HTTP ${HTTP_PORT}, HTTPS ${HTTPS_PORT}) ..."
@@ -92,6 +92,7 @@ docker run -d \
   -p "${HTTP_PORT}:80" \
   -p "${HTTPS_PORT}:443" \
   -v "${CONF_VOL}:/gateway-cfg:ro" \
+  -v "${CERTBOT_ABS}:/var/www/certbot" \
   "$NGINX_IMAGE" \
   sh -c 'mkdir -p /etc/nginx/certs /etc/nginx/conf.d /etc/nginx/apps /var/www/certbot && \
     cp /gateway-cfg/nginx.conf /etc/nginx/nginx.conf && \
@@ -99,7 +100,6 @@ docker run -d \
     if [ -d /gateway-cfg/apps ]; then cp -r /gateway-cfg/apps/. /etc/nginx/apps/; fi && \
     cp /gateway-cfg/certs/fullchain.pem /etc/nginx/certs/fullchain.pem && \
     cp /gateway-cfg/certs/privkey.pem /etc/nginx/certs/privkey.pem && \
-    cp -r /gateway-cfg/certbot/. /var/www/certbot/ 2>/dev/null || true && \
     exec nginx -g "daemon off;"' >/dev/null
 
 trap - EXIT