vibecarbon 0.1.0

package/src/lib/deploy/k8s/k3s.js

@@ -0,0 +1,2515 @@
+/**
+ * k3s-on-Hetzner deploy backend.
+ *
+ * The Pulumi program is `hetzner-k8s.js`; cloud-init scripts are
+ * `carbon/cloud-init/k3s/{master,supabase,worker}-init.sh`.
+ *
+ * End-to-end flow on a cold deploy:
+ *   1. Preflight: docker, ssh, kubectl, helm on PATH; server-type architecture.
+ *   2. Generate (or reuse) the SSH keypair under projectDir/.vibecarbon/ssh-<env>.
+ *   3. Pulumi up — cloud-init installs k3s + hcloud-ccm/csi on each node.
+ *   4. Wait for SSH on master + `/tmp/k3s-ready` marker.
+ *   5. scp `/etc/rancher/k3s/k3s.yaml` from master, patch server URL to public IP.
+ *   6. Build app image locally (10.0.1.1:5000/<project>:<sha>[-dirty]-<ts>).
+ *   7. Sideload via `docker save TAG | ssh node 'k3s ctr images import -'` per node.
+ *   8. `kubectl apply -k k8s/base/` (cert-manager via infra/, traefik, app).
+ *   9. (helm-render supabase values + helm install supabase.)
+ *   10. `kubectl set image deploy/app app=<tag>` + rollout-status.
+ *
+ * Warm path: skip 1-5; rebuild image (6); sideload (7) skipping unchanged tag;
+ * apply changed manifests (8); rollout (10).
+ *
+ * Target cold-deploy time: ≤ 3 min for single-master + supabase + 0 workers.
+ * Spike on 2026-04-25 measured 82s for cluster bring-up alone.
+ */
+
+import { execFileSync, spawn, spawnSync } from 'node:child_process';
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import * as p from '@clack/prompts';
+import { runCommandAsync } from '../../command.js';
+import { knownHostsPath } from '../../host-keys.js';
+import { loadCloudInit, renderScript } from '../../iac/cloud-init.js';
+import { perfAsync } from '../../perf.js';
+
+export const K3S_VERSION = 'v1.31.5+k3s1';
+
+/**
+ * Pattern of kube-apiserver transient signatures we've actually observed
+ * mid-RPC during k3s control-plane warm-up. See runKubectlWithRetry —
+ * exported so unit tests can exercise the regex without importing the
+ * helper itself (and so a future caller adding a new signature has one
+ * obvious place to do it).
+ */
+export const KUBECTL_TRANSIENT_PATTERN =
+  /http2: client connection lost|connection reset by peer|context deadline exceeded|EOF|i\/o timeout|TLS handshake|unexpected error when reading response body/i;
+
+/**
+ * Run a short-lived kubectl command with retry-on-transient-error.
+ *
+ * RCA from k8s-ha runs on 2026-04-29: a freshly-bootstrapped k3s control
+ * plane drops one http2 connection mid-RPC, kubectl fails the call (the
+ * error message itself ends with "Please retry. Original error: http2:
+ * client connection lost"), and with no retry one transient kills the
+ * entire HA deploy. Both `kubectl apply Secret/hcloud-cluster-config`
+ * and `kubectl patch deploy/cluster-autoscaler` crashed on consecutive
+ * runs at different points in applyK3sManifests — so the fix lives at
+ * the helper layer instead of being grafted onto each call site as it
+ * surfaces.
+ *
+ * Three attempts with 2s/4s back-off (~6s ceiling, well under deploy
+ * budget). Pattern matches against KUBECTL_TRANSIENT_PATTERN; non-
+ * transient errors throw on the first attempt with stderr tail attached
+ * for debuggability.
+ *
+ * Don't use this for `kubectl wait` / `kubectl rollout status` /
+ * `kubectl logs --follow`. They have their own --timeout and piping
+ * their stdio through buffered spawnSync would hide live progress.
+ *
+ * @param {string[]} args kubectl argv
+ * @param {{env: NodeJS.ProcessEnv, input?: string, captureStdout?: boolean, description?: string}} options
+ * @returns {Promise<string>} stdout (always returned; empty when nothing was emitted)
+ */
+export async function runKubectlWithRetry(
+  args,
+  { env, input, captureStdout = false, description } = {},
+) {
+  const ATTEMPTS = 3;
+  const desc = description ?? `kubectl ${args.slice(0, 4).join(' ')}`;
+  let result;
+  let lastStderr = '';
+  let lastStdout = '';
+  for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {
+    result = spawnSync('kubectl', args, {
+      env,
+      input,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    lastStdout = (result.stdout ?? '').toString();
+    lastStderr = (result.stderr ?? '').toString();
+    // Stream captured streams through to the operator's terminal so the
+    // success path looks identical to the legacy `stdio: 'inherit'` shape.
+    // captureStdout suppresses stdout streaming because the caller wants
+    // the bytes (e.g. `kubectl get -o jsonpath`) — stderr still streams
+    // so a real warning is still visible.
+    if (!captureStdout && lastStdout) process.stdout.write(lastStdout);
+    if (lastStderr) process.stderr.write(lastStderr);
+    if (result.status === 0) break;
+    if (!KUBECTL_TRANSIENT_PATTERN.test(lastStderr) || attempt === ATTEMPTS) break;
+    const backoffMs = 2000 * attempt;
+    console.log(
+      `${desc} hit transient error on attempt ${attempt}/${ATTEMPTS}, retrying in ${backoffMs}ms`,
+    );
+    await delay(backoffMs);
+  }
+  if (!result || result.status !== 0) {
+    throw new Error(
+      `${desc} failed with exit ${result?.status ?? '?'}: ${lastStderr.trim().slice(-500)}`,
+    );
+  }
+  return lastStdout;
+}
+
+/**
+ * Render the JSON cluster-config that cluster-autoscaler reads from its
+ * mounted Secret. The JSON describes the worker-pool node group: which
+ * cloud-init script to use, what serverLabels to attach to spawned
+ * servers, etc.
+ *
+ * Phase 5 writes the returned string into the `cluster-config.json`
+ * key of the `kube-system/hcloud-cluster-config` Secret. The
+ * cluster-autoscaler hcloud provider reads this at startup
+ * (HCLOUD_CLUSTER_CONFIG); on each scale-up, `serverLabels` are
+ * attached to the new Hetzner server and `cloudInit` (a plain UTF-8
+ * string) is base64-encoded into the user_data field of the
+ * `ServerCreate` API call.
+ *
+ * `serverLabels['cluster-autoscaler/node'] = 'worker-pool'` distinguishes
+ * CA-spawned workers from static workers (which carry
+ * `cluster-autoscaler/node: static` from the Pulumi program). Phase 7's
+ * destroy sweep relies on this distinction.
+ *
+ * CA-spawned workers join via the master's PRIVATE IP (default
+ * `10.0.1.1`) — the in-network join path is firewalled to the
+ * private-network range, so a public-IP join would be rejected and is
+ * also unnecessary since the spawned servers attach to the same Hetzner
+ * private network.
+ *
+ * @param {{
+ *   k3sVersion: string,
+ *   k3sToken: string,
+ *   masterPrivateIp?: string,
+ *   clusterName: string,
+ * }} args
+ * @returns {string} JSON-stringified cluster config
+ */
+export function renderCAClusterConfig({
+  k3sVersion,
+  k3sToken,
+  masterPrivateIp = '10.0.1.1',
+  clusterName,
+}) {
+  const tmpl = loadCloudInit('worker-init.sh');
+  const cloudInit = renderScript(tmpl, {
+    k3s_version: k3sVersion,
+    k3s_token: k3sToken,
+    master_ip: masterPrivateIp,
+    cluster_name: clusterName,
+  });
+  const config = {
+    imagesForArch: {
+      amd64: 'ubuntu-24.04',
+      arm64: 'ubuntu-24.04',
+    },
+    nodeConfigs: {
+      'worker-pool': {
+        cloudInit,
+        labels: {},
+        serverLabels: {
+          'cluster-autoscaler/node': 'worker-pool',
+        },
+        taints: [],
+      },
+    },
+  };
+  return JSON.stringify(config, null, 2);
+}
+
+/**
+ * Map (dnsProvider, acmeServer) → ClusterIssuer name that
+ * `applyK3sManifests` patches into `Certificate/vibecarbon-tls`.
+ *
+ * - cloudflare/hetzner: DNS-01 solver — works behind reverse proxies and
+ *   has no port-80 dependency. Requires per-provider Secret + (for
+ *   hetzner) the cert-manager-webhook-hetzner deployment.
+ * - manual (default): HTTP-01 solver — needs port 80 reachable from
+ *   Let's Encrypt. Used when no programmable DNS provider is wired up.
+ *
+ * Unknown providers fall back to `manual` so a future enum addition
+ * never deploys a Certificate that references a non-existent issuer.
+ *
+ * @param {{dnsProvider?: string|null, acmeServer?: string|null}} args
+ * @returns {string}
+ */
+export function pickIssuerName({ dnsProvider, acmeServer }) {
+  const stage = (acmeServer || '').includes('staging') ? 'staging' : 'prod';
+  const providerSuffix =
+    dnsProvider === 'cloudflare' ? 'cloudflare' : dnsProvider === 'hetzner' ? 'hetzner' : 'manual';
+  return `letsencrypt-${stage}-${providerSuffix}`;
+}
+
+/**
+ * Render the per-DNS-provider Secret YAML that cert-manager's DNS-01
+ * solvers read for API auth, or null when the provider doesn't need a
+ * Secret (`manual` uses HTTP-01).
+ *
+ * - cloudflare: `cloudflare-api-token` / key `api-token` — referenced by
+ *   `letsencrypt-{prod,staging}-cloudflare`.
+ * - hetzner: `hetzner` / key `token` — referenced by
+ *   `letsencrypt-{prod,staging}-hetzner` AND by the official
+ *   `cert-manager-webhook-hetzner` chart (matches the chart's expected
+ *   `tokenSecretKeyRef` shape). The token here is the SAME Cloud API
+ *   token used elsewhere — Hetzner shut down the separate DNS Console
+ *   API in May 2026 and folded zone management into the main Cloud
+ *   Console + api.hetzner.cloud/v1/zones.
+ *
+ * Throws when a token is required but missing — the early throw makes
+ * "Order pinned Pending forever" failures show up at deploy-start with
+ * an actionable error message instead.
+ *
+ * @param {{dnsProvider?: string|null, cloudflareApiToken?: string|null, hetznerApiToken?: string|null}} args
+ * @returns {{name: string, yaml: string}|null}
+ */
+export function buildDnsProviderSecret({ dnsProvider, cloudflareApiToken, hetznerApiToken }) {
+  if (dnsProvider === 'cloudflare') {
+    if (!cloudflareApiToken) {
+      throw new Error(
+        'Cloudflare DNS-01 requires a Cloudflare API token. ' +
+          'Set CLOUDFLARE_API_TOKEN or add cloudflare.apiToken to ~/.vibecarbon/credentials.json. ' +
+          'The token must have Zone:Zone:Read on all zones + Zone:DNS:Edit on the target zone.',
+      );
+    }
+    return {
+      name: 'cloudflare-api-token',
+      yaml: [
+        'apiVersion: v1',
+        'kind: Secret',
+        'metadata:',
+        '  name: cloudflare-api-token',
+        '  namespace: cert-manager',
+        'type: Opaque',
+        'stringData:',
+        `  api-token: ${JSON.stringify(cloudflareApiToken)}`,
+      ].join('\n'),
+    };
+  }
+  if (dnsProvider === 'hetzner') {
+    if (!hetznerApiToken) {
+      throw new Error(
+        'Hetzner DNS-01 requires a Cloud API token with read+write on the zones API. ' +
+          'Set HETZNER_API_TOKEN or add hetzner.apiToken to ~/.vibecarbon/credentials.json.',
+      );
+    }
+    return {
+      name: 'hetzner',
+      yaml: [
+        'apiVersion: v1',
+        'kind: Secret',
+        'metadata:',
+        '  name: hetzner',
+        '  namespace: cert-manager',
+        'type: Opaque',
+        'stringData:',
+        `  token: ${JSON.stringify(hetznerApiToken)}`,
+      ].join('\n'),
+    };
+  }
+  return null;
+}
+
+/**
+ * Compose the `--nodes=MIN:MAX:TYPE:LOCATION:NAME` argv slot the
+ * cluster-autoscaler hcloud provider parses on startup. CA's pool
+ * sits ON TOP of the Pulumi-static floor: total cluster workers =
+ * minWorkers (Pulumi-managed) + N CA-spawned, where 0 ≤ N ≤
+ * (maxWorkers - minWorkers). So CA's MIN is always 0 and its MAX is
+ * the headroom above the static floor. Server type is uppercased
+ * per the upstream cluster-autoscaler README — Hetzner's API accepts
+ * either case but the autoscaler's own validator is case-sensitive.
+ *
+ * @param {{minWorkers: number, maxWorkers: number, workerServerType: string, region: string, poolName?: string}} args
+ * @returns {string}
+ */
+export function formatCaNodesSpec({
+  minWorkers,
+  maxWorkers,
+  workerServerType,
+  region,
+  poolName = 'worker-pool',
+}) {
+  const caMin = 0;
+  const caMax = Math.max(0, maxWorkers - minWorkers);
+  return `${caMin}:${caMax}:${workerServerType.toUpperCase()}:${region}:${poolName}`;
+}
+
+/**
+ * @typedef {Object} K3sDeployOptions
+ * @property {string} projectName
+ * @property {string} environment
+ * @property {string} provider
+ * @property {string} region
+ * @property {string} [secondaryRegion]
+ * @property {string} masterServerType
+ * @property {string} supabaseServerType
+ * @property {string} workerServerType
+ * @property {string} serverType
+ * @property {number} minWorkers   Static floor (Pulumi-provisioned). Default 1.
+ * @property {number} maxWorkers   CA upper bound (CA pool max = maxWorkers - minWorkers). Default 3.
+ * @property {string} [domain]
+ * @property {string} apiToken     Hetzner Cloud API token (CCM/CSI + cluster-autoscaler).
+ * @property {Object} state
+ * @property {Object} [tracker]
+ * @property {boolean} [quietSuccess]
+ * @property {string} [sharedSshKeyPath]
+ * @property {string} [sharedSshPublicKey]
+ * @property {string|number} [sharedSshKeyId]
+ * @property {Object} [s3Config]
+ */
+
+/**
+ * Generate or reuse an SSH keypair for this environment.
+ * Stored under projectDir/.vibecarbon/ssh-<env>{,.pub}.
+ *
+ * @param {string} projectDir
+ * @param {string} environment
+ * @returns {{privateKeyPath: string, publicKey: string}}
+ */
+export function ensureSshKey(projectDir, environment) {
+  const dir = join(projectDir, '.vibecarbon');
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const privateKeyPath = join(dir, `ssh-${environment}`);
+  const publicKeyPath = `${privateKeyPath}.pub`;
+  if (existsSync(privateKeyPath) && existsSync(publicKeyPath)) {
+    return { privateKeyPath, publicKey: readFileSync(publicKeyPath, 'utf-8').trim() };
+  }
+  // ssh-keygen writes both halves: privateKeyPath in OpenSSH format, and
+  // privateKeyPath.pub as the OpenSSH public key. -N "" = empty passphrase,
+  // -q = quiet so we don't pollute the spinner output, -C tags the key with
+  // a comment for operator-side identification.
+  execFileSync(
+    'ssh-keygen',
+    ['-t', 'ed25519', '-N', '', '-f', privateKeyPath, '-C', `vibecarbon-${environment}`, '-q'],
+    { stdio: 'ignore' },
+  );
+  // ssh-keygen already creates the private key 0600, but defensively re-apply
+  // in case it was regenerated atop a permission-loose existing file.
+  chmodSync(privateKeyPath, 0o600);
+  const publicKey = readFileSync(publicKeyPath, 'utf-8').trim();
+  return { privateKeyPath, publicKey };
+}
+
+/**
+ * Build the standard SSH host-key opts for k3s deploy SSH calls.
+ *
+ * Pinned per-env known_hosts file under `.vibecarbon/known_hosts_<env>` —
+ * `accept-new` adds entries on first connect, then strict-checks on reuse.
+ * `GlobalKnownHostsFile=/dev/null` ignores the system-wide file. Crucially we
+ * also bypass the operator's `~/.ssh/known_hosts` — Hetzner recycles public
+ * IPs, and a stale ECDSA entry from an unrelated server at the same IP would
+ * otherwise reject the connection with "REMOTE HOST IDENTIFICATION HAS
+ * CHANGED" (observed during early k3s real-infra testing).
+ *
+ * @param {string} khPath - absolute path to .vibecarbon/known_hosts_<env>
+ * @returns {string[]} -o argv tokens for ssh/scp
+ */
+function sshHostKeyOpts(khPath) {
+  return [
+    '-o',
+    `UserKnownHostsFile=${khPath}`,
+    '-o',
+    'GlobalKnownHostsFile=/dev/null',
+    '-o',
+    'StrictHostKeyChecking=accept-new',
+    // BatchMode=yes is required so a key-auth failure aborts cleanly with a
+    // non-zero exit instead of falling back to interactive password prompt.
+    // Without this, running deploy in a TTY (e2e runner inherits one)
+    // can hang waiting for input — and on stdio:'inherit' pipelines like
+    // sideloadK3s, the prompt actually appears on the operator's terminal.
+    '-o',
+    'BatchMode=yes',
+  ];
+}
+
+/**
+ * Block until the cloud-init `/tmp/k3s-ready` marker appears via SSH.
+ * Cloud-init writes this after `kubectl get nodes` shows the master Ready.
+ *
+ * Exponential backoff (2s → 4s → 8s, capped at 15s) so early-ready nodes
+ * (~80s on the spike) aren't held back. Long tails still back off so we
+ * don't hammer SSH during slow boots.
+ *
+ * @param {string} masterIp
+ * @param {string} sshKeyPath
+ * @param {string} khPath - per-env known_hosts file path
+ * @param {number} [maxWaitSec]
+ * @returns {Promise<void>}
+ */
+export async function waitForK3sReady(masterIp, sshKeyPath, khPath, maxWaitSec = 600) {
+  const start = Date.now();
+  let waitMs = 2000;
+  let lastErr;
+  while ((Date.now() - start) / 1000 < maxWaitSec) {
+    try {
+      execFileSync(
+        'ssh',
+        [
+          '-i',
+          sshKeyPath,
+          ...sshHostKeyOpts(khPath),
+          '-o',
+          'ConnectTimeout=5',
+          '-o',
+          'BatchMode=yes',
+          `root@${masterIp}`,
+          'test -f /tmp/k3s-ready',
+        ],
+        { stdio: 'ignore' },
+      );
+      return;
+    } catch (err) {
+      lastErr = err;
+      await delay(waitMs);
+      waitMs = Math.min(waitMs * 2, 15000);
+    }
+  }
+  // Capture cloud-init + syslog before throwing so the failure log says
+  // WHY install timed out (apt slow? k3s download 5xx? script crashed?)
+  // instead of just "300s passed and the marker isn't there." Best-effort —
+  // if the box is unreachable, captureK3sInstallDiag returns an empty string
+  // and we surface only the SSH probe error.
+  const diag = captureK3sInstallDiag(masterIp, sshKeyPath, khPath);
+  throw new Error(
+    `k3s did not become ready on ${masterIp} within ${maxWaitSec}s. ` +
+      `Last SSH error: ${lastErr instanceof Error ? lastErr.message : 'unknown'}` +
+      (diag ? `\n\n--- install diagnostics from ${masterIp} ---\n${diag}` : ''),
+  );
+}
+
+/**
+ * scp the master's kubeconfig and patch its server URL to point at the
+ * public master IP (cloud-init writes `127.0.0.1:6443`, useless from
+ * the operator's laptop).
+ *
+ * @param {string} masterIp
+ * @param {string} sshKeyPath
+ * @param {string} khPath - per-env known_hosts file path
+ * @param {string} projectDir
+ * @param {string} environment
+ * @returns {string} absolute path to the local kubeconfig
+ */
+export function fetchKubeconfig(masterIp, sshKeyPath, khPath, projectDir, environment) {
+  const localPath = join(projectDir, '.vibecarbon', `kubeconfig-${environment}`);
+  execFileSync(
+    'scp',
+    [
+      '-i',
+      sshKeyPath,
+      ...sshHostKeyOpts(khPath),
+      `root@${masterIp}:/etc/rancher/k3s/k3s.yaml`,
+      localPath,
+    ],
+    { stdio: 'pipe' },
+  );
+  const kc = readFileSync(localPath, 'utf-8');
+  const patched = kc.replace(
+    /server: https:\/\/127\.0\.0\.1:6443/g,
+    `server: https://${masterIp}:6443`,
+  );
+  writeFileSync(localPath, patched);
+  // Tighten perms — kubeconfig embeds a cluster-admin client cert + key.
+  // scp inherits the operator's umask, so without this the file lands at
+  // 0644 and helm/kubectl warn "Kubernetes configuration file is
+  // world-readable" on every invocation.
+  chmodSync(localPath, 0o600);
+  return localPath;
+}
+
+const CERT_MANAGER_VERSION = 'v1.20.2';
+const CERT_MANAGER_URL = `https://github.com/cert-manager/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml`;
+const SUPABASE_HELM_REPO_URL = 'https://supabase-community.github.io/supabase-kubernetes';
+const SUPABASE_HELM_REPO_NAME = 'supabase-community';
+const SUPABASE_HELM_CHART = `${SUPABASE_HELM_REPO_NAME}/supabase`;
+const SUPABASE_HELM_RELEASE_NAME = 'supabase';
+const SUPABASE_HELM_TIMEOUT = '15m';
+
+// cert-manager-webhook-hetzner: third-party community chart for the
+// Hetzner DNS Console DNS-01 solver. Pinned explicitly — the chart's
+// API surface (groupName / solverName / values keys) is stable but
+// `latest` would silently drift on `helm repo update`. Bump in
+// lockstep with CERT_MANAGER_VERSION; the new e2e scenario
+// k8s-hetzner-dns is the canary that catches breakage.
+const HETZNER_WEBHOOK_HELM_REPO_URL = 'https://charts.hetzner.cloud';
+const HETZNER_WEBHOOK_HELM_REPO_NAME = 'hetzner-cloud';
+const HETZNER_WEBHOOK_CHART = `${HETZNER_WEBHOOK_HELM_REPO_NAME}/cert-manager-webhook-hetzner`;
+const HETZNER_WEBHOOK_CHART_VERSION = '0.7.0';
+const HETZNER_WEBHOOK_RELEASE_NAME = 'cert-manager-webhook-hetzner';
+
+/**
+ * Parse a `.env.local` file into a plain object.
+ *
+ * Strict subset of dotenv: `KEY=VALUE` per line, optional surrounding
+ * single/double quotes stripped, lines starting with `#` or blank ignored.
+ * Values with `=` in them are preserved (split-once on the first `=`).
+ *
+ * @param {string} envPath
+ * @returns {Record<string, string>}
+ */
+export function loadEnvLocal(envPath) {
+  if (!existsSync(envPath)) {
+    throw new Error(`loadEnvLocal: ${envPath} not found. Run 'vibecarbon create' to generate it.`);
+  }
+  const out = {};
+  const raw = readFileSync(envPath, 'utf-8');
+  for (const line of raw.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eq = trimmed.indexOf('=');
+    if (eq === -1) continue;
+    const key = trimmed.slice(0, eq).trim();
+    let val = trimmed.slice(eq + 1).trim();
+    if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
+      val = val.slice(1, -1);
+    }
+    out[key] = val;
+  }
+  return out;
+}
+
+/**
+ * Build the local container image and return its tag.
+ *
+ * Reuses the buildLocalImage primitive from src/lib/deploy/image.js, which
+ * already handles git sha + dirty-state + timestamp. Tag scheme:
+ * `10.0.1.1:5000/<projectName>:<sha>[-dirty]-<utcTimestamp>`.
+ *
+ * The `10.0.1.1:5000/` prefix points at the master's local OCI registry
+ * pod (Phase 1's `registries.yaml` configures kubelet on every node to
+ * resolve `10.0.1.1:5000` → that pod). The same tag is used at every
+ * step (build → sideload → push → Deployment), so:
+ *   - static workers get the image via `sideloadK3s` (containerd cache
+ *     hit, kubelet `imagePullPolicy: IfNotPresent` finds it),
+ *   - CA-spawned workers (which don't exist at sideload time) pull the
+ *     same tag from master's registry pod via `registries.yaml`.
+ *
+ * @param {string} projectDir
+ * @param {string} projectName
+ * @param {boolean} [rebuild]
+ * @returns {Promise<{tag: string, gitSha: string, isDirty: boolean}>}
+ */
+export async function buildAppImage(projectDir, projectName, rebuild = false) {
+  const { buildLocalImage } = await import('../image.js');
+  return buildLocalImage(projectDir, { projectName, rebuild, tagPrefix: '10.0.1.1:5000' });
+}
+
+/**
+ * Wait until the `k3s` binary is installed and on PATH on a given node.
+ *
+ * waitForK3sReady only polls the master's /tmp/k3s-ready marker. Workers
+ * and the supabase node run their own k3s install asynchronously
+ * (worker-init.sh / supabase-init.sh) which can take 1-3 min after the
+ * master is ready (apt update, dpkg locks, k3s install script). If
+ * sideload runs in that window, the remote `k3s ctr images import -`
+ * exits 127 ("command not found") and one node's import fails,
+ * cancelling the entire batch (observed 2026-04-26 e2e run #2 —
+ * worker @ 204.168.185.124 returned 127).
+ *
+ * We poll up to maxWaitSec for `command -v k3s` to succeed. Cheap (~10ms
+ * SSH per probe), idempotent, and resolves the race without changing
+ * cloud-init.
+ *
+ * @param {string} target - SSH target (e.g. root@1.2.3.4)
+ * @param {string} sshKeyPath
+ * @param {string} khPath - per-env known_hosts file
+ * @param {number} [maxWaitSec=300]
+ * @returns {Promise<void>}
+ */
+export async function waitForK3sBinary(target, sshKeyPath, khPath, maxWaitSec = 300) {
+  const start = Date.now();
+  let waitMs = 2000;
+  let lastErr;
+  while ((Date.now() - start) / 1000 < maxWaitSec) {
+    try {
+      execFileSync(
+        'ssh',
+        [
+          '-i',
+          sshKeyPath,
+          ...sshHostKeyOpts(khPath),
+          '-o',
+          'ConnectTimeout=5',
+          '-o',
+          'BatchMode=yes',
+          target,
+          'command -v k3s >/dev/null',
+        ],
+        { stdio: 'ignore' },
+      );
+      return;
+    } catch (err) {
+      lastErr = err;
+      await delay(waitMs);
+      waitMs = Math.min(waitMs * 2, 15000);
+    }
+  }
+  // Same diagnostic capture as waitForK3sReady — see that function for why.
+  // `target` here is `root@<ip>` form; strip the user@ prefix for the host.
+  const ip = String(target).includes('@') ? String(target).split('@').pop() : String(target);
+  const diag = captureK3sInstallDiag(ip, sshKeyPath, khPath);
+  throw new Error(
+    `k3s binary did not appear on ${target} within ${maxWaitSec}s. ` +
+      `Last SSH error: ${lastErr instanceof Error ? lastErr.message : 'unknown'}` +
+      (diag ? `\n\n--- install diagnostics from ${ip} ---\n${diag}` : ''),
+  );
+}
+
+/**
+ * Best-effort capture of cloud-init + syslog tails to answer "why didn't
+ * k3s install?" The caller has already exhausted its readiness budget;
+ * this runs ONE quick SSH per command (5s connect, 15s exec) and returns
+ * the joined output. Each command is wrapped in a short script so we get
+ * partial output when the box is sluggish but not dead.
+ *
+ * Probe order is "answer the most-likely question first":
+ *   1. cloud-init status — is user-data even done? (errored / running / done)
+ *   2. install-attempt grep — did our script try? how many attempts?
+ *   3. cloud-init-output.log (last 200) — install error or `exit 1` line
+ *   4. k3s binary presence — install never wrote /usr/local/bin/k3s
+ *   5. syslog tail — apt errors, kernel oopses
+ *   6. systemctl status k3s — service registered yet?
+ *   7. df -h — a full disk often masquerades as a hung install
+ *
+ * Without these, the 2026-04-27 morning-matrix k8s-ha primary failure
+ * read as "Unit k3s.service could not be found" with no signal as to
+ * whether the install script never started, errored mid-way, or hit
+ * the 5-attempt curl retry limit. The new probes localize the failure
+ * to one of: cloud-init didn't run, install script errored before line
+ * N, install retries exhausted, k3s binary present but service unit
+ * missing/disabled.
+ *
+ * Returns '' if even the first SSH fails (box unreachable / down).
+ */
+function captureK3sInstallDiag(host, sshKeyPath, khPath) {
+  const sshArgs = (cmd) => [
+    '-i',
+    sshKeyPath,
+    ...sshHostKeyOpts(khPath),
+    '-o',
+    'ConnectTimeout=5',
+    '-o',
+    'BatchMode=yes',
+    `root@${host}`,
+    cmd,
+  ];
+
+  // Each probe returns at most `slice` chars from the END of its output
+  // (so when the box dumps a wide cloud-init-output.log the tail — i.e. the
+  // failure point — is what survives truncation, not the early ci-info
+  // tables). Default is 4000; overridden per-probe where the relevant
+  // signal lives in a longer tail.
+  const probes = [
+    {
+      // `cloud-init status --long` reports `status: error|running|done` plus
+      // (on error) the failing module name. This single line tells us
+      // whether user-data even finished — the most useful signal of all.
+      label: 'cloud-init status',
+      cmd: 'cloud-init status --long 2>&1 | head -n 20 || echo "(cloud-init not installed)"',
+    },
+    {
+      // Grep for our master-init.sh markers so a long log boils down to
+      // "did install run, how many times, what was the result?" without
+      // having to read the whole tail. Markers come from the script:
+      //   "Installing k3s version: X"   → start of install section
+      //   "k3s install attempt N failed" → curl|sh retry fired
+      //   "k3s install failed after 5 attempts" → exhausted
+      //   "k3s master installation complete!" → success line
+      label: 'install attempts (cloud-init-output grep)',
+      cmd: 'grep -E "Installing k3s|install attempt|installation complete|FATAL: k3s" /var/log/cloud-init-output.log 2>&1 | tail -n 30 || echo "(no log)"',
+    },
+    {
+      // The output we *most* need to see in a failure: the lines right
+      // before cloud-init bailed out. Wide ci-info tables at the start of
+      // the file used to drown the failure tail under the 4k char cap;
+      // we now keep the last 16k chars of `tail -n 400` so the actual
+      // bash-level error survives truncation.
+      label: 'cloud-init-output.log (last 400 lines)',
+      cmd: 'echo "=== wc -l ===" && wc -l /var/log/cloud-init-output.log 2>&1; echo "=== tail ===" && tail -n 400 /var/log/cloud-init-output.log 2>&1 || echo "(not present)"',
+      slice: 16_000,
+    },
+    {
+      // The framework log (different file from cloud-init-output.log).
+      // When cloud-init's own modules error before user-data runs, the
+      // traceback lands here, not in -output.log. Critical when scripts_user
+      // status says "error" but -output.log shows no command failure.
+      label: 'cloud-init.log (errors + last 200 lines)',
+      cmd: 'echo "=== ERROR / WARNING grep ===" && grep -nE "ERROR|WARNING|CRITICAL|Traceback" /var/log/cloud-init.log 2>&1 | tail -n 60; echo "=== last 200 lines ===" && tail -n 200 /var/log/cloud-init.log 2>&1 || echo "(not present)"',
+      slice: 12_000,
+    },
+    {
+      // Show what command in part-001 actually ran. cloud-init writes the
+      // user-data script verbatim to /var/lib/cloud/instance/scripts/part-001,
+      // so seeing its first/last lines plus an md5 lets us confirm the
+      // generated script matches what we sent (no truncation in transit).
+      label: 'part-001 user-data script',
+      cmd: 'echo "=== md5 + size ===" && md5sum /var/lib/cloud/instance/scripts/*.sh /var/lib/cloud/instance/scripts/part-* 2>&1 | head -n 10; echo "=== head ===" && head -n 5 /var/lib/cloud/instance/scripts/part-* 2>/dev/null || echo "(not present)"; echo "=== tail ===" && tail -n 5 /var/lib/cloud/instance/scripts/part-* 2>/dev/null || echo "(not present)"',
+    },
+    {
+      // Did the install at least produce a binary? Distinguishes "install
+      // script never ran" (no binary) from "install ran but service unit
+      // is borked" (binary present, unit missing).
+      label: 'k3s binary',
+      cmd: 'ls -la /usr/local/bin/k3s 2>&1; which k3s 2>&1 || echo "(not in PATH)"',
+    },
+    {
+      // cloud-final is the systemd unit that runs scripts_user. Its
+      // journal output captures any stderr that didn't make it into
+      // cloud-init-output.log (e.g. when the script process is killed by
+      // OOM or the box reboots mid-script).
+      label: 'journalctl -u cloud-final',
+      cmd: 'journalctl -u cloud-final --no-pager -n 100 2>&1 || echo "(no journal)"',
+      slice: 8_000,
+    },
+    {
+      label: 'syslog (last 50 lines)',
+      cmd: 'tail -n 50 /var/log/syslog 2>&1 || journalctl -n 50 --no-pager 2>&1 || echo "(no log)"',
+    },
+    {
+      label: 'systemctl status k3s',
+      cmd: 'systemctl status k3s --no-pager 2>&1 | head -n 30 || echo "(no service)"',
+    },
+    { label: 'disk usage', cmd: 'df -h / 2>&1' },
+  ];
+
+  const out = [];
+  for (const probe of probes) {
+    const sliceLen = probe.slice ?? 4000;
+    try {
+      const result = execFileSync('ssh', sshArgs(probe.cmd), {
+        encoding: 'utf-8',
+        timeout: 20_000,
+        stdio: ['ignore', 'pipe', 'pipe'],
+        maxBuffer: 4 * 1024 * 1024,
+      });
+      // Keep the END of the output (failure tail), not the start. ci-info
+      // tables at the head of cloud-init-output.log used to push the
+      // failing line off the end; `slice(-sliceLen)` flips that.
+      const trimmed = result.trim();
+      const kept =
+        trimmed.length > sliceLen
+          ? `…(truncated; kept last ${sliceLen} chars)…\n${trimmed.slice(-sliceLen)}`
+          : trimmed;
+      out.push(`### ${probe.label}\n${kept}`);
+    } catch (err) {
+      // First probe fails -> box is dead, no point trying the others.
+      if (out.length === 0) return '';
+      out.push(
+        `### ${probe.label}\n(probe failed: ${err instanceof Error ? err.message.split('\n')[0] : 'unknown'})`,
+      );
+    }
+  }
+  return out.join('\n\n');
+}
+
+/**
+ * Sideload an image to every node in a k3s cluster via parallel SSH.
+ *
+ * Per node: `docker save TAG | ssh -i KEY -o ... TARGET 'k3s ctr images import -'`.
+ * Pipeline streams the tarball over SSH; no temp files on either end, no registry.
+ * Parallel via Promise.all + spawn — wall-clock dominated by the slowest node.
+ *
+ * Before each `docker save | ssh` we wait for `command -v k3s` on the target
+ * (workers/supabase finish their k3s install asynchronously after the master
+ * marker — see waitForK3sBinary). 5-min cap per node; total budget bounded
+ * by the deploy step's lifecycle timeout.
+ *
+ * @param {{tag: string, sshTargets: string[], sshKey: string, khPath: string}} args
+ * @returns {Promise<void>}
+ */
+export async function sideloadK3s({ tag, sshTargets, sshKey, khPath }) {
+  if (!sshTargets || sshTargets.length === 0) {
+    throw new Error('sideloadK3s: at least one SSH target is required');
+  }
+  if (!khPath) {
+    throw new Error('sideloadK3s: khPath is required');
+  }
+  const { spawn } = await import('node:child_process');
+  // We're going through `bash -c` because of the `docker save | ssh` pipe, so
+  // khPath/sshKey/target are interpolated into the shell. Single-quote the
+  // file paths so spaces in cwd don't break the command (and so a hostile
+  // env name — already validated upstream — couldn't smuggle metacharacters).
+  const q = (s) => `'${String(s).replace(/'/g, "'\\''")}'`;
+  // BatchMode=yes mirrors sshHostKeyOpts() — without it, a missing key on
+  // the target falls back to interactive password prompt and the docker
+  // save | gzip | ssh pipeline hangs (stdio:'inherit' surfaces the prompt
+  // on the operator's terminal).
+  const sshOptsStr = `-o UserKnownHostsFile=${q(khPath)} -o GlobalKnownHostsFile=/dev/null -o StrictHostKeyChecking=accept-new -o BatchMode=yes`;
+  await Promise.all(
+    sshTargets.map(async (target) => {
+      // Pre-flight: ensure k3s is on PATH before piping the tarball.
+      // Docker save streams ~600MB; if we ssh into a node that doesn't
+      // have k3s yet, the import errors mid-stream and we waste the
+      // bandwidth + cancel parallel imports on other healthy nodes.
+      // 600s mirrors waitForK3sReady — same cold-init story, same
+      // remedy as the comment there.
+      await waitForK3sBinary(target, sshKey, khPath, 600);
+
+      // Retry the docker-save | ssh pipe on transient failures.
+      // Observed in iter-wave1a (2026-05-01): standby k3s sideload failed
+      // with `kex_exchange_identification: read: Connection timed out`
+      // 21min into a successful deploy — cluster healthy, just an SSH
+      // socket flake. Retrying the pipe is the cheapest fix; the cost on
+      // the genuine-failure path is one extra ~5-10 min attempt, vs the
+      // current cost of throwing away an entire 21-min HA deploy.
+      const SIDELOAD_DELAYS_MS = [5_000, 15_000];
+      const runOnce = () =>
+        new Promise((resolve, reject) => {
+          // gzip on the wire: Node app images compress 3-5x (most layers
+          // are JS source + node_modules text). Without gzip, sideloading
+          // ~600MB to 6 nodes (HA) saturated the laptop's upload at ~25
+          // Mbps each, taking 23-29 min wall-clock (e2e run #1+#2).
+          // With gzip, the same payload is ~150-200MB → expected 5-10 min.
+          // We could reach for `pigz` for parallel compression, but plain
+          // `gzip -1` is universally present and dominant cost is network,
+          // not CPU. `gunzip` is available on Ubuntu 24.04 cloud-init by
+          // default (part of base image). pipefail propagates either side
+          // of the pipeline failing.
+          const cmd =
+            `set -o pipefail && docker save ${q(tag)} | gzip -1 | ` +
+            `ssh -i ${q(sshKey)} ${sshOptsStr} ${q(target)} ` +
+            `'gunzip | k3s ctr images import -'`;
+          const child = spawn('bash', ['-c', cmd], { stdio: 'inherit' });
+          child.on('error', (err) =>
+            reject(new Error(`sideloadK3s spawn failed for ${target}: ${err.message}`)),
+          );
+          child.on('exit', (code) => {
+            if (code === 0) resolve(undefined);
+            else reject(new Error(`sideloadK3s failed on ${target} with exit code ${code}`));
+          });
+        });
+
+      let lastErr;
+      for (let attempt = 0; attempt <= SIDELOAD_DELAYS_MS.length; attempt++) {
+        try {
+          await runOnce();
+          return;
+        } catch (err) {
+          lastErr = err;
+          if (attempt === SIDELOAD_DELAYS_MS.length) break;
+          console.warn(
+            `[sideloadK3s] ${target} attempt ${attempt + 1} failed (${err.message}); retrying in ${SIDELOAD_DELAYS_MS[attempt] / 1000}s`,
+          );
+          await new Promise((r) => setTimeout(r, SIDELOAD_DELAYS_MS[attempt]));
+        }
+      }
+      throw lastErr;
+    }),
+  );
+}
+
+/**
+ * Push a locally-built image to the master's local OCI registry via an
+ * SSH local-port-forward.
+ *
+ * Why an SSH tunnel: the registry pod (Phase 1) listens on the master's
+ * hostPort 5000 inside the cluster network — not exposed to the public
+ * internet (the firewall blocks 5000). Without DNS/L4 pointing the
+ * operator → master:5000, we forward locally:
+ *
+ *     ssh -L 5000:localhost:5000 -N -f <master>
+ *     docker tag 10.0.1.1:5000/<repo>:<tag> localhost:5000/<repo>:<tag>
+ *     docker push localhost:5000/<repo>:<tag>
+ *     pkill -f "ssh -L 5000:localhost:5000"
+ *
+ * Why retag to localhost first: docker resolves the registry hostname in
+ * a tag literally, so `docker push 10.0.1.1:5000/...` would dial
+ * 10.0.1.1 directly — bypassing the SSH tunnel (which is bound to
+ * `localhost:5000`) and timing out, since 10.0.1.1 is a Hetzner private
+ * IP unreachable from the operator. Pushing through `localhost:5000`
+ * routes through the tunnel AND inherits docker's loopback insecure-by-
+ * default behavior (the registry serves plain HTTP). The OCI registry
+ * stores content by `<repo>:<tag>` only — the host portion is just used
+ * to find the registry — so cluster-side pulls of the original
+ * `10.0.1.1:5000/<repo>:<tag>` reference resolve to the same content
+ * (Phase 1's `registries.yaml` maps that hostname to the registry pod).
+ *
+ * The `10.0.1.1:5000/` build prefix is set in `buildAppImage` (above);
+ * keep these two strings in sync.
+ *
+ * Sideload remains the primary distribution path for static workers; the
+ * registry push is additive — it lets CA-spawned workers (which don't
+ * exist at sideload time) pull the same tag on first schedule via
+ * Phase 1's `registries.yaml`.
+ *
+ * Idempotency: a stale forward from a prior crashed run blocks `-L 5000`
+ * binding. We tear down any existing tunnel first (best-effort), open a
+ * fresh one, push, then tear it down again. If the push fails because
+ * the tunnel didn't bind in time, we retry once with a fresh tunnel.
+ *
+ * Per `feedback_ssh_batchmode_required.md`, every ssh invocation
+ * includes `-o BatchMode=yes` so a key-auth failure aborts cleanly
+ * instead of falling back to interactive prompt. Mirrors the
+ * `sshHostKeyOpts` pattern used elsewhere.
+ *
+ * Throws on persistent push failure; tunnel-teardown and local-tag
+ * cleanup failures are non-fatal.
+ *
+ * @param {{
+ *   tag: string,
+ *   masterIp: string,
+ *   sshKey: string,
+ *   khPath: string,
+ *   settleDelaysMs?: number[],
+ * }} args
+ *   `settleDelaysMs` controls the per-attempt sleep between opening the
+ *   tunnel and dialing it. Length of the array is the attempt count.
+ *   Production callers should omit it (the default 1s/15s/30s schedule
+ *   is the considered budget — see comment in the loop). Tests pass
+ *   `[0, 0, 0]` to keep the suite fast.
+ * @returns {Promise<void>}
+ */
+export async function pushImageToLocalRegistry({
+  tag,
+  masterIp,
+  sshKey,
+  khPath,
+  settleDelaysMs,
+  localTunnelPort = 5000,
+}) {
+  if (!tag) throw new Error('pushImageToLocalRegistry: tag is required');
+  if (!masterIp) throw new Error('pushImageToLocalRegistry: masterIp is required');
+  if (!sshKey) throw new Error('pushImageToLocalRegistry: sshKey is required');
+  if (!khPath) throw new Error('pushImageToLocalRegistry: khPath is required');
+  if (!Number.isInteger(localTunnelPort) || localTunnelPort < 1 || localTunnelPort > 65535) {
+    throw new Error(
+      `pushImageToLocalRegistry: localTunnelPort must be a port in [1, 65535], got: ${localTunnelPort}`,
+    );
+  }
+
+  // Build prefix is hardcoded in buildAppImage as `10.0.1.1:5000`. Rewrite
+  // to localhost:<chosenPort> so the push hits THIS deploy's SSH tunnel
+  // and uses HTTP. The remote-side port is always 5000 (the local-registry
+  // hostPort); only the operator-side listen port varies.
+  const remotePrefix = '10.0.1.1:5000/';
+  if (!tag.startsWith(remotePrefix)) {
+    throw new Error(
+      `pushImageToLocalRegistry: expected tag prefixed with '${remotePrefix}', got: ${tag}`,
+    );
+  }
+
+  const target = `root@${masterIp}`;
+
+  // chosenPort + localTag get re-bound on each open-tunnel attempt below.
+  // We can't pick the port up-front because of a TOCTOU race: probing
+  // `isPortFree(5000)`, releasing the listener, then `ssh -L 5000:...` is
+  // not atomic — a sibling deploy in the matrix (k8s + k8s-ha primary
+  // both default to 5000) can win the bind in between. The fix is to try
+  // ssh directly and walk forward on failure, treating "tunnel open" as
+  // the only true free-port signal. Observed iter-wave1b 2026-05-01:
+  // single-k8s and k8s-ha-primary collided here.
+  let localTag = '';
+  let forwardSpec = '';
+  let tunnelPattern = '';
+  const setPort = (port) => {
+    forwardSpec = `${port}:localhost:5000`;
+    tunnelPattern = `ssh.*-L.*${forwardSpec}`;
+    localTag = `localhost:${port}/${tag.slice(remotePrefix.length)}`;
+  };
+  setPort(localTunnelPort);
+
+  /** Best-effort teardown of any existing tunnel on localhost:<port>. */
+  const teardown = () => {
+    try {
+      execFileSync('pkill', ['-f', tunnelPattern], { stdio: 'ignore' });
+    } catch {
+      // pkill exits 1 when no matching process — that's fine.
+    }
+  };
+
+  /**
+   * Try to open the SSH tunnel on the currently-set port; if SSH bind
+   * fails (sibling deploy holds the port), walk forward up to 20 ports.
+   * Does NOT call teardown() in the walk — the pkill pattern would
+   * match sibling processes' tunnels too. The walk is bind-race-resilient
+   * because ssh exits non-zero immediately on bind failure (via
+   * ExitOnForwardFailure=yes).
+   */
+  const openTunnelOrWalk = () => {
+    const errors = [];
+    for (let probe = 0; probe < 20; probe++) {
+      const candidate = localTunnelPort + probe;
+      if (candidate > 65535) break;
+      setPort(candidate);
+      try {
+        execFileSync(
+          'ssh',
+          [
+            '-i',
+            sshKey,
+            ...sshHostKeyOpts(khPath),
+            '-o',
+            'ExitOnForwardFailure=yes',
+            '-L',
+            forwardSpec,
+            '-N',
+            '-f',
+            target,
+          ],
+          { stdio: 'inherit' },
+        );
+        return;
+      } catch (err) {
+        errors.push(`port ${candidate}: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+    throw new Error(
+      `pushImageToLocalRegistry: failed to open SSH tunnel on any port in [${localTunnelPort}, ${localTunnelPort + 19}]. ` +
+        `Errors: ${errors.join(' | ')}`,
+    );
+  };
+  // openTunnel signature stays the same so the retry loop below is
+  // unchanged; it's now bind-race-resilient instead of TOCTOU-racy.
+  const openTunnel = openTunnelOrWalk;
+
+  /** Push the localhost-aliased tag through the local-forwarded port. */
+  const push = () => {
+    execFileSync('docker', ['push', localTag], { stdio: 'inherit' });
+  };
+
+  // Three attempts with progressive backoff. Failure modes we're
+  // covering, in priority order:
+  //   1. Stale tunnel from a parallel run holds :5000 → reopen tunnel.
+  //      Recovers in seconds (handled by the per-attempt teardown).
+  //   2. Registry pod restarted between back-to-back pushes (e.g.
+  //      probe-killed mid-prior-push) → manifest PUT returns EOF or
+  //      `manifest blob unknown to registry`. Pod recovery is ~10–20s,
+  //      so the second/third attempt has the longer backoff to let the
+  //      replacement pod become Ready before re-pushing.
+  //   3. S3 transient (Hetzner Object Storage 5xx during blob upload)
+  //      → registry returns the same `blob unknown` after the layer is
+  //      "uploaded" but not yet readable. Same recovery shape as (2).
+  // The 1s/15s/30s schedule keeps the happy path fast (single attempt
+  // ~6s end-to-end) while giving genuine restarts time to recover.
+  const settleDelays = settleDelaysMs ?? [1_000, 15_000, 30_000];
+  const maxAttempts = settleDelays.length;
+  const attemptErrors = [];
+  const taggedAliases = new Set();
+  let pushed = false;
+  // Per-attempt structured logging — without this, the only visible signal
+  // for a slow push is the final perf duration (which collapses retry
+  // count + per-attempt time into a single number). iter-perfwave2 showed
+  // primary registryPush.backup 2m53s vs standby 41s with no way to tell
+  // if the gap was 1 slow push or 3 retried pushes. The log lines below
+  // make next run's analysis trivial: count `[push] attempt=` lines for
+  // attempt-count, sum `dur=` for actual push wall-clock.
+  const startedAt = Date.now();
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const attemptStart = Date.now();
+    let outcome = 'ok';
+    try {
+      // openTunnel may walk to a different port on bind-collision; localTag
+      // is updated in lockstep so push targets the actual chosen port.
+      openTunnel();
+      // Add the localhost-prefixed alias for the chosen port. Skip the
+      // re-tag if we already aliased this port on an earlier attempt
+      // (same port → same alias, docker tag is idempotent but the call
+      // is wasteful). Track all aliases for cleanup so a port walk
+      // doesn't leak local image refs.
+      if (!taggedAliases.has(localTag)) {
+        execFileSync('docker', ['tag', tag, localTag], { stdio: 'inherit' });
+        taggedAliases.add(localTag);
+      }
+      await delay(settleDelays[attempt]);
+      push();
+      pushed = true;
+      break;
+    } catch (err) {
+      attemptErrors.push(err instanceof Error ? err.message : String(err));
+      outcome = `fail(${(err instanceof Error ? err.message : String(err)).split('\n')[0].slice(0, 80)})`;
+    } finally {
+      // Tear down between attempts so a stuck tunnel from this attempt
+      // doesn't bind-block the next attempt's openTunnel walk.
+      teardown();
+      const dur = Math.round((Date.now() - attemptStart) / 1000);
+      const totalDur = Math.round((Date.now() - startedAt) / 1000);
+      console.error(
+        `[push] tag=${tag.split('/').pop()} attempt=${attempt + 1}/${maxAttempts} port=${localTag.split('/')[0]} settle=${settleDelays[attempt] / 1000}s dur=${dur}s totalDur=${totalDur}s ${outcome}`,
+      );
+    }
+  }
+  // Cleanup every localhost alias we created. Underlying image content
+  // stays addressable as the original `tag` (used by sideload + Deployment).
+  for (const alias of taggedAliases) {
+    try {
+      execFileSync('docker', ['rmi', alias], { stdio: 'ignore' });
+    } catch {
+      // non-fatal — leftover alias just sits in the local docker cache.
+    }
+  }
+  if (!pushed) {
+    throw new Error(
+      `pushImageToLocalRegistry: docker push for ${tag} failed after ${maxAttempts} attempts. ` +
+        `Errors: ${attemptErrors.map((m, i) => `[#${i + 1}] ${m}`).join(' | ')}`,
+    );
+  }
+}
+
+/**
+ * Subset of .env.local keys that must land in the in-cluster
+ * vibecarbon-secrets Secret. The Supabase Helm chart reads JWT_SECRET +
+ * ANON_KEY + SERVICE_ROLE_KEY via secretRef from this Secret; the app
+ * Deployment reads DB_PASSWORD + the rest via envFrom.
+ *
+ * Defined as an explicit allowlist (not "everything in .env.local")
+ * because .env.local also carries client-side VITE_* values that don't
+ * belong in a Secret.
+ */
+const SECRET_KEYS = [
+  'DB_PASSWORD',
+  'JWT_SECRET',
+  'ANON_KEY',
+  'SERVICE_ROLE_KEY',
+  'REALTIME_SECRET',
+  'LOGFLARE_API_KEY',
+  'VAULT_ENC_KEY',
+  'PG_META_CRYPTO_KEY',
+  'DB_ENC_KEY',
+  'REPL_PASSWORD',
+  'ADMIN_EMAIL',
+  'ADMIN_PASSWORD',
+  'ADMIN_PASSWORD_HASH',
+];
+
+/**
+ * Apply the in-cluster `vibecarbon-secrets` Secret from the project's
+ * `.env.local`. Idempotent: rendered as a Secret manifest and piped to
+ * `kubectl apply -f -` (server-side apply semantics) so re-runs upsert
+ * cleanly.
+ *
+ * Must run BEFORE the base/ kustomization apply, because the app
+ * Deployment in base/ references this Secret via envFrom.
+ *
+ * S3 backup credentials (S3_ACCESS_KEY/SECRET_KEY/BUCKET/ENDPOINT/REGION)
+ * are merged in if `s3Config` is provided — backup CronJob reads them.
+ * `backupBucketName` is the dedicated backup bucket (separate from the
+ * storage bucket in s3Config.bucket); without it, backup.sh falls back to
+ * the storage bucket and uploads dumps next to user files (observed
+ * 2026-04-27 k8s-ha matrix #6 — restore couldn't find backups).
+ *
+ * @param {{kubeconfig: string, envLocal: Record<string,string>, s3Config?: {accessKey: string, secretKey: string, bucket: string, endpoint: string, region: string}, backupBucketName?: string|null}} args
+ */
+export async function applyVibecarbonSecrets({ kubeconfig, envLocal, s3Config, backupBucketName }) {
+  const env = { ...process.env, KUBECONFIG: kubeconfig };
+  /** @type {Record<string,string>} */
+  const stringData = {};
+  for (const key of SECRET_KEYS) {
+    if (envLocal[key]) stringData[key] = envLocal[key];
+  }
+  // .env.local stores Supabase keys with the SUPABASE_ prefix (matches the
+  // app's runtime env vars), but the supabase Helm chart's secretRefKey
+  // mapping reads them under their bare names from this Secret. Translate.
+  // Without this, kubelet fails kong/storage/studio/app pods with
+  // "couldn't find key ANON_KEY in Secret vibecarbon/vibecarbon-secrets".
+  if (!stringData.ANON_KEY && envLocal.SUPABASE_ANON_KEY) {
+    stringData.ANON_KEY = envLocal.SUPABASE_ANON_KEY;
+  }
+  if (!stringData.SERVICE_ROLE_KEY && envLocal.SUPABASE_SERVICE_ROLE_KEY) {
+    stringData.SERVICE_ROLE_KEY = envLocal.SUPABASE_SERVICE_ROLE_KEY;
+  }
+  // Supabase chart's db.secretRefKey.database maps to DB_NAME in this Secret.
+  // Neither .env.local nor SECRET_KEYS supplies it, so the chart-canonical
+  // default "postgres" is hardcoded — kubelet otherwise fails the supabase-db
+  // pod with "couldn't find key DB_NAME in Secret vibecarbon/vibecarbon-secrets".
+  if (!stringData.DB_NAME) stringData.DB_NAME = envLocal.POSTGRES_DB || 'postgres';
+  if (s3Config) {
+    stringData.S3_ACCESS_KEY = s3Config.accessKey;
+    stringData.S3_SECRET_KEY = s3Config.secretKey;
+    stringData.S3_BUCKET = s3Config.bucket;
+    stringData.S3_ENDPOINT = s3Config.endpoint;
+    stringData.S3_REGION = s3Config.region;
+  }
+  if (backupBucketName) {
+    stringData.S3_BACKUP_BUCKET = backupBucketName;
+  }
+  // Ensure the namespace exists before we try to land a Secret in it.
+  // base/ kustomization creates it too, but we run before base/.
+  // Two-step (render → apply) instead of `bash -c '… | kubectl apply -f -'`
+  // so each kubectl call goes through runKubectlWithRetry — apiserver
+  // transients during k3s warm-up otherwise kill the deploy here too.
+  const namespaceYaml = await runKubectlWithRetry(
+    ['create', 'namespace', 'vibecarbon', '--dry-run=client', '-o', 'yaml'],
+    { env, captureStdout: true, description: 'applyVibecarbonSecrets: render namespace yaml' },
+  );
+  await runKubectlWithRetry(['apply', '-f', '-'], {
+    env,
+    input: namespaceYaml,
+    description: 'applyVibecarbonSecrets: kubectl apply namespace/vibecarbon',
+  });
+  // Build secret YAML; pipe to kubectl apply via stdin (values never hit
+  // a shell argv). stringData → kubectl base64-encodes for us.
+  const secretYaml = [
+    'apiVersion: v1',
+    'kind: Secret',
+    'metadata:',
+    '  name: vibecarbon-secrets',
+    '  namespace: vibecarbon',
+    'type: Opaque',
+    'stringData:',
+    ...Object.entries(stringData).map(([k, v]) => `  ${k}: ${JSON.stringify(v)}`),
+  ].join('\n');
+  await runKubectlWithRetry(['apply', '-f', '-'], {
+    env,
+    input: secretYaml,
+    description: 'applyVibecarbonSecrets: kubectl apply Secret/vibecarbon-secrets',
+  });
+}
+
+/**
+ * Render the project's k8s/values/supabase.values.yaml template and
+ * `helm upgrade --install` the supabase community chart. Waits for the
+ * release to converge before returning.
+ *
+ * Direct `helm upgrade --install` keeps Flux off the deploy path; Flux
+ * is opt-in via `vibecarbon configure cicd`.
+ *
+ * @param {{kubeconfig: string, projectDir: string, projectName: string, domain: string, envLocal?: Record<string,string>}} args
+ */
+export async function installSupabase({
+  kubeconfig,
+  projectDir,
+  projectName,
+  domain,
+  s3Config,
+  envLocal,
+}) {
+  const env = { ...process.env, KUBECONFIG: kubeconfig };
+  // Render the values template — substitute every {{TOKEN}} the template
+  // uses so the rendered yaml is fully valid before helm sees it.
+  const valuesTemplatePath = join(projectDir, 'k8s/values/supabase.values.yaml');
+  if (!existsSync(valuesTemplatePath)) {
+    throw new Error(
+      `installSupabase: ${valuesTemplatePath} not found. Project must include the supabase values template.`,
+    );
+  }
+  const rendered = readFileSync(valuesTemplatePath, 'utf-8')
+    .replace(/\{\{DOMAIN\}\}/g, domain || 'localhost')
+    .replace(/\{\{PROJECT_NAME\}\}/g, projectName || 'vibecarbon')
+    .replace(/\{\{S3_BACKUP_BUCKET\}\}/g, s3Config?.bucket ?? '')
+    // Studio dashboard creds — sourced from .env.local at deploy time. Empty
+    // string fallback keeps yaml valid; chart inlines them into the
+    // chart-generated supabase-dashboard secret.
+    .replace(/\{\{ADMIN_EMAIL\}\}/g, envLocal?.ADMIN_EMAIL ?? '')
+    .replace(/\{\{ADMIN_PASSWORD\}\}/g, envLocal?.ADMIN_PASSWORD ?? '');
+  const tmpValues = join(tmpdir(), `vibecarbon-supabase-values-${process.pid}-${Date.now()}.yaml`);
+  writeFileSync(tmpValues, rendered);
+  try {
+    // Add the supabase-community helm repo (idempotent — non-zero if exists).
+    try {
+      execFileSync('helm', ['repo', 'add', SUPABASE_HELM_REPO_NAME, SUPABASE_HELM_REPO_URL], {
+        stdio: 'inherit',
+        env,
+      });
+    } catch {
+      // helm repo add fails if the repo already exists; `helm repo update`
+      // below refreshes either way.
+    }
+    execFileSync('helm', ['repo', 'update', SUPABASE_HELM_REPO_NAME], {
+      stdio: 'inherit',
+      env,
+    });
+    // --wait blocks until all pods are Ready. --create-namespace is a no-op
+    // if vibecarbon namespace already exists.
+    //
+    // Async (spawn-based via runCommandAsync) — `helm --wait --timeout 15m`
+    // can block for 1-15 minutes. A sync execFileSync would block Node's
+    // event loop the whole time, inflating every concurrent perfAsync's
+    // `Date.now()` measurement on parallel branches. In HA the standby
+    // cluster's installSupabase runs in parallel with primary's; without
+    // this, primary's blocking helm-wait makes standby's perf substep
+    // numbers unreadable (RCA from iter-validate6, where standby's
+    // certManager.wait reported 21 minutes despite kubectl wait finishing
+    // in seconds — the wall-clock was inflated by primary's blocking).
+    const helmStart = Date.now();
+    await runCommandAsync(
+      [
+        'helm',
+        'upgrade',
+        '--install',
+        SUPABASE_HELM_RELEASE_NAME,
+        SUPABASE_HELM_CHART,
+        '--namespace',
+        'vibecarbon',
+        '--create-namespace',
+        '-f',
+        tmpValues,
+        '--wait',
+        '--timeout',
+        SUPABASE_HELM_TIMEOUT,
+      ],
+      { silent: false, env },
+    );
+    const helmDur = Math.round((Date.now() - helmStart) / 1000);
+    // Snapshot pod state immediately after `--wait` returns. Each pod's
+    // age tells us how long IT took to become Ready (kubectl AGE = now -
+    // pod.creationTimestamp; for newly-created pods the age ≈ helm wait).
+    // Without this, the only signal for a slow installSupabase is the
+    // umbrella perfAsync number — which collapses 14+ pods into one
+    // duration. iter-perfwave2 showed primary 3m47s vs standby 1m43s
+    // with no way to localize which pod was the long pole.
+    try {
+      const podSnapshot = execFileSync(
+        'kubectl',
+        [
+          '-n',
+          'vibecarbon',
+          'get',
+          'pods',
+          '-l',
+          'app.kubernetes.io/instance=supabase',
+          '-o',
+          'custom-columns=NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,AGE:.metadata.creationTimestamp,NODE:.spec.nodeName',
+          '--no-headers',
+        ],
+        { encoding: 'utf8', env, stdio: ['ignore', 'pipe', 'pipe'] },
+      ).trim();
+      console.error(`[supabase] helm-wait ${helmDur}s; post-wait pods:\n${podSnapshot}`);
+    } catch (e) {
+      console.error(
+        `[supabase] helm-wait ${helmDur}s; pod snapshot failed: ${e.message?.slice(0, 80)}`,
+      );
+    }
+  } finally {
+    try {
+      unlinkSync(tmpValues);
+    } catch {
+      // non-fatal: leave tmp file, OS will clean /tmp eventually
+    }
+  }
+}
+
+/**
+ * Wait until the supabase storage schema has fully initialized — specifically
+ * until `storage.buckets.public` column exists. The supabase chart's
+ * storage-api container runs its own schema migrations on startup which lag
+ * helm's `--wait`-Ready signal by 30-90s on cold clusters; running our own
+ * migrations against an in-progress storage schema fails with errors like
+ * "column 'public' of relation 'buckets' does not exist" when an INSERT
+ * INTO storage.buckets references a column the storage container hasn't
+ * added yet (observed 2026-04-26 e2e run #4 — k8s-ha standby
+ * cluster).
+ *
+ * Polls every 5s for up to maxWaitSec. Idempotent: if the column already
+ * exists (warm path), returns immediately on first probe.
+ *
+ * Each kubectl exec is bounded to ~15s wall via `--request-timeout=15s` plus
+ * a Node-side kill backstop. A bare `spawnSync(kubectl exec)` with no
+ * timeout can hang for arbitrary time when the apiserver streaming
+ * connection wedges, which silently bypasses maxWaitSec — observed
+ * 2026-05-08 e2e run #3 where standby's poll ran ~1005s against a
+ * nominal maxWaitSec=300 and rolled into the parallel primary's 16.7-min
+ * rolloutApp window. Bounding each call also lets first-failure stderr
+ * surface so a real storage-api crashloop is distinguishable from an exec
+ * connectivity issue. Uses async spawn so the parent's event loop stays
+ * free for the parallel HA branch.
+ *
+ * @param {{kubeconfig: string, maxWaitSec?: number}} args
+ */
+export async function waitForSupabaseStorageSchema({ kubeconfig, maxWaitSec = 600 }) {
+  const env = { ...process.env, KUBECONFIG: kubeconfig };
+  const probe = `SELECT 1 FROM information_schema.columns WHERE table_schema='storage' AND table_name='buckets' AND column_name='public';`;
+  const start = Date.now();
+  const waitMs = 5000;
+  const perCallSec = 15;
+  let attempt = 0;
+  let firstErrLogged = false;
+  while ((Date.now() - start) / 1000 < maxWaitSec) {
+    attempt += 1;
+    const { status, stdout, stderr } = await execKubectlOnce(
+      [
+        '-n',
+        'vibecarbon',
+        'exec',
+        '-i',
+        `--request-timeout=${perCallSec}s`,
+        'supabase-supabase-db-0',
+        '--',
+        'psql',
+        '-U',
+        'supabase_admin',
+        '-d',
+        'postgres',
+        '-tA',
+      ],
+      { env, input: probe, killAfterMs: (perCallSec + 5) * 1000 },
+    );
+    if (status === 0 && stdout.trim() === '1') return;
+    if (!firstErrLogged && (status !== 0 || stderr)) {
+      // Surface the first failure so a real crashloop / connectivity issue is
+      // visible instead of silently looping. Subsequent polls stay quiet to
+      // keep the deploy log readable.
+      const msg = (stderr || stdout || '(no output)').trim().slice(-500);
+      console.log(
+        `[waitForSupabaseStorageSchema] attempt ${attempt} status=${status ?? '?'} — ${msg}`,
+      );
+      firstErrLogged = true;
+    }
+    await delay(waitMs);
+  }
+  throw new Error(
+    `Supabase storage schema didn't add storage.buckets.public column within ${maxWaitSec}s. ` +
+      `Storage-api container may have crashlooped — check kubectl logs supabase-supabase-storage-0`,
+  );
+}
+
+/**
+ * Single bounded kubectl exec: spawn-based (non-blocking), per-call kill
+ * backstop on top of kubectl's own `--request-timeout`. Caller passes argv
+ * starting after `kubectl`. Returns { status, stdout, stderr } with status
+ * `null` if killed by the backstop.
+ *
+ * @param {string[]} argv
+ * @param {{env: NodeJS.ProcessEnv, input?: string, killAfterMs: number}} opts
+ */
+function execKubectlOnce(argv, { env, input, killAfterMs }) {
+  return new Promise((resolveFn) => {
+    const child = spawn('kubectl', argv, {
+      env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (b) => {
+      stdout += b.toString();
+    });
+    child.stderr.on('data', (b) => {
+      stderr += b.toString();
+    });
+    const killTimer = setTimeout(() => {
+      try {
+        child.kill('SIGKILL');
+      } catch {
+        /* ignore */
+      }
+    }, killAfterMs);
+    child.on('error', () => {
+      clearTimeout(killTimer);
+      resolveFn({ status: null, stdout, stderr });
+    });
+    child.on('close', (code) => {
+      clearTimeout(killTimer);
+      resolveFn({ status: code, stdout, stderr });
+    });
+    if (input != null) {
+      child.stdin.write(input);
+      child.stdin.end();
+    } else {
+      child.stdin.end();
+    }
+  });
+}
+
+/**
+ * Apply the project's `supabase/migrations/*.sql` against the chart-installed
+ * supabase-postgres pod. Runs each file in lexicographic order via
+ * `kubectl exec -i ... psql -U supabase_admin -d postgres`. Errors per file
+ * are surfaced — unlike compose's `|| true` swallow — so a broken migration
+ * stops the deploy with a useful error instead of a 503 readiness probe
+ * loop downstream.
+ *
+ * The supabase-db StatefulSet pod must be Ready (helm `--wait` ensures this).
+ *
+ * @param {{kubeconfig: string, projectDir: string}} args
+ */
+export async function applyMigrations({ kubeconfig, projectDir }) {
+  const env = { ...process.env, KUBECONFIG: kubeconfig };
+  const migrationsDir = join(projectDir, 'supabase', 'migrations');
+  if (!existsSync(migrationsDir)) {
+    // Project has no migrations directory — skip silently. Matches compose
+    // behaviour (which `|| true`'s past a missing dir).
+    return;
+  }
+  const files = readdirSync(migrationsDir)
+    .filter((f) => f.endsWith('.sql'))
+    .sort();
+  if (files.length === 0) return;
+  // Wait for supabase's own storage-api schema to finish initializing —
+  // some project migrations (e.g. INSERT INTO storage.buckets with a
+  // `public` column) reference columns added by storage-api's startup
+  // migrations, which can lag helm's --wait Ready signal.
+  await waitForSupabaseStorageSchema({ kubeconfig });
+  for (const file of files) {
+    const sql = readFileSync(join(migrationsDir, file), 'utf-8');
+    const result = spawnSync(
+      'kubectl',
+      [
+        '-n',
+        'vibecarbon',
+        'exec',
+        '-i',
+        'supabase-supabase-db-0',
+        '--',
+        'psql',
+        '-U',
+        'supabase_admin',
+        '-d',
+        'postgres',
+        '-v',
+        'ON_ERROR_STOP=1',
+      ],
+      { env, input: sql, stdio: ['pipe', 'inherit', 'inherit'] },
+    );
+    if (result.status !== 0) {
+      throw new Error(
+        `applyMigrations: ${file} failed with exit ${result.status}. ` +
+          `Fix the migration or set ON_ERROR_STOP=0 manually.`,
+      );
+    }
+  }
+}
+
+/**
+ * Apply cert-manager + vibecarbon-secrets + the project's k8s/infra +
+ * k8s/base kustomizations, install supabase via helm, patch the app
+ * deployment to use the locally-sideloaded image, and wait for rollout.
+ *
+ * Local-first by default — no Flux on the deploy path; Flux is opt-in
+ * via `vibecarbon configure cicd`.
+ *
+ * @param {{kubeconfig: string, projectDir: string, projectName: string, imageTag: string, backupImageTag?: string, envLocal: Record<string,string>, domain: string, s3Config?: {accessKey: string, secretKey: string, bucket: string, endpoint: string, region: string}, backupBucketName?: string|null, dnsProvider?: string|null, cloudflareApiToken?: string|null, hetznerApiToken?: string|null, apiToken?: string, region?: string, environment?: string, minWorkers?: number, maxWorkers?: number, workerServerType?: string, k3sToken?: string, masterIp?: string, sshKeyPath?: string, khPath?: string}} args
+ */
+export async function applyK3sManifests({
+  kubeconfig,
+  projectDir,
+  projectName,
+  imageTag,
+  backupImageTag,
+  envLocal,
+  domain,
+  s3Config,
+  backupBucketName,
+  dnsProvider,
+  cloudflareApiToken,
+  hetznerApiToken,
+  // Phase 5: cluster-autoscaler wiring. `apiToken` is the Hetzner Cloud
+  // API token (CCM/CSI/CA), distinct from `hetznerApiToken` which only
+  // names the cert-manager DNS-01 token (same VALUE in practice since
+  // Hetzner consolidated APIs in May 2026, but kept separate so the
+  // wiring is explicit and a future split is a no-op).
+  apiToken,
+  region,
+  environment,
+  minWorkers,
+  maxWorkers,
+  workerServerType,
+  k3sToken,
+  // Phase 6: registry-push wiring. masterIp/sshKeyPath/khPath are the
+  // SSH coordinates needed to open a local-port-forward to master:5000
+  // for `docker push` (mirrors what sideloadK3s already received via a
+  // different shape — this set is operator → master only).
+  masterIp,
+  sshKeyPath,
+  khPath,
+  // HA runs primary+standby deploys in parallel; both call this
+  // function from the same operator host. Without distinct ports, the
+  // second `ssh -L 5000:localhost:5000` loses the bind race and
+  // ExitOnForwardFailure=yes aborts the deploy. Default 5000 is the
+  // single-cluster path; HA passes 5000 (primary) and 5001 (standby).
+  localTunnelPort = 5000,
+  // perfPrefix tags internal sub-stage perf timings — see deployK3s.
+  perfPrefix = 'k3s',
+}) {
+  const env = { ...process.env, KUBECONFIG: kubeconfig };
+  // 0. Validate DNS-01 prerequisites BEFORE any apply so the failure
+  //    surfaces at deploy-start, not 20 min in when an Order pins Pending.
+  //    Throws on missing token for cloudflare/hetzner; manual returns null.
+  const dnsSecret = buildDnsProviderSecret({
+    dnsProvider,
+    cloudflareApiToken,
+    hetznerApiToken,
+  });
+  // 1. Install cert-manager (raw YAML; not a kustomize base).
+  await perfAsync(`deploy.${perfPrefix}.certManager.install`, () =>
+    runKubectlWithRetry(['apply', '-f', CERT_MANAGER_URL], {
+      env,
+      description: 'applyK3sManifests: kubectl apply cert-manager',
+    }),
+  );
+  // 2. Wait for cert-manager to be ready before applying ClusterIssuers.
+  // Fan out the three deploys in parallel: kubectl wait blocks per-arg
+  // sequentially, and the cainjector + webhook usually become Available
+  // ~10-30s after the core deploy, so the serial form pays the longest
+  // arrival time three times in a row. Two-stage retry preserves the
+  // outer 180s+120s envelope of the previous --timeout=180s call.
+  const certManagerDeploys = [
+    'deploy/cert-manager',
+    'deploy/cert-manager-webhook',
+    'deploy/cert-manager-cainjector',
+  ];
+  await perfAsync(`deploy.${perfPrefix}.certManager.wait`, () =>
+    Promise.all(
+      certManagerDeploys.map(async (deploy) => {
+        const waitOnce = (timeoutSec) =>
+          new Promise((resolveFn, reject) => {
+            const child = spawn(
+              'kubectl',
+              [
+                '-n',
+                'cert-manager',
+                'wait',
+                '--for=condition=Available',
+                `--timeout=${timeoutSec}s`,
+                deploy,
+              ],
+              { stdio: 'inherit', env },
+            );
+            child.on('error', reject);
+            child.on('exit', (code) =>
+              code === 0 ? resolveFn() : reject(new Error(`kubectl wait ${deploy} exited ${code}`)),
+            );
+          });
+        try {
+          await waitOnce(90);
+        } catch {
+          await waitOnce(120);
+        }
+      }),
+    ),
+  );
+  // 2b. Land the per-DNS-provider Secret in the cert-manager namespace
+  //     BEFORE the kustomization that defines the issuers, so when the
+  //     ClusterIssuer is created the controller can immediately resolve
+  //     `apiTokenSecretRef`. Stdin pipe keeps the token out of argv.
+  if (dnsSecret) {
+    await runKubectlWithRetry(['apply', '-f', '-'], {
+      env,
+      input: dnsSecret.yaml,
+      description: `applyK3sManifests: kubectl apply Secret/${dnsSecret.name}`,
+    });
+  }
+  // 2c. Hetzner DNS-01 needs the cert-manager-webhook-hetzner deployment
+  //     because cert-manager core has no native Hetzner solver. The
+  //     webhook registers an APIService that cert-manager calls into for
+  //     dns01.webhook(groupName=acme.hetzner.com) — it must be
+  //     Available BEFORE any Certificate referencing the issuer is
+  //     reconciled, otherwise Orders fail with "no provider for solver".
+  //     Mirrors the Supabase pattern at step 7.
+  if (dnsProvider === 'hetzner') {
+    try {
+      execFileSync(
+        'helm',
+        ['repo', 'add', HETZNER_WEBHOOK_HELM_REPO_NAME, HETZNER_WEBHOOK_HELM_REPO_URL],
+        { stdio: 'inherit', env },
+      );
+    } catch {
+      // helm repo add fails if the repo already exists; `helm repo update`
+      // below refreshes either way.
+    }
+    execFileSync('helm', ['repo', 'update', HETZNER_WEBHOOK_HELM_REPO_NAME], {
+      stdio: 'inherit',
+      env,
+    });
+    try {
+      // Chart 0.7.0 reads zero Helm values that bind the issuer's
+      // hetzner-api-token Secret — that linkage lives entirely in the
+      // ClusterIssuer's `config:` block (secretName + apiUrl). The
+      // chart's `groupName` default `acme.hetzner.com` matches the
+      // ClusterIssuer's groupName, so we don't override it either.
+      // Async helm + kubectl wait — both can block for minutes; sync would
+      // block Node's event loop and inflate parallel-branch perf timings.
+      // See installSupabase above for the full RCA (iter-validate6 standby
+      // certManager.wait reporting 21min when actual was seconds).
+      await runCommandAsync(
+        [
+          'helm',
+          'upgrade',
+          '--install',
+          HETZNER_WEBHOOK_RELEASE_NAME,
+          HETZNER_WEBHOOK_CHART,
+          '--namespace',
+          'cert-manager',
+          '--version',
+          HETZNER_WEBHOOK_CHART_VERSION,
+          '--wait',
+          '--timeout',
+          '5m',
+        ],
+        { silent: false, env },
+      );
+      await runCommandAsync(
+        [
+          'kubectl',
+          '-n',
+          'cert-manager',
+          'wait',
+          '--for=condition=Available',
+          '--timeout=180s',
+          `deploy/${HETZNER_WEBHOOK_RELEASE_NAME}`,
+        ],
+        { silent: false, env },
+      );
+    } catch (err) {
+      // Surface webhook pod logs on failure — chart install rarely
+      // tells the operator anything useful when the actual cause is
+      // a CrashLoopBackOff or a missing dependency.
+      try {
+        execFileSync(
+          'kubectl',
+          ['-n', 'cert-manager', 'logs', `deploy/${HETZNER_WEBHOOK_RELEASE_NAME}`, '--tail=200'],
+          { stdio: 'inherit', env },
+        );
+      } catch {
+        // best-effort; continue to original throw
+      }
+      throw err;
+    }
+  }
+  // 3. Apply infra (cert-manager-resources/cluster-issuers + traefik-crds).
+  await runKubectlWithRetry(['apply', '-k', join(projectDir, 'k8s/infra/cert-manager-resources')], {
+    env,
+    description: 'applyK3sManifests: kubectl apply -k cert-manager-resources',
+  });
+  await runKubectlWithRetry(['apply', '-k', join(projectDir, 'k8s/infra/traefik-crds')], {
+    env,
+    description: 'applyK3sManifests: kubectl apply -k traefik-crds',
+  });
+  // 4. Create vibecarbon-secrets BEFORE base/ — base/'s Deployment
+  //    references it via envFrom; without it kubelet fails container
+  //    creation with `secret "vibecarbon-secrets" not found`.
+  await applyVibecarbonSecrets({ kubeconfig, envLocal, s3Config, backupBucketName });
+  // 4b. Create the kube-system/hcloud-cluster-config Secret that
+  //     cluster-autoscaler reads on startup. Two keys:
+  //     - `token`: Hetzner Cloud API token, exposed to CA via
+  //       valueFrom→HCLOUD_TOKEN in deployment.yaml.
+  //     - `cluster-config.json`: BASE64-ENCODED rendered node-pool config
+  //       from renderCAClusterConfig (Phase 4), exposed via
+  //       valueFrom→HCLOUD_CLUSTER_CONFIG. The autoscaler binary expects
+  //       HCLOUD_CLUSTER_CONFIG to be base64-encoded JSON (decoded inline
+  //       at startup) — NOT raw JSON. We must base64-encode here because
+  //       Kubernetes secretKeyRef injection auto-decodes the Secret data
+  //       to plaintext: storing raw JSON in stringData means the env var
+  //       receives raw JSON, which fails with "illegal base64 data at
+  //       input byte 0". Storing base64-encoded JSON in stringData means
+  //       the env var receives the base64 string the autoscaler wants.
+  //     RCA from k8s e2e fanout5 2026-05-01: cluster-autoscaler
+  //     crashlooped throughout deploy because the previous
+  //     HCLOUD_CLUSTER_CONFIG_FILE mechanism is not in v1.32.7 and the
+  //     env var path needed double-encoding to round-trip through K8s.
+  //     Stdin pipe keeps the token + JSON off argv (mirrors the
+  //     dnsSecret pattern at step 2b; enforced by
+  //     tests/unit/security/secrets-not-in-argv.test.ts).
+  if (!apiToken) {
+    throw new Error(
+      'applyK3sManifests: apiToken (Hetzner Cloud API token) required for cluster-autoscaler Secret/hcloud-cluster-config',
+    );
+  }
+  if (!k3sToken) {
+    throw new Error(
+      'applyK3sManifests: k3sToken required to render cluster-autoscaler cluster-config.json',
+    );
+  }
+  const caClusterConfigJson = renderCAClusterConfig({
+    k3sVersion: K3S_VERSION,
+    k3sToken,
+    clusterName: `${projectName}-${environment}`,
+  });
+  // See block comment above: cluster-autoscaler reads HCLOUD_CLUSTER_CONFIG
+  // as base64-encoded JSON. Storing the encoded form in stringData means the
+  // secretKeyRef→env injection delivers the base64 string verbatim.
+  const caClusterConfigB64 = Buffer.from(caClusterConfigJson, 'utf-8').toString('base64');
+  {
+    const caSecretYaml = [
+      'apiVersion: v1',
+      'kind: Secret',
+      'metadata:',
+      '  name: hcloud-cluster-config',
+      '  namespace: kube-system',
+      'type: Opaque',
+      'stringData:',
+      `  token: ${JSON.stringify(apiToken)}`,
+      `  cluster-config.json: ${JSON.stringify(caClusterConfigB64)}`,
+      '',
+    ].join('\n');
+    await runKubectlWithRetry(['apply', '-f', '-'], {
+      env,
+      input: caSecretYaml,
+      description: 'applyK3sManifests: kubectl apply Secret/hcloud-cluster-config',
+    });
+  }
+  // 5. Apply base (namespace, network-policies, app, backup, traefik, config).
+  await perfAsync(`deploy.${perfPrefix}.applyBase`, () =>
+    runKubectlWithRetry(['apply', '-k', join(projectDir, 'k8s/base')], {
+      env,
+      description: 'applyK3sManifests: kubectl apply -k k8s/base',
+    }),
+  );
+  // 5b'. Wait for the local registry pod (Phase 1) to be Ready before
+  //      pushing images. The pod is part of base/, so it just landed.
+  //      Without this wait, the SSH-tunnel push race-targets a Pending
+  //      pod (registry container not started → connection refused on
+  //      :5000). 120s mirrors other base-pod readiness budgets.
+  await perfAsync(`deploy.${perfPrefix}.localRegistry.wait`, async () =>
+    execFileSync(
+      'kubectl',
+      [
+        '-n',
+        'vibecarbon',
+        'wait',
+        '--for=condition=Ready',
+        'pod',
+        '-l',
+        'app=local-registry',
+        '--timeout=120s',
+      ],
+      { stdio: 'inherit', env },
+    ),
+  );
+  // 5b''. Push the app image to the local registry on master so
+  //       CA-spawned workers (which don't exist at sideload time) can
+  //       pull on first schedule. Sideload remains primary distribution
+  //       for static workers — this is additive. Both paths use the
+  //       same `10.0.1.1:5000/<project>:<tag>` reference (Phase 1's
+  //       registries.yaml maps that hostname to the registry pod).
+  if (!masterIp) {
+    throw new Error('applyK3sManifests: masterIp required for registry push (Phase 6)');
+  }
+  if (!sshKeyPath) {
+    throw new Error('applyK3sManifests: sshKeyPath required for registry push (Phase 6)');
+  }
+  if (!khPath) {
+    throw new Error('applyK3sManifests: khPath required for registry push (Phase 6)');
+  }
+  // Push app + backup images in parallel. Each opens its own SSH tunnel,
+  // so they need DISTINCT localTunnelPorts — pushImageToLocalRegistry's
+  // teardown is `pkill -f 'ssh.*-L.*<port>:localhost:5000'`, which would
+  // kill a sibling push's tunnel mid-operation if both used the same port.
+  // Offset the backup push by +100 so even the openTunnelOrWalk's 20-port
+  // fallback has 80 ports of headroom before colliding (plus HA's 5001
+  // standby cluster lands on +100=5101, well clear of primary's 5000+19).
+  // RCA: parallelizing without an offset would silently corrupt pushes
+  // (registry returns "blob unknown" on a teardown'd partial upload).
+  const appPushPort = localTunnelPort;
+  const backupPushPort = localTunnelPort + 100;
+  if (!masterIp) {
+    throw new Error('applyK3sManifests: masterIp required for registry push (Phase 6)');
+  }
+  if (!sshKeyPath) {
+    throw new Error('applyK3sManifests: sshKeyPath required for registry push (Phase 6)');
+  }
+  if (!khPath) {
+    throw new Error('applyK3sManifests: khPath required for registry push (Phase 6)');
+  }
+  const pushTasks = [
+    perfAsync(`deploy.${perfPrefix}.registryPush.app`, () =>
+      pushImageToLocalRegistry({
+        tag: imageTag,
+        masterIp,
+        sshKey: sshKeyPath,
+        khPath,
+        localTunnelPort: appPushPort,
+      }),
+    ),
+  ];
+  // 5b'''. Push the backup image too. The backup CronJob's pods may
+  //        eventually schedule onto a CA-spawned worker (no node
+  //        affinity pinning them to the static pool), so the same
+  //        registry-fallback story applies.
+  if (backupImageTag) {
+    pushTasks.push(
+      perfAsync(`deploy.${perfPrefix}.registryPush.backup`, () =>
+        pushImageToLocalRegistry({
+          tag: backupImageTag,
+          masterIp,
+          sshKey: sshKeyPath,
+          khPath,
+          localTunnelPort: backupPushPort,
+        }),
+      ),
+    );
+  }
+  await Promise.all(pushTasks);
+  // 5a'. Apply cluster-autoscaler manifests SEPARATELY. CA lives in
+  //      kube-system but base/'s parent kustomization sets
+  //      `namespace: vibecarbon`, whose namespace transformer would
+  //      override our kube-system metadata. Mirrors the CCM/CSI
+  //      separate-install pattern (those are commented in
+  //      base/kustomization.yaml and applied via cloud-init). The
+  //      child kustomization at carbon/k8s/base/cluster-autoscaler/
+  //      sets `namespace: kube-system` explicitly, which only takes
+  //      effect when applied standalone like this.
+  await runKubectlWithRetry(['apply', '-k', join(projectDir, 'k8s/base/cluster-autoscaler')], {
+    env,
+    description: 'applyK3sManifests: kubectl apply -k cluster-autoscaler',
+  });
+  // 5b. Patch the domain-specific resources that the base/ kustomization
+  //     ships with placeholder values (`app.example.com` + a placeholder
+  //     ClusterIssuer name). Without this, cert-manager tries to issue a Let's
+  //     Encrypt prod cert for `app.example.com` — ACME refuses (IANA-reserved
+  //     domain), the Order errors, the Certificate's secret never lands,
+  //     ingress TLS fails, and the app's progressDeadlineSeconds expires
+  //     during rollout-status (step 9) → deploy fails ~26 min in. The
+  //     placeholder is intentional in the template (the comment says
+  //     "patched at deploy time"); this is the patch.
+  //
+  // Issuer choice: `ACME_CA_SERVER` points at staging for e2e tests
+  // (and any dev env that wants to avoid LE rate limits). Read from
+  // .env.local first (per-project), then fall back to process.env so the
+  // e2e harness can inject the staging URL via env without writing
+  // it into the project's .env.local. Anything else gets the prod issuer.
+  // Provider suffix (cloudflare/hetzner/manual) determines which solver
+  // — DNS-01 vs HTTP-01 — cert-manager actually uses; see pickIssuerName.
+  const acmeServer = envLocal?.ACME_CA_SERVER || process.env.ACME_CA_SERVER || '';
+  const issuerName = pickIssuerName({ dnsProvider, acmeServer });
+  await runKubectlWithRetry(
+    [
+      '-n',
+      'vibecarbon',
+      'patch',
+      'certificate',
+      'vibecarbon-tls',
+      '--type=merge',
+      '-p',
+      JSON.stringify({
+        spec: { dnsNames: [domain], issuerRef: { name: issuerName, kind: 'ClusterIssuer' } },
+      }),
+    ],
+    { env, description: 'applyK3sManifests: kubectl patch certificate/vibecarbon-tls' },
+  );
+  await runKubectlWithRetry(
+    [
+      '-n',
+      'vibecarbon',
+      'patch',
+      'configmap',
+      'vibecarbon-config',
+      '--type=merge',
+      '-p',
+      JSON.stringify({ data: { SITE_URL: `https://${domain}` } }),
+    ],
+    { env, description: 'applyK3sManifests: kubectl patch configmap/vibecarbon-config' },
+  );
+  // 5d. Patch cluster-autoscaler Deployment placeholders. Phase 2's
+  //     deployment.yaml ships with `--nodes={{CA_NODES_SPEC}}` and
+  //     {{CLUSTER_NAME}}/{{REGION}} markers in the env vars. The
+  //     runtime values are computed here from operator-supplied
+  //     bounds + region. CA's pool sits ON TOP of the static floor
+  //     (see formatCaNodesSpec for the math). Patch via JSON-pointer
+  //     into the placeholder argv slot — index lookup keeps us
+  //     resilient to upstream argv reorder.
+  const clusterName = `${projectName}-${environment}`;
+  const caNodesSpec = formatCaNodesSpec({
+    minWorkers,
+    maxWorkers,
+    workerServerType,
+    region,
+  });
+  {
+    const argvJson = await runKubectlWithRetry(
+      [
+        '-n',
+        'kube-system',
+        'get',
+        'deploy/cluster-autoscaler',
+        '-o',
+        'jsonpath={.spec.template.spec.containers[0].args}',
+      ],
+      {
+        env,
+        captureStdout: true,
+        description: 'applyK3sManifests: kubectl get deploy/cluster-autoscaler args',
+      },
+    );
+    const argv = JSON.parse(argvJson);
+    const idx = argv.indexOf('--nodes={{CA_NODES_SPEC}}');
+    if (idx < 0) {
+      throw new Error(
+        'applyK3sManifests: cluster-autoscaler args missing --nodes={{CA_NODES_SPEC}} placeholder',
+      );
+    }
+    await runKubectlWithRetry(
+      [
+        '-n',
+        'kube-system',
+        'patch',
+        'deploy/cluster-autoscaler',
+        '--type=json',
+        '-p',
+        JSON.stringify([
+          {
+            op: 'replace',
+            path: `/spec/template/spec/containers/0/args/${idx}`,
+            value: `--nodes=${caNodesSpec}`,
+          },
+        ]),
+      ],
+      { env, description: 'applyK3sManifests: kubectl patch deploy/cluster-autoscaler --nodes' },
+    );
+  }
+  // Patch the three by-name env vars. HCLOUD_CLUSTER_CONFIG is sourced
+  // from the hcloud-cluster-config Secret (key cluster-config.json) at
+  // pod-start, so no patch needed for it.
+  await runKubectlWithRetry(
+    [
+      '-n',
+      'kube-system',
+      'set',
+      'env',
+      'deploy/cluster-autoscaler',
+      `HCLOUD_NETWORK=${clusterName}-network`,
+      `HCLOUD_FIREWALL=${clusterName}-firewall`,
+      `HCLOUD_SSH_KEY=${clusterName}-${region}-key`,
+    ],
+    { env, description: 'applyK3sManifests: kubectl set env deploy/cluster-autoscaler' },
+  );
+  // (Considered: setting HCLOUD_NETWORK on the Hetzner CCM so it populates
+  // each Node's InternalIP from the Hetzner private network. Reverted —
+  // because k3s installs with `--node-ip=<public>` per master-init.sh,
+  // the kubelet TLS cert SAN includes only the public IP. CCM then
+  // overwriting InternalIP with 10.0.1.x makes kubectl logs/exec fail with
+  // "tls: failed to verify certificate: x509: certificate is valid for
+  // 127.0.0.1, <public>, not 10.0.1.x", which silently breaks every
+  // diagnostic the lifecycle relies on. The actual fix for the
+  // traefik→apiserver "connection refused" loop turned out to be the
+  // NetworkPolicy egress widening in
+  // carbon/k8s/base/{traefik,app}/network-policy.yaml plus the explicit
+  // traefik rollout-restart below — kube-router's iptables can ship up
+  // stale on first apply, and forcing traefik to re-establish its watch
+  // after policies are in place clears it.)
+  // 5e. Roll out cluster-autoscaler so the patched Deployment lands.
+  //     `set env` already triggers a rolling update, but a deliberate
+  //     rollout-restart + status-wait gives us a clean failure mode if
+  //     CA is misconfigured (e.g. wrong network name) instead of
+  //     surfacing as a downstream "no nodes scaled" mystery.
+  await perfAsync(`deploy.${perfPrefix}.clusterAutoscaler.rollout`, async () => {
+    await runKubectlWithRetry(
+      ['-n', 'kube-system', 'rollout', 'restart', 'deploy/cluster-autoscaler'],
+      { env, description: 'applyK3sManifests: kubectl rollout restart cluster-autoscaler' },
+    );
+    // rollout-status has its own --timeout and benefits from live progress;
+    // leave on execFileSync. Internal apiserver-watch retries cover transient
+    // drops within the budget.
+    //
+    // 300s (was 180s): Phase 5 fires THREE rolling updates back-to-back on a
+    // `strategy: Recreate` Deployment (5d patch --nodes, 5d' set env,
+    // 5e rollout restart). Each cycle = old pod terminate + image-pull-or-cache
+    // + new pod start ≈ 30-60s. Three cycles + cold image pull from
+    // registry.k8s.io stretches the wall-clock right against 180s; primary
+    // finishes just under, standby hits the deadline (k8s-ha 2026-04-29 run).
+    // Capture pod state on failure so future timeouts surface root cause
+    // instead of just "rollout status timed out".
+    try {
+      execFileSync(
+        'kubectl',
+        ['-n', 'kube-system', 'rollout', 'status', 'deploy/cluster-autoscaler', '--timeout=300s'],
+        { stdio: 'inherit', env },
+      );
+    } catch (err) {
+      console.error('[k3s] cluster-autoscaler rollout failed — capturing diagnostics:');
+      for (const args of [
+        ['-n', 'kube-system', 'get', 'pods', '-l', 'app=cluster-autoscaler', '-o', 'wide'],
+        ['-n', 'kube-system', 'describe', 'deploy/cluster-autoscaler'],
+        ['-n', 'kube-system', 'logs', '-l', 'app=cluster-autoscaler', '--tail=80'],
+      ]) {
+        try {
+          execFileSync('kubectl', args, { stdio: 'inherit', env });
+        } catch {
+          // Best-effort diagnostics; original error is what matters
+        }
+      }
+      throw err;
+    }
+  });
+  // 6. Patch the app deployment to use the sideloaded image BEFORE installSupabase.
+  //    base/ ships image=ghcr.io/<owner>/<repo>:main — no pull secret in
+  //    local-first mode, so leaving it pointed at GHCR triggers
+  //    ImagePullBackOff that races against installSupabase's `--wait`.
+  //    Switching now means the app pods come up on the sideloaded image
+  //    immediately; we still wait for rollout after Supabase converges.
+  await runKubectlWithRetry(
+    ['-n', 'vibecarbon', 'set', 'image', 'deployment/app', `app=${imageTag}`],
+    { env, description: 'applyK3sManifests: kubectl set image deployment/app' },
+  );
+  // 6b. Patch the backup CronJob image. base/backup/cronjob.yaml ships
+  //     image=ghcr.io/<owner>/<project>-backup:latest — never pushed in
+  //     direct/local mode, so the scheduled job pod fails ImagePullBackOff.
+  //     deployK3s sideloaded the backup image to every node (step 7b);
+  //     point the CronJob at it. imagePullPolicy: Always in the template
+  //     would defeat the sideload — switch to IfNotPresent at the same
+  //     time so kubelet uses the local copy.
+  if (backupImageTag) {
+    await runKubectlWithRetry(
+      ['-n', 'vibecarbon', 'set', 'image', 'cronjob/backup', `backup=${backupImageTag}`],
+      { env, description: 'applyK3sManifests: kubectl set image cronjob/backup' },
+    );
+    await runKubectlWithRetry(
+      [
+        '-n',
+        'vibecarbon',
+        'patch',
+        'cronjob/backup',
+        '--type=json',
+        '-p',
+        JSON.stringify([
+          {
+            op: 'replace',
+            path: '/spec/jobTemplate/spec/template/spec/containers/0/imagePullPolicy',
+            value: 'IfNotPresent',
+          },
+        ]),
+      ],
+      { env, description: 'applyK3sManifests: kubectl patch cronjob/backup imagePullPolicy' },
+    );
+  }
+  // 7. Install Supabase via helm. Blocks until release converges (15m
+  //    timeout in chart). App pods may stay Pending on supabase URLs in
+  //    their config until kong/auth/db are Ready, but that's fine — they
+  //    won't churn ImagePullBackOff because step 6 already set them to
+  //    the sideloaded image.
+  await perfAsync(`deploy.${perfPrefix}.installSupabase`, async () =>
+    installSupabase({ kubeconfig, projectDir, projectName, domain, s3Config, envLocal }),
+  );
+  // 8. Apply project SQL migrations against the chart-installed postgres.
+  //    Helm's `--wait` returned only after supabase-db was Ready, so the
+  //    pod is guaranteed up here. Without this, the app's
+  //    /api/health/ready returns 503 forever ("Could not find table
+  //    public.organizations").
+  await perfAsync(`deploy.${perfPrefix}.applyMigrations`, async () =>
+    applyMigrations({ kubeconfig, projectDir }),
+  );
+  // 9. Wait for app rollout. The deployment template already has
+  //    imagePullPolicy: IfNotPresent, so kubelet uses the sideloaded image
+  //    without pulling. 300s wasn't enough on a cold deploy where the app
+  //    pods stay Pending behind kong/auth converge — bump to 10m to match
+  //    the helm `--wait` budget.
+  //
+  //    Retry on apiserver-warm-up transients. iter-postpush + iter-validate2
+  //    both saw `kubectl rollout status` fast-fail in ~10s on the k8s-ha
+  //    restore re-deploy with "Unable to connect to the server: net/http:
+  //    TLS handshake timeout" — the apiserver was still cycling certs after
+  //    the cluster came up. Pods rolled out fine within 2 minutes; kubectl
+  //    just bailed on the first connect. We can't use runKubectlWithRetry
+  //    here because rollout-status genuinely needs live stdout (operator
+  //    sees per-replica progress over a 10min window). Hand-rolled retry
+  //    keeps inherited stdout, captures stderr through a pipe so the
+  //    transient-error pattern can fire, and short-circuits non-transient
+  //    failures (real rollout failures or the deployment not existing
+  //    surface immediately, not after 3×retry).
+  await perfAsync(`deploy.${perfPrefix}.rolloutApp`, async () => {
+    const rolloutArgs = [
+      '-n',
+      'vibecarbon',
+      'rollout',
+      'status',
+      'deployment/app',
+      '--timeout=600s',
+    ];
+    const ATTEMPTS = 3;
+    // Async (spawn-based) — `kubectl rollout status --timeout=600s` can wait
+    // up to 10 minutes for pods to roll. spawnSync would block Node's event
+    // loop the whole time; in HA the parallel standby branch's perf substep
+    // numbers would all be inflated by primary's rollout-wait. Stream stdout
+    // to operator (inherit), pipe stderr so the transient-error classifier
+    // can fire (KUBECTL_TRANSIENT_PATTERN check below).
+    const runRollout = () =>
+      new Promise((resolve, reject) => {
+        const child = spawn('kubectl', rolloutArgs, {
+          env,
+          stdio: ['ignore', 'inherit', 'pipe'],
+        });
+        let stderrBuf = '';
+        child.stderr?.on('data', (chunk) => {
+          const s = chunk.toString();
+          stderrBuf += s;
+          process.stderr.write(s);
+        });
+        child.on('error', reject);
+        child.on('exit', (code) => resolve({ status: code, stderr: stderrBuf }));
+      });
+    let lastStderr = '';
+    let lastStatus;
+    for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {
+      const result = await runRollout();
+      lastStatus = result.status;
+      lastStderr = result.stderr;
+      if (lastStatus === 0) return;
+      if (!KUBECTL_TRANSIENT_PATTERN.test(lastStderr) || attempt === ATTEMPTS) break;
+      const backoffMs = 2000 * attempt;
+      console.log(
+        `kubectl rollout status hit transient error on attempt ${attempt}/${ATTEMPTS}, retrying in ${backoffMs}ms`,
+      );
+      await delay(backoffMs);
+    }
+    throw new Error(
+      `kubectl rollout status failed with exit ${lastStatus ?? '?'}: ${lastStderr.trim().slice(-500)}`,
+    );
+  });
+  // 10. Force-restart traefik so it starts AFTER kube-router has installed
+  //     iptables rules for the NetworkPolicies. RCA from k8s-hetzner
+  //     2026-04-28 deploy run (1di log): when traefik starts during
+  //     kustomize apply, kube-router's NetworkPolicy controller may not yet
+  //     have reconciled the new policies. traefik's first connection to
+  //     10.43.0.1:443 races the rule install — it gets stuck in a retry
+  //     loop on `connect: connection refused`, never loads IngressRoutes,
+  //     and serves its default 404 on every external request. Bouncing
+  //     traefik live fixed every observed instance (k8s-ha verify-1df-1dg
+  //     and k8s-hetzner restore probe). Forcing a clean rollout-restart
+  //     here closes the race deterministically — by the time we restart
+  //     traefik, all NetworkPolicies + ClusterIssuers + IngressRoutes have
+  //     landed and kube-router has had time to reconcile.
+  //     ignoreError=true is intentional: traefik uses hostPort 80/443 with
+  //     RollingUpdate, so the new pod can't start until the old one
+  //     terminates. We force-delete the existing pod (graceful would block
+  //     on the hostPort race), then wait briefly for the replacement to
+  //     stabilize. If anything in this block fails, the deploy still
+  //     succeeds — the public probe will catch a real issue.
+  try {
+    await perfAsync(`deploy.${perfPrefix}.traefik.restart`, async () => {
+      await runKubectlWithRetry(
+        [
+          '-n',
+          'vibecarbon',
+          'delete',
+          'pod',
+          '-l',
+          'app=vibecarbon-traefik',
+          '--force',
+          '--grace-period=0',
+        ],
+        { env, description: 'applyK3sManifests: kubectl delete pod traefik' },
+      );
+      execFileSync(
+        'kubectl',
+        ['-n', 'vibecarbon', 'rollout', 'status', 'deployment/traefik', '--timeout=120s'],
+        { stdio: 'inherit', env },
+      );
+    });
+  } catch (err) {
+    console.error(
+      `[k3s] traefik post-apply restart warning: ${err.message?.split('\n')[0] || err}`,
+    );
+  }
+}
+
+/**
+ * Deploy a k3s cluster on Hetzner.
+ *
+ * Cold-path implementation through to "kubeconfig saved + cluster reachable."
+ * App install + sideload land in a follow-up commit (Task 7).
+ *
+ * @param {K3sDeployOptions} options
+ * @returns {Promise<{masterIp: string, floatingIp: string, supabaseIp: string, workerIps: string[], networkId: string|number, kubeconfig: string, sshKeyPath: string}>}
+ */
+export async function deployK3s(options) {
+  const { state } = options;
+  const spin = (options.tracker ?? p).spinner.bind(options.tracker ?? p);
+  const s = spin();
+  const projectDir = process.cwd();
+  // perfPrefix tags all internal sub-stage perf timings. Default 'k3s' for
+  // single-cluster (`deploy.k3s.iac.upStack` etc); HA passes 'k3s.primary' /
+  // 'k3s.standby' so the perf_substep table can attribute primary/standby
+  // deploy time per sub-stage. Without this, both HA clusters' timings
+  // collide on identical sub-stage names and we can't tell which dominated.
+  const perfPrefix = options.perfPrefix ?? 'k3s';
+
+  // 1. Preflight
+  s.start('Preflight (docker + ssh + kubectl + helm)');
+  for (const tool of ['docker', 'ssh', 'kubectl', 'helm']) {
+    try {
+      execFileSync('which', [tool], { stdio: 'ignore' });
+    } catch {
+      throw new Error(`Required tool '${tool}' not found on PATH`);
+    }
+  }
+  s.stop('Preflight OK');
+
+  // 2. SSH key
+  const { privateKeyPath, publicKey } = options.sharedSshKeyPath
+    ? {
+        privateKeyPath: options.sharedSshKeyPath,
+        publicKey: options.sharedSshPublicKey ?? '',
+      }
+    : ensureSshKey(projectDir, options.environment);
+
+  // Per-env known_hosts file. host-keys.js: <projectDir>/.vibecarbon/known_hosts_<env>.
+  // Used by every SSH/SCP call below so we never read or pollute the
+  // operator's ~/.ssh/known_hosts (which would otherwise reject reconnects
+  // when Hetzner recycles a public IP).
+  const khPath = knownHostsPath(options.environment, projectDir);
+
+  // 3. Pulumi up — provisions network, firewall, master, supabase, workers,
+  //    floating IP. Cloud-init runs k3s install + hcloud-ccm/csi on each node.
+  // Phase 5: Pulumi sees `minWorkers` (static floor) only; `maxWorkers`
+  // is consumed by the cluster-autoscaler patch in applyK3sManifests
+  // and intentionally NOT passed to Pulumi. Defaults match the JSDoc on
+  // K8sStackConfig (1 / 3) — the orchestrator (Phase 8) populates from
+  // CLI flags + envConfig.
+  const minWorkers = options.minWorkers ?? 1;
+  const maxWorkers = options.maxWorkers ?? 3;
+  const infraInputs = {
+    region: options.region,
+    masterType: options.masterServerType,
+    supabaseType: options.supabaseServerType,
+    workerType: options.workerServerType,
+    minWorkers,
+    maxWorkers,
+    k3sVersion: K3S_VERSION,
+  };
+  if (!state.shouldSkip('k3s-infra', infraInputs)) {
+    state.startStep('k3s-infra', infraInputs);
+    const { buildHetznerK8sProgram } = await import('../../iac/programs/hetzner-k8s.js');
+    const { upStack } = await import('../../iac/index.js');
+
+    const allowedCidrs = (options.operatorCidrs ?? []).map((e) => e.cidr);
+    const program = buildHetznerK8sProgram({
+      projectName: options.projectName,
+      environment: options.environment,
+      sshPublicKey: publicKey,
+      existingSshKeyId: options.sharedSshKeyId ? String(options.sharedSshKeyId) : undefined,
+      location: options.region,
+      networkZone: 'eu-central',
+      masterServerType: options.masterServerType || options.serverType || 'cx23',
+      supabaseServerType: options.supabaseServerType || options.serverType || 'cx23',
+      workerServerType: options.workerServerType || options.serverType || 'cx23',
+      minWorkers,
+      maxWorkers,
+      k3sVersion: K3S_VERSION,
+      apiToken: options.apiToken,
+      labels: { 'managed-by': 'vibecarbon', 'os-flavor': 'k3s' },
+      allowedSshIps: allowedCidrs,
+      allowedK8sApiIps: allowedCidrs,
+    });
+
+    s.start('Provisioning k3s cluster (Pulumi up)');
+    const { outputs } = await perfAsync(`deploy.${perfPrefix}.iac.upStack`, () =>
+      upStack(options.environment, program, {
+        hcloudToken: options.apiToken,
+        s3Config: options.s3Config,
+      }),
+    );
+    if (!outputs.masterIp || !outputs.floatingIp) {
+      throw new Error('Pulumi outputs missing masterIp or floatingIp');
+    }
+    state.completeStep('k3s-infra', outputs);
+    s.stop(`Cluster provisioned — master=${outputs.masterIp} floating=${outputs.floatingIp}`);
+  }
+  const infraOutputs = state.getStepResult('k3s-infra');
+  const { masterIp, floatingIp, supabaseIp, workerIps, networkId } = infraOutputs;
+
+  // 4. Wait for cloud-init to finish + k3s to be Ready
+  if (!state.shouldSkip('k3s-ready', { masterIp })) {
+    state.startStep('k3s-ready', { masterIp });
+    s.start('Waiting for k3s on master (cloud-init + Ready)');
+    // 1200s (20 min) covers cold-init + apt-install + k3s-download + k3s-up.
+    // Earlier 600s budget tripped on 2026-04-27 morning matrix when apt was
+    // mid-install at the timeout (cloud-init-output.log diagnostics
+    // confirmed). The diagnostic capture inside waitForK3sReady means a
+    // genuine bug surfaces with cloud-init log content, so a longer budget
+    // costs us nothing — we still get the same actionable signal on real
+    // failures, just don't false-positive on slow apt mornings.
+    await perfAsync(`deploy.${perfPrefix}.waitForReady`, () =>
+      waitForK3sReady(masterIp, privateKeyPath, khPath, 1200),
+    );
+    state.completeStep('k3s-ready');
+    s.stop('k3s reachable');
+  }
+
+  // 5. Fetch + patch kubeconfig
+  let kubeconfig;
+  if (!state.shouldSkip('k3s-kubeconfig', { masterIp })) {
+    state.startStep('k3s-kubeconfig', { masterIp });
+    s.start('Retrieving kubeconfig');
+    kubeconfig = fetchKubeconfig(masterIp, privateKeyPath, khPath, projectDir, options.environment);
+    state.completeStep('k3s-kubeconfig', { kubeconfig });
+    s.stop('Kubeconfig saved');
+  } else {
+    kubeconfig = state.getStepResult('k3s-kubeconfig').kubeconfig;
+  }
+
+  // 6. Build the app image locally
+  const buildInputs = { projectName: options.projectName };
+  if (!state.shouldSkip('k3s-build', buildInputs)) {
+    state.startStep('k3s-build', buildInputs);
+    s.start('Building app image (10.0.1.1:5000/...)');
+    const built = await buildAppImage(projectDir, options.projectName);
+    state.completeStep('k3s-build', {
+      tag: built.tag,
+      gitSha: built.gitSha,
+      isDirty: built.isDirty,
+    });
+    s.stop(`Image built: ${built.tag}`);
+  }
+  const { tag: imageTag } = state.getStepResult('k3s-build');
+
+  // 7. Build the backup image up-front so we can fan out BOTH sideloads in
+  //    parallel. base/'s CronJob ships image=ghcr.io/<owner>/<project>-backup
+  //    — unreachable in direct/local mode (no GHCR push). Build the
+  //    <projectDir>/backup context locally, then later patch the CronJob
+  //    image in applyK3sManifests (step 5c).
+  const backupBuildInputs = { projectName: `${options.projectName}-backup` };
+  if (!state.shouldSkip('k3s-backup-build', backupBuildInputs)) {
+    state.startStep('k3s-backup-build', backupBuildInputs);
+    s.start('Building backup image');
+    const built = await buildAppImage(join(projectDir, 'backup'), `${options.projectName}-backup`);
+    state.completeStep('k3s-backup-build', {
+      tag: built.tag,
+      gitSha: built.gitSha,
+      isDirty: built.isDirty,
+    });
+    s.stop(`Backup image built: ${built.tag}`);
+  }
+  const { tag: backupImageTag } = state.getStepResult('k3s-backup-build');
+
+  // 7b. Sideload to every node (parallel SSH within sideloadK3s) AND fan out
+  //     the app+backup sideloads against each other. Previously sequential:
+  //     sideload.app then sideload.backup ≈ 1m4s + 57s on k8s-ha primary.
+  //     Both writes go through `docker save | ssh ... gunzip | k3s ctr import`
+  //     against disjoint image refs on the same target nodes — kernel disk
+  //     buffer + sshd are happy to multiplex two parallel streams. Cuts the
+  //     critical path to max(app, backup) instead of sum.
+  const sshTargets = [`root@${masterIp}`];
+  if (supabaseIp) sshTargets.push(`root@${supabaseIp}`);
+  for (const wIp of workerIps ?? []) sshTargets.push(`root@${wIp}`);
+  const sideloadInputs = { imageTag, targets: sshTargets.join(',') };
+  const backupSideloadInputs = { backupImageTag, targets: sshTargets.join(',') };
+  const sideloadTasks = [];
+  if (!state.shouldSkip('k3s-sideload', sideloadInputs)) {
+    state.startStep('k3s-sideload', sideloadInputs);
+    sideloadTasks.push(
+      perfAsync(`deploy.${perfPrefix}.sideload.app`, () =>
+        sideloadK3s({ tag: imageTag, sshTargets, sshKey: privateKeyPath, khPath }),
+      ).then(() => state.completeStep('k3s-sideload')),
+    );
+  }
+  if (!state.shouldSkip('k3s-backup-sideload', backupSideloadInputs)) {
+    state.startStep('k3s-backup-sideload', backupSideloadInputs);
+    sideloadTasks.push(
+      perfAsync(`deploy.${perfPrefix}.sideload.backup`, () =>
+        sideloadK3s({ tag: backupImageTag, sshTargets, sshKey: privateKeyPath, khPath }),
+      ).then(() => state.completeStep('k3s-backup-sideload')),
+    );
+  }
+  if (sideloadTasks.length > 0) {
+    s.start(`Sideloading app + backup to ${sshTargets.length} node(s) in parallel`);
+    await Promise.all(sideloadTasks);
+    s.stop('Sideload complete (app + backup)');
+  }
+
+  // 8. Apply manifests + supabase Helm release + roll out the app.
+  if (!state.shouldSkip('k3s-apply', { imageTag, backupImageTag })) {
+    state.startStep('k3s-apply', { imageTag, backupImageTag });
+    s.start('Applying k8s manifests + installing Supabase + rolling out app');
+    const envLocal = loadEnvLocal(join(projectDir, '.env.local'));
+    await perfAsync(`deploy.${perfPrefix}.applyManifests`, () =>
+      applyK3sManifests({
+        perfPrefix,
+        kubeconfig,
+        projectDir,
+        projectName: options.projectName,
+        imageTag,
+        backupImageTag,
+        envLocal,
+        domain: options.domain || 'localhost',
+        s3Config: options.s3Config,
+        backupBucketName: options.backupBucketName,
+        dnsProvider: options.dnsProvider,
+        cloudflareApiToken: options.cloudflareApiToken,
+        // Hetzner DNS-01 uses the SAME Cloud API token (api.hetzner.cloud)
+        // — the separate dns.hetzner.com console + token were retired in
+        // May 2026 (zone management folded into the main Cloud Console).
+        hetznerApiToken: options.apiToken,
+        // Phase 5: cluster-autoscaler wiring. apiToken (the Hetzner Cloud
+        // token CCM/CSI/Pulumi already use) flows in here under its own
+        // name to make the CA Secret apply unambiguous; the k3sToken
+        // came back from Pulumi outputs in `k3s-infra` step.
+        apiToken: options.apiToken,
+        region: options.region,
+        environment: options.environment,
+        minWorkers,
+        maxWorkers,
+        workerServerType: options.workerServerType || options.serverType || 'cx23',
+        k3sToken: infraOutputs.k3sToken,
+        // Phase 6: registry-push wiring. operator opens an SSH tunnel
+        // to master:5000 inside applyK3sManifests; the same SSH coords
+        // sideloadK3s already used (privateKeyPath + per-env khPath).
+        masterIp,
+        sshKeyPath: privateKeyPath,
+        khPath,
+        // HA passes a per-cluster port so primary + standby don't race
+        // for `localhost:5000` on the operator host. Single-cluster
+        // deploys leave it undefined → applyK3sManifests defaults to 5000.
+        localTunnelPort: options.localTunnelPort,
+      }),
+    );
+    state.completeStep('k3s-apply');
+    s.stop('App rolled out');
+  }
+
+  // 9. Cluster details
+  return {
+    masterIp,
+    floatingIp,
+    supabaseIp,
+    workerIps,
+    networkId,
+    kubeconfig,
+    sshKeyPath: privateKeyPath,
+    imageTag,
+  };
+}