vibecarbon 0.9.0 → 0.11.0

package/src/lib/deploy/k8s/k3s.js

@@ -23,7 +23,7 @@
  * Spike on 2026-04-25 measured 82s for cluster bring-up alone.
  */
 
-import { execFileSync, spawn, spawnSync } from 'node:child_process';
+import { execFileSync, spawn } from 'node:child_process';
 import {
   chmodSync,
   existsSync,
@@ -37,12 +37,13 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { setTimeout as delay } from 'node:timers/promises';
 import * as p from '@clack/prompts';
-import { runCommandAsync } from '../../command.js';
+import { runCommandAsync, writeSecretFile } from '../../command.js';
 import { featureConfigKeys, featureSecretKeys } from '../../config-registry.js';
-import { knownHostsPath } from '../../host-keys.js';
+import { knownHostsPath, seedKnownHosts } from '../../host-keys.js';
 import { loadCloudInit, renderScript } from '../../iac/cloud-init.js';
 import { dbImageRef } from '../../images.js';
 import { perfAsync } from '../../perf.js';
+import { pollUntil, runWithRetry } from '../../retry.js';
 import { postAdminUser, waitForGotrueHealth } from '../admin-user.js';
 
 export const K3S_VERSION = 'v1.31.5+k3s1';
@@ -97,40 +98,41 @@ export async function runKubectlWithRetry(
   args,
   { env, input, captureStdout = false, description } = {},
 ) {
-  const ATTEMPTS = 3;
+  const RETRY_DELAYS_MS = [2000, 4000];
+  const ATTEMPTS = RETRY_DELAYS_MS.length + 1;
   const desc = description ?? `kubectl ${args.slice(0, 4).join(' ')}`;
-  let result;
-  let lastStderr = '';
-  let lastStdout = '';
-  for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {
-    result = spawnSync('kubectl', args, {
-      env,
-      input,
-      stdio: ['pipe', 'pipe', 'pipe'],
-    });
-    lastStdout = (result.stdout ?? '').toString();
-    lastStderr = (result.stderr ?? '').toString();
-    // Stream captured streams through to the operator's terminal so the
-    // success path looks identical to the legacy `stdio: 'inherit'` shape.
-    // captureStdout suppresses stdout streaming because the caller wants
-    // the bytes (e.g. `kubectl get -o jsonpath`) — stderr still streams
-    // so a real warning is still visible.
-    if (!captureStdout && lastStdout) process.stdout.write(lastStdout);
-    if (lastStderr) process.stderr.write(lastStderr);
-    if (result.status === 0) break;
-    if (!KUBECTL_TRANSIENT_PATTERN.test(lastStderr) || attempt === ATTEMPTS) break;
-    const backoffMs = 2000 * attempt;
-    console.log(
-      `${desc} hit transient error on attempt ${attempt}/${ATTEMPTS}, retrying in ${backoffMs}ms`,
+  try {
+    return await runWithRetry(
+      async () => {
+        try {
+          const out = await runCommandAsync(['kubectl', ...args], { env, input, silent: true });
+          // Stream captured stdout through to the operator's terminal so the
+          // success path looks identical to the legacy `stdio: 'inherit'` shape.
+          // captureStdout suppresses stdout streaming because the caller wants
+          // the bytes (e.g. `kubectl get -o jsonpath`).
+          if (!captureStdout && out) process.stdout.write(out);
+          return out;
+        } catch (err) {
+          if (!captureStdout && err.stdout) process.stdout.write(err.stdout);
+          if (err.stderr) process.stderr.write(err.stderr);
+          throw err;
+        }
+      },
+      {
+        delaysMs: RETRY_DELAYS_MS,
+        isTransient: (err) => KUBECTL_TRANSIENT_PATTERN.test(String(err.stderr ?? err.message)),
+        onRetry: (_err, attempt) => {
+          console.log(
+            `${desc} hit transient error on attempt ${attempt}/${ATTEMPTS}, retrying in ${RETRY_DELAYS_MS[attempt - 1]}ms`,
+          );
+        },
+      },
     );
-    await delay(backoffMs);
-  }
-  if (result?.status !== 0) {
+  } catch (err) {
     throw new Error(
-      `${desc} failed with exit ${result?.status ?? '?'}: ${lastStderr.trim().slice(-500)}`,
+      `${desc} failed with exit ${err.status ?? '?'}: ${(err.stderr ?? '').toString().trim().slice(-500)}`,
     );
   }
-  return lastStdout;
 }
 
 /**
@@ -447,44 +449,42 @@ function sshHostKeyOpts(khPath) {
  * @returns {Promise<void>}
  */
 export async function waitForK3sReady(masterIp, sshKeyPath, khPath, maxWaitSec = 600) {
-  const start = Date.now();
-  let waitMs = 2000;
-  let lastErr;
-  while ((Date.now() - start) / 1000 < maxWaitSec) {
-    try {
-      execFileSync(
-        'ssh',
-        [
-          '-i',
-          sshKeyPath,
-          ...sshHostKeyOpts(khPath),
-          '-o',
-          'ConnectTimeout=5',
-          '-o',
-          'BatchMode=yes',
-          `root@${masterIp}`,
-          'test -f /tmp/k3s-ready',
-        ],
-        { stdio: 'ignore' },
-      );
-      return;
-    } catch (err) {
-      lastErr = err;
-      await delay(waitMs);
-      waitMs = Math.min(waitMs * 2, 15000);
-    }
+  try {
+    await pollUntil(
+      async () => {
+        await runCommandAsync(
+          [
+            'ssh',
+            '-i',
+            sshKeyPath,
+            ...sshHostKeyOpts(khPath),
+            '-o',
+            'ConnectTimeout=5',
+            '-o',
+            'BatchMode=yes',
+            `root@${masterIp}`,
+            'test -f /tmp/k3s-ready',
+          ],
+          { silent: true },
+        );
+        return true;
+      },
+      { budgetMs: maxWaitSec * 1000, description: 'k3s ready flag' },
+    );
+  } catch (err) {
+    // Capture cloud-init + syslog before throwing so the failure log says
+    // WHY install timed out (apt slow? k3s download 5xx? script crashed?)
+    // instead of just "300s passed and the marker isn't there." Best-effort —
+    // if the box is unreachable, captureK3sInstallDiag returns an empty string
+    // and we surface only the SSH probe error.
+    const lastErr = err.cause ?? err;
+    const diag = await captureK3sInstallDiag(masterIp, sshKeyPath, khPath);
+    throw new Error(
+      `k3s did not become ready on ${masterIp} within ${maxWaitSec}s. ` +
+        `Last SSH error: ${lastErr instanceof Error ? lastErr.message : 'unknown'}` +
+        (diag ? `\n\n--- install diagnostics from ${masterIp} ---\n${diag}` : ''),
+    );
   }
-  // Capture cloud-init + syslog before throwing so the failure log says
-  // WHY install timed out (apt slow? k3s download 5xx? script crashed?)
-  // instead of just "300s passed and the marker isn't there." Best-effort —
-  // if the box is unreachable, captureK3sInstallDiag returns an empty string
-  // and we surface only the SSH probe error.
-  const diag = captureK3sInstallDiag(masterIp, sshKeyPath, khPath);
-  throw new Error(
-    `k3s did not become ready on ${masterIp} within ${maxWaitSec}s. ` +
-      `Last SSH error: ${lastErr instanceof Error ? lastErr.message : 'unknown'}` +
-      (diag ? `\n\n--- install diagnostics from ${masterIp} ---\n${diag}` : ''),
-  );
 }
 
 /**
@@ -497,20 +497,20 @@ export async function waitForK3sReady(masterIp, sshKeyPath, khPath, maxWaitSec =
  * @param {string} khPath - per-env known_hosts file path
  * @param {string} projectDir
  * @param {string} environment
- * @returns {string} absolute path to the local kubeconfig
+ * @returns {Promise<string>} absolute path to the local kubeconfig
  */
-export function fetchKubeconfig(masterIp, sshKeyPath, khPath, projectDir, environment) {
+export async function fetchKubeconfig(masterIp, sshKeyPath, khPath, projectDir, environment) {
   const localPath = join(projectDir, '.vibecarbon', `kubeconfig-${environment}`);
-  execFileSync(
-    'scp',
+  await runCommandAsync(
     [
+      'scp',
       '-i',
       sshKeyPath,
       ...sshHostKeyOpts(khPath),
       `root@${masterIp}:/etc/rancher/k3s/k3s.yaml`,
       localPath,
     ],
-    { stdio: 'pipe' },
+    { silent: true },
   );
   const kc = readFileSync(localPath, 'utf-8');
   const patched = kc.replace(
@@ -611,7 +611,7 @@ export async function buildAppImage(projectDir, projectName, rebuild = false, do
     const { collectComposeBuildArgs } = await import('../compose/build-args.js');
     buildArgs = collectComposeBuildArgs(projectDir, { projectName, domain });
   }
-  return buildLocalImage(projectDir, {
+  return await buildLocalImage(projectDir, {
     projectName,
     rebuild,
     tagPrefix: '10.0.1.1:5000',
@@ -642,42 +642,40 @@ export async function buildAppImage(projectDir, projectName, rebuild = false, do
  * @returns {Promise<void>}
  */
 export async function waitForK3sBinary(target, sshKeyPath, khPath, maxWaitSec = 300) {
-  const start = Date.now();
-  let waitMs = 2000;
-  let lastErr;
-  while ((Date.now() - start) / 1000 < maxWaitSec) {
-    try {
-      execFileSync(
-        'ssh',
-        [
-          '-i',
-          sshKeyPath,
-          ...sshHostKeyOpts(khPath),
-          '-o',
-          'ConnectTimeout=5',
-          '-o',
-          'BatchMode=yes',
-          target,
-          'command -v k3s >/dev/null',
-        ],
-        { stdio: 'ignore' },
-      );
-      return;
-    } catch (err) {
-      lastErr = err;
-      await delay(waitMs);
-      waitMs = Math.min(waitMs * 2, 15000);
-    }
+  try {
+    await pollUntil(
+      async () => {
+        await runCommandAsync(
+          [
+            'ssh',
+            '-i',
+            sshKeyPath,
+            ...sshHostKeyOpts(khPath),
+            '-o',
+            'ConnectTimeout=5',
+            '-o',
+            'BatchMode=yes',
+            target,
+            'command -v k3s >/dev/null',
+          ],
+          { silent: true },
+        );
+        return true;
+      },
+      { budgetMs: maxWaitSec * 1000, description: 'k3s binary on PATH' },
+    );
+  } catch (err) {
+    // Same diagnostic capture as waitForK3sReady — see that function for why.
+    // `target` here is `root@<ip>` form; strip the user@ prefix for the host.
+    const lastErr = err.cause ?? err;
+    const ip = String(target).includes('@') ? String(target).split('@').pop() : String(target);
+    const diag = await captureK3sInstallDiag(ip, sshKeyPath, khPath);
+    throw new Error(
+      `k3s binary did not appear on ${target} within ${maxWaitSec}s. ` +
+        `Last SSH error: ${lastErr instanceof Error ? lastErr.message : 'unknown'}` +
+        (diag ? `\n\n--- install diagnostics from ${ip} ---\n${diag}` : ''),
+    );
   }
-  // Same diagnostic capture as waitForK3sReady — see that function for why.
-  // `target` here is `root@<ip>` form; strip the user@ prefix for the host.
-  const ip = String(target).includes('@') ? String(target).split('@').pop() : String(target);
-  const diag = captureK3sInstallDiag(ip, sshKeyPath, khPath);
-  throw new Error(
-    `k3s binary did not appear on ${target} within ${maxWaitSec}s. ` +
-      `Last SSH error: ${lastErr instanceof Error ? lastErr.message : 'unknown'}` +
-      (diag ? `\n\n--- install diagnostics from ${ip} ---\n${diag}` : ''),
-  );
 }
 
 /**
@@ -706,7 +704,7 @@ export async function waitForK3sBinary(target, sshKeyPath, khPath, maxWaitSec =
  *
  * Returns '' if even the first SSH fails (box unreachable / down).
  */
-function captureK3sInstallDiag(host, sshKeyPath, khPath) {
+async function captureK3sInstallDiag(host, sshKeyPath, khPath) {
   const sshArgs = (cmd) => [
     '-i',
     sshKeyPath,
@@ -801,11 +799,9 @@ function captureK3sInstallDiag(host, sshKeyPath, khPath) {
   for (const probe of probes) {
     const sliceLen = probe.slice ?? 4000;
     try {
-      const result = execFileSync('ssh', sshArgs(probe.cmd), {
-        encoding: 'utf-8',
+      const result = await runCommandAsync(['ssh', ...sshArgs(probe.cmd)], {
+        silent: true,
         timeout: 20_000,
-        stdio: ['ignore', 'pipe', 'pipe'],
-        maxBuffer: 4 * 1024 * 1024,
       });
       // Keep the END of the output (failure tail), not the start. ci-info
       // tables at the head of cloud-init-output.log used to push the
@@ -904,21 +900,14 @@ export async function sideloadK3s({ tag, sshTargets, sshKey, khPath }) {
           });
         });
 
-      let lastErr;
-      for (let attempt = 0; attempt <= SIDELOAD_DELAYS_MS.length; attempt++) {
-        try {
-          await runOnce();
-          return;
-        } catch (err) {
-          lastErr = err;
-          if (attempt === SIDELOAD_DELAYS_MS.length) break;
+      await runWithRetry(runOnce, {
+        delaysMs: SIDELOAD_DELAYS_MS,
+        onRetry: (err, attempt) => {
           console.warn(
-            `[sideloadK3s] ${target} attempt ${attempt + 1} failed (${err.message}); retrying in ${SIDELOAD_DELAYS_MS[attempt] / 1000}s`,
+            `[sideloadK3s] ${target} attempt ${attempt} failed (${err.message}); retrying in ${SIDELOAD_DELAYS_MS[attempt - 1] / 1000}s`,
           );
-          await new Promise((r) => setTimeout(r, SIDELOAD_DELAYS_MS[attempt]));
-        }
-      }
-      throw lastErr;
+        },
+      });
     }),
   );
 }
@@ -1034,14 +1023,54 @@ export async function pushImageToLocalRegistry({
   setPort(localTunnelPort);
 
   /** Best-effort teardown of any existing tunnel on localhost:<port>. */
-  const teardown = () => {
-    try {
-      execFileSync('pkill', ['-f', tunnelPattern], { stdio: 'ignore' });
-    } catch {
-      // pkill exits 1 when no matching process — that's fine.
-    }
+  const teardown = async () => {
+    // pkill exits 1 when no matching process — that's fine, hence ignoreError.
+    await runCommandAsync(['pkill', '-f', tunnelPattern], { silent: true, ignoreError: true });
   };
 
+  /**
+   * Open a backgrounding SSH tunnel (`-N -f`) and settle on the child's
+   * 'exit' event — NOT 'close'.
+   *
+   * OpenSSH's `-f` forks a daemon that inherits the tunnel's stdout/stderr
+   * fds and holds them open for the tunnel's entire lifetime. With piped
+   * stdio the parent process therefore never sees 'close' on SUCCESS (the
+   * fds stay open in the daemon), so a runCommandAsync-style settle-on-close
+   * would hang the whole deploy. 'exit' fires when the foreground ssh
+   * process exits, which happens right after the successful fork (code 0)
+   * or immediately on a pre-fork bind failure (nonzero under
+   * ExitOnForwardFailure=yes). stdout is ignored; only stderr is piped so a
+   * bind failure's diagnostic text survives for the port-walk catch below.
+   * The rejected error mirrors runCommandAsync's shape (.status + .stderr +
+   * `Command failed: …` message) so the surrounding catch is unchanged. The
+   * daemon reaps itself, so we don't register it in command.js's
+   * activeChildren (which isn't exported anyway).
+   */
+  const openSshTunnel = (argv) =>
+    new Promise((resolve, reject) => {
+      const child = spawn(argv[0], argv.slice(1), { stdio: ['ignore', 'ignore', 'pipe'] });
+      // The -f daemon inherits the stderr pipe for its lifetime; without
+      // unref a missed pkill teardown would hold the event loop open.
+      child.unref();
+      let stderr = '';
+      child.stderr?.on('data', (d) => {
+        stderr += d;
+      });
+      child.on('error', (err) => reject(err));
+      child.on('exit', (code, signal) => {
+        if (code === 0) return resolve(undefined);
+        const cmdStr = argv.join(' ');
+        const trimmed = stderr.trim();
+        const error = new Error(
+          trimmed ? `Command failed: ${cmdStr}\n${trimmed}` : `Command failed: ${cmdStr}`,
+        );
+        error.status = code;
+        error.stderr = stderr;
+        error.signal = signal;
+        reject(error);
+      });
+    });
+
   /**
    * Try to open the SSH tunnel on the currently-set port; if SSH bind
    * fails (sibling deploy holds the port), walk forward up to 20 ports.
@@ -1050,29 +1079,26 @@ export async function pushImageToLocalRegistry({
    * because ssh exits non-zero immediately on bind failure (via
    * ExitOnForwardFailure=yes).
    */
-  const openTunnelOrWalk = () => {
+  const openTunnelOrWalk = async () => {
     const errors = [];
     for (let probe = 0; probe < 20; probe++) {
       const candidate = localTunnelPort + probe;
       if (candidate > 65535) break;
       setPort(candidate);
       try {
-        execFileSync(
+        await openSshTunnel([
           'ssh',
-          [
-            '-i',
-            sshKey,
-            ...sshHostKeyOpts(khPath),
-            '-o',
-            'ExitOnForwardFailure=yes',
-            '-L',
-            forwardSpec,
-            '-N',
-            '-f',
-            target,
-          ],
-          { stdio: 'inherit' },
-        );
+          '-i',
+          sshKey,
+          ...sshHostKeyOpts(khPath),
+          '-o',
+          'ExitOnForwardFailure=yes',
+          '-L',
+          forwardSpec,
+          '-N',
+          '-f',
+          target,
+        ]);
         return;
       } catch (err) {
         errors.push(`port ${candidate}: ${err instanceof Error ? err.message : String(err)}`);
@@ -1101,25 +1127,31 @@ export async function pushImageToLocalRegistry({
       );
     });
 
-  // Three attempts with progressive backoff. Failure modes we're
-  // covering, in priority order:
+  // Progressive backoff across attempts. Failure modes we're covering, in
+  // priority order:
   //   1. Stale tunnel from a parallel run holds :5000 → reopen tunnel.
   //      Recovers in seconds (handled by the per-attempt teardown).
   //   2. Registry pod restarted between back-to-back pushes (e.g.
   //      probe-killed mid-prior-push) → manifest PUT returns EOF or
   //      `manifest blob unknown to registry`. Pod recovery is ~10–20s,
-  //      so the second/third attempt has the longer backoff to let the
+  //      so the later attempts have the longer backoff to let the
   //      replacement pod become Ready before re-pushing.
-  //   3. S3 transient (Hetzner Object Storage 5xx during blob upload)
-  //      → registry returns the same `blob unknown` after the layer is
-  //      "uploaded" but not yet readable. Same recovery shape as (2).
-  // The 1s/15s/30s schedule keeps the happy path fast (single attempt
-  // ~6s end-to-end) while giving genuine restarts time to recover.
-  const settleDelays = settleDelaysMs ?? [1_000, 15_000, 30_000];
+  //   3. S3 throttle (Hetzner Object Storage 503 SlowDown during blob
+  //      upload) → the registry propagates a `503 Service Unavailable` to
+  //      the docker client. This is LOAD-induced: k8s-HA deploys the primary
+  //      and standby clusters in parallel, and BOTH registries write layers
+  //      to the same Hetzner S3 account at once, so the account throttles.
+  //      3 short attempts (~46s of settle) couldn't ride it out. The longer
+  //      tail (60s/120s) extends the window past the throttle window AND past
+  //      the sibling cluster's push draining, which halves the S3 write load.
+  // The 1s/15s head keeps the happy path fast (single attempt ~6s end-to-end)
+  // while the 30s/60s/120s tail rides out genuine restarts + S3 throttling.
+  // Note docker push is content-addressable: layers whose S3 write already
+  // succeeded are skipped on retry, so later attempts get progressively cheaper.
+  const settleDelays = settleDelaysMs ?? [1_000, 15_000, 30_000, 60_000, 120_000];
   const maxAttempts = settleDelays.length;
   const attemptErrors = [];
   const taggedAliases = new Set();
-  let pushed = false;
   // Per-attempt structured logging — without this, the only visible signal
   // for a slow push is the final perf duration (which collapses retry
   // count + per-attempt time into a single number). iter-perfwave2 showed
@@ -1128,48 +1160,63 @@ export async function pushImageToLocalRegistry({
   // make next run's analysis trivial: count `[push] attempt=` lines for
   // attempt-count, sum `dur=` for actual push wall-clock.
   const startedAt = Date.now();
-  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+
+  // The per-attempt work, folded onto runWithRetry below. The settle delay
+  // stays INSIDE this function (keyed by the same `attempt` index
+  // runWithRetry hands us) because it must apply before every push,
+  // including the very first — the freshly-opened tunnel needs a moment to
+  // bind, it's not just a backoff between failures. runWithRetry's own
+  // inter-attempt delay is left at 0ms (see below) so the total attempt
+  // count matches settleDelays.length exactly, same as the old hand-rolled
+  // loop; the meaningful delay is the one computed here.
+  const runAttempt = async (attempt) => {
     const attemptStart = Date.now();
     let outcome = 'ok';
     try {
       // openTunnel may walk to a different port on bind-collision; localTag
       // is updated in lockstep so push targets the actual chosen port.
-      openTunnel();
+      await openTunnel();
       // Add the localhost-prefixed alias for the chosen port. Skip the
       // re-tag if we already aliased this port on an earlier attempt
       // (same port → same alias, docker tag is idempotent but the call
       // is wasteful). Track all aliases for cleanup so a port walk
       // doesn't leak local image refs.
       if (!taggedAliases.has(localTag)) {
-        execFileSync('docker', ['tag', tag, localTag], { stdio: 'inherit' });
+        await runCommandAsync(['docker', 'tag', tag, localTag], { silent: true });
         taggedAliases.add(localTag);
       }
       await delay(settleDelays[attempt]);
       await push();
-      pushed = true;
-      break;
     } catch (err) {
       attemptErrors.push(err instanceof Error ? err.message : String(err));
       outcome = `fail(${(err instanceof Error ? err.message : String(err)).split('\n')[0].slice(0, 80)})`;
+      throw err;
     } finally {
       // Tear down between attempts so a stuck tunnel from this attempt
       // doesn't bind-block the next attempt's openTunnel walk.
-      teardown();
+      await teardown();
       const dur = Math.round((Date.now() - attemptStart) / 1000);
       const totalDur = Math.round((Date.now() - startedAt) / 1000);
       console.error(
         `[push] tag=${tag.split('/').pop()} attempt=${attempt + 1}/${maxAttempts} port=${localTag.split('/')[0]} settle=${settleDelays[attempt] / 1000}s dur=${dur}s totalDur=${totalDur}s ${outcome}`,
       );
     }
+  };
+
+  let pushed = false;
+  try {
+    await runWithRetry(runAttempt, {
+      delaysMs: new Array(Math.max(maxAttempts - 1, 0)).fill(0),
+    });
+    pushed = true;
+  } catch {
+    // attemptErrors was already populated inside runAttempt; the
+    // exhausted-attempts error below is what the caller actually sees.
   }
   // Cleanup every localhost alias we created. Underlying image content
   // stays addressable as the original `tag` (used by sideload + Deployment).
   for (const alias of taggedAliases) {
-    try {
-      execFileSync('docker', ['rmi', alias], { stdio: 'ignore' });
-    } catch {
-      // non-fatal — leftover alias just sits in the local docker cache.
-    }
+    await runCommandAsync(['docker', 'rmi', alias], { silent: true, ignoreError: true });
   }
   if (!pushed) {
     throw new Error(
@@ -1325,6 +1372,7 @@ export async function installSupabase({
   envLocal,
   dbImageTag,
   backupBucketName,
+  walgRole = 'primary',
 }) {
   const env = { ...process.env, KUBECONFIG: kubeconfig };
   // Render the values template — substitute every {{TOKEN}} the template
@@ -1351,6 +1399,15 @@ export async function installSupabase({
   // WAL-G S3 prefix — dedicated backup bucket if provided, else the storage
   // bucket (mirrors compose's S3_BACKUP_BUCKET:-S3_BUCKET fallback). Never
   // leave it as s3:/// (wal-g would error).
+  //
+  // SINGLE canonical prefix (NO role segment). Reads (backup-fetch/restore/
+  // reseed) and writes (backup-push/wal-push) must all agree on ONE prefix — a
+  // role-segmented prefix made the standby read an empty `…/walg/standby` and
+  // fail restore/reseed with "No backups found" (caught by compose-ha scale
+  // e2e). Anti-collision (finding #3: a standby / bring-up-phase independent
+  // primary must never WRITE here) is enforced by the WALG_ROLE WRITE-GUARD:
+  // wal-archive.sh and the backup CronJob both no-op when WALG_ROLE=standby.
+  // walgRole is rendered into the db env below so those write guards can read it.
   const walgBucket = backupBucketName || s3Config?.bucket || '';
   const walgS3Prefix = walgBucket
     ? `s3://${walgBucket}/backups/${projectName || 'vibecarbon'}/walg`
@@ -1360,6 +1417,7 @@ export async function installSupabase({
     .replace(/\{\{PROJECT_NAME\}\}/g, projectName || 'vibecarbon')
     .replace(/\{\{S3_BACKUP_BUCKET\}\}/g, walgBucket)
     .replace(/\{\{WALG_S3_PREFIX\}\}/g, walgS3Prefix)
+    .replace(/\{\{WALG_ROLE\}\}/g, walgRole)
     .replace(/\{\{S3_ENDPOINT\}\}/g, s3Config?.endpoint ?? '')
     .replace(/\{\{S3_REGION\}\}/g, s3Config?.region ?? '')
     .replace(/\{\{DB_IMAGE\}\}/g, dbImageRepo)
@@ -1370,20 +1428,23 @@ export async function installSupabase({
     .replace(/\{\{ADMIN_EMAIL\}\}/g, envLocal?.ADMIN_EMAIL ?? '')
     .replace(/\{\{ADMIN_PASSWORD\}\}/g, envLocal?.ADMIN_PASSWORD ?? '');
   const tmpValues = join(tmpdir(), `vibecarbon-supabase-values-${process.pid}-${Date.now()}.yaml`);
-  writeFileSync(tmpValues, rendered);
+  // SECURITY: the rendered values contain ADMIN_EMAIL/ADMIN_PASSWORD and live
+  // for the whole `helm upgrade --wait` window — write 0o600, never 0o644.
+  writeSecretFile(tmpValues, rendered);
   try {
     // Add the supabase-community helm repo (idempotent — non-zero if exists).
-    try {
-      execFileSync('helm', ['repo', 'add', SUPABASE_HELM_REPO_NAME, SUPABASE_HELM_REPO_URL], {
-        stdio: 'inherit',
+    // helm repo add fails if the repo already exists; `helm repo update`
+    // below refreshes either way, hence ignoreError.
+    await runCommandAsync(
+      ['helm', 'repo', 'add', SUPABASE_HELM_REPO_NAME, SUPABASE_HELM_REPO_URL],
+      {
+        silent: true,
+        ignoreError: true,
         env,
-      });
-    } catch {
-      // helm repo add fails if the repo already exists; `helm repo update`
-      // below refreshes either way.
-    }
-    execFileSync('helm', ['repo', 'update', SUPABASE_HELM_REPO_NAME], {
-      stdio: 'inherit',
+      },
+    );
+    await runCommandAsync(['helm', 'repo', 'update', SUPABASE_HELM_REPO_NAME], {
+      silent: true,
       env,
     });
     // --wait blocks until all pods are Ready. --create-namespace is a no-op
@@ -1426,20 +1487,22 @@ export async function installSupabase({
     // duration. iter-perfwave2 showed primary 3m47s vs standby 1m43s
     // with no way to localize which pod was the long pole.
     try {
-      const podSnapshot = execFileSync(
-        'kubectl',
-        [
-          '-n',
-          'vibecarbon',
-          'get',
-          'pods',
-          '-l',
-          'app.kubernetes.io/instance=supabase',
-          '-o',
-          'custom-columns=NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,AGE:.metadata.creationTimestamp,NODE:.spec.nodeName',
-          '--no-headers',
-        ],
-        { encoding: 'utf8', env, stdio: ['ignore', 'pipe', 'pipe'] },
+      const podSnapshot = (
+        await runCommandAsync(
+          [
+            'kubectl',
+            '-n',
+            'vibecarbon',
+            'get',
+            'pods',
+            '-l',
+            'app.kubernetes.io/instance=supabase',
+            '-o',
+            'custom-columns=NAME:.metadata.name,STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,AGE:.metadata.creationTimestamp,NODE:.spec.nodeName',
+            '--no-headers',
+          ],
+          { env, silent: true },
+        )
       ).trim();
       console.error(`[supabase] helm-wait ${helmDur}s; post-wait pods:\n${podSnapshot}`);
     } catch (e) {
@@ -1608,28 +1671,34 @@ export async function applyMigrations({ kubeconfig, projectDir }) {
   await waitForSupabaseStorageSchema({ kubeconfig });
   for (const file of files) {
     const sql = readFileSync(join(migrationsDir, file), 'utf-8');
-    const result = spawnSync(
-      'kubectl',
-      [
-        '-n',
-        'vibecarbon',
-        'exec',
-        '-i',
-        'supabase-supabase-db-0',
-        '--',
-        'psql',
-        '-U',
-        'supabase_admin',
-        '-d',
-        'postgres',
-        '-v',
-        'ON_ERROR_STOP=1',
-      ],
-      { env, input: sql, stdio: ['pipe', 'inherit', 'inherit'] },
-    );
-    if (result.status !== 0) {
+    try {
+      await runCommandAsync(
+        [
+          'kubectl',
+          '-n',
+          'vibecarbon',
+          'exec',
+          '-i',
+          'supabase-supabase-db-0',
+          '--',
+          'psql',
+          '-U',
+          'supabase_admin',
+          '-d',
+          'postgres',
+          '-v',
+          'ON_ERROR_STOP=1',
+        ],
+        { env, input: sql, silent: true },
+      );
+    } catch (err) {
+      // silent:true buffers stdout/stderr instead of streaming them
+      // (unlike the old stdio: [...,'inherit','inherit']) — print them here
+      // on failure so the operator still sees the psql error output.
+      if (err.stdout) process.stdout.write(err.stdout);
+      if (err.stderr) process.stderr.write(err.stderr);
       throw new Error(
-        `applyMigrations: ${file} failed with exit ${result.status}. ` +
+        `applyMigrations: ${file} failed with exit ${err.status}. ` +
           `Fix the migration or set ON_ERROR_STOP=0 manually.`,
       );
     }
@@ -1936,18 +2005,14 @@ export async function applyK3sManifests({
   //     reconciled, otherwise Orders fail with "no provider for solver".
   //     Mirrors the Supabase pattern at step 7.
   if (dnsProvider === 'hetzner') {
-    try {
-      execFileSync(
-        'helm',
-        ['repo', 'add', HETZNER_WEBHOOK_HELM_REPO_NAME, HETZNER_WEBHOOK_HELM_REPO_URL],
-        { stdio: 'inherit', env },
-      );
-    } catch {
-      // helm repo add fails if the repo already exists; `helm repo update`
-      // below refreshes either way.
-    }
-    execFileSync('helm', ['repo', 'update', HETZNER_WEBHOOK_HELM_REPO_NAME], {
-      stdio: 'inherit',
+    // helm repo add fails if the repo already exists; `helm repo update`
+    // below refreshes either way, hence ignoreError.
+    await runCommandAsync(
+      ['helm', 'repo', 'add', HETZNER_WEBHOOK_HELM_REPO_NAME, HETZNER_WEBHOOK_HELM_REPO_URL],
+      { silent: true, ignoreError: true, env },
+    );
+    await runCommandAsync(['helm', 'repo', 'update', HETZNER_WEBHOOK_HELM_REPO_NAME], {
+      silent: true,
       env,
     });
     try {
@@ -1992,16 +2057,19 @@ export async function applyK3sManifests({
     } catch (err) {
       // Surface webhook pod logs on failure — chart install rarely
       // tells the operator anything useful when the actual cause is
-      // a CrashLoopBackOff or a missing dependency.
-      try {
-        execFileSync(
+      // a CrashLoopBackOff or a missing dependency. Best-effort: diag
+      // output should stream to the operator regardless of outcome.
+      await runCommandAsync(
+        [
           'kubectl',
-          ['-n', 'cert-manager', 'logs', `deploy/${HETZNER_WEBHOOK_RELEASE_NAME}`, '--tail=200'],
-          { stdio: 'inherit', env },
-        );
-      } catch {
-        // best-effort; continue to original throw
-      }
+          '-n',
+          'cert-manager',
+          'logs',
+          `deploy/${HETZNER_WEBHOOK_RELEASE_NAME}`,
+          '--tail=200',
+        ],
+        { silent: false, ignoreError: true, env },
+      );
       throw err;
     }
   }
@@ -2089,10 +2157,10 @@ export async function applyK3sManifests({
   //      Without this wait, the SSH-tunnel push race-targets a Pending
   //      pod (registry container not started → connection refused on
   //      :5000). 120s mirrors other base-pod readiness budgets.
-  await perfAsync(`deploy.${perfPrefix}.localRegistry.wait`, async () =>
-    execFileSync(
-      'kubectl',
+  await perfAsync(`deploy.${perfPrefix}.localRegistry.wait`, async () => {
+    const ok = await runCommandAsync(
       [
+        'kubectl',
         '-n',
         'vibecarbon',
         'wait',
@@ -2102,9 +2170,10 @@ export async function applyK3sManifests({
         'app=local-registry',
         '--timeout=120s',
       ],
-      { stdio: 'inherit', env },
-    ),
-  );
+      { silent: false, env },
+    );
+    if (ok === false) throw new Error('local-registry pod not Ready');
+  });
   // 5b''. Push the app image to the local registry on master so
   //       CA-spawned workers (which don't exist at sideload time) can
   //       pull on first schedule. Sideload remains primary distribution
@@ -2297,8 +2366,8 @@ export async function applyK3sManifests({
       { env, description: 'applyK3sManifests: kubectl rollout restart cluster-autoscaler' },
     );
     // rollout-status has its own --timeout and benefits from live progress;
-    // leave on execFileSync. Internal apiserver-watch retries cover transient
-    // drops within the budget.
+    // runs via runCommandAsync (kubectl rollout status). Internal
+    // apiserver-watch retries cover transient drops within the budget.
     //
     // 300s (was 180s): Phase 5 fires THREE rolling updates back-to-back on a
     // `strategy: Recreate` Deployment (5d patch --nodes, 5d' set env,
@@ -2309,11 +2378,19 @@ export async function applyK3sManifests({
     // Capture pod state on failure so future timeouts surface root cause
     // instead of just "rollout status timed out".
     try {
-      execFileSync(
-        'kubectl',
-        ['-n', 'kube-system', 'rollout', 'status', 'deploy/cluster-autoscaler', '--timeout=300s'],
-        { stdio: 'inherit', env },
+      const ok = await runCommandAsync(
+        [
+          'kubectl',
+          '-n',
+          'kube-system',
+          'rollout',
+          'status',
+          'deploy/cluster-autoscaler',
+          '--timeout=300s',
+        ],
+        { silent: false, env },
       );
+      if (ok === false) throw new Error('cluster-autoscaler rollout status failed (timeout 300s)');
     } catch (err) {
       console.error('[k3s] cluster-autoscaler rollout failed — capturing diagnostics:');
       for (const args of [
@@ -2321,11 +2398,7 @@ export async function applyK3sManifests({
         ['-n', 'kube-system', 'describe', 'deploy/cluster-autoscaler'],
         ['-n', 'kube-system', 'logs', '-l', 'app=cluster-autoscaler', '--tail=80'],
       ]) {
-        try {
-          execFileSync('kubectl', args, { stdio: 'inherit', env });
-        } catch {
-          // Best-effort diagnostics; original error is what matters
-        }
+        await runCommandAsync(['kubectl', ...args], { silent: false, ignoreError: true, env });
       }
       throw err;
     }
@@ -2344,6 +2417,12 @@ export async function applyK3sManifests({
   //    timeout in chart). image.db points at the wal-g-equipped db image
   //    (sideloaded in step 7b), and the chart mounts wal-archive.sh + the
   //    non-secret WALG_* env from supabase.values.yaml.
+  // Derive the wal-g WRITE-GUARD role (NOT a path segment). deployK8sHA calls
+  // deployK3s with environment `${env}-primary` / `${env}-standby`; single-
+  // cluster deploys pass the plain env name → 'primary'. Rendered into the db
+  // container env as WALG_ROLE so wal-archive.sh + the backup CronJob no-op on
+  // the standby. The S3 prefix itself stays canonical (no role segment).
+  const walgRole = environment?.endsWith('-standby') ? 'standby' : 'primary';
   await perfAsync(`deploy.${perfPrefix}.installSupabase`, async () =>
     installSupabase({
       kubeconfig,
@@ -2354,6 +2433,7 @@ export async function applyK3sManifests({
       envLocal,
       dbImageTag,
       backupBucketName,
+      walgRole,
     }),
   );
   // 7b. Enable continuous WAL archiving on the supabase-db. The chart can't
@@ -2499,7 +2579,8 @@ export async function applyK3sManifests({
       'deployment/app',
       '--timeout=600s',
     ];
-    const ATTEMPTS = 3;
+    const ROLLOUT_RETRY_DELAYS_MS = [2000, 4000];
+    const ATTEMPTS = ROLLOUT_RETRY_DELAYS_MS.length + 1;
     // Async (spawn-based) — `kubectl rollout status --timeout=600s` can wait
     // up to 10 minutes for pods to roll. spawnSync would block Node's event
     // loop the whole time; in HA the parallel standby branch's perf substep
@@ -2521,22 +2602,27 @@ export async function applyK3sManifests({
         child.on('error', reject);
         child.on('exit', (code) => resolve({ status: code, stderr: stderrBuf }));
       });
-    let lastStderr = '';
-    let lastStatus;
-    for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {
-      const result = await runRollout();
-      lastStatus = result.status;
-      lastStderr = result.stderr;
-      if (lastStatus === 0) return;
-      if (!KUBECTL_TRANSIENT_PATTERN.test(lastStderr) || attempt === ATTEMPTS) break;
-      const backoffMs = 2000 * attempt;
-      console.log(
-        `kubectl rollout status hit transient error on attempt ${attempt}/${ATTEMPTS}, retrying in ${backoffMs}ms`,
-      );
-      await delay(backoffMs);
-    }
-    throw new Error(
-      `kubectl rollout status failed with exit ${lastStatus ?? '?'}: ${lastStderr.trim().slice(-500)}`,
+    await runWithRetry(
+      async () => {
+        const result = await runRollout();
+        if (result.status !== 0) {
+          const err = new Error(
+            `kubectl rollout status failed with exit ${result.status ?? '?'}: ${result.stderr.trim().slice(-500)}`,
+          );
+          err.stderr = result.stderr;
+          err.status = result.status;
+          throw err;
+        }
+      },
+      {
+        delaysMs: ROLLOUT_RETRY_DELAYS_MS,
+        isTransient: (err) => KUBECTL_TRANSIENT_PATTERN.test(String(err.stderr ?? err.message)),
+        onRetry: (_err, attempt) => {
+          console.log(
+            `kubectl rollout status hit transient error on attempt ${attempt}/${ATTEMPTS}, retrying in ${ROLLOUT_RETRY_DELAYS_MS[attempt - 1]}ms`,
+          );
+        },
+      },
     );
   });
   // 10. Force-restart traefik so it starts AFTER kube-router has installed
@@ -2573,11 +2659,27 @@ export async function applyK3sManifests({
         ],
         { env, description: 'applyK3sManifests: kubectl delete pod traefik' },
       );
-      execFileSync(
-        'kubectl',
-        ['-n', 'vibecarbon', 'rollout', 'status', 'deployment/traefik', '--timeout=120s'],
-        { stdio: 'inherit', env },
+      // ignoreError: true means this resolves null/false on failure instead
+      // of throwing — check explicitly and warn so the non-fatal-failure
+      // signal isn't silently dropped (the outer catch below only fires on
+      // a THROWN error, e.g. from the delete-pod call above).
+      const ok = await runCommandAsync(
+        [
+          'kubectl',
+          '-n',
+          'vibecarbon',
+          'rollout',
+          'status',
+          'deployment/traefik',
+          '--timeout=120s',
+        ],
+        { silent: false, ignoreError: true, env },
       );
+      if (ok === null || ok === false) {
+        console.error(
+          '[k3s] traefik post-apply restart warning: kubectl rollout status deployment/traefik did not complete within 120s',
+        );
+      }
     });
   } catch (err) {
     console.error(
@@ -2593,7 +2695,7 @@ export async function applyK3sManifests({
  * App install + sideload land in a follow-up commit (Task 7).
  *
  * @param {K3sDeployOptions} options
- * @returns {Promise<{masterIp: string, floatingIp: string, supabaseIp: string, workerIps: string[], networkId: string|number, kubeconfig: string, sshKeyPath: string}>}
+ * @returns {Promise<{masterIp: string, floatingIp: string, supabaseIp: string, supabasePrivateIp: string, workerIps: string[], networkId: string|number, kubeconfig: string, sshKeyPath: string}>}
  */
 export async function deployK3s(options) {
   const { state } = options;
@@ -2689,7 +2791,14 @@ export async function deployK3s(options) {
     s.stop(`Cluster provisioned — master=${outputs.masterIp} floating=${outputs.floatingIp}`);
   }
   const infraOutputs = state.getStepResult('k3s-infra');
-  const { masterIp, floatingIp, supabaseIp, workerIps, networkId } = infraOutputs;
+  const {
+    masterIp,
+    floatingIp,
+    supabaseIp,
+    supabasePrivateIp = '10.0.1.2',
+    workerIps,
+    networkId,
+  } = infraOutputs;
 
   // 4. Wait for cloud-init to finish + k3s to be Ready
   if (!state.shouldSkip('k3s-ready', { masterIp })) {
@@ -2709,12 +2818,30 @@ export async function deployK3s(options) {
     s.stop('k3s reachable');
   }
 
+  // Trusted host-key seed. The SSH opts above pin per-env with accept-new
+  // (TOFU on first connect); seeding the known_hosts from a real ssh-keyscan
+  // now — on (re)provision, once SSH is reachable — turns accept-new into a
+  // strict pin for the rest of this deploy AND every later command, and
+  // re-pins a Hetzner-recycled IP cleanly (seedKnownHosts drops the stale
+  // per-IP line). Best-effort: if ssh-keyscan can't reach a node we fall back
+  // to accept-new TOFU. See host-keys.js seedKnownHosts for the full security
+  // rationale (why only provisioning re-seeds).
+  for (const seedIp of [masterIp, supabaseIp, ...(workerIps ?? [])].filter(Boolean)) {
+    await seedKnownHosts(khPath, seedIp);
+  }
+
   // 5. Fetch + patch kubeconfig
   let kubeconfig;
   if (!state.shouldSkip('k3s-kubeconfig', { masterIp })) {
     state.startStep('k3s-kubeconfig', { masterIp });
     s.start('Retrieving kubeconfig');
-    kubeconfig = fetchKubeconfig(masterIp, privateKeyPath, khPath, projectDir, options.environment);
+    kubeconfig = await fetchKubeconfig(
+      masterIp,
+      privateKeyPath,
+      khPath,
+      projectDir,
+      options.environment,
+    );
     state.completeStep('k3s-kubeconfig', { kubeconfig });
     s.stop('Kubeconfig saved');
   } else {
@@ -2827,6 +2954,11 @@ export async function deployK3s(options) {
     masterIp,
     floatingIp,
     supabaseIp,
+    // The supabase node's Hetzner private-network IP (deterministic in the IaC
+    // program, but threaded through the output rather than hardcoded at the use
+    // site). setupReplication renders it into the repl-gateway egress
+    // NetworkPolicy + socat bind for this cluster.
+    supabasePrivateIp,
     workerIps,
     networkId,
     kubeconfig,