vibecarbon 0.9.0 → 0.11.0

package/src/lib/deploy/remote-build.js

@@ -9,6 +9,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import * as p from '@clack/prompts';
 import { runCommand } from '../command.js';
+import { knownHostsPathForKey } from '../host-keys.js';
 import { perfTimer } from '../perf.js';
 
 /**
@@ -41,8 +42,13 @@ export async function buildRemote(ip, sshKeyPath, imageTag, cwd, buildArgs = {})
   // session helper drops with "error reading preface ... file already closed".
   // Keepalives let ssh detect a dead peer (~60s) and fail the attempt cleanly
   // so the retry below can re-establish, instead of hanging to the build timeout.
+  // Host-key pinned against the per-env known_hosts (passed via env var so no
+  // shell-quoting concern), NOT /dev/null + no. accept-new TOFU's an
+  // ephemeral/recycled Hetzner IP but rejects a changed key for an already-
+  // pinned host (MITM on an established env fails); GlobalKnownHostsFile=
+  // /dev/null ignores the system file and never touches ~/.ssh/known_hosts.
   const sshWrapper = `#!/bin/bash
-exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 "$@"
+exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$VIBECARBON_KNOWN_HOSTS" -o GlobalKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 "$@"
 `;
   writeFileSync(sshWrapperPath, sshWrapper, { mode: 0o755 });
 
@@ -52,6 +58,7 @@ exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKn
     DOCKER_BUILDKIT: '1',
     PATH: `${tempBinDir}:${process.env.PATH}`, // Prepend our wrapper
     VIBECARBON_SSH_KEY: sshKeyPath,
+    VIBECARBON_KNOWN_HOSTS: knownHostsPathForKey(sshKeyPath),
   };
 
   try {

package/src/lib/deploy/replication.js

@@ -0,0 +1,540 @@
+/**
+ * Shared PostgreSQL replication / re-seed / promotion primitives.
+ *
+ * The standby re-seed, streaming verification, and promotion logic used to be
+ * duplicated (and slowly diverging) across:
+ *   - src/lib/deploy/k8s/ha/index.js   (setupReplication — deploy-time seed)
+ *   - src/failover.js                  (reseed + promote at failover)
+ *   - src/lib/deploy/compose/ha.js     (configureStandbyReplication + verify)
+ *   - src/restore.js                   (post-restore standby resync)
+ *
+ * This module houses the reusable, exec-backend-parameterized primitives so the
+ * call sites share ONE hardened implementation instead of forking it. The two
+ * exec backends differ only in HOW a command reaches postgres:
+ *   - compose: `docker compose exec -T db psql …`  (via sshRun)
+ *   - k8s:     `sshKubectl exec -n vibecarbon <pod> -- …`
+ * so the primitives take an injected runner / readState closure rather than
+ * hardcoding either transport.
+ */
+
+import { getPostgresPod, sshKubectl } from '../ssh.js';
+
+// The physical replication slot name is a project-wide constant — the primary
+// creates it, the standby's pg_basebackup -S references it, and streaming uses
+// it so the primary holds WAL for this standby specifically.
+export const REPLICATION_SLOT = 'vibecarbon_standby_slot';
+
+// The single replication port for BOTH HA modes. compose publishes db:5432 →
+// host:5433 (supavisor owns host 5432); k8s uses a hostPort 5433 → container
+// 5432. One port keeps primary_conninfo / pg_basebackup byte-identical.
+export const REPL_PORT = 5433;
+
+/**
+ * The `primary_conninfo` the standby uses to stream from the primary. The
+ * transport is a point-to-point WireGuard tunnel (see wireguard.js) — the
+ * wire is already encrypted at the network layer, so Postgres itself runs
+ * plaintext (`sslmode=disable`, no CA to verify against). `primaryHost` is
+ * the local tunnel-side endpoint the transport layer supplies (e.g. the
+ * WireGuard gateway IP), not a public address. `port` defaults to REPL_PORT
+ * (5433) for the compose path where the standby dials the primary db directly;
+ * the k8s-ha path passes the repl-gateway relay port (15433) since it dials its
+ * local socat gateway rather than postgres directly.
+ * @param {{primaryHost:string, replPassword:string, applicationName?:string, port?:string|number}} o
+ * @returns {string}
+ */
+export function buildPrimaryConninfo({
+  primaryHost,
+  replPassword,
+  applicationName = 'standby',
+  port = REPL_PORT,
+}) {
+  return (
+    `host=${primaryHost} port=${port} user=replicator ` +
+    `password=${replPassword} sslmode=disable ` +
+    `application_name=${applicationName}`
+  );
+}
+
+/**
+ * pg_hba.conf line allowing replication from the tunnel subnet. Plain `host`
+ * (not `hostssl`) — WireGuard encrypts the wire, so Postgres does not also
+ * require TLS. Scoped to the WireGuard tunnel's /30 (WG_SUBNET_CIDR), which
+ * covers both peers regardless of which side is currently primary/standby.
+ * @param {string} peerCidr
+ * @returns {string}
+ */
+export function buildReplicationHbaLine(peerCidr) {
+  return `host replication replicator ${peerCidr} scram-sha-256`;
+}
+
+/**
+ * Compute the updated Hetzner firewall rule set admitting the peer supabase
+ * node's WireGuard tunnel endpoint (udp/51821), or null when no update is
+ * needed. Pure: deployK8sHA fetches the firewall, calls this, and PUTs the
+ * result — for BOTH clusters with the peer swapped (the post-failover
+ * reverse re-seed dials the promoted standby, so its firewall must admit the
+ * old primary too; neither failover.js nor restore.js touches firewalls).
+ *
+ * Replication moved from a public-IP TCP connection to a point-to-point
+ * WireGuard tunnel (wireguard.js — WG_PORT=51821; 51820 is flannel-wg's), so
+ * this also drops the stale TCP rules from that earlier architecture: the
+ * direct 5432 rule (never worked — no hostPort listener), the retired
+ * NodePort 30432 rule (SNAT masked the source IP, breaking the /32 pg_hba
+ * scope), and the superseded 5433 rule — plus any udp/51821 rule pointing at
+ * a previous peer IP (server replacement).
+ *
+ * @param {Array<object>} existingRules - The firewall's current rules.
+ * @param {string} peerIp - The peer supabase node's public IPv4.
+ * @returns {Array<object>|null} Updated rules, or null if already correct.
+ */
+export function buildReplicationFirewallRules(existingRules, peerIp) {
+  const wgPort = '51821';
+  const staleTcpPorts = new Set(['5432', '5433', '30432']);
+  const isStaleTcp = (r) => r.protocol === 'tcp' && staleTcpPorts.has(r.port);
+  const isWgRule = (r) => r.protocol === 'udp' && r.port === wgPort;
+
+  const hasExactRule = existingRules.some(
+    (r) => isWgRule(r) && r.source_ips?.includes(`${peerIp}/32`),
+  );
+  const hasStale = existingRules.some((r) => isStaleTcp(r) || (isWgRule(r) && !hasExactRule));
+  if (hasExactRule && !hasStale) return null;
+  const cleaned = existingRules.filter((r) => !isStaleTcp(r) && !isWgRule(r));
+  return [
+    ...cleaned,
+    {
+      direction: 'in',
+      protocol: 'udp',
+      port: wgPort,
+      source_ips: [`${peerIp}/32`],
+      destination_ips: [],
+    },
+  ];
+}
+
+/**
+ * Build the hardened staged-basebackup + atomic-swap bash script.
+ *
+ * This is the SINGLE source for the k8s re-seed script used at deploy time
+ * (k8s/ha/index.js setupReplication) AND at failover/restore time (failover.js,
+ * restore.js). Hardenings preserved from the previous inline copies:
+ *   1. `set -e -o pipefail` — a partial pg_basebackup leaves PGDATA non-empty
+ *      without PG_VERSION; the supabase-db entrypoint then runs initdb on next
+ *      restart, aborts on "directory exists but is not empty", and CrashLoops.
+ *   2. Stage into a tmp dir + atomic swap — a failed backup never leaves PGDATA
+ *      half-formed; the original survives at data.prev until the swap succeeds.
+ *   3. Verify PG_VERSION present before swapping in.
+ *   4. `probeFirst` embeds a pg_isready guard that aborts (exit 0) BEFORE any
+ *      destructive work when the primary is unreachable — the standby then keeps
+ *      its self-bootstrapped postgres instead of a permanent CrashLoopBackOff.
+ *      (Used by the deploy-time k8s seed, where the probe is inline; the
+ *      failover/restore path probes separately, so it passes probeFirst=false.)
+ *
+ * The replication password is embedded in PGPASSWORD inside the returned script,
+ * which callers MUST deliver over stdin (never argv) so it stays out of
+ * /proc/cmdline and the ssh command string.
+ *
+ * The connection itself is plaintext (no PGSSLMODE/PGSSLROOTCERT) — the
+ * transport is a point-to-point WireGuard tunnel (wireguard.js) that already
+ * encrypts the wire, so app-layer TLS is redundant.
+ *
+ * @param {object} opts
+ * @param {string} opts.replPassword
+ * @param {string} opts.primaryHost
+ * @param {string} [opts.primaryPort='5432']
+ * @param {string} [opts.slotName=REPLICATION_SLOT]
+ * @param {boolean} [opts.probeFirst=false]
+ * @param {string} [opts.label='reseed'] - tag for the script's stderr breadcrumbs
+ * @returns {string}
+ */
+export function buildStagedBasebackupScript({
+  replPassword,
+  primaryHost,
+  primaryPort = String(REPL_PORT),
+  slotName = REPLICATION_SLOT,
+  probeFirst = false,
+  label = 'reseed',
+}) {
+  const probeBlock = probeFirst
+    ? `
+# Probe primary's postgres from inside the standby pod. If unreachable
+# (cross-cluster network gap is the historical case), abort cleanly so the
+# standby keeps its self-bootstrapped postgres rather than entering a
+# permanent CrashLoopBackOff via half-wiped PGDATA.
+if ! pg_isready -h ${primaryHost} -p ${primaryPort} -t 5 -U replicator -d postgres > /dev/null 2>&1; then
+  echo "[${label}] primary postgres at ${primaryHost}:${primaryPort} not reachable from standby — skipping pg_basebackup. Standby keeps its independent postgres; failover serves the app but data does not sync from primary." >&2
+  exit 0
+fi
+`
+    : '';
+
+  return `
+set -e -o pipefail
+${probeBlock}
+# Stage into a tmp dir; atomic swap only after basebackup verifies good.
+rm -rf /var/lib/postgresql/data.staging
+mkdir -p /var/lib/postgresql/data.staging
+chown postgres:postgres /var/lib/postgresql/data.staging
+
+PGPASSWORD='${replPassword}' pg_basebackup \\
+  -h ${primaryHost} -p ${primaryPort} \\
+  -U replicator -D /var/lib/postgresql/data.staging \\
+  -Fp -Xs -R -c fast -S ${slotName}
+
+# A successful pg_basebackup always writes PG_VERSION (and standby.signal via -R).
+if [ ! -f /var/lib/postgresql/data.staging/PG_VERSION ]; then
+  echo "[${label}] pg_basebackup produced no PG_VERSION — refusing to swap" >&2
+  rm -rf /var/lib/postgresql/data.staging
+  exit 1
+fi
+# -R should already have created standby.signal; touch belt-and-suspenders.
+touch /var/lib/postgresql/data.staging/standby.signal
+echo "primary_slot_name = '${slotName}'" >> /var/lib/postgresql/data.staging/postgresql.auto.conf
+chown -R postgres:postgres /var/lib/postgresql/data.staging
+
+# Replace PGDATA CONTENTS in place. /var/lib/postgresql/data is a container
+# volume MOUNTPOINT in both compose (db_data:/var/lib/postgresql/data) and k8s —
+# it CANNOT be renamed (mv of a mountpoint fails with EBUSY/EXDEV), so we clear
+# the mountpoint and move the verified staged basebackup in. The staged copy is
+# already verified good (PG_VERSION check above) before this destructive clear,
+# so a known-good replacement exists before we touch the live dir.
+find /var/lib/postgresql/data -mindepth 1 -delete
+find /var/lib/postgresql/data.staging -mindepth 1 -maxdepth 1 -exec mv -t /var/lib/postgresql/data/ {} +
+rmdir /var/lib/postgresql/data.staging
+chown -R postgres:postgres /var/lib/postgresql/data
+`.trim();
+}
+
+/**
+ * Poll a "read the standby's replication state on the primary" closure until it
+ * reports `streaming`, or the attempt budget lapses. Injected `readState`
+ * (returns pg_stat_replication.state, or '' when no replica is connected) keeps
+ * this transport-agnostic — compose passes a `docker compose exec` reader, k8s a
+ * `sshKubectl exec` reader.
+ *
+ * @param {object} opts
+ * @param {() => Promise<string>} opts.readState
+ * @param {number} [opts.attempts=15]
+ * @param {number[]} [opts.delaysMs]
+ * @param {(ms: number) => Promise<void>} [opts.sleep]
+ * @returns {Promise<{ streaming: boolean, lastState: string }>}
+ */
+export async function verifyStreaming({
+  readState,
+  attempts = 15,
+  delaysMs = [250, 500, 1000, 2000, 3000],
+  sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+}) {
+  let lastState = '';
+  let sawValidRead = false;
+  for (let i = 0; i < attempts; i++) {
+    try {
+      const state = await readState();
+      sawValidRead = true;
+      lastState = typeof state === 'string' ? state.trim() : '';
+      if (lastState === 'streaming') return { streaming: true, lastState };
+    } catch {
+      // Retry — the standby may still be catching up, or the read itself
+      // (SSH/kubectl exec) transiently failed. A read that never yields a valid
+      // answer is handled below so the abort message stays honest.
+    }
+    if (i < attempts - 1) {
+      await sleep(delaysMs[Math.min(i, delaysMs.length - 1)] ?? 3000);
+    }
+  }
+  // Distinguish "connected but not streaming" (reads succeeded, state != streaming)
+  // from "could not read at all" (every read threw — e.g. the primary was
+  // unreachable for the whole window). Without this, the latter reports as a bare
+  // "not streaming" which hides a connectivity failure.
+  if (!sawValidRead) lastState = 'unreadable (could not query pg_stat_replication)';
+  return { streaming: false, lastState };
+}
+
+/**
+ * Finding #1 — the hard-gate policy, shared by the compose-HA and k8s-HA deploy
+ * paths so they behave identically. Given the verified streaming result:
+ *   - streaming            → returns { degraded: false } (deploy proceeds).
+ *   - !streaming + allowDegraded → returns { degraded: true, reason } (proceed,
+ *     but the caller must record + surface a DEGRADED / warm-standby state).
+ *   - !streaming + !allowDegraded → THROWS an actionable Error (abort the deploy):
+ *     it states WHY (standby not streaming, last observed state) and that
+ *     `-allow-degraded` will proceed with warm-standby DR.
+ *
+ * DECISION: hard-gate-by-default was the chosen policy — a green `deployed` HA
+ * env must have a verifiably streaming standby. Warm-standby DR is opt-in only.
+ *
+ * @param {object} opts
+ * @param {boolean} opts.streaming
+ * @param {string} [opts.lastState]
+ * @param {boolean} [opts.allowDegraded=false]
+ * @param {string} [opts.fixHint] - mode-specific remediation appended to the error
+ * @returns {{ degraded: boolean, reason?: string }}
+ */
+export function assertReplicationStreamingOrDegraded({
+  streaming,
+  lastState = '',
+  allowDegraded = false,
+  fixHint = '',
+}) {
+  if (streaming) return { degraded: false };
+
+  const stateDesc = lastState ? JSON.stringify(lastState) : 'no replica connected';
+  const reason = `standby not streaming (last pg_stat_replication.state=${stateDesc})`;
+
+  if (allowDegraded) {
+    return { degraded: true, reason };
+  }
+
+  throw new Error(
+    `HA deploy aborted: the standby is not verifiably streaming from the primary, so ` +
+      `disaster recovery is not guaranteed.\n  Reason: ${reason}.\n` +
+      `${fixHint ? `  ${fixHint}\n` : ''}` +
+      `  Re-run \`vibecarbon deploy\` after fixing replication, or re-run with ` +
+      `\`-allow-degraded\` to finalize a warm-standby (DEGRADED) deployment that accepts ` +
+      `reduced DR.`,
+  );
+}
+
+/**
+ * Read REPL_PASSWORD from the `vibecarbon-secrets` k8s Secret — the SAME source
+ * the db uses. k8s nodes have NO compose-style `/opt/<project>/.env.local`; the
+ * old grep of that path always returned empty, so a re-seed silently no-op'd and
+ * the caller promoted a possibly-stale (or empty) standby. jsonpath returns the
+ * base64-encoded value; decode it locally.
+ *
+ * Returns the plaintext password, or null if the key is absent/undecodable.
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @returns {Promise<string|null>}
+ */
+export async function getReplPasswordFromSecret(ip, sshKeyPath) {
+  const b64 = await sshKubectl(ip, sshKeyPath, [
+    'get',
+    'secret',
+    'vibecarbon-secrets',
+    '-n',
+    'vibecarbon',
+    '-o',
+    'jsonpath={.data.REPL_PASSWORD}',
+  ]);
+  const trimmed = (typeof b64 === 'string' ? b64 : '').trim();
+  if (!trimmed) return null;
+  try {
+    const decoded = Buffer.from(trimmed, 'base64').toString('utf-8').trim();
+    return decoded || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Re-seed the standby Postgres from the primary via pg_basebackup (k8s transport).
+ * Closes the drift window that opens when streaming silently falls behind or the
+ * standby pod restarted and missed WAL. Used by BOTH failover (before promotion)
+ * and restore (post-restore standby resync).
+ *
+ * Safety contract:
+ *   - REPL_PASSWORD is read from the `vibecarbon-secrets` Secret. If it is not
+ *     found we CANNOT safely re-seed → throw a HARD ERROR so the caller aborts
+ *     rather than acting on a stale standby. Distinct from "primary unreachable".
+ *   - Reachability is probed from inside the standby pod BEFORE any destructive
+ *     work. If the primary is not reachable we return 'skipped' WITHOUT touching
+ *     PGDATA.
+ *   - The basebackup stages into a tmp dir and atomically swaps in only after it
+ *     verifies good (PG_VERSION present) — a failed backup never leaves a
+ *     half-wiped PGDATA that would CrashLoopBackOff on initdb.
+ *
+ * @param {string} standbyIp
+ * @param {string} sshKeyPath
+ * @param {string} primarySupabaseIp
+ * @returns {Promise<'reseeded'|'skipped'>} 'reseeded' on success, 'skipped' when
+ *   the primary is unreachable. Throws on password-missing or a real failure —
+ *   NEVER silently returns as if skipped.
+ */
+export async function reseedStandbyFromPrimary(standbyIp, sshKeyPath, primarySupabaseIp) {
+  if (!primarySupabaseIp) {
+    throw new Error(
+      're-seed aborted: primary supabase IP is unknown, so pg_basebackup has no ' +
+        'source. Cannot safely re-seed — redeploy to persist the primary supabase IP.',
+    );
+  }
+
+  const replPassword = await getReplPasswordFromSecret(standbyIp, sshKeyPath);
+  if (!replPassword) {
+    throw new Error(
+      're-seed aborted: REPL_PASSWORD not found in the vibecarbon-secrets Secret. ' +
+        'Cannot re-seed the standby from the primary — refusing to act on a ' +
+        'possibly-stale standby. Verify the standby cluster was deployed with `vibecarbon deploy --ha`.',
+    );
+  }
+
+  const pod = await getPostgresPod(standbyIp, sshKeyPath);
+
+  // Connectivity pre-check FROM THE STANDBY POD, before any destructive work.
+  // pg_isready exits non-zero when the primary is not accepting connections;
+  // sshKubectl throws on non-zero → treat as "unreachable" and skip cleanly.
+  let reachable = false;
+  try {
+    await sshKubectl(standbyIp, sshKeyPath, [
+      'exec',
+      '-n',
+      'vibecarbon',
+      pod,
+      '--',
+      'pg_isready',
+      '-h',
+      primarySupabaseIp,
+      '-p',
+      '5432',
+      '-t',
+      '5',
+      '-U',
+      'replicator',
+      '-d',
+      'postgres',
+    ]);
+    reachable = true;
+  } catch {
+    reachable = false;
+  }
+  if (!reachable) {
+    return 'skipped';
+  }
+
+  // Stop the standby postgres so PGDATA can be swapped safely, then stage a
+  // fresh basebackup and atomically swap it in.
+  await sshKubectl(
+    standbyIp,
+    sshKeyPath,
+    [
+      'exec',
+      '-n',
+      'vibecarbon',
+      pod,
+      '--',
+      'su',
+      'postgres',
+      '-c',
+      'pg_ctl stop -D /var/lib/postgresql/data -m fast',
+    ],
+    {},
+  ).catch(() => {
+    // Already stopped / not running — the swap below is still safe.
+  });
+
+  const script = buildStagedBasebackupScript({
+    replPassword,
+    primaryHost: primarySupabaseIp,
+    primaryPort: '5432',
+    probeFirst: false,
+    label: 'reseed',
+  });
+  await sshKubectl(standbyIp, sshKeyPath, ['exec', '-i', '-n', 'vibecarbon', pod, '--', 'bash'], {
+    input: script,
+  });
+
+  // Bring the freshly-seeded standby back up (in recovery — streaming). Any
+  // promotion is the caller's next step (failover); restore leaves it streaming.
+  await sshKubectl(standbyIp, sshKeyPath, [
+    'exec',
+    '-n',
+    'vibecarbon',
+    pod,
+    '--',
+    'su',
+    'postgres',
+    '-c',
+    'pg_ctl start -D /var/lib/postgresql/data -l /var/log/postgresql/postgresql.log',
+  ]);
+  return 'reseeded';
+}
+
+/**
+ * Promote the standby database to primary (k8s transport) and confirm it exited
+ * recovery. Returns true only when pg_is_in_recovery() flips to 'f' — the caller
+ * MUST abort the failover on false rather than flip DNS onto a read-only standby.
+ *
+ * @param {string} standbyIp
+ * @param {string} sshKeyPath
+ * @returns {Promise<boolean>}
+ */
+export async function promoteStandby(standbyIp, sshKeyPath) {
+  const pod = await getPostgresPod(standbyIp, sshKeyPath);
+
+  // Promote PostgreSQL on standby. pg_ctl promote may return non-zero even on
+  // success, so we confirm via the pg_is_in_recovery() poll below.
+  try {
+    await sshKubectl(standbyIp, sshKeyPath, [
+      'exec',
+      '-n',
+      'vibecarbon',
+      pod,
+      '--',
+      'su',
+      'postgres',
+      '-c',
+      'pg_ctl promote -D /var/lib/postgresql/data',
+    ]);
+  } catch {
+    // pg_ctl promote may return non-zero even on success — confirm via poll.
+  }
+
+  for (let i = 0; i < 30; i++) {
+    try {
+      const result = await sshKubectl(standbyIp, sshKeyPath, [
+        'exec',
+        '-n',
+        'vibecarbon',
+        pod,
+        '--',
+        'psql',
+        '-U',
+        'supabase_admin',
+        '-tAc',
+        'SELECT pg_is_in_recovery()',
+      ]);
+      if (result.trim() === 'f') return true;
+    } catch {
+      // Retry
+    }
+    await new Promise((r) => setTimeout(r, 2000));
+  }
+
+  return false;
+}
+
+/**
+ * Read the standby's replication state as seen on the PRIMARY (k8s transport).
+ * Returns pg_stat_replication.state ('streaming'/'catchup'/…) or '' when no
+ * replica is connected. Suitable as the `readState` closure for verifyStreaming.
+ *
+ * @param {string} primaryIp
+ * @param {string} sshKeyPath
+ * @returns {Promise<string>}
+ */
+export async function readK8sReplicationState(primaryIp, sshKeyPath) {
+  const pod = await getPostgresPod(primaryIp, sshKeyPath);
+  const result = await sshKubectl(primaryIp, sshKeyPath, [
+    'exec',
+    '-n',
+    'vibecarbon',
+    pod,
+    '--',
+    'psql',
+    '-U',
+    'supabase_admin',
+    '-tAc',
+    "SELECT state FROM pg_stat_replication ORDER BY (state = 'streaming') DESC LIMIT 1",
+  ]);
+  const out = (typeof result === 'string' ? result : '').trim();
+  // `psql -tAc` yields exactly one token: a pg_stat_replication.state value, or
+  // '' when no replica is connected. Anything else means the read itself failed —
+  // an SSH/kubectl error banner (e.g. "banner exchange: Connection timed out")
+  // leaked through as stdout. Throw so verifyStreaming RETRIES rather than letting
+  // the error text masquerade as a replication state in the deploy-abort message.
+  const VALID_STATES = new Set(['', 'startup', 'catchup', 'streaming', 'backup', 'stopping']);
+  if (!VALID_STATES.has(out)) {
+    throw new Error(`could not read pg_stat_replication (unexpected output: ${out.slice(0, 120)})`);
+  }
+  return out;
+}

package/src/lib/deploy/state.js

@@ -7,6 +7,18 @@ import { createHash } from 'node:crypto';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 
+// Schema version for the persisted deploy-state file. Bump this whenever the
+// on-disk shape or the meaning of a step's inputs/result changes. `load()`
+// treats any file whose `version` doesn't match as fully-unstarted, so a CLI
+// upgrade can't resume a deploy on top of an incompatible state shape (which
+// would skip steps whose semantics have since changed).
+//
+// v2: introduced the `version` field itself + the dedicated-Pulumi-state
+// bucket. Pre-v2 files (no `version`) are auto-invalidated — their `s3-setup`
+// step predates the state bucket, so re-running it (to create + migrate the
+// state bucket) is exactly what we want.
+export const STATE_VERSION = 2;
+
 export class StateTracker {
   constructor(projectName, environment) {
     this.projectName = projectName;
@@ -17,17 +29,25 @@ export class StateTracker {
   }
 
   /**
-   * Load state from disk
+   * Load state from disk. Auto-invalidates (returns a fresh, empty state) when
+   * the persisted `version` doesn't match STATE_VERSION, so an upgraded CLI
+   * never resumes on an incompatible state shape.
    */
   load() {
     if (existsSync(this.stateFile)) {
       try {
-        return JSON.parse(readFileSync(this.stateFile, 'utf-8'));
+        const parsed = JSON.parse(readFileSync(this.stateFile, 'utf-8'));
+        if (!parsed || parsed.version !== STATE_VERSION) {
+          // Version mismatch (or a pre-versioning file) — discard so every
+          // step is treated as not-completed and re-runs against reality.
+          return { version: STATE_VERSION, steps: {} };
+        }
+        return parsed;
       } catch {
-        return { steps: {} };
+        return { version: STATE_VERSION, steps: {} };
       }
     }
-    return { steps: {} };
+    return { version: STATE_VERSION, steps: {} };
   }
 
   /**
@@ -37,6 +57,9 @@ export class StateTracker {
     if (!existsSync(this.stateDir)) {
       mkdirSync(this.stateDir, { recursive: true });
     }
+    // Always stamp the current schema version so a later load() can detect
+    // incompatibility after a CLI upgrade.
+    this.state.version = STATE_VERSION;
     writeFileSync(this.stateFile, JSON.stringify(this.state, null, 2));
   }
 
@@ -66,6 +89,40 @@ export class StateTracker {
     return false;
   }
 
+  /**
+   * Like shouldSkip, but consults an optional async remote-existence probe
+   * before agreeing to skip. Local hash equality only proves the step ran
+   * with these inputs *once* — it says nothing about whether the remote
+   * resource still exists (a bucket / server may have been deleted
+   * out-of-band, e.g. by a prior `destroy`). If `verifyFn` resolves falsy,
+   * the remote is gone and the step must re-run even on a hash match, so the
+   * reconciling step runs instead of a downstream step failing on the missing
+   * resource.
+   *
+   * `verifyFn` is only invoked when the hash already matches (the cheap local
+   * check gates the network probe). A probe that THROWS is treated as
+   * "can't confirm the remote is gone" → fall back to the hash decision
+   * (skip), so a transient S3/API blip doesn't force an expensive re-run.
+   *
+   * @param {string} stepId
+   * @param {object} inputs
+   * @param {() => Promise<boolean>} [verifyFn] - resolves true if the remote
+   *   resource still exists, false if it's gone.
+   * @returns {Promise<boolean>}
+   */
+  async shouldSkipWithVerify(stepId, inputs, verifyFn) {
+    if (!this.shouldSkip(stepId, inputs)) return false;
+    if (typeof verifyFn !== 'function') return true;
+    try {
+      const stillExists = await verifyFn();
+      return !!stillExists;
+    } catch {
+      // Probe failed — we can't prove the remote is gone. Honor the hash
+      // decision (skip) rather than forcing a re-run on a transient error.
+      return true;
+    }
+  }
+
   /**
    * Record a step as started
    * @param {string} stepId
@@ -110,7 +167,7 @@ export class StateTracker {
    * Clear state for a new deployment
    */
   clear() {
-    this.state = { steps: {} };
+    this.state = { version: STATE_VERSION, steps: {} };
     if (existsSync(this.stateFile)) {
       try {
         writeFileSync(this.stateFile, JSON.stringify(this.state, null, 2));