vibecarbon 0.1.0

package/src/lib/deploy/compose/ha.js

@@ -0,0 +1,1874 @@
+/**
+ * Docker Compose HA Deployment Module
+ *
+ * Deploys Vibecarbon to 2 VPS in different regions with:
+ * - Docker Compose on both servers
+ * - PostgreSQL streaming replication (primary → standby)
+ * - Cloudflare health checks + failover (or manual DNS instructions)
+ *
+ * Available on Diamond and Fullerene tiers.
+ */
+
+import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import * as p from '@clack/prompts';
+import { setupHA as setupCloudflareHA } from '../../cloudflare.js';
+import { c } from '../../colors.js';
+import { runCommand } from '../../command.js';
+import {
+  loadCredentials,
+  loadProjectConfig,
+  registerProject,
+  saveProjectConfig,
+} from '../../config.js';
+import { fetchWithRetry } from '../../fetch-retry.js';
+import { perfAsync } from '../../perf.js';
+import { getProvider } from '../../providers/index.js';
+import {
+  createAdminUser,
+  dockerLoginOnServer,
+  isLocalOnlyImageTag,
+  pullComposeImages,
+  runMigrations,
+  setupServer,
+  setupServerFiles,
+  sshRunAsync,
+  startComposeStack,
+  waitForSSH,
+} from './index.js';
+
+const SSH_OPTS = '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes';
+
+/**
+ * Run a command on a remote server via SSH
+ */
+function sshRun(ip, sshKeyPath, command, options = {}) {
+  const { timeout = 120_000 } = options;
+  return runCommand(['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command], {
+    stdio: 'pipe',
+    timeout,
+    encoding: 'utf-8',
+    ...options,
+  });
+}
+
+/**
+ * Look up a Hetzner firewall by exact name and DELETE it. Polls afterwards
+ * until the GET returns nothing (firewall actually gone) or the deadline
+ * lapses. Uses the `?name=<exact>` filter so the listing is one row deep
+ * — pagination (Hetzner default per_page=25) was the original bug: when
+ * 25+ leaked firewalls accumulated in the project, the project's firewalls
+ * weren't on page 1, the destroy filter found nothing, and the next deploy
+ * collided on "name is already used (uniqueness_error)".
+ *
+ * Returns true if the firewall was deleted (or never existed), false on
+ * timeout. Non-throwing — destroy paths swallow the failure but it's
+ * surfaced to the caller for verification.
+ */
+/**
+ * Resolve the VPS set that destroyComposeHA should delete.
+ *
+ * envConfig.servers is the canonical source when deploy completed and wrote
+ * it back. But when deploy gets canceled after Pulumi creates the VMs and
+ * before envConfig is persisted (the 2026-05-16 compose-ha matrix stall at
+ * "Waiting for primary services to initialize"), .vibecarbon.json never
+ * gets the IDs — destroy then sees `servers: []`, deletes nothing, and the
+ * test runner's orphan sweep has to mop up the leak.
+ *
+ * Fall back to name-based discovery: the Pulumi program names servers
+ * deterministically as `${projectName}-${environment}-(primary|standby)`,
+ * so a fresh Hetzner lookup recovers what envConfig is missing. Mirrors
+ * the pattern already used for firewall cleanup below.
+ *
+ * Exported for unit-test access; in production it's called only from
+ * destroyComposeHA.
+ *
+ * @param {object} args
+ * @param {string} args.projectName
+ * @param {string} args.environment
+ * @param {object} args.envConfig
+ * @param {string|null} args.hetznerToken
+ * @param {object} args.hetznerHeaders
+ * @returns {Promise<Array<{id: number|string, name?: string, hetznerName?: string}>>}
+ */
+export async function resolveHaServers({
+  projectName,
+  environment,
+  envConfig,
+  hetznerToken,
+  hetznerHeaders,
+}) {
+  const expectedNames = [
+    `${projectName}-${environment}-primary`,
+    `${projectName}-${environment}-standby`,
+  ];
+
+  const knownById = new Map(
+    (envConfig?.servers || []).filter((s) => s?.id && hetznerToken).map((s) => [String(s.id), s]),
+  );
+
+  if (!hetznerToken) return Array.from(knownById.values());
+
+  for (const name of expectedNames) {
+    const alreadyKnown = [...knownById.values()].some(
+      (s) => s.hetznerName === name || s.name === name,
+    );
+    if (alreadyKnown) continue;
+    try {
+      const resp = await fetchWithRetry(
+        `https://api.hetzner.cloud/v1/servers?name=${encodeURIComponent(name)}`,
+        { headers: hetznerHeaders },
+      );
+      const { servers } = await resp.json();
+      for (const found of servers || []) {
+        knownById.set(String(found.id), { id: found.id, name, hetznerName: name });
+      }
+    } catch {
+      // Discovery is best-effort — keep moving with whatever envConfig had.
+    }
+  }
+
+  return Array.from(knownById.values());
+}
+
+async function deleteFirewallByName(name, hetznerHeaders, deadlineMs = 30_000) {
+  const lookupUrl = `https://api.hetzner.cloud/v1/firewalls?name=${encodeURIComponent(name)}`;
+  const start = Date.now();
+  while (Date.now() - start < deadlineMs) {
+    const listResp = await fetchWithRetry(lookupUrl, { headers: hetznerHeaders });
+    const { firewalls } = await listResp.json();
+    const fw = firewalls?.[0];
+    if (!fw) return true;
+    if (fw.applied_to?.length > 0) {
+      const serverResources = fw.applied_to
+        .filter((a) => a.type === 'server' && a.server?.id)
+        .map((a) => ({ type: 'server', server: { id: a.server.id } }));
+      if (serverResources.length > 0) {
+        try {
+          await fetchWithRetry(
+            `https://api.hetzner.cloud/v1/firewalls/${fw.id}/actions/remove_from_resources`,
+            {
+              method: 'POST',
+              headers: { ...hetznerHeaders, 'Content-Type': 'application/json' },
+              body: JSON.stringify({ remove_from: serverResources }),
+            },
+          );
+        } catch {
+          // Servers may already be gone — proceed to DELETE anyway
+        }
+      }
+    }
+    const delResp = await fetchWithRetry(`https://api.hetzner.cloud/v1/firewalls/${fw.id}`, {
+      method: 'DELETE',
+      headers: hetznerHeaders,
+    });
+    if (delResp.ok || delResp.status === 404) return true;
+    // 409 = still attached; back off and retry the whole lookup so we get
+    // a fresh applied_to list (servers may have finished detaching).
+    await new Promise((r) => setTimeout(r, 3_000));
+  }
+  return false;
+}
+
+/**
+ * Read REPL_PASSWORD — process.env first (CI may export it), then .env.local
+ * (where `vibecarbon create` writes it at project-init time). Every HA code
+ * path funnels through this so one fix handles all three call sites that
+ * previously threw `REPL_PASSWORD is not set` when run without the env var.
+ * Returns null if the value is not present anywhere.
+ */
+function readReplPassword(cwd = process.cwd()) {
+  if (process.env.REPL_PASSWORD) return process.env.REPL_PASSWORD;
+  const envLocalPath = join(cwd, '.env.local');
+  if (!existsSync(envLocalPath)) return null;
+  const content = readFileSync(envLocalPath, 'utf-8');
+  // Accept both double-quoted (machine secrets) and single-quoted
+  // (escapeDotenv'd user secrets) forms, matching create.js output.
+  const m =
+    content.match(/^REPL_PASSWORD="([^"]+)"/m) || content.match(/^REPL_PASSWORD='([^']+)'/m);
+  return m ? m[1] : null;
+}
+
+/**
+ * Generate SSH key pair for deployment
+ */
+function generateSSHKeyPair(sshKeyPath) {
+  if (!existsSync(sshKeyPath)) {
+    runCommand(
+      ['ssh-keygen', '-t', 'ed25519', '-f', sshKeyPath, '-N', '', '-C', 'vibecarbon-deploy'],
+      { stdio: 'pipe' },
+    );
+  }
+}
+
+// ============================================================================
+// POSTGRESQL STREAMING REPLICATION SETUP
+// ============================================================================
+
+/**
+ * Configure the primary PostgreSQL server for streaming replication.
+ *
+ * 1. Create a dedicated replication user
+ * 2. Allow the standby IP in pg_hba.conf
+ * 3. Set wal_level=replica and max_wal_senders
+ * 4. Restart PostgreSQL to apply changes
+ */
+function configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, projectName) {
+  const remoteDir = `/opt/${projectName}`;
+  // replPassword is generated at create time from crypto.randomBytes and is
+  // restricted to base64url characters — no shell escaping needed.
+  // Read from process.env first (CI), then .env.local (the `vibecarbon create`
+  // default write target). Without the .env.local fallback, HA deploys fail
+  // the moment the CLI is invoked without the env var pre-exported.
+  const replPassword = readReplPassword();
+  if (!replPassword) {
+    throw new Error(
+      'REPL_PASSWORD is not set in process.env or .env.local. Regenerate your .env.local or run `vibecarbon configure` — HA deploys require a replication password generated at create time.',
+    );
+  }
+
+  // Create or update replication user + physical replication slot.
+  //
+  // Without a replication slot, the standby's WAL follow position isn't
+  // persisted on the primary — if the standby falls behind by more than
+  // `wal_keep_size` (512MB), primary recycles WAL segments and streaming
+  // breaks silently. The standby keeps appearing "up" but stops receiving
+  // updates. A physical slot (vibecarbon_standby_slot) tells primary to
+  // hold WAL until the standby confirms receipt, closing the drift
+  // window.
+  //
+  // The SQL is base64-encoded so the password never appears in argv —
+  // the remote shell only sees the base64 blob.
+  const createRoleSql = `
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'replicator') THEN
+    CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD '${replPassword}';
+  ELSE
+    ALTER ROLE replicator WITH PASSWORD '${replPassword}';
+  END IF;
+END $$;
+
+-- Create the physical replication slot if missing. pg_create_physical_replication_slot
+-- returns a row when successful, so wrap in a DO block that swallows the
+-- 'already exists' error for idempotent reruns.
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_replication_slots WHERE slot_name = 'vibecarbon_standby_slot'
+  ) THEN
+    PERFORM pg_create_physical_replication_slot('vibecarbon_standby_slot');
+  END IF;
+END $$;
+`;
+  const createRoleB64 = Buffer.from(createRoleSql).toString('base64');
+  sshRun(
+    primaryIp,
+    sshKeyPath,
+    `cd ${remoteDir} && echo ${createRoleB64} | base64 -d | docker compose exec -T db psql -U supabase_admin -d postgres`,
+    { timeout: 30_000 },
+  );
+
+  // Update pg_hba.conf to allow replication from standby. The supabase/postgres
+  // image overrides hba_file to /etc/postgresql/pg_hba.conf (NOT the default
+  // /var/lib/postgresql/data/pg_hba.conf). RCA: compose-ha 2026-05-01 fanout12
+  // — appended to the wrong file, postgres never saw the rule, every
+  // replication connection was rejected with "no pg_hba.conf entry" while
+  // pg_hba_file_rules reflected the actual /etc loaded contents (no replicator
+  // entry). Resolve the path at runtime via SHOW hba_file so we don't break
+  // again if the image moves the file.
+  sshRun(
+    primaryIp,
+    sshKeyPath,
+    `cd ${remoteDir} && HBA_FILE=$(docker compose exec -T db psql -U supabase_admin -d postgres -tAc 'SHOW hba_file;' | tr -d '[:space:]') && ` +
+      `docker compose exec -T db sh -c "echo 'host replication replicator ${standbyIp}/32 md5' >> $HBA_FILE && ` +
+      `psql -U supabase_admin -d postgres -tAc 'SELECT pg_reload_conf();'"`,
+    { timeout: 15_000 },
+  );
+
+  // Set WAL settings for replication
+  sshRun(
+    primaryIp,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose exec -T db psql -U supabase_admin -d postgres -c "ALTER SYSTEM SET wal_level = 'replica';" && docker compose exec -T db psql -U supabase_admin -d postgres -c "ALTER SYSTEM SET max_wal_senders = 5;" && docker compose exec -T db psql -U supabase_admin -d postgres -c "ALTER SYSTEM SET wal_keep_size = '512MB';"`,
+    { timeout: 30_000 },
+  );
+
+  // Restart PostgreSQL to apply WAL changes
+  sshRun(primaryIp, sshKeyPath, `cd ${remoteDir} && docker compose restart db`, {
+    timeout: 60_000,
+  });
+
+  // Wait for PostgreSQL to be ready again
+  for (let i = 0; i < 15; i++) {
+    try {
+      sshRun(
+        primaryIp,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db pg_isready -U postgres`,
+        { timeout: 10_000 },
+      );
+      return;
+    } catch {
+      // Retry
+    }
+  }
+}
+
+/**
+ * Configure the standby PostgreSQL server as a hot standby replica.
+ *
+ * 1. Stop the standby database
+ * 2. Wipe the data directory and run pg_basebackup from the primary
+ * 3. Write standby.signal and configure primary_conninfo
+ * 4. Restart as hot standby
+ */
+async function configureStandbyReplication(standbyIp, primaryIp, sshKeyPath, projectName) {
+  const remoteDir = `/opt/${projectName}`;
+  // replPassword is generated at create time from crypto.randomBytes and is
+  // restricted to base64url characters — no shell escaping needed.
+  // Read from process.env first (CI), then .env.local (the `vibecarbon create`
+  // default write target). Without the .env.local fallback, HA deploys fail
+  // the moment the CLI is invoked without the env var pre-exported.
+  const replPassword = readReplPassword();
+  if (!replPassword) {
+    throw new Error(
+      'REPL_PASSWORD is not set in process.env or .env.local. Regenerate your .env.local or run `vibecarbon configure` — HA deploys require a replication password generated at create time.',
+    );
+  }
+
+  // Stop the standby database
+  sshRun(standbyIp, sshKeyPath, `cd ${remoteDir} && docker compose stop db`, {
+    timeout: 60_000,
+  });
+
+  // Run pg_basebackup from primary into standby's data directory.
+  //
+  // -Fp: plain format (writes directly to PGDATA)
+  // -Xs: stream WAL during backup
+  // -R:  auto-create standby.signal and write primary_conninfo to postgresql.auto.conf
+  // -C --slot: use the physical replication slot created by
+  //            configurePrimaryReplication. This makes the primary hold
+  //            WAL for this standby specifically — so streaming stays
+  //            caught up indefinitely instead of silently falling behind
+  //            when wal_keep_size runs out.
+  //
+  // After pg_basebackup, append `primary_slot_name` to postgresql.auto.conf
+  // so the continuous-streaming connection (after initial backup) also uses
+  // the slot. `-R` writes primary_conninfo but not primary_slot_name.
+  //
+  // `set -e -o pipefail` is load-bearing: without it, a failed
+  // pg_basebackup leaves the data dir with only `postgresql.auto.conf`
+  // (because the trailing `>>` succeeds creating it from scratch), no
+  // `PG_VERSION`. The next `docker compose start db` then loops forever
+  // on `initdb: directory exists but is not empty` (RCA from compose-ha
+  // failover failure 2026-04-30). With `set -e`, basebackup failure
+  // exits the script non-zero, sshRun throws, the caller can react.
+  const basebackupScript = `
+set -e -o pipefail
+rm -rf /var/lib/postgresql/data/*
+PGPASSWORD='${replPassword}' pg_basebackup -h ${primaryIp} -p 5433 -U replicator -D /var/lib/postgresql/data -Fp -Xs -P -R -S vibecarbon_standby_slot -v
+# pg_basebackup -R is supposed to (a) create standby.signal and (b)
+# write primary_conninfo to postgresql.auto.conf. RCA from compose-ha
+# fanout9 2026-05-01: in supabase/postgres:15.8.1.085, neither happened
+# — postgresql.auto.conf came back at 88 bytes (just the ALTER SYSTEM
+# header) and standby.signal was missing. Postgres on next start saw
+# no standby.signal, came up as PRIMARY, started archiving WAL via
+# wal-g, and pg_is_in_recovery() returned 'f' for the entire 60s probe
+# window. Write everything ourselves — don't trust -R.
+touch /var/lib/postgresql/data/standby.signal
+# Replace postgresql.auto.conf wholesale (don't append — the upstream
+# image's first start can ALTER SYSTEM RESET ALL and wipe whatever -R
+# claimed to write). Both primary_conninfo and primary_slot_name are
+# load-bearing: missing primary_conninfo means recovery has nowhere
+# to read WAL from, missing primary_slot_name means primary won't
+# hold WAL for this standby and streaming silently falls behind.
+cat > /var/lib/postgresql/data/postgresql.auto.conf <<EOF
+# Do not edit this file manually!
+# It will be overwritten by the ALTER SYSTEM command.
+primary_conninfo = 'host=${primaryIp} port=5433 user=replicator password=${replPassword} application_name=standby'
+primary_slot_name = 'vibecarbon_standby_slot'
+EOF
+chown -R postgres:postgres /var/lib/postgresql/data
+# Final sanity checks. Each one is the postcondition for a different
+# downstream failure mode: PG_VERSION → docker compose start initdb
+# loop; standby.signal → postgres comes up as primary; primary_conninfo
+# in auto.conf → recovery has no source.
+test -f /var/lib/postgresql/data/PG_VERSION
+test -f /var/lib/postgresql/data/standby.signal
+grep -q '^primary_conninfo' /var/lib/postgresql/data/postgresql.auto.conf
+grep -q '^primary_slot_name' /var/lib/postgresql/data/postgresql.auto.conf
+`;
+  const basebackupB64 = Buffer.from(basebackupScript).toString('base64');
+  try {
+    sshRun(
+      standbyIp,
+      sshKeyPath,
+      // Use bash, not sh: the supabase/postgres image symlinks /bin/sh →
+      // dash, which rejects `set -o pipefail` with "Illegal option" and
+      // — because the offending line *is* the one that activates `set -e`
+      // — silently neuters every subsequent failure check (chown, grep,
+      // test). The basebackup itself still succeeds, but the "verify
+      // recovery conf is in place" sanity checks become no-ops, and a
+      // missing standby.signal / wiped postgresql.auto.conf goes
+      // undetected until pg_is_in_recovery() returns 'f' an entire
+      // basebackup later. RCA: compose-ha 2026-05-01 fanout10 failover.
+      `cd ${remoteDir} && echo ${basebackupB64} | base64 -d | docker compose run --rm -T --entrypoint bash db 2>&1`,
+      // silent:true makes sshRun THROW on non-zero exit instead of returning
+      // false. Without it, a failed basebackup (e.g. pg_hba.conf rejecting
+      // the connection) is silently swallowed: try-catch never enters, code
+      // proceeds to docker compose start db, probe loop times out, and the
+      // operator sees "did not enter recovery mode within 60s" with no clue
+      // that pg_basebackup never even ran. RCA: fanout12 2026-05-01.
+      { timeout: 300_000, silent: true },
+    );
+  } catch (err) {
+    // Surface the actual basebackup output. sshRun on failure attaches
+    // stdout to the error; without re-raising it deliberately the caller
+    // only sees "Command failed" and has to ssh in to root-cause.
+    const detail = err.stdout?.toString?.()?.trim?.() || err.message;
+    throw new Error(
+      `configureStandbyReplication: pg_basebackup failed on ${standbyIp}. ` +
+        `The standby data dir may now be in an inconsistent state ` +
+        `(non-empty without PG_VERSION) and require manual recovery.\n` +
+        `--- pg_basebackup output ---\n${detail}`,
+    );
+  }
+
+  // Start the standby database
+  sshRun(standbyIp, sshKeyPath, `cd ${remoteDir} && docker compose start db`, {
+    timeout: 60_000,
+  });
+
+  // Verify the standby is in recovery mode.
+  //
+  // Two independent signals — either is sufficient:
+  //   1. standby's pg_is_in_recovery() returns 't' (requires hot_standby on
+  //      AND consistent recovery state — can take 60-180s after fresh
+  //      basebackup of a Supabase cluster, especially under fanout-3 load)
+  //   2. primary's pg_stat_replication has a row with client_addr matching
+  //      this standby AND state in ('streaming','catchup','startup')
+  //
+  // The standby probe alone is fragile: with hot_standby still applying
+  // initial WAL, psql gets "Hot standby mode is disabled / database system
+  // is not accepting connections" — sshRun returns false (without
+  // silent:true), result becomes empty, the 60s budget burns through, and
+  // failover (which calls this function for re-seed) hard-fails with
+  // "did not enter recovery mode" even though the standby is fine.
+  //
+  // RCA: compose-ha 2026-05-01 fanout13 — replication was actively
+  // streaming (verifyReplication on the primary saw "Streaming replication
+  // active") but failover's re-seed still aborted because the standby's
+  // psql probe returned empty for the entire 60s. Bumped the budget to
+  // 180s and added the primary-side signal.
+  const PROBE_BUDGET_S = 180;
+  const PROBE_INTERVAL_S = 3;
+  let lastProbeResult = '';
+  let lastPrimarySignal = '';
+  for (let i = 0; i < PROBE_BUDGET_S / PROBE_INTERVAL_S; i++) {
+    try {
+      const result = sshRun(
+        standbyIp,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc 'SELECT pg_is_in_recovery()'`,
+        { timeout: 10_000 },
+      );
+      lastProbeResult = typeof result === 'string' ? result.trim() : '';
+      if (lastProbeResult === 't') return true;
+    } catch {
+      // Retry — database may still be applying initial WAL, hot_standby off
+    }
+    try {
+      const primaryResult = sshRun(
+        primaryIp,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "SELECT state FROM pg_stat_replication WHERE client_addr = '${standbyIp}'::inet"`,
+        { timeout: 10_000 },
+      );
+      lastPrimarySignal = typeof primaryResult === 'string' ? primaryResult.trim() : '';
+      if (
+        lastPrimarySignal === 'streaming' ||
+        lastPrimarySignal === 'catchup' ||
+        lastPrimarySignal === 'startup'
+      ) {
+        return true;
+      }
+    } catch {
+      // Retry
+    }
+    await new Promise((r) => setTimeout(r, PROBE_INTERVAL_S * 1000));
+  }
+
+  throw new Error(
+    `configureStandbyReplication: standby ${standbyIp} did not enter recovery mode ` +
+      `within ${PROBE_BUDGET_S}s after pg_basebackup. Last standby pg_is_in_recovery() = ` +
+      `${JSON.stringify(lastProbeResult)}, last primary pg_stat_replication.state for this ` +
+      `standby = ${JSON.stringify(lastPrimarySignal)}. SSH in and check: docker compose exec ` +
+      `db psql -U supabase_admin -d postgres -tAc 'SELECT pg_is_in_recovery()' and ` +
+      `ls /var/lib/postgresql/data/standby.signal`,
+  );
+}
+
+// ============================================================================
+// FIREWALL: OPEN REPLICATION PORT BETWEEN SERVERS
+// ============================================================================
+
+/**
+ * Open PostgreSQL port (5432) between the two servers via UFW.
+ * Only allows traffic from the peer's IP — not from the public internet.
+ */
+/**
+ * Open the dedicated replication port between HA peers.
+ *
+ * Port 5433 maps directly to the db container's PostgreSQL (bypassing supavisor
+ * on port 5432). This is required because supavisor doesn't support PostgreSQL
+ * replication protocol, and pg_basebackup / streaming replication need a raw
+ * PostgreSQL connection.
+ */
+async function openReplicationPort(ip, peerIp, sshKeyPath) {
+  await sshRunAsync(ip, sshKeyPath, `ufw allow from ${peerIp} to any port 5433 proto tcp`, {
+    timeout: 15_000,
+  });
+}
+
+/**
+ * Add an `in tcp 5433` rule scoped to the peer's IP on a server's Hetzner Cloud
+ * firewall. UFW alone isn't enough — Hetzner Cloud Firewall sits in front of
+ * the host and blocks SYN before it ever reaches UFW. Without this, every
+ * pg_basebackup attempt from the peer hits a TCP `Connection timed out`,
+ * configureStandbyReplication's bash script (with set -e + bash) fails on
+ * pg_basebackup, the catch path throws "pg_basebackup failed", but historically
+ * the script ran under dash (no `set -e -o pipefail` support) and silently
+ * proceeded — so the warning surfaced two stages later as "did not enter
+ * recovery mode within 60s" with the standby actually running its first-init
+ * primary-mode state. RCA: compose-ha 2026-05-01 fanout11.
+ *
+ * The Pulumi-managed firewall (src/lib/iac/programs/hetzner-compose.js) only
+ * declares 22/80/443 because the standby's IP isn't known at primary stack-up
+ * time. Patch in 5433 here, after both stacks are up.
+ *
+ * Idempotent: skips if a matching rule already exists. Non-fatal on error —
+ * the deploy continues with a warning so the operator can iterate the rest of
+ * the scenario; HA replication just won't actually work until 5433 is open.
+ */
+async function openReplicationPortHetznerFirewall(serverName, peerIp, hetznerToken) {
+  if (!hetznerToken) return;
+  const firewallName = `${serverName}-firewall`;
+  const headers = {
+    Authorization: `Bearer ${hetznerToken}`,
+    'Content-Type': 'application/json',
+  };
+  const fwListRes = await fetchWithRetry(
+    `https://api.hetzner.cloud/v1/firewalls?name=${encodeURIComponent(firewallName)}`,
+    { headers },
+  );
+  const fwListData = await fwListRes.json();
+  const firewall = fwListData.firewalls?.find((fw) => fw.name === firewallName);
+  if (!firewall) return;
+  const existingRules = firewall.rules || [];
+  const peerCidr = `${peerIp}/32`;
+  const hasReplRule = existingRules.some(
+    (r) => r.port === '5433' && r.source_ips?.includes(peerCidr),
+  );
+  if (hasReplRule) return;
+  const updatedRules = [
+    ...existingRules,
+    {
+      direction: 'in',
+      protocol: 'tcp',
+      port: '5433',
+      source_ips: [peerCidr],
+      description: 'PostgreSQL replication from HA peer',
+    },
+  ];
+  await fetchWithRetry(`https://api.hetzner.cloud/v1/firewalls/${firewall.id}/actions/set_rules`, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify({ rules: updatedRules }),
+  });
+}
+
+/**
+ * Check whether the primary server is reachable and its Postgres accepting
+ * connections. Short timeout — the failover flow needs to decide in seconds
+ * whether to attempt a re-seed or skip it and promote with current state.
+ *
+ * Returns true only when both the SSH connection works AND Postgres inside
+ * the db container reports ready. A firewall-level reachable + Postgres-
+ * unhealthy state should be treated the same as unreachable, since a
+ * re-seed would stall.
+ */
+async function isPrimaryReachable(primaryIp, sshKeyPath, remoteDir) {
+  try {
+    const result = sshRun(
+      primaryIp,
+      sshKeyPath,
+      `cd ${remoteDir} && docker compose exec -T db pg_isready -U postgres -t 3`,
+      { timeout: 10_000 },
+    );
+    return result?.includes('accepting connections') ?? false;
+  } catch {
+    return false;
+  }
+}
+
+// ============================================================================
+// DEPLOY
+// ============================================================================
+
+/**
+ * Deploy Docker Compose HA (2 VPS, 2 regions, PostgreSQL replication)
+ *
+ * @param {object} options
+ * @param {string} options.projectName
+ * @param {string} options.environment
+ * @param {string} options.sshKeyPath
+ * @param {string} options.domain
+ * @param {string} options.region - Primary region
+ * @param {string} options.serverType
+ * @param {string} options.apiToken - Hetzner API token
+ * @param {string} [options.cloudflareApiToken]
+ * @param {string} [options.cloudflareZoneId]
+ * @param {string} [options.dnsProvider]
+ * @param {object} [options.services]
+ * @param {object} [options.envConfig] - Existing environment config
+ * @param {Function} [options.onProgress]
+ */
+export async function deployComposeHA(options) {
+  const {
+    projectName,
+    environment,
+    sshKeyPath,
+    domain,
+    region,
+    secondaryRegion: requestedSecondaryRegion,
+    serverType = 'cpx21',
+    apiToken,
+    cloudflareApiToken,
+    cloudflareZoneId,
+    dnsProvider,
+    services = {},
+    envConfig = {},
+    dockerHubCreds = null,
+    ghcrPullCreds = null,
+    imageRef,
+    s3Config = null,
+    backupS3Config = null,
+    backupConfig = null,
+    bundlePath = null,
+    operatorCidrs = [],
+    onProgress = () => {},
+  } = options;
+
+  if (!imageRef) {
+    throw new Error(
+      'deployComposeHA requires options.imageRef (ghcr.io/<owner>/<repo>:<tag>) — build pipeline changed in Lever 1',
+    );
+  }
+
+  const _provider = getProvider('hetzner', apiToken);
+
+  // Use the caller-specified secondary region, or pick the first region that differs from primary
+  const HETZNER_REGIONS = ['nbg1', 'hel1', 'fsn1'];
+  const standbyRegion =
+    requestedSecondaryRegion || HETZNER_REGIONS.find((r) => r !== region) || 'hel1';
+
+  // Generate SSH key pair
+  generateSSHKeyPair(sshKeyPath);
+  const publicKey = readFileSync(`${sshKeyPath}.pub`, 'utf-8').trim();
+
+  // Upload SSH key to Hetzner once, then share its ID across both Pulumi
+  // stacks — otherwise primary + standby race to create the same key and
+  // one side hits HTTP 409 "SSH key not unique" (same public-key bytes
+  // cannot be registered twice in a Hetzner project, regardless of name).
+  const sharedSshKeyName = `${projectName}-${environment}-key`;
+  const sharedSshKeyId = String(await _provider.createSSHKey(sharedSshKeyName, publicKey));
+
+  // Provision both VPS via Pulumi — one stack per cluster, concurrent up().
+  // Each stack declares SSH key + firewall + server in a single reconcile, so
+  // partial failures on one cluster are cleaned up automatically on re-run.
+  const { initBackend, upStack } = await import('../../iac/index.js');
+  const { buildHetznerComposeProgram } = await import('../../iac/programs/hetzner-compose.js');
+  onProgress(`Provisioning VPS in ${region} and ${standbyRegion}...`);
+
+  // Pre-initialize the Pulumi S3 backend before launching parallel stack ops.
+  // RCA from compose-ha iter-perfwave3 (2026-05-01): destroy deletes the S3
+  // bucket; restore re-creates it; the parallel upStack(primary)+upStack(standby)
+  // then race to `pulumi stack select` against a freshly-created bucket and
+  // both fail in <2s with `could not list bucket: blob (code=NotFound):
+  // NoSuchBucket` — Hetzner Ceph hasn't propagated the new bucket for list
+  // operations yet. K8s-HA already does this same pre-init for the same reason
+  // (src/lib/deploy/k8s/ha/index.js:201-207); compose-HA was missing the call.
+  await perfAsync('deploy.ha.compose.initBackend', () =>
+    initBackend({ s3Config, hcloudToken: apiToken }),
+  );
+  const allowedSshIps = operatorCidrs.map((e) => e.cidr);
+  const primaryProgram = buildHetznerComposeProgram({
+    projectName,
+    environment: `${environment}-primary`,
+    sshPublicKey: publicKey,
+    existingSshKeyId: sharedSshKeyId,
+    location: region,
+    serverType,
+    labels: { 'managed-by': 'vibecarbon', role: 'primary' },
+    allowedSshIps,
+  });
+  const standbyProgram = buildHetznerComposeProgram({
+    projectName,
+    environment: `${environment}-standby`,
+    sshPublicKey: publicKey,
+    existingSshKeyId: sharedSshKeyId,
+    location: standbyRegion,
+    serverType,
+    labels: { 'managed-by': 'vibecarbon', role: 'standby' },
+    allowedSshIps,
+  });
+  // Pass s3Config so Pulumi state lives in the project's S3 bucket rather
+  // than the GLOBAL local backend at ~/.vibecarbon/pulumi-state/. Without
+  // this, parallel test scenarios share a backend, and another scenario's
+  // orphan-detect path can find compose-ha's mid-deploy stacks and destroy
+  // them — observed 2026-04-26 batch run #5 where compose's final-destroy
+  // wiped compose-ha's e2-primary/e2-standby while compose-ha was still
+  // mid-flight in verify-deploy.
+  // Pulumi up of two stacks in parallel — typically the largest single
+  // wall-clock chunk in compose-ha deploy (~2-3 min cold; image+firewall
+  // creation in two regions concurrently). Per-stack timers expose which
+  // region/stack is the slow one in regression analysis.
+  const [primaryResult, standbyResult] = await perfAsync('deploy.ha.compose.iac.upStack.fan', () =>
+    Promise.all([
+      perfAsync('deploy.ha.compose.iac.upStack.primary', () =>
+        upStack(`${environment}-primary`, primaryProgram, { hcloudToken: apiToken, s3Config }),
+      ),
+      perfAsync('deploy.ha.compose.iac.upStack.standby', () =>
+        upStack(`${environment}-standby`, standbyProgram, { hcloudToken: apiToken, s3Config }),
+      ),
+    ]),
+  );
+  const primaryIp = primaryResult.outputs.serverIp;
+  const primaryServerId = primaryResult.outputs.serverId;
+  const standbyIp = standbyResult.outputs.serverIp;
+  const standbyServerId = standbyResult.outputs.serverId;
+
+  p.log.success(`Primary: ${c.info(primaryIp)} (${region})`);
+  p.log.success(`Standby: ${c.info(standbyIp)} (${standbyRegion})`);
+
+  // Persist server info immediately so `destroy` can clean up if deploy fails later
+  const projectConfig = loadProjectConfig() || { projectName, environments: {} };
+  const pendingEnvConfig = {
+    ...envConfig,
+    status: 'deploying',
+    deployMode: 'compose-ha',
+    provider: 'hetzner',
+    region,
+    secondaryRegion: standbyRegion,
+    serverType,
+    ha: { enabled: true, failoverRegion: standbyRegion },
+    servers: [
+      {
+        name: `${projectName}-${environment}-primary`,
+        id: primaryServerId,
+        ip: primaryIp,
+        region,
+        serverType,
+        role: 'primary',
+      },
+      {
+        name: `${projectName}-${environment}-standby`,
+        id: standbyServerId,
+        ip: standbyIp,
+        region: standbyRegion,
+        serverType,
+        role: 'standby',
+      },
+    ],
+    domain: domain || null,
+    dnsProvider: dnsProvider || null,
+    dns:
+      dnsProvider === 'cloudflare' && cloudflareZoneId
+        ? { provider: 'cloudflare', cloudflareZoneId }
+        : dnsProvider === 'hetzner' && options.hetznerDnsZoneId
+          ? { provider: 'hetzner', hetznerZoneId: options.hetznerDnsZoneId }
+          : null,
+    // s3.accessKey is intentionally NOT persisted — the secret scanner
+    // gates every `vibecarbon deploy` invocation against tracked + untracked
+    // files in cwd, and .vibecarbon.json sits at the project root with
+    // neither a gitignore entry nor an isUniversalLocalEnv match. The
+    // orchestrator's skeleton save (src/lib/deploy/orchestrator.js:316)
+    // already omits accessKey; compose-ha was the lone outlier that wrote
+    // it back here, which is what made warm-deploy fast-fail with
+    // "Refusing to deploy: secrets detected" while single-region compose
+    // and both k8s modes passed. prompts.js:472-475 falls back to
+    // getS3Credentials() (reading ~/.vibecarbon/credentials.json) when
+    // envConfig.s3.accessKey is missing, so runtime still resolves.
+    ...(s3Config && {
+      s3: {
+        bucket: s3Config.bucket,
+        region: s3Config.region,
+        endpoint: s3Config.endpoint,
+      },
+    }),
+    ...(backupS3Config && { backupS3: backupS3Config }),
+    ...(backupConfig && { backup: backupConfig }),
+    services,
+    lastAttempt: new Date().toISOString(),
+  };
+  saveProjectConfig({
+    ...projectConfig,
+    provider: 'hetzner',
+    environments: {
+      ...projectConfig.environments,
+      [environment]: pendingEnvConfig,
+    },
+  });
+
+  // Wait for SSH on both servers
+  onProgress('Waiting for servers to become available...');
+  const [primaryReady, standbyReady] = await perfAsync('deploy.ha.compose.waitForSSH.both', () =>
+    Promise.all([waitForSSH(primaryIp, sshKeyPath, 30), waitForSSH(standbyIp, sshKeyPath, 30)]),
+  );
+
+  if (!primaryReady || !standbyReady) {
+    throw new Error('Timed out waiting for SSH on one or both servers');
+  }
+
+  // Ensure Docker is running, configure firewall, enable auto-updates
+  onProgress('Configuring servers...');
+  await perfAsync('deploy.ha.compose.cloudInitReady.both', () =>
+    Promise.all([setupServer(primaryIp, sshKeyPath), setupServer(standbyIp, sshKeyPath)]),
+  );
+
+  // Wait for Docker daemon to be ready on both servers before logins/pulls/builds.
+  // dockerLoginOnServer can fire before Docker is listening, silently "succeed"
+  // (errors are swallowed with a warning), then reconcile.sh hits `unauthorized`
+  // when pulling from ghcr. The 03:55 UTC run's compose-ha failure was exactly
+  // this race on standby — primary completed login first, standby's login ran
+  // against a not-yet-ready dockerd, ghcr credentials never persisted, pull
+  // failed.
+  const { waitForDockerReady } = await import('./index.js');
+  await perfAsync('deploy.ha.compose.dockerReady.both', () =>
+    Promise.all([
+      waitForDockerReady(primaryIp, sshKeyPath),
+      waitForDockerReady(standbyIp, sshKeyPath),
+    ]),
+  );
+
+  // --- Local-image build + sideload (HA) ---
+  // When the image tag is local-only (no `:` registry prefix), it was NOT
+  // pushed to a registry. The pre-PR-5 deploy gated this on `direct`; the
+  // CLI sweep collapsed `direct` and `local` build modes into a single
+  // auto-detected default and `direct` is now mostly false for compose-HA.
+  // The trigger that matters here is whether the image will be pullable
+  // from `docker compose up -d` on each server — which is purely a
+  // function of the tag. So we fire whenever the imageRef is local-only,
+  // regardless of how it got there. Without this sideload, standby's
+  // reconcile fails with `pull access denied for <project>-app, repository
+  // does not exist`.
+  //
+  // We build natively on BOTH servers in parallel (over DOCKER_HOST=ssh://)
+  // rather than building once locally and piping `docker save` cross-region.
+  // The previous docker-save-pipe hung at 1475s (= 24.6 min, past the
+  // deploy timeout) on the 2026-04-27 matrix when streaming a ~500MB tarball
+  // over a single TCP stream between Hetzner regions; per-region builds run
+  // in parallel against a tiny build context (~tens of KB) and the resulting
+  // image is byte-identical (same Dockerfile, same source). The same swap
+  // was made for compose scale; this completes it for compose-ha deploy.
+  if (isLocalOnlyImageTag(imageRef)) {
+    const { buildRemote } = await import('../remote-build.js');
+    // Plumb VITE_* build args through to docker build on each host so the
+    // browser bundle ships with real Supabase URL + project name etc.
+    // (mirrors the single-region fix in orchestrator.js; without these the
+    // bundle gets empty strings and the client throws "Missing Supabase
+    // environment variables" at page load).
+    const { collectComposeBuildArgs } = await import('./build-args.js');
+    const haBuildArgs = collectComposeBuildArgs(process.cwd(), { projectName, domain });
+    onProgress('Building image natively on primary + standby (parallel)...');
+    const [primaryOk, standbyOk] = await perfAsync('deploy.ha.compose.remoteBuild.fan', () =>
+      Promise.all([
+        buildRemote(primaryIp, sshKeyPath, imageRef, process.cwd(), haBuildArgs),
+        buildRemote(standbyIp, sshKeyPath, imageRef, process.cwd(), haBuildArgs),
+      ]),
+    );
+    if (!primaryOk) throw new Error('Remote build on primary failed');
+    if (!standbyOk) throw new Error('Remote build on standby failed');
+  }
+
+  // Docker login and file copy — all independent, run in parallel
+  onProgress('Copying project files...');
+  const serviceOpts = {
+    domain,
+    image: imageRef,
+    observability: services.observability,
+    n8n: services.n8n,
+    metabase: services.metabase,
+    redis: services.redis,
+    bundlePath,
+  };
+  const setupTasks = [
+    perfAsync('deploy.ha.compose.setupFiles.primary', () =>
+      setupServerFiles(primaryIp, sshKeyPath, projectName, serviceOpts),
+    ),
+    perfAsync('deploy.ha.compose.setupFiles.standby', () =>
+      setupServerFiles(standbyIp, sshKeyPath, projectName, serviceOpts),
+    ),
+    perfAsync('deploy.ha.compose.openReplPort.primary', () =>
+      openReplicationPort(primaryIp, standbyIp, sshKeyPath),
+    ),
+    perfAsync('deploy.ha.compose.openReplPort.standby', () =>
+      openReplicationPort(standbyIp, primaryIp, sshKeyPath),
+    ),
+    // Open 5433 in the Hetzner Cloud firewall for both peers. UFW is the
+    // host-level layer and was already configured above; without these two
+    // calls the Cloud Firewall (Pulumi-declared with only 22/80/443) silently
+    // drops every replication packet at the edge. Both directions because
+    // post-failover the standby becomes primary and needs its own 5433 open
+    // for the new standby (now the old primary) to reconnect.
+    perfAsync('deploy.ha.compose.cloudFirewall5433.primary', () =>
+      openReplicationPortHetznerFirewall(
+        `${projectName}-${environment}-primary`,
+        standbyIp,
+        apiToken,
+      ),
+    ),
+    perfAsync('deploy.ha.compose.cloudFirewall5433.standby', () =>
+      openReplicationPortHetznerFirewall(
+        `${projectName}-${environment}-standby`,
+        primaryIp,
+        apiToken,
+      ),
+    ),
+  ];
+  if (dockerHubCreds) {
+    setupTasks.push(
+      perfAsync('deploy.ha.compose.dockerLogin.primary', () =>
+        dockerLoginOnServer(primaryIp, sshKeyPath, dockerHubCreds),
+      ),
+      perfAsync('deploy.ha.compose.dockerLogin.standby', () =>
+        dockerLoginOnServer(standbyIp, sshKeyPath, dockerHubCreds),
+      ),
+    );
+  }
+  if (ghcrPullCreds?.owner && ghcrPullCreds?.token) {
+    const ghcrLogin = {
+      username: ghcrPullCreds.owner,
+      token: ghcrPullCreds.token,
+      registry: 'ghcr.io',
+    };
+    setupTasks.push(
+      perfAsync('deploy.ha.compose.ghcrLogin.primary', () =>
+        dockerLoginOnServer(primaryIp, sshKeyPath, ghcrLogin),
+      ),
+      perfAsync('deploy.ha.compose.ghcrLogin.standby', () =>
+        dockerLoginOnServer(standbyIp, sshKeyPath, ghcrLogin),
+      ),
+    );
+  }
+  await perfAsync('deploy.ha.compose.setupFan', () => Promise.all(setupTasks));
+
+  // Pull app + base images on both servers in parallel. No remote build —
+  // image comes from ghcr.io (CI-built, multi-arch manifest; each node
+  // pulls its arch). Each server is independent; no cross-cluster dep.
+  onProgress('Pulling images on both servers...');
+  await perfAsync('deploy.ha.compose.pullFan', () =>
+    Promise.all([
+      perfAsync('deploy.ha.compose.pullImages.primary', () =>
+        pullComposeImages(primaryIp, sshKeyPath, projectName, serviceOpts),
+      ),
+      perfAsync('deploy.ha.compose.pullImages.standby', () =>
+        pullComposeImages(standbyIp, sshKeyPath, projectName, serviceOpts),
+      ),
+    ]),
+  );
+
+  // Set up DNS BEFORE starting services so Traefik's ACME challenges can succeed
+  // immediately (subdomains use DNS-only records that Let's Encrypt validates directly)
+  if (domain && cloudflareApiToken && cloudflareZoneId) {
+    onProgress('Setting up Cloudflare HA (health checks + load balancer)...');
+    try {
+      await perfAsync('deploy.ha.compose.dns.cloudflare', () =>
+        setupCloudflareHA(cloudflareApiToken, cloudflareZoneId, domain, [
+          { name: `${projectName}-${environment}-primary`, ip: primaryIp, region },
+          { name: `${projectName}-${environment}-standby`, ip: standbyIp, region: standbyRegion },
+        ]),
+      );
+    } catch (error) {
+      p.log.warn(`Cloudflare HA setup failed: ${error.message}`);
+      p.log.info('You can configure DNS manually.');
+    }
+  } else if (domain && options.hetznerDnsZoneId && apiToken) {
+    onProgress('Setting up Hetzner DNS for HA...');
+    try {
+      const { setupHA: setupHetznerDnsHA } = await import('../../hetzner-dns.js');
+      await perfAsync('deploy.ha.compose.dns.hetzner', () =>
+        setupHetznerDnsHA(apiToken, options.hetznerDnsZoneId, domain, [
+          { name: `${projectName}-${environment}-primary`, ip: primaryIp, region },
+          { name: `${projectName}-${environment}-standby`, ip: standbyIp, region: standbyRegion },
+        ]),
+      );
+    } catch (error) {
+      p.log.warn(`Hetzner DNS HA setup failed: ${error.message}`);
+      p.log.info('You can configure DNS manually.');
+    }
+  } else if (domain && dnsProvider === 'manual') {
+    p.log.info(c.bold('Manual DNS Configuration:'));
+    p.log.message(`  Point ${c.bold(domain)} to ${c.bold(primaryIp)} (primary)`);
+    p.log.message(`  Standby server: ${c.bold(standbyIp)} (for manual failover)`);
+  }
+
+  // Block compose-up until public resolvers see the apex domain. Traefik's
+  // ACME burst (~7 attempts in ~30s) fails if LE can't resolve the host
+  // when challenges fire. HA has historically hidden this race by accident
+  // because there's substantial setup work between the DNS write above and
+  // compose-up below; making the wait explicit eliminates the surprise on
+  // future deploys that shrink that gap. Same fix lives in orchestrator.js
+  // for single-region compose where the race was actively biting (RCA from
+  // vibecarbon.com cold-deploy 2026-05-19).
+  if (domain && dnsProvider && dnsProvider !== 'manual') {
+    // Surface this in the enclosing tracker spinner so the operator sees
+    // why we're waiting (up to 120s on a cold edge). Without the message
+    // the HA spinner sits on the prior "Configuring servers..." text for
+    // potentially two minutes, looking hung.
+    onProgress(`Waiting for ${domain} to propagate to ${primaryIp}...`);
+    const { waitForDNSPropagation } = await import('../../dns-propagation.js');
+    await perfAsync('deploy.ha.compose.dns.waitForPropagation', () =>
+      waitForDNSPropagation(domain, primaryIp, 120_000),
+    );
+  }
+
+  // Start compose stacks on both servers in parallel
+  onProgress('Starting services on both servers...');
+  await perfAsync('deploy.ha.compose.composeUp.both', () =>
+    Promise.all([
+      startComposeStack(primaryIp, sshKeyPath, projectName, serviceOpts),
+      startComposeStack(standbyIp, sshKeyPath, projectName, serviceOpts),
+    ]),
+  );
+
+  // Run migrations on primary only
+  onProgress('Running database migrations on primary...');
+  try {
+    await perfAsync('deploy.ha.compose.migrations', () =>
+      runMigrations(primaryIp, sshKeyPath, projectName),
+    );
+  } catch {
+    // Migrations may partially fail on first run, non-fatal
+  }
+
+  // Create admin user in production Supabase (same as single-server deploy)
+  onProgress('Creating admin user...');
+  const adminResult = await perfAsync('deploy.ha.compose.createAdminUser', () =>
+    createAdminUser(primaryIp, sshKeyPath, projectName),
+  );
+  if (adminResult.success) {
+    p.log.success(adminResult.message);
+  } else {
+    p.log.warn(`${adminResult.message}. Run \`vibecarbon deploy\` again to retry.`);
+  }
+
+  // Wait for primary Postgres to accept connections before configuring
+  // replication. pg_basebackup needs the PG socket reachable; it doesn't
+  // need GoTrue/Storage to have finished their own migrations.
+  //
+  // Earlier versions of this loop polled GoTrue's /health endpoint on the
+  // assumption that pg_basebackup running before app migrations completed
+  // would leave the standby with an incomplete schema. That's a
+  // misreading of how physical replication works: pg_basebackup takes a
+  // consistent snapshot at LSN T_0 and the standby replays the WAL stream
+  // from T_0 onwards via the replication slot, so any GoTrue/Storage
+  // migrations committed after the snapshot land on the standby
+  // automatically. The previous run (2026-05-18) measured the GoTrue
+  // probe at 2:25 wall-clock on a healthy primary; swapping to
+  // pg_isready collapses that to <10s.
+  //
+  // Wrapped in perfAsync so the distribution stays visible in
+  // perf_substep — same metric name as before for trend continuity, even
+  // though the probe semantics have tightened.
+  onProgress('Waiting for primary Postgres to accept connections...');
+  const healthConfirmed = await perfAsync('deploy.ha.compose.primaryHealthProbe', async () => {
+    for (let i = 0; i < 30; i++) {
+      try {
+        const health = sshRun(
+          primaryIp,
+          sshKeyPath,
+          `cd /opt/${projectName} && docker compose exec -T db pg_isready -h localhost -p 5432 2>/dev/null`,
+          { timeout: 10_000 },
+        );
+        if (health?.includes('accepting connections')) return true;
+      } catch {
+        // Retry — non-zero exit from pg_isready (rejecting / no response)
+        // bubbles up as a thrown error from sshRun; same handling either way.
+      }
+      if (i === 29) break;
+      const ms = i < 5 ? 1000 : 3000;
+      await new Promise((r) => setTimeout(r, ms));
+    }
+    return false;
+  });
+  if (!healthConfirmed) {
+    // Breadcrumb so a downstream replication failure isn't surprising —
+    // the 2026-05-16 e2e matrix's compose-ha cancel left the deploy
+    // log with no signal that the wait had given up unconfirmed before
+    // the SSH retries downstream blew up on a flaky local network.
+    // Proceeding anyway: PG may still come up in time for the
+    // replication step, and hard-failing here would mask genuine
+    // late-init successes.
+    console.error(
+      `[ha] primary Postgres did not accept connections within ~6min; proceeding optimistically — configurePrimaryReplication is the next failure surface if PG is truly dead`,
+    );
+  }
+
+  // Expose raw PostgreSQL on port 5433 for replication (supavisor owns 5432).
+  // Add a docker-compose.replication.yml overlay that maps db:5432 → host:5433.
+  // Fan out across primary + standby — the overlay write + db-only `compose up`
+  // touch disjoint VPS hosts and need to land on both before
+  // configurePrimaryReplication grants pub access. Previously serial via a
+  // for-loop using sync sshRun (~4 ssh calls × 2-3s each = ~8-12s blocking the
+  // event loop); Promise.all halves wall-clock and unblocks any concurrent
+  // perfAsync timers from being charged for the dead time.
+  const replOverlay = `version: "3.8"\nservices:\n  db:\n    ports:\n      - "5433:5432"\n`;
+  await Promise.all(
+    [primaryIp, standbyIp].map(async (ip) => {
+      await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cat > /opt/${projectName}/docker-compose.replication.yml << 'REPL'\n${replOverlay}REPL`,
+        { timeout: 10_000 },
+      );
+      // Recreate only the db service with the new port mapping
+      await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cd /opt/${projectName} && docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.replication.yml up -d db 2>&1`,
+        { timeout: 60_000 },
+      );
+    }),
+  );
+
+  // Configure PostgreSQL streaming replication.
+  // Wrap the entire setup (primary + standby + slot creation + pg_basebackup)
+  // so the perf trace shows the headline replication cost — typical 30-90s,
+  // dominated by pg_basebackup for the standby seed.
+  await perfAsync('deploy.ha.replication.setup', async () => {
+    onProgress('Configuring PostgreSQL replication on primary...');
+    configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, projectName);
+
+    onProgress('Configuring standby as hot replica...');
+    let replicationOk = false;
+    try {
+      replicationOk = await configureStandbyReplication(
+        standbyIp,
+        primaryIp,
+        sshKeyPath,
+        projectName,
+      );
+    } catch (err) {
+      // Soft-fail in deploy: replication has been chronically broken
+      // (see project_replication_broken.md) and hard-failing every
+      // compose-ha deploy here would block iteration on the rest of
+      // the scenario. Surface the captured pg_basebackup output so
+      // we can root-cause it, then continue with a clear warning.
+      // Failover-path still hard-fails (see line ~1185) so we don't
+      // cascade into a data-wipe + CrashLoopBackOff there.
+      p.log.warn(`Replication setup error (continuing with warning): ${err.message}`);
+    }
+    if (!replicationOk) {
+      p.log.warn('Replication setup may not have completed — verify manually');
+    }
+  });
+
+  // Verify streaming replication is active on the primary. Once the standby
+  // catches up, pg_stat_replication.state flips to 'streaming' usually
+  // within 1-2s of standby start. Exponential pacing instead of the old
+  // hardcoded 1s/3s shell-out: fast happy path (probe at 250ms, 750ms,
+  // 1.75s, …) without giving up too early under load. Total budget ~25s.
+  onProgress('Verifying replication...');
+  const REPL_DELAYS_MS = [250, 500, 1000, 2000, 3000, 3000, 3000, 3000, 3000, 3000];
+  let replActive = false;
+  for (let i = 0; i < REPL_DELAYS_MS.length; i++) {
+    try {
+      const count = sshRun(
+        primaryIp,
+        sshKeyPath,
+        `cd /opt/${projectName} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "SELECT count(*) FROM pg_stat_replication WHERE state = 'streaming'"`,
+        { timeout: 10_000 },
+      );
+      if (Number.parseInt(count?.trim(), 10) > 0) {
+        replActive = true;
+        break;
+      }
+    } catch {
+      // Retry
+    }
+    if (i === REPL_DELAYS_MS.length - 1) break;
+    await new Promise((r) => setTimeout(r, REPL_DELAYS_MS[i]));
+  }
+  if (replActive) {
+    p.log.success('Streaming replication active');
+  } else {
+    p.log.warn('Streaming replication not detected — standby may fall behind');
+  }
+
+  // Finalize config — promote pending → deployed
+  saveProjectConfig({
+    ...projectConfig,
+    provider: 'hetzner',
+    environments: {
+      ...projectConfig.environments,
+      [environment]: {
+        ...pendingEnvConfig,
+        status: 'deployed',
+        deployedAt: new Date().toISOString(),
+        lastAttempt: undefined,
+      },
+    },
+  });
+  registerProject(projectName, process.cwd());
+
+  // Summary
+  p.log.success('');
+  if (domain) {
+    p.log.message(`  ${c.dim('URL')}            ${c.info(`https://${domain}`)}`);
+  }
+  p.log.message(`  ${c.dim('Primary')}        ${c.info(primaryIp)} (${region})`);
+  p.log.message(`  ${c.dim('Standby')}        ${c.info(standbyIp)} (${standbyRegion})`);
+  p.log.message(`  ${c.dim('Deploy method')}  Docker Compose HA`);
+  p.log.message(`  ${c.dim('Replication')}    PostgreSQL streaming`);
+  p.log.message('');
+
+  return {
+    success: true,
+    primaryIp,
+    standbyIp,
+    domain,
+    deployMode: 'compose-ha',
+  };
+}
+
+// ============================================================================
+// DESTROY
+// ============================================================================
+
+/**
+ * Destroy a Compose HA environment (both VPS, firewalls, SSH keys, DNS/LB)
+ *
+ * @param {object} options
+ * @param {string} options.projectName
+ * @param {string} options.environment
+ * @param {object} options.envConfig
+ * @param {string} options.hetznerToken
+ * @param {string} [options.cloudflareToken]
+ * @param {Function} [options.onProgress]
+ */
+export async function destroyComposeHA(options) {
+  const {
+    projectName,
+    environment,
+    envConfig,
+    hetznerToken,
+    cloudflareToken,
+    onProgress = () => {},
+  } = options;
+
+  const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
+  const hetznerHeaders = { Authorization: `Bearer ${hetznerToken}` };
+
+  // Stop compose services on both servers in parallel
+  onProgress('Stopping services on all servers...');
+  if (existsSync(sshKeyPath)) {
+    await Promise.allSettled(
+      (envConfig.servers || [])
+        .filter((s) => s.ip)
+        .map((server) =>
+          sshRunAsync(
+            server.ip,
+            sshKeyPath,
+            `cd /opt/${projectName} && docker compose down -v --remove-orphans 2>/dev/null; rm -rf /opt/${projectName}`,
+            { timeout: 120_000, ignoreError: true },
+          ),
+        ),
+    );
+  }
+
+  // Delete both VPS in parallel.
+  //
+  // Hetzner's DELETE /servers/{id} is async: it returns 202 immediately
+  // and the actual VM removal continues for several seconds afterwards.
+  // The previous code only awaited the 202 — Pulumi state was wiped
+  // moments later when the S3 bucket got deleted, then compose-ha
+  // restore re-ran the deploy with a fresh Pulumi stack and tried to
+  // create a server with the same `${projectName}-${environment}-primary`
+  // name. Hetzner rejected with HTTP 409 'server name is already used
+  // (uniqueness_error)' because the old VM was still draining
+  // (observed in compose-ha restore matrix runs).
+  //
+  // Issue DELETE first, then poll GET /servers/{id} on each one until
+  // it returns 404. 90s deadline is comfortable: empirical Hetzner
+  // delete time is 5-30s on cx23.
+  onProgress('Deleting VPS...');
+  const serversToDelete = await resolveHaServers({
+    projectName,
+    environment,
+    envConfig,
+    hetznerToken,
+    hetznerHeaders,
+  });
+  await Promise.allSettled(
+    serversToDelete.map((server) =>
+      fetchWithRetry(`https://api.hetzner.cloud/v1/servers/${server.id}`, {
+        method: 'DELETE',
+        headers: hetznerHeaders,
+      }).catch(() => {}),
+    ),
+  );
+  if (serversToDelete.length > 0) {
+    onProgress('Waiting for VPS deletion to complete...');
+    const deadline = Date.now() + 90_000;
+    const remaining = new Map(serversToDelete.map((s) => [s.id, s]));
+    while (remaining.size > 0 && Date.now() < deadline) {
+      await Promise.allSettled(
+        Array.from(remaining.entries()).map(async ([id]) => {
+          try {
+            const resp = await fetch(`https://api.hetzner.cloud/v1/servers/${id}`, {
+              headers: hetznerHeaders,
+            });
+            if (resp.status === 404) remaining.delete(id);
+          } catch {
+            // Transient — retry next loop iteration
+          }
+        }),
+      );
+      if (remaining.size > 0) await new Promise((r) => setTimeout(r, 2_000));
+    }
+  }
+
+  // Delete firewalls — fetch list once, delete matching in parallel.
+  // The Pulumi program (hetzner-compose.js) names firewalls
+  //   `${projectName}-${stackEnv}-firewall`
+  // where stackEnv is `${environment}-primary` or `${environment}-standby`
+  // (set by deployComposeHA above). Earlier the names tried below were
+  // `vibecarbon-${projectName}-${environment}[-primary|-standby]` — none of
+  // those match what Pulumi created, so destroy left every firewall behind
+  // and the next deploy hit "name is already used (uniqueness_error)" on
+  // pulumi up. Match the actual Pulumi naming convention.
+  //
+  // Detach from servers first — server deletion above is async (Hetzner
+  // returns 202 and finishes the delete behind the scenes), so the firewall
+  // can still be `applied_to` a deleting server when we hit DELETE. That
+  // returns 409 "still attached" which the previous `.catch(() => {})` was
+  // silently swallowing, leaving the firewall behind. The next re-deploy
+  // (compose-ha restore step) then hit "name is already used
+  // (uniqueness_error)" on Pulumi up. Mirror the detach + retry logic
+  // from `hetznerDeleteFirewall` in src/destroy.js.
+  // The previous implementation listed `/v1/firewalls` without `?name=` and
+  // without pagination. Hetzner returns `per_page=25` by default, so once
+  // the project (or test matrix) accumulates 25+ firewalls the project's
+  // firewalls aren't in page 1 → the destroy silently no-ops → the next
+  // deploy hits "name is already used (uniqueness_error)" on the firewall
+  // create. Look up each expected name with the exact-name filter (no
+  // pagination concern), then poll until DELETE actually removes it.
+  onProgress('Deleting firewalls...');
+  const fwNames = [
+    `${projectName}-${environment}-primary-firewall`,
+    `${projectName}-${environment}-standby-firewall`,
+    `${projectName}-${environment}-firewall`,
+  ];
+  await Promise.allSettled(fwNames.map((name) => deleteFirewallByName(name, hetznerHeaders)));
+
+  // Delete SSH key + DNS resources in parallel
+  onProgress('Cleaning up resources...');
+  await Promise.allSettled([
+    // SSH key deletion. deployComposeHA creates a SHARED ssh key named
+    //   `${projectName}-${environment}-key`
+    // (see line ~385 — `_provider.createSSHKey(sharedSshKeyName, ...)`)
+    // and passes its ID into both Pulumi programs as existingSshKeyId so
+    // Pulumi never creates per-cluster keys. The old name pattern here
+    // (`vibecarbon-${projectName}-${environment}`) didn't match anything
+    // Pulumi or HA actually created — the shared key always leaked.
+    (async () => {
+      const sshKeyName = `${projectName}-${environment}-key`;
+      try {
+        const listResponse = await fetchWithRetry('https://api.hetzner.cloud/v1/ssh_keys', {
+          headers: hetznerHeaders,
+        });
+        const { ssh_keys } = await listResponse.json();
+        const key = ssh_keys?.find((k) => k.name === sshKeyName);
+        if (key) {
+          await fetchWithRetry(`https://api.hetzner.cloud/v1/ssh_keys/${key.id}`, {
+            method: 'DELETE',
+            headers: hetznerHeaders,
+          });
+        }
+      } catch {
+        // Non-fatal
+      }
+    })(),
+    // DNS resource deletion (ownership-filtered — only records pointing at
+    // this env's server IPs are removed; shared zones keep their other
+    // tenants' records intact).
+    (async () => {
+      const ownedIps = (envConfig.servers || []).map((srv) => srv.ip).filter(Boolean);
+      if (cloudflareToken && envConfig.dns?.cloudflareZoneId && envConfig.domain) {
+        const { deleteDNSRecord, deleteHealthCheck } = await import('../../cloudflare.js');
+        const zoneId = envConfig.dns.cloudflareZoneId;
+        try {
+          await deleteDNSRecord(cloudflareToken, zoneId, envConfig.domain, ownedIps);
+          await Promise.allSettled(
+            (envConfig.servers || []).map((server) =>
+              deleteHealthCheck(cloudflareToken, zoneId, `${server.name}-health`),
+            ),
+          );
+        } catch {
+          // Non-fatal
+        }
+      } else if (
+        envConfig.dns?.provider === 'hetzner' &&
+        envConfig.dns?.hetznerZoneId &&
+        envConfig.domain
+      ) {
+        const { deleteDNSRecord: hetznerDeleteDNS, getZone: hetznerGetZone } = await import(
+          '../../hetzner-dns.js'
+        );
+        const hToken = hetznerToken || process.env.HETZNER_API_TOKEN;
+        if (hToken) {
+          try {
+            const zone = await hetznerGetZone(hToken, envConfig.dns.hetznerZoneId);
+            await Promise.allSettled([
+              hetznerDeleteDNS(
+                hToken,
+                envConfig.dns.hetznerZoneId,
+                zone.name,
+                envConfig.domain,
+                ownedIps,
+              ),
+              hetznerDeleteDNS(
+                hToken,
+                envConfig.dns.hetznerZoneId,
+                zone.name,
+                `*.${envConfig.domain}`,
+                ownedIps,
+              ),
+            ]);
+          } catch {
+            // Non-fatal
+          }
+        }
+      }
+    })(),
+  ]);
+
+  // Clean up local SSH key files
+  for (const f of [sshKeyPath, `${sshKeyPath}.pub`]) {
+    if (existsSync(f)) {
+      try {
+        rmSync(f);
+      } catch {
+        // Non-fatal
+      }
+    }
+  }
+}
+
+// ============================================================================
+// STATUS
+// ============================================================================
+
+/**
+ * Get status of both Compose HA servers in parallel
+ */
+export async function getComposeHAStatus(envConfig, sshKeyPath, projectName) {
+  const results = await Promise.all(
+    (envConfig.servers || []).map(async (server) => {
+      try {
+        const remoteDir = `/opt/${projectName}`;
+        const output = await sshRunAsync(
+          server.ip,
+          sshKeyPath,
+          `cd ${remoteDir} && docker compose ps --format json`,
+          { timeout: 15_000 },
+        );
+        if (!output?.trim()) {
+          return { ...server, running: false, services: [] };
+        }
+        const services = output
+          .trim()
+          .split('\n')
+          .filter(Boolean)
+          .map((line) => {
+            try {
+              return JSON.parse(line);
+            } catch {
+              return null;
+            }
+          })
+          .filter(Boolean);
+        return {
+          ...server,
+          running: services.some((s) => s.State === 'running'),
+          services,
+        };
+      } catch {
+        return { ...server, running: false, error: 'Unreachable' };
+      }
+    }),
+  );
+  return results;
+}
+
+// ============================================================================
+// FAILOVER
+// ============================================================================
+
+/**
+ * Perform failover for a Compose HA environment.
+ * Promotes the standby database and updates DNS if Cloudflare configured.
+ */
+export async function failoverComposeHA(envName, envConfig, projectConfig, parsed, tracker) {
+  const servers = envConfig.servers || [];
+  const primaryServer = servers.find((s) => s.role === 'primary');
+  const standbyServer = servers.find((s) => s.role === 'standby');
+
+  if (!primaryServer || !standbyServer) {
+    p.log.error('Could not identify primary and standby servers from config');
+    p.log.info('Ensure Compose HA deployment was set up with vibecarbon deploy --ha');
+    process.exit(1);
+  }
+
+  const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${envName}`);
+  if (!existsSync(sshKeyPath)) {
+    p.log.error(`SSH key not found: ${sshKeyPath}`);
+    process.exit(1);
+  }
+
+  const projectName = projectConfig.projectName;
+  const remoteDir = `/opt/${projectName}`;
+
+  p.log.info(c.bold('Current Compose HA state:'));
+  p.log.message(`  ${c.dim(`Primary (${primaryServer.region})`.padEnd(22))} ${primaryServer.ip}`);
+  p.log.message(`  ${c.dim(`Standby (${standbyServer.region})`.padEnd(22))} ${standbyServer.ip}`);
+
+  p.log.info(c.bold('This will:'));
+  p.log.message('  1. Promote the standby database to primary (docker compose exec)');
+  p.log.message('  2. Update Cloudflare DNS (if configured)');
+  p.log.message('  3. Print DNS update instructions (if manual DNS)');
+
+  if (parsed.dryRun) {
+    p.log.info(c.dim('Dry run — no changes made'));
+    return;
+  }
+
+  if (!parsed.yes) {
+    const confirm = await p.confirm({
+      message: `Execute failover for ${c.bold(envName)}?`,
+    });
+    if (p.isCancel(confirm) || !confirm) {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    }
+  }
+
+  const s = tracker.spinner();
+
+  // Step 0: Re-seed standby from primary if primary is still reachable.
+  //
+  // Streaming replication is best-effort; without a replication slot (which
+  // we don't configure), the standby can fall behind if primary churns
+  // through WAL faster than `wal_keep_size`. A stale standby yields
+  // "undefined column" (PG code 42703) errors post-promotion because recent
+  // migrations on primary never streamed over.
+  //
+  // For planned failover (the e2e-test case), primary is still alive
+  // — run pg_basebackup to guarantee schema parity, then promote. For a
+  // real disaster failover where primary is unreachable, skip the re-seed
+  // and promote whatever state standby has (best-effort recovery).
+  const primaryReachable = await isPrimaryReachable(primaryServer.ip, sshKeyPath, remoteDir);
+  if (primaryReachable) {
+    s.start('Re-seeding standby from primary (pg_basebackup)');
+    try {
+      await configureStandbyReplication(
+        standbyServer.ip,
+        primaryServer.ip,
+        sshKeyPath,
+        projectName,
+      );
+      s.stop('Standby re-seeded with primary state');
+    } catch (err) {
+      // Was: catch + warn + proceed. That was actively misleading —
+      // configureStandbyReplication starts with `rm -rf .../data/*`,
+      // so on failure there is NO "existing standby state" to fall
+      // back to; the dir is wiped and (with the script's `set -e`)
+      // basebackup didn't complete. Promoting from this state hits
+      // a CrashLoopBackOff (initdb-refuses-non-empty-dir loop). Hard
+      // fail so the operator sees the issue instead of a confusing
+      // pg_promote failure 5 minutes later.
+      s.stop('Re-seed failed — aborting failover');
+      throw new Error(
+        `Failover aborted: re-seed failed and the standby's data directory ` +
+          `is now in an inconsistent state. Manual recovery required ` +
+          `(redeploy the standby or restore from backup).\nUnderlying error: ${err.message}`,
+      );
+    }
+  } else {
+    p.log.warn('Primary unreachable — skipping re-seed. Standby will promote with current state.');
+  }
+
+  // Step 1: Promote standby database.
+  //
+  // Use SQL `pg_promote(wait => true, wait_seconds => 120)` instead of
+  // `pg_ctl promote`. Why: pg_ctl is fire-and-forget — it writes the promote
+  // signal file and returns. If the postmaster hasn't fully bound the socket
+  // yet (configureStandbyReplication's `docker compose start db` finished
+  // mere seconds earlier), pg_ctl can fail or no-op silently, leaving the
+  // standby in recovery mode. The downstream poll loop then sees `t` for its
+  // entire budget and we declare "Promotion may still be in progress" while
+  // the function continues to restart app-tier + DNS-flip — verify-failover
+  // ends up hitting a read-only DB and fails with 500 "Database error finding
+  // user" (auth) + 544 "DatabaseTimeout" (storage).
+  // (Observed compose-ha 2026-04-29 — 2m 40s of polling with promoted=false.)
+  //
+  // pg_promote() is server-side and synchronous: with `wait=>true` it blocks
+  // until the standby has actually exited recovery (or `wait_seconds` lapses)
+  // and returns the result. We retry a handful of times because the very
+  // first call can race the postmaster startup ("connection refused" /
+  // "the database system is starting up").
+  // `silent: true` is load-bearing: ha.js's local sshRun wraps runCommand,
+  // which on non-zero exit returns `false` (not throws) unless silent — so
+  // without it, a still-booting postmaster's exit-1 manifests downstream as
+  // `false?.trim is not a function`, the catch records it, and all 5 attempts
+  // fail with the same TypeError instead of actually retrying the SQL call.
+  // Helper: probe pg_is_in_recovery() and return 't'/'f'/'' (unknown).
+  // Used after pg_promote to disambiguate three cases:
+  //   - pg_promote returned 't' → success
+  //   - pg_promote returned 'f' (timed out internally) → check probe
+  //   - pg_promote threw (psql exit 1, e.g. "recovery is not in progress"
+  //     because a previous attempt actually promoted) → check probe
+  // Without the catch-side probe, the first attempt's success was silently
+  // converted to "did not exit recovery mode after 5 attempts" because
+  // attempts 2-5 hit the same not-in-recovery error and never re-checked.
+  // RCA: compose-ha 2026-05-01 fanout4b failover.
+  const probeRecovery = () => {
+    try {
+      const r = sshRun(
+        standbyServer.ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc 'SELECT pg_is_in_recovery()'`,
+        { silent: true, timeout: 10_000 },
+      );
+      return typeof r === 'string' ? r.trim() : '';
+    } catch {
+      return '';
+    }
+  };
+
+  s.start('Promoting standby database');
+  // Use SQL pg_promote(wait => true). hot_standby=on (set in carbon/
+  // docker-compose.yml) guarantees the standby accepts read connections,
+  // so psql can reach pg_promote even before promotion finishes. Verified
+  // 2026-05-01 fanout13 — without hot_standby the standby rejects every
+  // psql connection with "Hot standby mode is disabled" and pg_promote
+  // can't be invoked at all.
+  let promoted = false;
+  let lastPromoteError = null;
+  const MAX_PROMOTE_ATTEMPTS = 5;
+  for (let attempt = 1; attempt <= MAX_PROMOTE_ATTEMPTS && !promoted; attempt++) {
+    try {
+      const result = sshRun(
+        standbyServer.ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "SELECT pg_promote(wait => true, wait_seconds => 120)"`,
+        { silent: true, timeout: 150_000 },
+      );
+      const trimmed = typeof result === 'string' ? result.trim() : '';
+      if (trimmed === 't') {
+        promoted = true;
+        break;
+      }
+      const probeTrimmed = probeRecovery();
+      if (probeTrimmed === 'f') {
+        promoted = true;
+        break;
+      }
+      lastPromoteError = new Error(
+        `pg_promote returned ${JSON.stringify(trimmed)}; pg_is_in_recovery=${JSON.stringify(probeTrimmed)}`,
+      );
+    } catch (err) {
+      const probeTrimmed = probeRecovery();
+      if (probeTrimmed === 'f') {
+        promoted = true;
+        break;
+      }
+      lastPromoteError = err;
+    }
+    if (!promoted && attempt < MAX_PROMOTE_ATTEMPTS) {
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+  }
+
+  if (!promoted) {
+    // Capture diagnostics so the operator can see why the promotion failed
+    // instead of guessing. Best-effort — failover-step abort is what matters.
+    let dbLogs = '';
+    try {
+      dbLogs = sshRun(
+        standbyServer.ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose logs db --tail=60`,
+        { silent: true, timeout: 30_000 },
+      );
+      if (typeof dbLogs !== 'string') dbLogs = '(db logs returned non-string)';
+    } catch (err) {
+      dbLogs = `(failed to capture db logs: ${err.message})`;
+    }
+    s.stop('Promotion failed — standby still in recovery mode');
+    throw new Error(
+      `Failover aborted: standby ${standbyServer.ip} did not exit recovery mode after ${MAX_PROMOTE_ATTEMPTS} pg_promote attempts. ` +
+        `Last error: ${lastPromoteError?.message || 'unknown'}.\n` +
+        `--- db container logs (last 60 lines) ---\n${dbLogs}`,
+    );
+  }
+
+  s.stop('Standby database promoted to primary');
+
+  // Step 1b: Restart app-tier services on the new primary so they re-pool
+  // against the now-read-write DB. These services were started during deploy
+  // while the local DB was in recovery — supavisor's tenant pools and the
+  // app/auth/rest/realtime/storage DB handles can hold stale connections that
+  // surface as 500 "Database error" / 544 DatabaseTimeout right after
+  // promotion. Restart is cheap (~10s) and forces a clean reconnect.
+  s.start('Restarting new-primary app-tier');
+  try {
+    sshRun(
+      standbyServer.ip,
+      sshKeyPath,
+      `cd ${remoteDir} && docker compose restart supavisor auth rest realtime storage app`,
+      { timeout: 120_000 },
+    );
+  } catch (err) {
+    p.log.warn(`Restart on new primary failed: ${err.message}`);
+  }
+  s.stop('New-primary app-tier restarted');
+
+  // Step 2: Stop old primary services to prevent split-brain
+  s.start('Stopping old primary services');
+  try {
+    sshRun(
+      primaryServer.ip,
+      sshKeyPath,
+      `cd ${remoteDir} && docker compose stop app auth rest realtime storage supavisor`,
+      { timeout: 60_000 },
+    );
+  } catch {
+    // Old primary may be unreachable — that's often why we're failing over
+    p.log.warn('Could not reach old primary to stop services');
+  }
+  s.stop('Old primary services stopped');
+
+  // Step 3: Update DNS if Cloudflare configured
+  if (envConfig.dns?.provider === 'cloudflare' && envConfig.dns?.cloudflareZoneId) {
+    let cfToken = process.env.CLOUDFLARE_API_TOKEN;
+    if (!cfToken) {
+      const creds = loadCredentials();
+      if (creds.cloudflare?.apiToken) {
+        cfToken = creds.cloudflare.apiToken;
+      }
+    }
+    if (cfToken) {
+      const domain = envConfig.domain;
+      const zoneId = envConfig.dns.cloudflareZoneId;
+
+      // Clean up legacy load balancer if present from older deployments
+      try {
+        const { createDNSRecord } = await import('../../cloudflare.js');
+
+        // Update DNS A record to standby IP. proxied:false matches
+        // the deploy-time setup in cloudflare.js setupHA (Cloudflare
+        // LBs were removed 2026-03-26 — orange cloud is no longer
+        // needed and was triggering 404 caching at CF's edge during
+        // cert provisioning).
+        s.start('Updating DNS A record');
+        await createDNSRecord(cfToken, zoneId, {
+          type: 'A',
+          name: domain,
+          content: standbyServer.ip,
+          proxied: false,
+        });
+        await createDNSRecord(cfToken, zoneId, {
+          type: 'A',
+          name: `*.${domain}`,
+          content: standbyServer.ip,
+          proxied: false,
+        });
+        s.stop(`DNS updated: ${domain} → ${standbyServer.ip}`);
+      } catch (error) {
+        s.stop(`DNS update failed: ${error.message}`);
+        p.log.warn(`Manually update the A record for ${domain} to ${standbyServer.ip}`);
+      }
+    } else {
+      p.log.warn('CLOUDFLARE_API_TOKEN not set — update DNS A record manually');
+    }
+  } else if (envConfig.dns?.provider === 'hetzner' && envConfig.dns?.hetznerZoneId) {
+    let hetznerToken = process.env.HETZNER_API_TOKEN;
+    if (!hetznerToken) {
+      const creds = loadCredentials();
+      if (creds.hetzner?.apiToken) {
+        hetznerToken = creds.hetzner.apiToken;
+      }
+    }
+    if (hetznerToken) {
+      const { createDNSRecord, getZone } = await import('../../hetzner-dns.js');
+      const zone = await getZone(hetznerToken, envConfig.dns.hetznerZoneId);
+      s.start('Updating DNS A record');
+      try {
+        await createDNSRecord(hetznerToken, envConfig.dns.hetznerZoneId, zone.name, {
+          type: 'A',
+          name: envConfig.domain,
+          value: standbyServer.ip,
+          ttl: 60,
+        });
+        await createDNSRecord(hetznerToken, envConfig.dns.hetznerZoneId, zone.name, {
+          type: 'A',
+          name: `*.${envConfig.domain}`,
+          value: standbyServer.ip,
+          ttl: 60,
+        });
+        s.stop(`DNS updated: ${envConfig.domain} → ${standbyServer.ip}`);
+      } catch (error) {
+        s.stop(`DNS update failed: ${error.message}`);
+        p.log.warn(`Manually update the A record for ${envConfig.domain} to ${standbyServer.ip}`);
+      }
+    } else {
+      p.log.warn('Hetzner API token not found — update DNS A record manually');
+    }
+  } else {
+    // Manual DNS instructions
+    const domain = envConfig.domain || '<your-domain>';
+    p.log.info(c.bold('Update DNS to complete failover:'));
+    p.log.message(`  ${c.dim('Domain'.padEnd(14))} ${c.bold(domain)}`);
+    p.log.message(`  ${c.dim('New IP'.padEnd(14))} ${c.bold(standbyServer.ip)} (promoted standby)`);
+  }
+
+  // Update config: swap primary/standby roles so subsequent commands reflect reality
+  const updatedServers = servers.map((s) => ({
+    ...s,
+    role: s.role === 'primary' ? 'standby' : 'primary',
+  }));
+  const updatedEnvConfig = {
+    ...envConfig,
+    servers: updatedServers,
+    region: standbyServer.region,
+    secondaryRegion: primaryServer.region,
+    ha: { ...envConfig.ha, failoverRegion: primaryServer.region },
+    lastFailover: new Date().toISOString(),
+  };
+  saveProjectConfig({
+    ...projectConfig,
+    environments: {
+      ...projectConfig.environments,
+      [envName]: updatedEnvConfig,
+    },
+  });
+
+  p.log.success('Failover complete');
+  p.log.info(c.bold('Recovery instructions:'));
+  p.log.message(`  1. Fix the issue on the original primary (${primaryServer.ip})`);
+  p.log.message('  2. Re-initialize it as a standby replica');
+  p.log.message('  3. Redeploy to update configuration');
+}