vibecarbon 0.3.1 → 0.5.0

package/src/lib/deploy/bundle.js

@@ -8,17 +8,26 @@ import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { escapeDotenv } from '../shell.js';
+import { DNS01_OVERRIDE_FILE, dnsChallengeEnv } from './acme.js';
 
 /**
  * Render the reconcile.sh script
  */
-function renderReconcileScript(projectName, composeFlags, isFast = false) {
+export function renderReconcileScript(projectName, composeFlags, isFast = false) {
+  // On compose-ha servers the replication overlay (docker-compose.replication.yml)
+  // publishes port 5433 → db:5432 for pg_basebackup / streaming replication.
+  // reconcile.sh is baked with static composeFlags at deploy time, so we embed a
+  // shell conditional here that appends the overlay at runtime when present.
+  // On single-server compose the file never exists, so this is a safe no-op.
+  const replOverlayFlag =
+    "$([ -f docker-compose.replication.yml ] && echo '-f docker-compose.replication.yml')";
+
   const pullStep = isFast
     ? ''
     : `
 # 1. Pull latest images if needed
 echo "Pulling images..."
-docker compose ${composeFlags} pull --quiet --policy missing --ignore-buildable || true
+docker compose ${composeFlags} ${replOverlayFlag} pull --quiet --policy missing --ignore-buildable || true
 `;
 
   return `#!/bin/bash
@@ -46,7 +55,7 @@ fi
 ${pullStep}
 # 2. Re-create containers if state differs
 echo "Updating containers..."
-docker compose ${composeFlags} up -d --remove-orphans
+docker compose ${composeFlags} ${replOverlayFlag} up -d --remove-orphans
 
 # 3. Ensure systemd service is active (if applicable)
 if [ -f "/etc/systemd/system/${projectName}.service" ]; then
@@ -86,6 +95,10 @@ WantedBy=multi-user.target
  */
 function composeFileFlags(options = {}) {
   const files = ['-f', 'docker-compose.yml', '-f', 'docker-compose.prod.yml'];
+  // DNS-01 override must follow prod.yml so it replaces the Traefik command.
+  if (options.dnsChallenge) {
+    files.push('-f', DNS01_OVERRIDE_FILE);
+  }
   if (options.observability) {
     files.push('-f', 'docker-compose.observability.yml');
     files.push('-f', 'docker-compose.observability.prod.yml');
@@ -102,6 +115,90 @@ function composeFileFlags(options = {}) {
   return files.join(' ');
 }
 
+/**
+ * Render the Traefik dynamic-config file that instructs Traefik to obtain and
+ * serve ONE cert for the entire deploy as the TLS default store entry.
+ *
+ * With `defaultGeneratedCert`, Traefik proactively requests a single cert via
+ * the named resolver at startup and serves it by default for every TLS router
+ * that does NOT carry an explicit `tls.certresolver` label. All our production
+ * routers use `tls=true` without a resolver — they all get this cert via SNI.
+ *
+ * Challenge-type differences:
+ *   - DNS-01 (managed DNS — cloudflare/hetzner): `sans: ["*.${domain}"]`.
+ *     One wildcard cert covers all subdomains, no per-subdomain ACME orders.
+ *   - HTTP-01 (manual DNS): wildcards require DNS-01 (LE policy), so we list
+ *     the explicit subdomains that are active for this deploy instead.
+ *     `main` stays `${domain}` (the app root); `sans` lists api/studio/dashboard
+ *     plus any enabled feature subdomains (grafana/prometheus, n8n, metabase).
+ *
+ * The file is written into `volumes/traefik/` which is already bind-mounted
+ * as `/etc/traefik/dynamic:ro`. Traefik hot-reloads files from that dir.
+ * Multiple files in the dir coexist fine; this file adds only a top-level
+ * `tls:` key that has no overlap with the `http:` keys in middlewares.yml.
+ *
+ * @param {{domain: string, dnsChallenge: boolean, features?: {observability?: boolean, n8n?: boolean, metabase?: boolean}}} opts
+ * @returns {string} YAML string suitable for writing to `volumes/traefik/tls-default.yml`
+ */
+export function renderTraefikDefaultCert({ domain, dnsChallenge, features = {} }) {
+  // The default store cert covers the SUBDOMAINS only. The app at the bare apex
+  // (`${domain}`) gets its OWN single-domain cert via the app router's
+  // `tls.certresolver` label — it is deliberately NOT in this cert.
+  //
+  // Why split apex off the wildcard: a cert for [`${domain}`, `*.${domain}`] has
+  // TWO ACME authorizations that BOTH validate via the SAME TXT record
+  // `_acme-challenge.${domain}`, so both challenge values must be present there
+  // simultaneously. The Hetzner lego DNS provider REPLACES the TXT rrset instead
+  // of holding both values, so one authorization always fails with
+  // "403 unauthorized :: Incorrect TXT record". (Cloudflare holds both and
+  // worked — RCA 2026-06-01, compose e2e wildcard verify-deploy.) As two
+  // SEPARATE single-authz certs (apex via the app router, `*.${domain}` here),
+  // Traefik obtains them serially, so `_acme-challenge.${domain}` only ever
+  // needs one value at a time — robust on Hetzner AND Cloudflare.
+  let main;
+  let sans;
+  if (dnsChallenge) {
+    // DNS-01: one wildcard covers every subdomain (most general level).
+    main = `*.${domain}`;
+    sans = [];
+  } else {
+    // HTTP-01: wildcards require DNS-01 (LE policy), so enumerate the active
+    // subdomains. `api` (kong), `studio`, `dashboard` always exist; the rest are
+    // feature-gated. The apex app is covered by its own router cert, not here.
+    const subs = ['studio', 'dashboard'];
+    if (features.observability) subs.push('grafana', 'prometheus');
+    if (features.n8n) subs.push('n8n');
+    if (features.metabase) subs.push('metabase');
+    main = `api.${domain}`;
+    sans = subs.map((s) => `${s}.${domain}`);
+  }
+
+  const lines = [
+    'tls:',
+    '  stores:',
+    '    default:',
+    '      defaultGeneratedCert:',
+    '        resolver: letsencrypt',
+    '        domain:',
+    `          main: "${main}"`,
+  ];
+  if (sans.length) {
+    lines.push('          sans:');
+    for (const d of sans) lines.push(`            - "${d}"`);
+  }
+
+  return `# Traefik default TLS certificate — generated by Vibecarbon at deploy time.
+#
+# Serves as the default cert for ALL TLS routers carrying \`tls=true\` without an
+# explicit certresolver (SNI fallback). Covers the SUBDOMAINS; the apex app gets
+# its own cert via the app router's \`tls.certresolver\`.
+#
+# DNS-01 (cloudflare/hetzner): one wildcard cert covering every subdomain.
+# HTTP-01 (manual DNS): explicit subdomain SANs (LE policy forbids wildcards).
+${lines.join('\n')}
+`;
+}
+
 /**
  * Render the deployment bundle to a local temp directory
  * @param {string} projectName
@@ -129,6 +226,7 @@ export function renderBundle(projectName, options = {}) {
     rootFiles.push('docker-compose.metabase.yml', 'docker-compose.metabase.prod.yml');
   }
   if (options.redis) rootFiles.push('docker-compose.redis.yml', 'docker-compose.redis.prod.yml');
+  if (options.dnsChallenge) rootFiles.push(DNS01_OVERRIDE_FILE);
 
   for (const file of rootFiles) {
     const src = join(cwd, file);
@@ -168,6 +266,17 @@ export function renderBundle(projectName, options = {}) {
   if (projectName) {
     envOverrides.PROJECT_NAME = projectName;
   }
+  // DNS-01: lego reads ACME_DNS_PROVIDER + the matching provider token from the
+  // server .env (consumed by docker-compose.dns01.prod.yml's Traefik env).
+  if (options.dnsChallenge) {
+    Object.assign(
+      envOverrides,
+      dnsChallengeEnv(options.dnsProvider, {
+        cloudflareApiToken: options.cloudflareApiToken,
+        hetznerApiToken: options.hetznerApiToken,
+      }),
+    );
+  }
 
   // Merge overrides into local copy
   const lines = envContent.split('\n');
@@ -216,13 +325,36 @@ export function renderBundle(projectName, options = {}) {
   // template ships a stub at functions/main/index.ts. RCA: prod-1 2026-05-26.
   copyDirIfExist('functions', 'functions');
 
-  // volumes/traefik/middlewares.yml
+  // volumes/traefik/ — dynamic-config directory mounted into Traefik.
+  // Always ensure the dir exists in the bundle so Traefik's file provider
+  // has a valid mount target even if the source dir is absent.
+  mkdirSync(join(stageDir, 'volumes', 'traefik'), { recursive: true });
+
   const traefikMiddlewares = join(cwd, 'volumes', 'traefik', 'middlewares.yml');
   if (existsSync(traefikMiddlewares)) {
-    mkdirSync(join(stageDir, 'volumes', 'traefik'), { recursive: true });
     cpSync(traefikMiddlewares, join(stageDir, 'volumes', 'traefik', 'middlewares.yml'));
   }
 
+  // Render tls-default.yml only when a domain is known (production deploys).
+  // This instructs Traefik to proactively obtain ONE cert for the deploy and
+  // serve it as the default for all TLS routers via SNI. DNS-01 deploys get
+  // a wildcard SAN (*.${domain}); HTTP-01 deploys get explicit subdomain SANs.
+  // The file coexists with middlewares.yml — both live under the same `tls:`
+  // / `http:` top-level keys respectively; Traefik merges all files in the dir.
+  if (options.domain) {
+    const tlsDefaultYaml = renderTraefikDefaultCert({
+      domain: options.domain,
+      dnsChallenge: Boolean(options.dnsChallenge),
+      features: {
+        observability: options.observability,
+        n8n: options.n8n,
+        metabase: options.metabase,
+      },
+    });
+    writeFileSync(join(stageDir, 'volumes', 'traefik', 'tls-default.yml'), tlsDefaultYaml);
+    if (options.verbose) console.log('[bundle] Rendered volumes/traefik/tls-default.yml');
+  }
+
   copyDirIfExist('backup', 'backup');
   if (options.n8n) copyDirIfExist('volumes/n8n', 'volumes/n8n');
   if (options.metabase) copyDirIfExist('volumes/metabase', 'volumes/metabase');

package/src/lib/deploy/compose/ha.js

@@ -14,7 +14,6 @@ import { join } from 'node:path';
 import * as p from '@clack/prompts';
 import { setupHA as setupCloudflareHA } from '../../cloudflare.js';
 import { c } from '../../colors.js';
-import { runCommand } from '../../command.js';
 import {
   loadCredentials,
   loadProjectConfig,
@@ -24,34 +23,23 @@ import {
 import { fetchWithRetry } from '../../fetch-retry.js';
 import { perfAsync } from '../../perf.js';
 import { getProvider } from '../../providers/index.js';
+import { useDnsChallenge } from '../acme.js';
+import { generateSSHKeyPair, readReplPassword } from '../utils.js';
 import {
   createAdminUser,
   dockerLoginOnServer,
   isLocalOnlyImageTag,
   pullComposeImages,
   runMigrations,
+  setupComposeBackupCron,
   setupServer,
   setupServerFiles,
+  sshRun,
   sshRunAsync,
   startComposeStack,
   waitForSSH,
 } from './index.js';
 
-const SSH_OPTS = '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes';
-
-/**
- * Run a command on a remote server via SSH
- */
-function sshRun(ip, sshKeyPath, command, options = {}) {
-  const { timeout = 120_000 } = options;
-  return runCommand(['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command], {
-    stdio: 'pipe',
-    timeout,
-    encoding: 'utf-8',
-    ...options,
-  });
-}
-
 /**
  * Look up a Hetzner firewall by exact name and DELETE it. Polls afterwards
  * until the GET returns nothing (firewall actually gone) or the deadline
@@ -170,41 +158,55 @@ async function deleteFirewallByName(name, hetznerHeaders, deadlineMs = 30_000) {
   return false;
 }
 
-/**
- * Read REPL_PASSWORD — process.env first (CI may export it), then .env.local
- * (where `vibecarbon create` writes it at project-init time). Every HA code
- * path funnels through this so one fix handles all three call sites that
- * previously threw `REPL_PASSWORD is not set` when run without the env var.
- * Returns null if the value is not present anywhere.
- */
-function readReplPassword(cwd = process.cwd()) {
-  if (process.env.REPL_PASSWORD) return process.env.REPL_PASSWORD;
-  const envLocalPath = join(cwd, '.env.local');
-  if (!existsSync(envLocalPath)) return null;
-  const content = readFileSync(envLocalPath, 'utf-8');
-  // Accept both double-quoted (machine secrets) and single-quoted
-  // (escapeDotenv'd user secrets) forms, matching create.js output.
-  const m =
-    content.match(/^REPL_PASSWORD="([^"]+)"/m) || content.match(/^REPL_PASSWORD='([^']+)'/m);
-  return m ? m[1] : null;
-}
-
-/**
- * Generate SSH key pair for deployment
- */
-function generateSSHKeyPair(sshKeyPath) {
-  if (!existsSync(sshKeyPath)) {
-    runCommand(
-      ['ssh-keygen', '-t', 'ed25519', '-f', sshKeyPath, '-N', '', '-C', 'vibecarbon-deploy'],
-      { stdio: 'pipe' },
-    );
-  }
-}
-
 // ============================================================================
 // POSTGRESQL STREAMING REPLICATION SETUP
 // ============================================================================
 
+// WAL-G / ARCHIVING DOVETAIL INVARIANTS (wal-g unify, 2026-05-31)
+//
+// Invariant 1 — archiving survives failover.
+//   carbon/docker-compose.yml hardcodes `archive_mode=on` and
+//   `archive_command=bash /etc/postgresql/wal-archive.sh %p` in the db
+//   container's `command:`. A promoted standby runs the SAME image/command,
+//   so it starts archiving WAL on the new timeline immediately after promotion.
+//   No extra step is needed.
+//   WALG_S3_PREFIX is derived from S3_BACKUP_BUCKET + PROJECT_NAME (see
+//   docker-compose.yml:197) and is identical on both nodes — same .env is
+//   deployed to both servers via setupServerFiles. The WAL timeline is therefore
+//   continuous across failover: pre-promotion WAL is on TL1, post-promotion WAL
+//   is on TL2, both under the same S3 prefix.
+//   Note: a PostgreSQL standby with archive_mode=on does NOT archive while it is
+//   in recovery — only the primary archives. There is no double-archive risk
+//   before failover.
+//
+// Invariant 2 — backup guard means only the active primary backs up.
+//   carbon/backup/compose-backup.sh checks `pg_is_in_recovery()='f'` before
+//   calling `wal-g backup-push` (and self-skips with exit 0 when no S3 backup
+//   target / credentials are configured). Before failover: standby cron is a
+//   no-op; primary cron backs up. After failover: promoted standby cron backs
+//   up; old primary (if still alive) is now a standby, so its cron is a no-op.
+//   The scheduled backup cron is installed on BOTH nodes at the end of
+//   deployComposeHA (see the setupComposeBackupCron fan-out after replication
+//   verification). The standby's cron is a guarded no-op until it is promoted —
+//   installing on both means the survivor always has the cron after a failover,
+//   so scheduled backups don't silently stop when DNS flips to the standby.
+//
+// Invariant 3 — restore seeds the primary; standby is REBUILT afterward.
+//   compose restore (restoreCompose in compose/index.js) rewinds the primary's
+//   data directory and replays WAL from S3. A standby that was streaming from the
+//   old (pre-restore) primary then has WAL *ahead* of the restored primary's LSN
+//   and CANNOT resume streaming — PG would reject it with "requested timeline
+//   does not contain minimum recovery point" (timeline divergence). The correct
+//   post-restore sequence is:
+//     1. Restore the primary (restoreCompose → backup-fetch + archive recovery).
+//     2. Re-seed the standby via configureStandbyReplication (pg_basebackup
+//        from the now-restored primary). This wipes the diverged standby state
+//        and brings it in sync with the new primary timeline.
+//   This is wired in restore.js:runComposeRestore — after restoreCompose on the
+//   primary, it re-runs configureStandbyReplication against the standby whenever
+//   deployMode === 'compose-ha' (skipped for single-region compose, which has no
+//   standby). configureStandbyReplication is exported from this file for that.
+
 /**
  * Configure the primary PostgreSQL server for streaming replication.
  *
@@ -323,7 +325,7 @@ END $$;
  * 3. Write standby.signal and configure primary_conninfo
  * 4. Restart as hot standby
  */
-async function configureStandbyReplication(standbyIp, primaryIp, sshKeyPath, projectName) {
+export async function configureStandbyReplication(standbyIp, primaryIp, sshKeyPath, projectName) {
   const remoteDir = `/opt/${projectName}`;
   // replPassword is generated at create time from crypto.randomBytes and is
   // restricted to base64url characters — no shell escaping needed.
@@ -337,6 +339,33 @@ async function configureStandbyReplication(standbyIp, primaryIp, sshKeyPath, pro
     );
   }
 
+  // Ensure the physical replication slot exists on the PRIMARY before the
+  // standby's pg_basebackup -S below. configurePrimaryReplication creates it at
+  // deploy time, but a wal-g RESTORE overwrites the primary's PGDATA — which
+  // includes pg_replslot — so the slot is wiped. Without re-creating it,
+  // pg_basebackup fails late with `replication slot "vibecarbon_standby_slot"
+  // does not exist` (it connects + checkpoints, then the -Xs WAL receiver's
+  // START_REPLICATION errors). This bit the failover re-seed after a wal-g
+  // restore — the pg_dump restore never had this problem because it loaded
+  // logical data without touching pg_replslot. Idempotent (IF NOT EXISTS), so
+  // it's a safe no-op on the deploy path where the slot already exists.
+  // RCA 2026-06-01: compose-ha failover after wal-g restore.
+  const ensureSlotSql = `DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_replication_slots WHERE slot_name = 'vibecarbon_standby_slot'
+  ) THEN
+    PERFORM pg_create_physical_replication_slot('vibecarbon_standby_slot');
+  END IF;
+END $$;`;
+  const ensureSlotB64 = Buffer.from(ensureSlotSql).toString('base64');
+  sshRun(
+    primaryIp,
+    sshKeyPath,
+    `cd ${remoteDir} && echo ${ensureSlotB64} | base64 -d | docker compose exec -T db psql -U supabase_admin -d postgres`,
+    { timeout: 30_000 },
+  );
+
   // Stop the standby database
   sshRun(standbyIp, sshKeyPath, `cd ${remoteDir} && docker compose stop db`, {
     timeout: 60_000,
@@ -1022,7 +1051,11 @@ export async function deployComposeHA(options) {
   // future deploys that shrink that gap. Same fix lives in orchestrator.js
   // for single-region compose where the race was actively biting (RCA from
   // vibecarbon.com cold-deploy 2026-05-19).
-  if (domain && dnsProvider && dnsProvider !== 'manual') {
+  // HTTP-01 only: managed-DNS HA deploys issue certs via DNS-01 (the shared
+  // bundle ships docker-compose.dns01.prod.yml to both nodes), which doesn't
+  // gate on the A record propagating — so skip the wait there. `manual` keeps
+  // HTTP-01 but isn't reached here (the guard already excludes it).
+  if (domain && dnsProvider && dnsProvider !== 'manual' && !useDnsChallenge(dnsProvider)) {
     // Surface this in the enclosing tracker spinner so the operator sees
     // why we're waiting (up to 120s on a cold edge). Without the message
     // the HA spinner sits on the prior "Configuring servers..." text for
@@ -1210,6 +1243,39 @@ export async function deployComposeHA(options) {
     p.log.warn('Streaming replication not detected — standby may fall behind');
   }
 
+  // Install the scheduled wal-g backup cron on BOTH nodes. Invariant 2: the
+  // standby's cron is a harmless no-op — compose-backup.sh guards on
+  // `pg_is_in_recovery()='f'` and exits 0 on a standby without pushing. We
+  // install on both so that after a failover the SURVIVOR (whichever node was
+  // promoted) already has the cron and keeps backing up — without it, a
+  // failover to the standby would silently lose scheduled backups until the
+  // operator re-ran deploy. setupComposeBackupCron defaults schedule + retain
+  // when backupConfig is absent; compose-backup.sh self-skips (exit 0) when no
+  // S3 backup target is configured, so an always-installed cron never hard-fails.
+  //
+  // allSettled (not all) + per-node warn: a cron-install hiccup on one node
+  // must not fail an already-serving HA deploy, nor mask the other node. The
+  // app + replication are already up by this point.
+  onProgress('Installing backup cron on both servers...');
+  const cronResults = await perfAsync('deploy.ha.compose.backupCron.both', () =>
+    Promise.allSettled([
+      perfAsync('deploy.ha.compose.backupCron.primary', async () =>
+        setupComposeBackupCron(primaryIp, sshKeyPath, projectName, backupConfig),
+      ),
+      perfAsync('deploy.ha.compose.backupCron.standby', async () =>
+        setupComposeBackupCron(standbyIp, sshKeyPath, projectName, backupConfig),
+      ),
+    ]),
+  );
+  for (const [i, result] of cronResults.entries()) {
+    if (result.status === 'rejected') {
+      const node = i === 0 ? `primary (${primaryIp})` : `standby (${standbyIp})`;
+      p.log.warn(
+        `Scheduled backup cron install failed on ${node} (deploy still succeeded): ${result.reason?.message ?? result.reason}`,
+      );
+    }
+  }
+
   // Finalize config — promote pending → deployed
   saveProjectConfig({
     ...projectConfig,
@@ -1471,54 +1537,80 @@ export async function destroyComposeHA(options) {
 }
 
 // ============================================================================
-// STATUS
+// FAILOVER
 // ============================================================================
 
 /**
- * Get status of both Compose HA servers in parallel
+ * Build the curl used to probe the new primary's real API endpoint locally.
+ *
+ * `--resolve <domain>:443:127.0.0.1` pins the hostname to loopback, so the
+ * probe hits Traefik on the new-primary host itself and is independent of DNS
+ * propagation (the DNS flip happens later in failover, and the operator host's
+ * resolver may still be cached). `-k` skips the LE-staging cert. The path is
+ * the *exact* endpoint verify-failover checks, so a 200 here means the whole
+ * app → Kong → PostgREST → DB path is serving — including PostgREST's schema
+ * cache, the thing that's still cold right after `docker compose restart`.
+ *
+ * @param {string} domain
+ * @returns {string}
  */
-export async function getComposeHAStatus(envConfig, sshKeyPath, projectName) {
-  const results = await Promise.all(
-    (envConfig.servers || []).map(async (server) => {
-      try {
-        const remoteDir = `/opt/${projectName}`;
-        const output = await sshRunAsync(
-          server.ip,
-          sshKeyPath,
-          `cd ${remoteDir} && docker compose ps --format json`,
-          { timeout: 15_000 },
-        );
-        if (!output?.trim()) {
-          return { ...server, running: false, services: [] };
-        }
-        const services = output
-          .trim()
-          .split('\n')
-          .filter(Boolean)
-          .map((line) => {
-            try {
-              return JSON.parse(line);
-            } catch {
-              return null;
-            }
-          })
-          .filter(Boolean);
-        return {
-          ...server,
-          running: services.some((s) => s.State === 'running'),
-          services,
-        };
-      } catch {
-        return { ...server, running: false, error: 'Unreachable' };
-      }
-    }),
-  );
-  return results;
+export function newPrimaryApiProbeCmd(domain) {
+  const url = `https://${domain}/api/v1/notifications`;
+  return `curl -sk -o /dev/null -w '%{http_code}' --max-time 5 --resolve ${domain}:443:127.0.0.1 ${url}`;
 }
 
-// ============================================================================
-// FAILOVER
-// ============================================================================
+/**
+ * Poll the new primary until it actually SERVES a DB-backed API request.
+ *
+ * Root-cause fix for the compose-ha failover-readiness race: Step 1b runs
+ * `docker compose restart … rest app …` to force a clean reconnect against the
+ * now-read-write DB, but `restart` returns when containers *start*, not when
+ * they're *ready*. PostgREST then spends ~10-30s reconnecting and rebuilding
+ * its schema cache; until it does, /api/v1/notifications 500s with a
+ * PGRST schema-cache miss. failover used to return straight into that window,
+ * and verify-failover (which does NOT retry on 500) caught it. Gating on the
+ * real endpoint closes the race for the e2e *and* real operators (a failover
+ * that reports success before the app serves traffic is itself a bug).
+ *
+ * Best-effort: returns false (and logs the last status) on timeout rather than
+ * throwing — a not-yet-confirmed new primary is still better than aborting a
+ * promotion that already happened.
+ *
+ * @param {string} ip - new primary IP
+ * @param {string} sshKeyPath
+ * @param {string} domain
+ * @param {object} [opts]
+ * @param {number} [opts.attempts=40]
+ * @param {number} [opts.intervalMs=3000]
+ * @param {(ip: string, key: string, cmd: string, o: object) => string|false} [opts.runner=sshRun]
+ * @param {(ms: number) => Promise<void>} [opts.sleep]
+ * @returns {Promise<boolean>}
+ */
+export async function waitForNewPrimaryApi(ip, sshKeyPath, domain, opts = {}) {
+  const {
+    attempts = 40,
+    intervalMs = 3000,
+    runner = sshRun,
+    sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+  } = opts;
+  const cmd = newPrimaryApiProbeCmd(domain);
+  let lastCode = '';
+  for (let i = 0; i < attempts; i++) {
+    try {
+      const code = runner(ip, sshKeyPath, cmd, { silent: true, timeout: 12_000 });
+      lastCode = typeof code === 'string' ? code.trim() : '';
+      if (lastCode === '200') return true;
+    } catch {
+      lastCode = 'error';
+    }
+    if (i < attempts - 1) await sleep(intervalMs);
+  }
+  console.error(
+    `[ha] new primary did not serve /api/v1/notifications (last HTTP ${lastCode || 'none'}) ` +
+      `after ${attempts} probes — proceeding, but verify-failover may race PostgREST readiness`,
+  );
+  return false;
+}
 
 /**
  * Perform failover for a Compose HA environment.
@@ -1582,6 +1674,13 @@ export async function failoverComposeHA(envName, envConfig, projectConfig, parse
   // — run pg_basebackup to guarantee schema parity, then promote. For a
   // real disaster failover where primary is unreachable, skip the re-seed
   // and promote whatever state standby has (best-effort recovery).
+  //
+  // WAL-G archiving invariant (Invariant 1): the promoted node starts
+  // archiving on the new timeline immediately after pg_promote() — the
+  // docker-compose.yml command hardcodes archive_mode=on + archive_command
+  // on the db image. WALG_S3_PREFIX is the same on both nodes (same .env),
+  // so the WAL timeline is continuous in S3 across the failover. No extra
+  // action needed here.
   const primaryReachable = await isPrimaryReachable(primaryServer.ip, sshKeyPath, remoteDir);
   if (primaryReachable) {
     s.start('Re-seeding standby from primary (pg_basebackup)');
@@ -1632,7 +1731,7 @@ export async function failoverComposeHA(envName, envConfig, projectConfig, parse
   // and returns the result. We retry a handful of times because the very
   // first call can race the postmaster startup ("connection refused" /
   // "the database system is starting up").
-  // `silent: true` is load-bearing: ha.js's local sshRun wraps runCommand,
+  // `silent: true` is load-bearing: the shared compose sshRun wraps runCommand,
   // which on non-zero exit returns `false` (not throws) unless silent — so
   // without it, a still-booting postmaster's exit-1 manifests downstream as
   // `false?.trim is not a function`, the catch records it, and all 5 attempts
@@ -1749,6 +1848,23 @@ export async function failoverComposeHA(envName, envConfig, projectConfig, parse
   }
   s.stop('New-primary app-tier restarted');
 
+  // Step 1c: Gate on the new primary actually SERVING a DB-backed request
+  // before we stop the old primary and flip DNS. The Step 1b restart only
+  // *starts* the app tier; PostgREST's schema cache is cold for ~10-30s after,
+  // during which /api/v1/notifications 500s. Without this wait, failover
+  // returned into that window and verify-failover (no retry on 500) failed.
+  // Probed locally via curl --resolve so readiness is independent of the DNS
+  // flip below. Best-effort — see waitForNewPrimaryApi.
+  if (envConfig.domain) {
+    s.start('Waiting for new primary to serve API traffic');
+    const apiReady = await waitForNewPrimaryApi(standbyServer.ip, sshKeyPath, envConfig.domain);
+    s.stop(
+      apiReady
+        ? 'New primary serving API traffic'
+        : 'New primary API not confirmed — proceeding (verify may race readiness)',
+    );
+  }
+
   // Step 2: Stop old primary services to prevent split-brain
   s.start('Stopping old primary services');
   try {