vibecarbon 0.1.0

package/src/failover.js

@@ -0,0 +1,1032 @@
+/**
+ * Vibecarbon Failover Command
+ *
+ * Promotes the standby region of an HA deployment to primary. Handles
+ * four scenarios: HA+Cloudflare DNS, HA+Hetzner DNS, HA+manual DNS,
+ * and single-server (where failover is a no-op + recovery instructions).
+ *
+ * Form rule: vibecarbon uses single-dash flags only — see
+ * memory:feedback_cli_single_dash_flags. `-dry` (not `-dry-run` / `-n`)
+ * previews the failover plan without executing.
+ */
+
+import { existsSync } from 'node:fs';
+import * as p from '@clack/prompts';
+import { renderHelp } from './lib/cli/help.js';
+import { parseFlags } from './lib/cli/parse-flags.js';
+import { selectEnvironment } from './lib/cli/select-environment.js';
+import { requireTTYOrFlags } from './lib/cli/tty-guard.js';
+import { createDNSRecord } from './lib/cloudflare.js';
+import { c, printBanner } from './lib/colors.js';
+import { runCommand } from './lib/command.js';
+import { loadCredentials, loadProjectConfig } from './lib/config.js';
+import {
+  createDNSRecord as hetznerDnsCreateRecord,
+  getZone as hetznerDnsGetZone,
+} from './lib/hetzner-dns.js';
+import { requireLicense } from './lib/licensing/index.js';
+import { ensureOperatorIpAccess } from './lib/operator-ip.js';
+import { perfTimer } from './lib/perf.js';
+import { assertInProjectDir } from './lib/project-guard.js';
+import { getPostgresPod, getSSHKeyPath, sshKubectl, sshRun } from './lib/ssh.js';
+import { createTracker } from './lib/tracker.js';
+import { VERSION } from './lib/version.js';
+
+// ============================================================================
+// COMMAND SPEC — single source of truth for argv parsing AND help output.
+// ============================================================================
+
+/** @type {import('./lib/cli/parse-flags.js').CommandSpec & { summary?: string, description?: string, examples?: Array<{ command: string, description?: string }> }} */
+const SPEC = {
+  name: 'failover',
+  summary: 'Promote the standby region to primary',
+  description: [
+    'Scenarios:',
+    '  HA + Cloudflare DNS   Promotes standby DB, switches DNS A record automatically',
+    '  HA + Hetzner DNS      Promotes standby DB, switches DNS A record automatically',
+    '  HA + Manual DNS       Promotes standby DB, prints DNS update instructions',
+    '  Single Server         Prints backup/restore recovery steps',
+  ].join('\n'),
+  positional: [
+    {
+      name: 'env',
+      optional: true,
+      description: 'Environment to fail over (skips the env prompt)',
+    },
+  ],
+  flags: [
+    { name: 'h', boolean: true, description: 'Show this help' },
+    { name: 'v', boolean: true, description: 'Show version' },
+    { name: 'y', boolean: true, description: 'Skip confirmation prompts' },
+    { name: 'dry', boolean: true, description: 'Show plan without executing' },
+    { name: 'env', value: '<name>', description: 'Environment seed (alternative to positional)' },
+  ],
+  examples: [
+    { command: 'vibecarbon failover', description: 'prompts for env' },
+    { command: 'vibecarbon failover prod -dry', description: 'preview the failover plan' },
+    { command: 'vibecarbon failover prod -y', description: 'execute without confirmation' },
+  ],
+};
+
+// ============================================================================
+// SCENARIO DETECTION
+// ============================================================================
+
+export function detectScenario(envConfig) {
+  const ha = envConfig.ha?.enabled || envConfig.ha === true;
+  const cloudflare = envConfig.dns?.provider === 'cloudflare' && envConfig.dns?.cloudflareZoneId;
+  const hetznerDns = envConfig.dns?.provider === 'hetzner' && envConfig.dns?.hetznerZoneId;
+  if (ha && cloudflare) return 'ha_cloudflare';
+  if (ha && hetznerDns) return 'ha_hetzner_dns';
+  if (ha) return 'ha_manual';
+  return 'single_server';
+}
+
+/**
+ * Identify primary and standby servers from the environment config.
+ * HA deployments store server info in ha.primary/ha.standby within the environment,
+ * or as separate entries in the servers array with region identifiers in their names.
+ */
+export function identifyServers(envName, envConfig, _projectConfig) {
+  // Check for HA config with primary/standby info nested in the environment
+  if (envConfig.ha?.primary && envConfig.ha?.standby) {
+    return {
+      primary: {
+        ip: envConfig.ha.primary.masterIp,
+        supabaseIp: envConfig.ha.primary.supabaseIp,
+        floatingIp: envConfig.ha.primary.floatingIp || envConfig.ha.primary.masterIp,
+        region: envConfig.ha.primary.region,
+        envKey: envName,
+      },
+      standby: {
+        ip: envConfig.ha.standby.masterIp,
+        supabaseIp: envConfig.ha.standby.supabaseIp,
+        floatingIp: envConfig.ha.standby.floatingIp || envConfig.ha.standby.masterIp,
+        region: envConfig.ha.standby.region,
+        envKey: envName,
+      },
+    };
+  }
+
+  // Fall back to servers array with role/region matching
+  const servers = envConfig.servers || [];
+  const primaryRegion = envConfig.region;
+  const standbyRegion = envConfig.secondaryRegion || envConfig.ha?.failoverRegion;
+
+  if (servers.length >= 2 && primaryRegion && standbyRegion) {
+    // Prefer name-based match (k8s HA stores { name: 'primary' / 'standby' });
+    // fall back to region match (pre-gitops compose-HA used server names
+    // that embedded the region).
+    const byName = (role) => servers.find((s) => s.name === role);
+    const byRegion = (region) => servers.find((s) => s.name?.includes(region));
+    const primary = byName('primary') || byRegion(primaryRegion) || servers[0];
+    const standby = byName('standby') || byRegion(standbyRegion) || servers[1];
+
+    return {
+      primary: {
+        ip: primary.ip,
+        supabaseIp: primary.supabaseIp,
+        region: primaryRegion,
+        envKey: envName,
+      },
+      standby: {
+        ip: standby.ip,
+        supabaseIp: standby.supabaseIp,
+        region: standbyRegion,
+        envKey: envName,
+      },
+    };
+  }
+
+  return null;
+}
+
+// ============================================================================
+// PROMOTION + SERVICE SCALE HELPERS
+// ============================================================================
+
+/**
+ * Check whether the primary cluster's postgres is reachable. Used to decide
+ * whether to attempt a pg_basebackup re-seed before promoting the standby.
+ * Primary-dead failovers skip the re-seed (can't; best-effort promote).
+ */
+function isPrimaryPostgresReady(primaryIp, sshKeyPath) {
+  try {
+    const pod = getPostgresPod(primaryIp, sshKeyPath);
+    const result = sshKubectl(primaryIp, sshKeyPath, [
+      'exec',
+      '-n',
+      'vibecarbon',
+      pod,
+      '--',
+      'pg_isready',
+      '-U',
+      'postgres',
+      '-t',
+      '3',
+    ]);
+    return result?.includes('accepting connections') ?? false;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Re-seed the standby Postgres from the primary via pg_basebackup before
+ * promotion. Closes the drift window that opens when streaming replication
+ * silently falls behind (no replication slot) or when the standby's pod
+ * restarted and missed WAL. Without re-seeding, promoted standby can serve
+ * requests against a schema that's missing recent migrations, yielding
+ * "undefined column" (PG 42703) errors in auth/storage after failover.
+ *
+ * Requires `primarySupabaseIp` — the Hetzner private IP of the primary's
+ * supabase node (where the postgres pod runs with hostPort 5432). Returns
+ * true if re-seed succeeded, false if primary was unreachable or re-seed
+ * itself failed (caller proceeds to promote with existing state).
+ */
+function reseedStandbyFromPrimary(standbyIp, sshKeyPath, primarySupabaseIp) {
+  if (!primarySupabaseIp) return false;
+  try {
+    // REPL_PASSWORD is written to both primary and standby .env.local at
+    // create time; read it from the standby's filesystem (easier than
+    // threading through the failover args).
+    const replPassword = sshRun(standbyIp, sshKeyPath, [
+      'bash',
+      '-c',
+      'grep -oE "^REPL_PASSWORD=[\\"\']?[^\\"\'\\n]+" /opt/*/.env.local 2>/dev/null | head -1 | sed -E "s/^REPL_PASSWORD=[\\"\']?//; s/[\\"\']?$//"',
+    ]);
+    if (!replPassword?.trim()) return false;
+
+    const pod = getPostgresPod(standbyIp, sshKeyPath);
+    const script = [
+      `rm -rf /var/lib/postgresql/data/*`,
+      `PGPASSWORD='${replPassword.trim()}' pg_basebackup -h ${primarySupabaseIp} -p 5432 -U replicator -D /var/lib/postgresql/data -Fp -Xs -R -c fast -S vibecarbon_standby_slot`,
+      `echo "primary_slot_name = 'vibecarbon_standby_slot'" >> /var/lib/postgresql/data/postgresql.auto.conf`,
+      `chown -R postgres:postgres /var/lib/postgresql/data`,
+    ].join('\n');
+    sshKubectl(standbyIp, sshKeyPath, ['exec', '-i', '-n', 'vibecarbon', pod, '--', 'bash'], {
+      input: script,
+    });
+    // Restart the pod to pick up the new PGDATA + recovery config.
+    sshKubectl(standbyIp, sshKeyPath, [
+      'exec',
+      '-n',
+      'vibecarbon',
+      pod,
+      '--',
+      'su',
+      'postgres',
+      '-c',
+      'pg_ctl restart -D /var/lib/postgresql/data -m fast',
+    ]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function promoteStandbyDatabase(standbyIp, sshKeyPath) {
+  const _t = perfTimer('failover.promoteStandby');
+  try {
+    return _promoteStandbyDatabaseImpl(standbyIp, sshKeyPath);
+  } finally {
+    _t.end();
+  }
+}
+
+function _promoteStandbyDatabaseImpl(standbyIp, sshKeyPath) {
+  // Get the postgres pod name via a separate SSH call (avoids local shell expansion of $())
+  const pod = getPostgresPod(standbyIp, sshKeyPath);
+
+  // Promote PostgreSQL on standby
+  try {
+    sshKubectl(standbyIp, sshKeyPath, [
+      'exec',
+      '-n',
+      'vibecarbon',
+      pod,
+      '--',
+      'su',
+      'postgres',
+      '-c',
+      'pg_ctl promote -D /var/lib/postgresql/data',
+    ]);
+  } catch {
+    // pg_ctl promote may return non-zero even on success
+  }
+
+  // Poll until standby is no longer in recovery
+  for (let i = 0; i < 30; i++) {
+    try {
+      const result = sshKubectl(standbyIp, sshKeyPath, [
+        'exec',
+        '-n',
+        'vibecarbon',
+        pod,
+        '--',
+        'psql',
+        '-U',
+        'supabase_admin',
+        '-tAc',
+        'SELECT pg_is_in_recovery()',
+      ]);
+      if (result.trim() === 'f') return true;
+    } catch {
+      // Retry
+    }
+    runCommand(['sleep', '2'], { silent: true, ignoreError: true });
+  }
+
+  return false;
+}
+
+function scaleUpServices(ip, sshKeyPath) {
+  const _t = perfTimer('failover.scaleUpServices');
+  try {
+    return _scaleUpServicesImpl(ip, sshKeyPath);
+  } finally {
+    _t.end();
+  }
+}
+
+/**
+ * Deployments scaled during failover. The user's `app` deployment is
+ * named verbatim in carbon/k8s/base/app/deployment.yaml. Supabase
+ * back-ends are owned by the Helm chart with `<release>-<chart>-<comp>`
+ * naming — release `supabase`, chart `supabase` → `supabase-supabase-*`.
+ *
+ * The previous implementation scaled `auth`/`rest`/`realtime`/`storage`
+ * (the bare component names) — kubectl returned "no objects passed to
+ * scale" and the calls became silent no-ops. Result: failover scaled
+ * ONLY the `app` deployment up on standby, leaving Supabase back-ends
+ * stuck at whatever replica count the chart left them at, and the
+ * old primary's auth/rest were never scaled to 0 — split-brain
+ * possible. Caught by k8s-ha matrix verify-failover.
+ */
+const FAILOVER_DEPLOYMENTS = [
+  { name: 'app', upReplicas: 2 },
+  { name: 'supabase-supabase-auth', upReplicas: 2 },
+  { name: 'supabase-supabase-rest', upReplicas: 2 },
+  { name: 'supabase-supabase-realtime', upReplicas: 2 },
+  { name: 'supabase-supabase-storage', upReplicas: 1 },
+];
+
+/**
+ * Scale a list of deployments to N replicas via SSH+kubectl, swallowing
+ * per-deployment "not found" so one missing chart-name doesn't abort
+ * the whole failover. Reports each result so the operator can see what
+ * actually scaled.
+ */
+function scaleDeployments(ip, sshKeyPath, deployments, replicas, label) {
+  for (const dep of deployments) {
+    try {
+      sshKubectl(ip, sshKeyPath, [
+        'scale',
+        'deployment',
+        dep.name,
+        '-n',
+        'vibecarbon',
+        `--replicas=${replicas}`,
+      ]);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      // "no objects passed to scale" / "deployments.apps not found" → log and continue.
+      // Anything else (auth failure, network) we still want to surface.
+      const isMissing = /not found|no objects passed/i.test(msg);
+      if (isMissing) {
+        p.log.warn(`${label}: ${dep.name} not found — skipped`);
+      } else {
+        p.log.warn(`${label}: ${dep.name} failed: ${msg.split('\n')[0]}`);
+      }
+    }
+  }
+}
+
+function _scaleUpServicesImpl(ip, sshKeyPath) {
+  // Group by upReplicas so we don't touch the same deployment twice.
+  const byReplicas = new Map();
+  for (const dep of FAILOVER_DEPLOYMENTS) {
+    if (!byReplicas.has(dep.upReplicas)) byReplicas.set(dep.upReplicas, []);
+    byReplicas.get(dep.upReplicas).push(dep);
+  }
+  for (const [replicas, deps] of byReplicas.entries()) {
+    scaleDeployments(ip, sshKeyPath, deps, replicas, `scale-up standby (replicas=${replicas})`);
+  }
+}
+
+function waitForHealth(ip, sshKeyPath) {
+  const _t = perfTimer('failover.waitForHealth');
+  try {
+    return _waitForHealthImpl(ip, sshKeyPath);
+  } finally {
+    _t.end();
+  }
+}
+
+function _waitForHealthImpl(ip, sshKeyPath) {
+  // The previous implementation curled http://localhost:3000/api/health from
+  // the standby master — but K8s app pods don't run on the master's host
+  // network (no hostPort:3000), so localhost:3000 has nothing listening and
+  // every probe returned "FAIL". 30 retries × 2s = 60s wasted. Use kubectl
+  // rollout status against the actual deployment, which is the cluster's
+  // own readiness signal — gives `app` time to come up after scaleUp and
+  // surfaces the real failure (ImagePullBackOff, CrashLoop, missing secret)
+  // in the kubectl error rather than a generic timeout.
+  try {
+    sshKubectl(ip, sshKeyPath, [
+      'rollout',
+      'status',
+      'deployment/app',
+      '-n',
+      'vibecarbon',
+      '--timeout=300s',
+    ]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ============================================================================
+// FAILOVER: HA + CLOUDFLARE
+// ============================================================================
+
+async function failoverHACloudflare(envName, envConfig, projectConfig, parsed, tracker) {
+  const servers = identifyServers(envName, envConfig, projectConfig);
+  if (!servers) {
+    p.log.error('Could not identify primary and standby servers');
+    p.log.info('Ensure HA deployment was set up with vibecarbon deploy');
+    process.exit(1);
+  }
+
+  const sshKeyPath = getSSHKeyPath(envName);
+  if (!existsSync(sshKeyPath)) {
+    p.log.error(`SSH key not found: ${sshKeyPath}`);
+    process.exit(1);
+  }
+
+  // Get Cloudflare token — check env var, then credentials file, then prompt
+  let cfToken = process.env.CLOUDFLARE_API_TOKEN;
+  if (!cfToken) {
+    const creds = loadCredentials();
+    if (creds.cloudflare?.apiToken) {
+      cfToken = creds.cloudflare.apiToken;
+      p.log.info('Using Cloudflare API token from ~/.vibecarbon/credentials.json');
+    }
+  }
+  if (!cfToken) {
+    cfToken = await p.text({
+      message: 'Cloudflare API token:',
+      validate: (v) => (!v ? 'Required' : undefined),
+    });
+    if (p.isCancel(cfToken)) {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    }
+  }
+
+  const zoneId = envConfig.dns.cloudflareZoneId;
+  const domain = envConfig.domain || envConfig.dns?.domain;
+  const standbyFloatingIp = servers.standby.floatingIp || servers.standby.ip;
+
+  const s = tracker.spinner();
+
+  // Display current state
+  p.log.info(c.bold('Current HA state:'));
+  p.log.message(
+    `  ${c.dim(`Primary (${servers.primary.region})`.padEnd(22))} ${servers.primary.floatingIp || servers.primary.ip}`,
+  );
+  p.log.message(
+    `  ${c.dim(`Standby (${servers.standby.region})`.padEnd(22))} ${standbyFloatingIp}`,
+  );
+
+  p.log.info(c.bold('This will:'));
+  p.log.message('  1. Promote the standby database to primary');
+  p.log.message('  2. Scale down old primary services');
+  p.log.message('  3. Scale up standby services');
+  p.log.message(`  4. Update DNS A record → ${standbyFloatingIp}`);
+
+  // Dry run stops here
+  if (parsed.dryRun) {
+    p.log.info(c.dim('Dry run — no changes made'));
+    return;
+  }
+
+  // Confirmation
+  if (!parsed.yes) {
+    const confirm = await p.confirm({
+      message: `Execute failover for ${c.bold(envName)}?`,
+    });
+    if (p.isCancel(confirm) || !confirm) {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    }
+  }
+
+  // Step 0: Re-seed standby from primary (if primary is still reachable).
+  // Streaming replication can silently fall behind; a stale standby promotes
+  // to serve queries against an out-of-date schema and hits "undefined
+  // column" (PG 42703) errors. Skipped when primary is unreachable (true
+  // disaster failover) — promotion proceeds with whatever state standby
+  // has. Requires primarySupabaseIp from identifyServers (persisted in
+  // envConfig.servers by orchestrator.js post-deploy).
+  if (isPrimaryPostgresReady(servers.primary.ip, sshKeyPath)) {
+    s.start('Re-seeding standby from primary (pg_basebackup)');
+    const reseeded = reseedStandbyFromPrimary(
+      servers.standby.ip,
+      sshKeyPath,
+      servers.primary.supabaseIp,
+    );
+    s.stop(reseeded ? 'Standby re-seeded with primary state' : 'Re-seed skipped (proceeding)');
+  } else {
+    p.log.warn('Primary unreachable — skipping re-seed, promoting standby with current state');
+  }
+
+  // Step 1: Promote standby database
+  s.start('Promoting standby database');
+  const promoted = promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
+  s.stop(
+    promoted
+      ? 'Standby database promoted to primary'
+      : 'Database promotion may still be in progress',
+  );
+  if (!promoted) {
+    p.log.warn('Could not confirm promotion completed — verify manually');
+  }
+
+  // Step 2: Scale down old primary services to prevent split-brain.
+  // Per-deployment errors (not-found, unreachable) are logged inside
+  // scaleDeployments — don't abort the whole failover on one missing
+  // chart name.
+  s.start('Scaling down old primary services');
+  scaleDeployments(servers.primary.ip, sshKeyPath, FAILOVER_DEPLOYMENTS, 0, 'scale-down primary');
+  s.stop('Old primary services scaled down');
+
+  // Step 3: Scale up standby services
+  s.start('Scaling up standby services');
+  try {
+    scaleUpServices(servers.standby.ip, sshKeyPath);
+  } catch (error) {
+    s.stop('Some services failed to scale');
+    p.log.warn(`Scale error: ${error.message}`);
+  }
+  s.stop('Standby services scaled up');
+
+  // Wait for health endpoint
+  s.start('Waiting for standby health check');
+  const healthy = waitForHealth(servers.standby.ip, sshKeyPath);
+  s.stop(healthy ? 'Standby health check passed' : 'Health check timed out');
+
+  // Step 4: Switch DNS A record to standby floating IP. proxied:false
+  // matches the deploy-time setup (Cloudflare LBs removed 2026-03-26).
+  s.start('Updating DNS A record');
+  try {
+    await createDNSRecord(cfToken, zoneId, {
+      type: 'A',
+      name: domain,
+      content: standbyFloatingIp,
+      proxied: false,
+    });
+    await createDNSRecord(cfToken, zoneId, {
+      type: 'A',
+      name: `*.${domain}`,
+      content: standbyFloatingIp,
+      proxied: false,
+    });
+    s.stop(`DNS updated: ${domain} → ${standbyFloatingIp}`);
+  } catch (error) {
+    s.stop('DNS update failed');
+    p.log.error(error.message);
+    p.log.warn(`Manually update the A record for ${domain} to ${standbyFloatingIp}`);
+  }
+
+  // Success
+  p.log.success('Failover complete');
+  p.log.info(c.bold('Recovery instructions:'));
+  p.log.message('  To resync the original primary as a new standby:');
+  p.log.message(`  1. Fix the issue on the original primary (${servers.primary.ip})`);
+  p.log.message('  2. Re-initialize it as a standby replica');
+  p.log.message('  3. Redeploy to update configuration');
+}
+
+// ============================================================================
+// FAILOVER: HA + HETZNER DNS
+// ============================================================================
+
+async function failoverHAHetznerDNS(envName, envConfig, projectConfig, parsed, tracker) {
+  const servers = identifyServers(envName, envConfig, projectConfig);
+  if (!servers) {
+    p.log.error('Could not identify primary and standby servers');
+    p.log.info('Ensure HA deployment was set up with vibecarbon deploy');
+    process.exit(1);
+  }
+
+  const sshKeyPath = getSSHKeyPath(envName);
+  if (!existsSync(sshKeyPath)) {
+    p.log.error(`SSH key not found: ${sshKeyPath}`);
+    process.exit(1);
+  }
+
+  // Get Hetzner API token (same token used for cloud operations)
+  let apiToken = process.env.HETZNER_API_TOKEN;
+  if (!apiToken) {
+    const creds = loadCredentials();
+    apiToken = creds.hetzner?.apiToken;
+  }
+  if (!apiToken) {
+    apiToken = await p.text({
+      message: 'Hetzner API token:',
+      validate: (v) => (!v ? 'Required' : undefined),
+    });
+    if (p.isCancel(apiToken)) {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    }
+  }
+
+  const zoneId = envConfig.dns.hetznerZoneId;
+  const domain = envConfig.domain || envConfig.dns?.domain;
+  const standbyFloatingIp = servers.standby.floatingIp || servers.standby.ip;
+
+  const s = tracker.spinner();
+
+  // Display current state
+  p.log.info(c.bold('Current HA state:'));
+  p.log.message(
+    `  ${c.dim(`Primary (${servers.primary.region})`.padEnd(22))} ${servers.primary.floatingIp || servers.primary.ip}`,
+  );
+  p.log.message(
+    `  ${c.dim(`Standby (${servers.standby.region})`.padEnd(22))} ${standbyFloatingIp}`,
+  );
+
+  p.log.info(c.bold('This will:'));
+  p.log.message('  1. Promote the standby database to primary');
+  p.log.message('  2. Scale down old primary services');
+  p.log.message('  3. Scale up standby services');
+  p.log.message(`  4. Update DNS A record → ${standbyFloatingIp}`);
+
+  if (parsed.dryRun) {
+    p.log.info(c.dim('Dry run — no changes made'));
+    return;
+  }
+
+  if (!parsed.yes) {
+    const confirm = await p.confirm({
+      message: `Execute failover for ${c.bold(envName)}?`,
+    });
+    if (p.isCancel(confirm) || !confirm) {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    }
+  }
+
+  // Step 0: Re-seed standby from primary (if primary is still reachable).
+  // Streaming replication can silently fall behind; a stale standby promotes
+  // to serve queries against an out-of-date schema and hits "undefined
+  // column" (PG 42703) errors. Skipped when primary is unreachable (true
+  // disaster failover) — promotion proceeds with whatever state standby
+  // has. Requires primarySupabaseIp from identifyServers (persisted in
+  // envConfig.servers by orchestrator.js post-deploy).
+  if (isPrimaryPostgresReady(servers.primary.ip, sshKeyPath)) {
+    s.start('Re-seeding standby from primary (pg_basebackup)');
+    const reseeded = reseedStandbyFromPrimary(
+      servers.standby.ip,
+      sshKeyPath,
+      servers.primary.supabaseIp,
+    );
+    s.stop(reseeded ? 'Standby re-seeded with primary state' : 'Re-seed skipped (proceeding)');
+  } else {
+    p.log.warn('Primary unreachable — skipping re-seed, promoting standby with current state');
+  }
+
+  // Step 1: Promote standby database
+  s.start('Promoting standby database');
+  const promoted = promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
+  s.stop(
+    promoted
+      ? 'Standby database promoted to primary'
+      : 'Database promotion may still be in progress',
+  );
+  if (!promoted) {
+    p.log.warn('Could not confirm promotion completed — verify manually');
+  }
+
+  // Step 2: Scale down old primary services to prevent split-brain.
+  // See FAILOVER_DEPLOYMENTS for the chart-canonical names — bare
+  // `auth`/`rest`/etc. silently no-op'd in an earlier version.
+  s.start('Scaling down old primary services');
+  scaleDeployments(servers.primary.ip, sshKeyPath, FAILOVER_DEPLOYMENTS, 0, 'scale-down primary');
+  s.stop('Old primary services scaled down');
+
+  // Step 3: Scale up standby services
+  s.start('Scaling up standby services');
+  try {
+    scaleUpServices(servers.standby.ip, sshKeyPath);
+  } catch (error) {
+    s.stop('Some services failed to scale');
+    p.log.warn(`Scale error: ${error.message}`);
+  }
+  s.stop('Standby services scaled up');
+
+  // Wait for health endpoint
+  s.start('Waiting for standby health check');
+  const healthy = waitForHealth(servers.standby.ip, sshKeyPath);
+  s.stop(healthy ? 'Standby health check passed' : 'Health check timed out');
+
+  // Step 4: Update DNS A record via Hetzner DNS API
+  s.start('Updating DNS A record');
+  try {
+    const zone = await hetznerDnsGetZone(apiToken, zoneId);
+    await hetznerDnsCreateRecord(apiToken, zoneId, zone.name, {
+      type: 'A',
+      name: domain,
+      value: standbyFloatingIp,
+      ttl: 60,
+    });
+    await hetznerDnsCreateRecord(apiToken, zoneId, zone.name, {
+      type: 'A',
+      name: `*.${domain}`,
+      value: standbyFloatingIp,
+      ttl: 60,
+    });
+    s.stop(`DNS updated: ${domain} → ${standbyFloatingIp}`);
+  } catch (error) {
+    s.stop('DNS update failed');
+    p.log.error(error.message);
+    p.log.warn(`Manually update the A record for ${domain} to ${standbyFloatingIp}`);
+  }
+
+  // Success
+  p.log.success('Failover complete');
+  p.log.info(c.bold('Recovery instructions:'));
+  p.log.message('  To resync the original primary as a new standby:');
+  p.log.message(`  1. Fix the issue on the original primary (${servers.primary.ip})`);
+  p.log.message('  2. Re-initialize it as a standby replica');
+  p.log.message('  3. Redeploy to update configuration');
+}
+
+// ============================================================================
+// FAILOVER: HA + MANUAL DNS
+// ============================================================================
+
+async function failoverHAManual(envName, envConfig, projectConfig, parsed, tracker) {
+  const servers = identifyServers(envName, envConfig, projectConfig);
+  if (!servers) {
+    p.log.error('Could not identify primary and standby servers');
+    p.log.info('Ensure HA deployment was set up with vibecarbon deploy');
+    process.exit(1);
+  }
+
+  const sshKeyPath = getSSHKeyPath(envName);
+  if (!existsSync(sshKeyPath)) {
+    p.log.error(`SSH key not found: ${sshKeyPath}`);
+    process.exit(1);
+  }
+
+  const domain = envConfig.domain || envConfig.dns?.domain || '<your-domain>';
+
+  p.log.info(c.bold('Current HA state:'));
+  p.log.message(
+    `  ${c.dim(`Primary (${servers.primary.region})`.padEnd(22))} ${servers.primary.ip}`,
+  );
+  p.log.message(
+    `  ${c.dim(`Standby (${servers.standby.region})`.padEnd(22))} ${servers.standby.ip}`,
+  );
+  p.log.info(c.bold('This will:'));
+  p.log.message('  1. Promote the standby database to primary');
+  p.log.message('  2. Scale up standby services');
+  p.log.message('  3. Print DNS update instructions for you to complete');
+
+  if (parsed.dryRun) {
+    p.log.info(c.dim('Dry run — no changes made'));
+    return;
+  }
+
+  if (!parsed.yes) {
+    const confirm = await p.confirm({
+      message: `Execute failover for ${c.bold(envName)}?`,
+    });
+    if (p.isCancel(confirm) || !confirm) {
+      p.cancel('Operation cancelled.');
+      process.exit(0);
+    }
+  }
+
+  const s = tracker.spinner();
+
+  // Promote standby database
+  s.start('Promoting standby database');
+  const promoted = promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
+  s.stop(promoted ? 'Standby database promoted' : 'Promotion may still be in progress');
+
+  // Scale down old primary services to prevent split-brain
+  s.start('Scaling down old primary services');
+  try {
+    sshKubectl(servers.primary.ip, sshKeyPath, [
+      'scale',
+      'deployment',
+      'app',
+      '-n',
+      'vibecarbon',
+      '--replicas=0',
+    ]);
+    sshKubectl(servers.primary.ip, sshKeyPath, [
+      'scale',
+      'deployment',
+      'auth',
+      '-n',
+      'vibecarbon',
+      '--replicas=0',
+    ]);
+    sshKubectl(servers.primary.ip, sshKeyPath, [
+      'scale',
+      'deployment',
+      'rest',
+      '-n',
+      'vibecarbon',
+      '--replicas=0',
+    ]);
+    sshKubectl(servers.primary.ip, sshKeyPath, [
+      'scale',
+      'deployment',
+      'realtime',
+      '-n',
+      'vibecarbon',
+      '--replicas=0',
+    ]);
+  } catch {
+    p.log.warn('Could not reach old primary to scale down services');
+  }
+  s.stop('Old primary services scaled down');
+
+  // Scale up services
+  s.start('Scaling up standby services');
+  try {
+    scaleUpServices(servers.standby.ip, sshKeyPath);
+  } catch (error) {
+    s.stop('Some services failed to scale');
+    p.log.warn(`Scale error: ${error.message}`);
+  }
+  s.stop('Standby services scaled up');
+
+  // Wait for health
+  s.start('Waiting for standby health check');
+  const healthy = waitForHealth(servers.standby.ip, sshKeyPath);
+  s.stop(healthy ? 'Standby health check passed' : 'Health check timed out');
+
+  // Print DNS instructions
+  p.log.success('Standby region activated');
+  p.log.info(c.bold('Update DNS to complete failover:'));
+  p.log.message(`  ${c.dim('Domain'.padEnd(14))} ${c.bold(domain)}`);
+  p.log.message(`  ${c.dim('New IP'.padEnd(14))} ${c.bold(servers.standby.ip)} (standby master)`);
+  p.log.info(c.bold('Steps:'));
+  p.log.message('  1. Log into your DNS provider');
+  p.log.message(`  2. Update the A record for ${domain} to ${c.bold(servers.standby.ip)}`);
+  p.log.message(`  3. Wait for DNS propagation (check: ${c.dim(`dig ${domain}`)})`);
+}
+
+// ============================================================================
+// FAILOVER: SINGLE SERVER (RECOVERY GUIDE)
+// ============================================================================
+
+async function failoverSingleServer(envName, envConfig) {
+  const serverIp = envConfig.servers?.[0]?.ip;
+  const sshKeyPath = getSSHKeyPath(envName);
+
+  p.log.warn('This environment has no standby region');
+  p.log.info('Recovery requires deploying a new server and restoring from backup');
+
+  // Check for available backups
+  if (serverIp && existsSync(sshKeyPath)) {
+    const s = p.spinner();
+    s.start('Checking for available backups');
+    try {
+      const pod = sshKubectl(serverIp, sshKeyPath, [
+        'get',
+        'pods',
+        '-n',
+        'vibecarbon',
+        '-l',
+        'app.kubernetes.io/name=supabase-db',
+        '-o',
+        'jsonpath={.items[0].metadata.name}',
+      ]);
+      const output = sshKubectl(serverIp, sshKeyPath, [
+        'exec',
+        '-n',
+        'vibecarbon',
+        pod,
+        '--',
+        'sh',
+        '-c',
+        'ls -lt /backups/*_full.tar.gz /backups/*.sql.gz 2>/dev/null | head -5 || echo NO_BACKUPS',
+      ]);
+      s.stop('Backup check complete');
+
+      if (output !== 'NO_BACKUPS' && output) {
+        p.log.info(c.bold('Available backups:'));
+        for (const line of output
+          .split('\n')
+          .filter((l) => l.includes('.tar.gz') || l.includes('.sql.gz'))) {
+          p.log.message(`  ${line.trim()}`);
+        }
+      } else {
+        p.log.info('No backups found on the server');
+        p.log.info(`Create one with: ${c.info(`vibecarbon backup ${envName}`)}`);
+      }
+    } catch {
+      s.stop('Could not reach server');
+      p.log.warn('Server may be unreachable — backups may not be accessible');
+    }
+  }
+
+  p.log.info(c.bold('Recovery steps:'));
+  p.log.message(`  1. ${c.info(`vibecarbon deploy ${envName}`)}`);
+  p.log.message(`  2. ${c.info(`vibecarbon restore ${envName}`)}`);
+  p.log.message(`  3. ${c.info(`vibecarbon status ${envName}`)}`);
+  p.log.info('To enable HA for one-command failover in the future:');
+  p.log.message(`  ${c.info(`vibecarbon deploy ${envName} --ha`)}`);
+}
+
+// ============================================================================
+// MAIN
+// ============================================================================
+
+export async function run(args) {
+  const { values, positional, errors } = parseFlags(args, SPEC);
+
+  if (errors.length > 0) {
+    for (const e of errors) {
+      process.stderr.write(`${c.error('✗')} ${e}\n`);
+    }
+    process.stderr.write(`Run ${c.info('vibecarbon failover -h')} for usage.\n`);
+    process.exit(1);
+  }
+
+  if (values.h) {
+    process.stdout.write(renderHelp(SPEC));
+    return;
+  }
+  if (values.v) {
+    console.log(`vibecarbon v${VERSION}`);
+    return;
+  }
+
+  // Project guard runs before banner so an accidental `vibecarbon
+  // failover` from a parent directory emits the canonical message.
+  assertInProjectDir();
+
+  requireLicense('failover');
+  printBanner();
+  p.intro(`${c.bold('vibecarbon failover')} ${c.dim(`v${VERSION}`)}`);
+
+  const projectConfig = loadProjectConfig();
+  const envCount = Object.keys(projectConfig.environments || {}).length;
+  const envSeed =
+    /** @type {string|undefined} */ (positional.env) ||
+    /** @type {string|null} */ (values.env) ||
+    null;
+
+  requireTTYOrFlags({
+    requirements: [
+      {
+        flag: 'env',
+        description: 'name an environment to fail over',
+        satisfied: !!envSeed || envCount <= 1,
+      },
+    ],
+  });
+
+  const { envName, envConfig } = await selectEnvironment(projectConfig, {
+    actionVerb: 'fail over',
+    seed: envSeed,
+  });
+
+  // Compatibility shim — the existing failover orchestration helpers
+  // read `parsed.dryRun` and `parsed.yes` from the legacy struct shape.
+  // Build that shape from the new flag values without changing the
+  // helpers (each is hundreds of lines and well-tested).
+  const parsed = {
+    env: envName,
+    dryRun: !!values.dry,
+    yes: !!values.y,
+  };
+
+  // Failover hits the active and standby clusters via SSH + kubectl, so the
+  // operator's IP must be in the firewall allowlist for both.
+  const apiToken = loadCredentials()?.hetzner?.apiToken;
+  if (apiToken) {
+    try {
+      const result = await ensureOperatorIpAccess({
+        projectConfig,
+        environment: envName,
+        isHA: true,
+        apiToken,
+        yes: !!parsed.yes,
+        onMessage: (msg) => p.log.info(msg),
+      });
+      if (result.added) {
+        p.log.success(`Firewall updated: SSH/k8s API now allow ${result.cidr}`);
+      }
+    } catch (err) {
+      p.log.error(`Operator-IP check failed: ${err.message}`);
+      process.exit(1);
+    }
+  }
+
+  // Single-server compose has no failover target
+  if (envConfig.deployMode === 'compose') {
+    p.log.info('Failover is not available for single-server Compose deployments.');
+    p.log.info(`Environment "${envName}" uses Docker Compose on a single VPS.`);
+    p.log.info('');
+    p.log.info(`${c.dim('To enable failover, redeploy with --ha:')}`);
+    p.log.info(`  ${c.info(`vibecarbon deploy ${envName} --ha`)}`);
+    p.outro('');
+    return;
+  }
+
+  // Compose-HA failover uses docker compose exec instead of kubectl
+  if (envConfig.deployMode === 'compose-ha') {
+    const composeHATracker = createTracker('failover', {
+      environment: envName,
+      scenario: 'compose_ha',
+    });
+    const { failoverComposeHA } = await import('./lib/deploy/compose/ha.js');
+    await failoverComposeHA(envName, envConfig, projectConfig, parsed, composeHATracker);
+    composeHATracker.finish();
+    p.outro(c.success('Done'));
+    return;
+  }
+
+  // Detect scenario
+  const scenario = detectScenario(envConfig);
+  const tracker = createTracker('failover', { environment: envName, scenario });
+
+  switch (scenario) {
+    case 'ha_cloudflare':
+      p.log.info(`Scenario: ${c.bold('HA + Cloudflare')} (DNS failover)`);
+      await failoverHACloudflare(envName, envConfig, projectConfig, parsed, tracker);
+      break;
+
+    case 'ha_hetzner_dns':
+      p.log.info(`Scenario: ${c.bold('HA + Hetzner DNS')} (automatic DNS failover)`);
+      await failoverHAHetznerDNS(envName, envConfig, projectConfig, parsed, tracker);
+      break;
+
+    case 'ha_manual':
+      p.log.info(`Scenario: ${c.bold('HA + Manual DNS')}`);
+      await failoverHAManual(envName, envConfig, projectConfig, parsed, tracker);
+      break;
+
+    case 'single_server':
+      p.log.info(`Scenario: ${c.bold('Single Server')} (no HA configured)`);
+      await failoverSingleServer(envName, envConfig);
+      break;
+  }
+
+  tracker.finish();
+  p.outro(c.success('Done'));
+}
+
+// Exported for tests.
+export { SPEC };