vibecarbon 0.8.0 → 0.10.0

package/src/failover.js

@@ -12,6 +12,7 @@
 
 import { existsSync } from 'node:fs';
 import * as p from '@clack/prompts';
+import { formatInstant } from './lib/backup-format.js';
 import { renderHelp } from './lib/cli/help.js';
 import { parseFlags } from './lib/cli/parse-flags.js';
 import { selectEnvironment } from './lib/cli/select-environment.js';
@@ -19,17 +20,23 @@ import { requireTTYOrFlags } from './lib/cli/tty-guard.js';
 import { createDNSRecord } from './lib/cloudflare.js';
 import { c, printBanner } from './lib/colors.js';
 import { loadCredentials, loadProjectConfig } from './lib/config.js';
+import {
+  getReplPasswordFromSecret,
+  promoteStandby,
+  reseedStandbyFromPrimary,
+} from './lib/deploy/replication.js';
 import {
   createDNSRecord as hetznerDnsCreateRecord,
   getZone as hetznerDnsGetZone,
 } from './lib/hetzner-dns.js';
-import { requireLicense } from './lib/licensing/index.js';
 import { ensureOperatorIpAccess } from './lib/operator-ip.js';
 import { perfTimer } from './lib/perf.js';
+import { confirmProdOrExit } from './lib/prod-confirm.js';
 import { assertInProjectDir } from './lib/project-guard.js';
-import { getPostgresPod, getSSHKeyPath, sshKubectl, sshRun } from './lib/ssh.js';
+import { getPostgresPod, getSSHKeyPath, sshKubectl } from './lib/ssh.js';
 import { createTracker } from './lib/tracker.js';
 import { VERSION } from './lib/version.js';
+import { listWalgBackups } from './lib/walg-backups.js';
 
 // ============================================================================
 // COMMAND SPEC — single source of truth for argv parsing AND help output.
@@ -170,113 +177,69 @@ async function isPrimaryPostgresReady(primaryIp, sshKeyPath) {
   }
 }
 
-/**
- * Re-seed the standby Postgres from the primary via pg_basebackup before
- * promotion. Closes the drift window that opens when streaming replication
- * silently falls behind (no replication slot) or when the standby's pod
- * restarted and missed WAL. Without re-seeding, promoted standby can serve
- * requests against a schema that's missing recent migrations, yielding
- * "undefined column" (PG 42703) errors in auth/storage after failover.
- *
- * Requires `primarySupabaseIp` — the Hetzner private IP of the primary's
- * supabase node (where the postgres pod runs with hostPort 5432). Returns
- * true if re-seed succeeded, false if primary was unreachable or re-seed
- * itself failed (caller proceeds to promote with existing state).
- */
-async function reseedStandbyFromPrimary(standbyIp, sshKeyPath, primarySupabaseIp) {
-  if (!primarySupabaseIp) return false;
-  try {
-    // REPL_PASSWORD is written to both primary and standby .env.local at
-    // create time; read it from the standby's filesystem (easier than
-    // threading through the failover args).
-    const replPassword = await sshRun(standbyIp, sshKeyPath, [
-      'bash',
-      '-c',
-      'grep -oE "^REPL_PASSWORD=[\\"\']?[^\\"\'\\n]+" /opt/*/.env.local 2>/dev/null | head -1 | sed -E "s/^REPL_PASSWORD=[\\"\']?//; s/[\\"\']?$//"',
-    ]);
-    if (!replPassword?.trim()) return false;
-
-    const pod = await getPostgresPod(standbyIp, sshKeyPath);
-    const script = [
-      `rm -rf /var/lib/postgresql/data/*`,
-      `PGPASSWORD='${replPassword.trim()}' pg_basebackup -h ${primarySupabaseIp} -p 5432 -U replicator -D /var/lib/postgresql/data -Fp -Xs -R -c fast -S vibecarbon_standby_slot`,
-      `echo "primary_slot_name = 'vibecarbon_standby_slot'" >> /var/lib/postgresql/data/postgresql.auto.conf`,
-      `chown -R postgres:postgres /var/lib/postgresql/data`,
-    ].join('\n');
-    await sshKubectl(standbyIp, sshKeyPath, ['exec', '-i', '-n', 'vibecarbon', pod, '--', 'bash'], {
-      input: script,
-    });
-    // Restart the pod to pick up the new PGDATA + recovery config.
-    await sshKubectl(standbyIp, sshKeyPath, [
-      'exec',
-      '-n',
-      'vibecarbon',
-      pod,
-      '--',
-      'su',
-      'postgres',
-      '-c',
-      'pg_ctl restart -D /var/lib/postgresql/data -m fast',
-    ]);
-    return true;
-  } catch {
-    return false;
-  }
-}
+// getReplPasswordFromSecret + reseedStandbyFromPrimary + promoteStandby now live
+// in src/lib/deploy/replication.js (finding #2 — shared with restore.js and the
+// k8s deploy-time seed). They are imported at the top of this file. The perfTimer
+// wrapper for promotion stays here since it is a failover-flow concern.
 
 async function promoteStandbyDatabase(standbyIp, sshKeyPath) {
   const _t = perfTimer('failover.promoteStandby');
   try {
-    return await _promoteStandbyDatabaseImpl(standbyIp, sshKeyPath);
+    return await promoteStandby(standbyIp, sshKeyPath);
   } finally {
     _t.end();
   }
 }
 
-async function _promoteStandbyDatabaseImpl(standbyIp, sshKeyPath) {
-  // Get the postgres pod name via a separate SSH call (avoids local shell expansion of $())
-  const pod = await getPostgresPod(standbyIp, sshKeyPath);
-
-  // Promote PostgreSQL on standby
-  try {
-    await sshKubectl(standbyIp, sshKeyPath, [
-      'exec',
-      '-n',
-      'vibecarbon',
-      pod,
-      '--',
-      'su',
-      'postgres',
-      '-c',
-      'pg_ctl promote -D /var/lib/postgresql/data',
-    ]);
-  } catch {
-    // pg_ctl promote may return non-zero even on success
-  }
-
-  // Poll until standby is no longer in recovery
-  for (let i = 0; i < 30; i++) {
+/**
+ * Re-seed the standby from the primary (when reachable) then promote it.
+ * ABORTS the whole failover (throws) if the re-seed fails or the promotion
+ * cannot be confirmed — we must NEVER scale down the old primary and flip DNS
+ * onto a standby that was left half-seeded or is still in recovery. Brings the
+ * k8s failover path to parity with compose (compose/ha.js hard-fails both).
+ */
+async function reseedAndPromoteOrAbort(servers, sshKeyPath, s) {
+  // Step 0: Re-seed standby from primary (only when the primary is reachable).
+  if (await isPrimaryPostgresReady(servers.primary.ip, sshKeyPath)) {
+    s.start('Re-seeding standby from primary (pg_basebackup)');
+    let result;
     try {
-      const result = await sshKubectl(standbyIp, sshKeyPath, [
-        'exec',
-        '-n',
-        'vibecarbon',
-        pod,
-        '--',
-        'psql',
-        '-U',
-        'supabase_admin',
-        '-tAc',
-        'SELECT pg_is_in_recovery()',
-      ]);
-      if (result.trim() === 'f') return true;
-    } catch {
-      // Retry
+      result = await reseedStandbyFromPrimary(
+        servers.standby.ip,
+        sshKeyPath,
+        servers.primary.supabaseIp,
+      );
+    } catch (err) {
+      s.stop('Re-seed failed — aborting failover');
+      throw new Error(
+        `Failover aborted before promotion: ${err.message}\n` +
+          `The standby was NOT promoted and the old primary is untouched — no ` +
+          `split-brain and no DNS flip. Resolve the issue (or redeploy the ` +
+          `standby) and retry.`,
+      );
     }
-    await new Promise((r) => setTimeout(r, 2000));
+    s.stop(
+      result === 'reseeded'
+        ? 'Standby re-seeded with primary state'
+        : 'Primary unreachable mid-probe — promoting standby with current state',
+    );
+  } else {
+    p.log.warn('Primary unreachable — skipping re-seed, promoting standby with current state');
   }
 
-  return false;
+  // Step 1: Promote standby database. A failure here is FATAL: continuing would
+  // scale down the old primary and flip DNS onto a read-only standby.
+  s.start('Promoting standby database');
+  const promoted = await promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
+  if (!promoted) {
+    s.stop('Promotion could not be confirmed — aborting failover');
+    throw new Error(
+      `Failover aborted: standby ${servers.standby.ip} did not exit recovery mode. ` +
+        `Refusing to scale down the old primary or flip DNS onto an unpromoted ` +
+        `standby (that would take the app read-only). Verify the standby db pod and retry.`,
+    );
+  }
+  s.stop('Standby database promoted to primary');
 }
 
 async function scaleUpServices(ip, sshKeyPath) {
@@ -468,36 +431,13 @@ async function failoverHACloudflare(envName, envConfig, projectConfig, parsed, t
     }
   }
 
-  // Step 0: Re-seed standby from primary (if primary is still reachable).
-  // Streaming replication can silently fall behind; a stale standby promotes
-  // to serve queries against an out-of-date schema and hits "undefined
-  // column" (PG 42703) errors. Skipped when primary is unreachable (true
-  // disaster failover) — promotion proceeds with whatever state standby
-  // has. Requires primarySupabaseIp from identifyServers (persisted in
-  // envConfig.servers by orchestrator.js post-deploy).
-  if (await isPrimaryPostgresReady(servers.primary.ip, sshKeyPath)) {
-    s.start('Re-seeding standby from primary (pg_basebackup)');
-    const reseeded = await reseedStandbyFromPrimary(
-      servers.standby.ip,
-      sshKeyPath,
-      servers.primary.supabaseIp,
-    );
-    s.stop(reseeded ? 'Standby re-seeded with primary state' : 'Re-seed skipped (proceeding)');
-  } else {
-    p.log.warn('Primary unreachable — skipping re-seed, promoting standby with current state');
-  }
-
-  // Step 1: Promote standby database
-  s.start('Promoting standby database');
-  const promoted = await promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
-  s.stop(
-    promoted
-      ? 'Standby database promoted to primary'
-      : 'Database promotion may still be in progress',
-  );
-  if (!promoted) {
-    p.log.warn('Could not confirm promotion completed — verify manually');
-  }
+  // Steps 0 + 1: Re-seed standby from primary (if reachable) then promote.
+  // Streaming replication can silently fall behind; a stale standby would
+  // promote to serve queries against an out-of-date schema and hit "undefined
+  // column" (PG 42703) errors. A re-seed failure or an unconfirmed promotion
+  // ABORTS the failover (throws) — see reseedAndPromoteOrAbort — so we never
+  // scale down the old primary or flip DNS onto a half-seeded/unpromoted node.
+  await reseedAndPromoteOrAbort(servers, sshKeyPath, s);
 
   // Step 2: Scale down old primary services to prevent split-brain.
   // Per-deployment errors (not-found, unreachable) are logged inside
@@ -631,36 +571,11 @@ async function failoverHAHetznerDNS(envName, envConfig, projectConfig, parsed, t
     }
   }
 
-  // Step 0: Re-seed standby from primary (if primary is still reachable).
-  // Streaming replication can silently fall behind; a stale standby promotes
-  // to serve queries against an out-of-date schema and hits "undefined
-  // column" (PG 42703) errors. Skipped when primary is unreachable (true
-  // disaster failover) — promotion proceeds with whatever state standby
-  // has. Requires primarySupabaseIp from identifyServers (persisted in
-  // envConfig.servers by orchestrator.js post-deploy).
-  if (await isPrimaryPostgresReady(servers.primary.ip, sshKeyPath)) {
-    s.start('Re-seeding standby from primary (pg_basebackup)');
-    const reseeded = await reseedStandbyFromPrimary(
-      servers.standby.ip,
-      sshKeyPath,
-      servers.primary.supabaseIp,
-    );
-    s.stop(reseeded ? 'Standby re-seeded with primary state' : 'Re-seed skipped (proceeding)');
-  } else {
-    p.log.warn('Primary unreachable — skipping re-seed, promoting standby with current state');
-  }
-
-  // Step 1: Promote standby database
-  s.start('Promoting standby database');
-  const promoted = await promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
-  s.stop(
-    promoted
-      ? 'Standby database promoted to primary'
-      : 'Database promotion may still be in progress',
-  );
-  if (!promoted) {
-    p.log.warn('Could not confirm promotion completed — verify manually');
-  }
+  // Steps 0 + 1: Re-seed standby from primary (if reachable) then promote.
+  // A re-seed failure or an unconfirmed promotion ABORTS the failover (throws)
+  // — see reseedAndPromoteOrAbort — so we never scale down the old primary or
+  // flip DNS onto a half-seeded/unpromoted node.
+  await reseedAndPromoteOrAbort(servers, sshKeyPath, s);
 
   // Step 2: Scale down old primary services to prevent split-brain.
   // See FAILOVER_DEPLOYMENTS for the chart-canonical names — bare
@@ -771,49 +686,22 @@ async function failoverHAManual(envName, envConfig, projectConfig, parsed, track
 
   const s = tracker.spinner();
 
-  // Promote standby database
-  s.start('Promoting standby database');
-  const promoted = await promoteStandbyDatabase(servers.standby.ip, sshKeyPath);
-  s.stop(promoted ? 'Standby database promoted' : 'Promotion may still be in progress');
+  // Re-seed standby from primary (if reachable) then promote — aborts on
+  // failure so we never activate a half-seeded/unpromoted standby.
+  await reseedAndPromoteOrAbort(servers, sshKeyPath, s);
 
-  // Scale down old primary services to prevent split-brain
+  // Scale down old primary services to prevent split-brain. Uses the
+  // chart-canonical deployment names (FAILOVER_DEPLOYMENTS) — the previous bare
+  // `auth`/`rest`/`realtime` names silently no-op'd against the Helm-owned
+  // `supabase-supabase-*` deployments, so the old primary was never scaled down.
   s.start('Scaling down old primary services');
-  try {
-    await sshKubectl(servers.primary.ip, sshKeyPath, [
-      'scale',
-      'deployment',
-      'app',
-      '-n',
-      'vibecarbon',
-      '--replicas=0',
-    ]);
-    await sshKubectl(servers.primary.ip, sshKeyPath, [
-      'scale',
-      'deployment',
-      'auth',
-      '-n',
-      'vibecarbon',
-      '--replicas=0',
-    ]);
-    await sshKubectl(servers.primary.ip, sshKeyPath, [
-      'scale',
-      'deployment',
-      'rest',
-      '-n',
-      'vibecarbon',
-      '--replicas=0',
-    ]);
-    await sshKubectl(servers.primary.ip, sshKeyPath, [
-      'scale',
-      'deployment',
-      'realtime',
-      '-n',
-      'vibecarbon',
-      '--replicas=0',
-    ]);
-  } catch {
-    p.log.warn('Could not reach old primary to scale down services');
-  }
+  await scaleDeployments(
+    servers.primary.ip,
+    sshKeyPath,
+    FAILOVER_DEPLOYMENTS,
+    0,
+    'scale-down primary',
+  );
   s.stop('Old primary services scaled down');
 
   // Scale up services
@@ -853,47 +741,28 @@ async function failoverSingleServer(envName, envConfig) {
   p.log.warn('This environment has no standby region');
   p.log.info('Recovery requires deploying a new server and restoring from backup');
 
-  // Check for available backups
+  // Check for available backups. wal-g is the SINGLE SOURCE OF TRUTH for what is
+  // actually restorable — the legacy `/backups/*_full.tar.gz` + `*.sql.gz` S3
+  // objects are NOT wal-g backups and `vibecarbon restore` cannot replay them,
+  // so listing them here would tempt an operator to pick a non-restorable
+  // artifact mid-disaster. List wal-g base backups only.
   if (serverIp && existsSync(sshKeyPath)) {
     const s = p.spinner();
-    s.start('Checking for available backups');
-    try {
-      const pod = await sshKubectl(serverIp, sshKeyPath, [
-        'get',
-        'pods',
-        '-n',
-        'vibecarbon',
-        '-l',
-        'app.kubernetes.io/name=supabase-db',
-        '-o',
-        'jsonpath={.items[0].metadata.name}',
-      ]);
-      const output = await sshKubectl(serverIp, sshKeyPath, [
-        'exec',
-        '-n',
-        'vibecarbon',
-        pod,
-        '--',
-        'sh',
-        '-c',
-        'ls -lt /backups/*_full.tar.gz /backups/*.sql.gz 2>/dev/null | head -5 || echo NO_BACKUPS',
-      ]);
-      s.stop('Backup check complete');
-
-      if (output !== 'NO_BACKUPS' && output) {
-        p.log.info(c.bold('Available backups:'));
-        for (const line of output
-          .split('\n')
-          .filter((l) => l.includes('.tar.gz') || l.includes('.sql.gz'))) {
-          p.log.message(`  ${line.trim()}`);
-        }
-      } else {
-        p.log.info('No backups found on the server');
-        p.log.info(`Create one with: ${c.info(`vibecarbon backup ${envName}`)}`);
+    s.start('Checking for available backups (wal-g)');
+    const projectConfig = loadProjectConfig();
+    const projectName = projectConfig?.projectName || 'project';
+    const isCompose = envConfig.deployMode === 'compose' || envConfig.deployMode === 'compose-ha';
+    const backups = await listWalgBackups({ serverIp, sshKeyPath, projectName, isCompose });
+    s.stop('Backup check complete');
+
+    if (backups.length > 0) {
+      p.log.info(c.bold(`Available wal-g base backups (${backups.length}, newest first):`));
+      for (const b of backups.slice(0, 5)) {
+        p.log.message(`  ${formatInstant(b.time).padEnd(22)} ${c.dim(b.name)}`);
       }
-    } catch {
-      s.stop('Could not reach server');
-      p.log.warn('Server may be unreachable — backups may not be accessible');
+    } else {
+      p.log.info('No wal-g base backups found on the server');
+      p.log.info(`Create one with: ${c.info(`vibecarbon backup ${envName}`)}`);
     }
   }
 
@@ -933,7 +802,6 @@ export async function run(args) {
   // failover` from a parent directory emits the canonical message.
   assertInProjectDir();
 
-  requireLicense('failover');
   printBanner();
   p.intro(`${c.bold('vibecarbon failover')} ${c.dim(`v${VERSION}`)}`);
 
@@ -1002,6 +870,19 @@ export async function run(args) {
     return;
   }
 
+  // SECURITY: production type-to-confirm runs UNCONDITIONALLY here — even with
+  // -y — for any DESTRUCTIVE failover (compose-HA or an HA k8s scenario), so a
+  // scripted `failover -y prod` can never silently promote/flip production. This
+  // is deliberately OUTSIDE every `if (!parsed.yes)` soft-confirm block (those
+  // are skippable with -y; this hard gate is not). Mirrors restore/destroy.
+  // Skipped for -dry (previews nothing destructive) and for the single-server
+  // recovery guide (informational only — no promotion/DNS change).
+  const isDestructiveFailover =
+    envConfig.deployMode === 'compose-ha' || detectScenario(envConfig) !== 'single_server';
+  if (!parsed.dryRun && isDestructiveFailover) {
+    await confirmProdOrExit(envName, { actionLabel: 'failover', yes: !!parsed.yes });
+  }
+
   // Compose-HA failover uses docker compose exec instead of kubectl
   if (envConfig.deployMode === 'compose-ha') {
     const composeHATracker = createTracker('failover', {
@@ -1046,4 +927,4 @@ export async function run(args) {
 }
 
 // Exported for tests.
-export { SPEC };
+export { getReplPasswordFromSecret, reseedStandbyFromPrimary, SPEC };

package/src/lib/command.js

@@ -11,7 +11,7 @@
  */
 
 import { spawn, spawnSync } from 'node:child_process';
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import * as p from '@clack/prompts';
@@ -359,3 +359,21 @@ export function runCommandThroughTaskLog(cmd, options) {
 export function checkDependency(cmd) {
   return which.sync(cmd, { nothrow: true }) !== null;
 }
+
+/**
+ * Write a file that contains secrets (rendered helm values, credentials, etc.)
+ * with owner-only 0o600 permissions, ALWAYS — even when overwriting an existing
+ * file (writeFileSync only applies `mode` on creation, so we chmod after).
+ *
+ * SECURITY: use this for any temp file holding secret material. A plain
+ * writeFileSync defaults to 0o644 (world-readable), which leaks credentials to
+ * every local user for the lifetime of the file (e.g. a rendered helm values
+ * file lives for the whole `helm upgrade --wait` window).
+ *
+ * @param {string} path
+ * @param {string | NodeJS.ArrayBufferView} data
+ */
+export function writeSecretFile(path, data) {
+  writeFileSync(path, data, { mode: 0o600 });
+  chmodSync(path, 0o600);
+}