vibecarbon 0.4.0 → 0.5.0

package/src/lib/deploy/compose/index.js

@@ -11,16 +11,15 @@
 
 import { spawn } from 'node:child_process';
 import { randomBytes } from 'node:crypto';
-import { existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'node:fs';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import * as p from '@clack/prompts';
-import { uploadS3Backup } from '../../backup-s3.js';
 import { runCommand, runCommandAsync } from '../../command.js';
 import { perfAsync, perfTimer } from '../../perf.js';
 import { parseDotenv, shEscape } from '../../shell.js';
-import { NO_PIN_SSH_OPTS, scpDownload } from '../../ssh.js';
+import { NO_PIN_SSH_OPTS, sshRunScript } from '../../ssh.js';
 import { postAdminUser, waitForGotrueHealth } from '../admin-user.js';
 
 const __filename = fileURLToPath(import.meta.url);
@@ -825,216 +824,283 @@ export function destroyCompose(ip, sshKeyPath, projectName) {
 }
 
 /**
- * Create a full backup via Docker Compose.
- * Produces a .tar.gz archive containing:
- *   - postgres.sql.gz (main Supabase database, always)
- *   - n8n.sql.gz (if n8n database exists)
- *   - metabase.sql.gz (if metabase database exists)
- *   - storage.tar.gz (if file-backend storage files exist)
+ * Build the shell command that runs the wal-g base backup on a Compose VPS.
+ *
+ * `carbon/backup/compose-backup.sh` is the SINGLE source of truth for the
+ * backup logic (guard + PGUSER=supabase_admin + wal-g backup-push + delete
+ * retain, plus a flock and set -e). This builder is a thin, quote-safe
+ * wrapper so the on-demand path and the cron path run byte-identical commands.
+ *
+ * The `RETAIN=<n>` prefix binds to `bash` (a real command, so the assignment
+ * is exported into the script's environment) — NOT to `cd` (a builtin, where
+ * it would be discarded). The script reads it via `${RETAIN:-7}`.
+ *
+ * Invoking via `bash backup/compose-backup.sh` (relative to remoteDir, which
+ * we cd into) means the script's exec bit is irrelevant — same lesson as
+ * wal-archive.sh. The bundle ships it to `${remoteDir}/backup/`.
+ *
+ * @param {string} remoteDir  Absolute path to the project directory on the VPS
+ *                            (e.g. /opt/myapp)
+ * @param {number} [retain=7]  Number of full base backups to keep. Validated
+ *                             to a positive integer (injection-safe + sane).
+ * @returns {string}
  */
-export function backupCompose(ip, sshKeyPath, projectName, options = {}) {
-  const { clean = false } = options;
-  const remoteDir = `/opt/${projectName}`;
-  const timestamp = new Date().toISOString().replace(/T/, '_').replace(/[:.]/g, '').slice(0, 15);
-  const archiveName = `${projectName}_${timestamp}_full.tar.gz`;
-  const cleanFlags = clean ? '--clean --if-exists' : '';
-
-  // Step 1: Dump postgres database
-  const t1 = perfTimer('backup.pgDump.main');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir} && docker compose exec -T db pg_dump -U postgres ${cleanFlags} postgres | gzip > backups/_postgres.sql.gz`,
-    { timeout: 300_000 },
-  );
-  t1.end();
-
-  // Step 2: Dump optional service databases (n8n, metabase) if they exist
-  const t2 = perfTimer('backup.pgDump.features');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir} && for db in n8n metabase; do docker compose exec -T db psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='\${db}'" 2>/dev/null | grep -q 1 && docker compose exec -T db pg_dump -U postgres "\${db}" | gzip > "backups/_\${db}.sql.gz" || true; done`,
-    { timeout: 300_000 },
-  );
-  t2.end();
-
-  // Step 3: Backup storage files if using file backend
-  const t3 = perfTimer('backup.storageFiles');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir} && FCOUNT=$(docker compose exec -T storage sh -c 'find /var/lib/storage -type f 2>/dev/null | wc -l' 2>/dev/null || echo 0) && [ "$FCOUNT" -gt 0 ] 2>/dev/null && docker compose exec -T storage tar czf - -C / var/lib/storage > backups/_storage.tar.gz || true`,
-    { timeout: 300_000 },
-  );
-  t3.end();
-
-  // Step 4: Create combined archive from work files (prefixed with _)
-  const t4 = perfTimer('backup.combineArchive');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir}/backups && tar czf ${archiveName} $(ls _postgres.sql.gz _n8n.sql.gz _metabase.sql.gz _storage.tar.gz 2>/dev/null) && rm -f _postgres.sql.gz _n8n.sql.gz _metabase.sql.gz _storage.tar.gz`,
-    { timeout: 120_000 },
-  );
-  t4.end();
-
-  return archiveName;
+export function composeBackupCmd(remoteDir, retain = 7) {
+  const r = Number.isInteger(retain) && retain > 0 ? retain : 7;
+  return `cd ${remoteDir} && RETAIN=${r} bash backup/compose-backup.sh`;
 }
 
 /**
- * Upload a compose backup to S3.
+ * Trigger a wal-g base backup on a Compose VPS via compose-backup.sh.
+ *
+ * wal-g pushes the backup directly to S3 from inside the db container —
+ * there is no local archive to scp or upload. Success = exit 0.
  *
- * Downloads the archive from the VPS to a local temp file via SCP, then uploads
- * to S3 using the Node.js SDK. This replaces the previous approach of installing
- * awscli on the VPS (which failed silently on many server images).
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {string} projectName
+ * @param {object} [options]
+ * @param {number} [options.retain]  Full base backups to keep (default 7).
  */
-export async function uploadComposeBackupToS3(
-  ip,
-  sshKeyPath,
-  projectName,
-  archiveName,
-  backupS3Config,
-) {
-  const localTmpPath = join(tmpdir(), `vc-backup-${archiveName}`);
-  try {
-    await perfAsync('backup.scpFromServer', async () =>
-      scpDownload(ip, sshKeyPath, `/opt/${projectName}/backups/${archiveName}`, localTmpPath),
-    );
-    await perfAsync('backup.uploadS3', async () =>
-      uploadS3Backup(backupS3Config, localTmpPath, `backups/${archiveName}`),
-    );
-  } finally {
-    try {
-      unlinkSync(localTmpPath);
-    } catch {
-      // Ignore cleanup errors
-    }
-  }
+export function backupCompose(ip, sshKeyPath, projectName, options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  const t = perfTimer('backup.walgPush');
+  sshRun(ip, sshKeyPath, composeBackupCmd(remoteDir, options.retain ?? 7), { timeout: 900_000 });
+  t.end();
 }
 
 /**
- * Set up automated backup cron on the VPS.
+ * Set up the automated wal-g backup cron on the VPS.
+ *
+ * Installs a crontab entry that runs compose-backup.sh on the configured
+ * schedule — the SAME builder the on-demand path uses, so there is exactly
+ * one quoting/invocation path. No awscli required — wal-g already has S3
+ * credentials via env vars in docker-compose.yml.
+ *
+ * The cron line contains no single quotes (the builder is plain
+ * `cd … && RETAIN=N bash …`), so it is installed quote-safely by piping the
+ * new crontab through stdin (heredoc) rather than wrapping it in `echo '…'`.
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {string} projectName
+ * @param {object} [backupConfig]
+ * @param {string} [backupConfig.schedule]
+ * @param {number} [backupConfig.retentionDays]
+ * @param {object} [opts]
+ * @param {(ip: string, key: string, script: string, o: object) => unknown} [opts.runScript=sshRunScript]
+ *   Injectable script runner — defaults to the real SSH runner; overridden in unit tests.
  */
-export function setupComposeBackupCron(ip, sshKeyPath, projectName, backupS3Config, backupConfig) {
+export function setupComposeBackupCron(ip, sshKeyPath, projectName, backupConfig, opts = {}) {
+  const { runScript = sshRunScript } = opts;
   const remoteDir = `/opt/${projectName}`;
-  const schedule = backupConfig?.schedule || '0 */6 * * *';
-  const retentionDays = backupConfig?.retentionDays || 30;
-
-  // Build environment file for cron
-  const envLines = [`PROJECT_NAME=${projectName}`, `BACKUP_RETENTION_DAYS=${retentionDays}`];
-  if (backupS3Config) {
-    envLines.push(
-      `S3_BACKUP_BUCKET=${backupS3Config.bucket}`,
-      `S3_ACCESS_KEY=${backupS3Config.accessKey}`,
-      `S3_SECRET_KEY=${backupS3Config.secretKey}`,
-      `S3_ENDPOINT=${backupS3Config.endpoint}`,
-      `S3_REGION=${backupS3Config.region}`,
-    );
-  }
+  const schedule = backupConfig?.schedule || '0 2 * * *';
+  const retain =
+    Number.isInteger(backupConfig?.retentionDays) && backupConfig.retentionDays > 0
+      ? backupConfig.retentionDays
+      : 7;
+
+  const cronLine = `${schedule} ${composeBackupCmd(remoteDir, retain)} >> ${remoteDir}/backups/backup.log 2>&1`;
+
+  // Install quote-safely: build the next crontab on the server (existing minus
+  // any prior compose-backup.sh line, plus our line) and pipe via stdin. The
+  // cron line is fed to `cat` through a heredoc so no shell quoting can mangle
+  // it — there are no single quotes in it to collide with anyway.
+  const installScript = [
+    'set -e',
+    `mkdir -p ${remoteDir}/backups`,
+    'TMP_CRON=$(mktemp)',
+    'crontab -l 2>/dev/null | grep -v \'compose-backup.sh\' > "$TMP_CRON" || true',
+    'cat >> "$TMP_CRON" <<\'VC_CRON_EOF\'',
+    cronLine,
+    'VC_CRON_EOF',
+    'crontab "$TMP_CRON"',
+    'rm -f "$TMP_CRON"',
+  ].join('\n');
+
+  runScript(ip, sshKeyPath, installScript, { timeout: 60_000 });
+}
 
-  // Install awscli for automated S3 uploads (fail loudly if it can't be installed)
-  if (backupS3Config) {
-    sshRun(
-      ip,
-      sshKeyPath,
-      'command -v aws >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y awscli || pip3 install awscli)',
-      { timeout: 120_000 },
+// ISO-8601 datetime: YYYY-MM-DDTHH:MM:SS followed by Z or ±HH:MM
+const ISO_DATETIME_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$/;
+
+/**
+ * Build the shell script that fetches a wal-g base backup and writes the
+ * archive-recovery configuration so postgres replays WAL from S3 and
+ * promotes to read-write on completion (or at a PITR target).
+ *
+ * This is a pure function (no I/O) so it can be unit-tested without SSH.
+ * The script is SCP'd to the server and bind-mounted into the db container
+ * via `docker compose run --rm -v …:/restore.sh:ro db bash /restore.sh`,
+ * which avoids the quoting collision that would arise from embedding
+ * single-quoted postgres config values in a deeply-nested ssh command.
+ *
+ * Mirrors the k8s walg-restore init container in carbon/k8s/values/supabase.values.yaml.
+ *
+ * @param {'latest' | string} target  'latest' for the most-recent base backup,
+ *   or an ISO-8601 datetime string for point-in-time recovery.
+ * @returns {string} The bash script body.
+ */
+export function composeRestoreScript(target) {
+  if (target !== 'latest' && !ISO_DATETIME_RE.test(target)) {
+    throw new Error(
+      `Invalid target ${JSON.stringify(target)}: must be "latest" or an ISO-8601 datetime (e.g. 2026-05-31T12:00:00Z)`,
     );
   }
 
-  // Write env file and install cron
-  const envContent = envLines.join('\\n');
-  sshRun(
-    ip,
-    sshKeyPath,
-    [
-      `printf '${envContent}\\n' > ${remoteDir}/backups/.backup-env`,
-      `chmod 600 ${remoteDir}/backups/.backup-env`,
-      `chmod +x ${remoteDir}/backup/compose-backup.sh`,
-      `(crontab -l 2>/dev/null | grep -v 'compose-backup.sh' ; echo '${schedule} set -a && . ${remoteDir}/backups/.backup-env && set +a && ${remoteDir}/backup/compose-backup.sh >> ${remoteDir}/backups/backup.log 2>&1') | crontab -`,
-    ].join(' && '),
-    { timeout: 120_000 },
-  );
+  // pitrLine goes inside the { ... } block — no separate redirect needed
+  const pitrLine = target !== 'latest' ? `  echo "recovery_target_time = '${target}'"` : null;
+
+  return [
+    '#!/bin/bash',
+    'set -euo pipefail',
+    'PGDATA=/var/lib/postgresql/data',
+    '',
+    '# Empty PGDATA — wal-g backup-fetch requires an empty target dir. We are',
+    '# inside the db container, which mounts the real <project>_db_data volume',
+    '# at $PGDATA, so this clears the correct data (unlike a `docker compose run',
+    "# -v db_data:/data` flag, which bypasses Compose's project-volume naming).",
+    'if [ -f "$PGDATA/PG_VERSION" ]; then',
+    // biome-ignore lint/suspicious/noTemplateCurlyInString: ${PGDATA:?} is a deliberate bash "fail if unset" guard, not a JS template placeholder
+    '  rm -rf "${PGDATA:?}/"* "${PGDATA:?}/".* 2>/dev/null || true',
+    'fi',
+    '',
+    '# Fetch the base backup from S3 (wal-g reads S3 credentials from container env)',
+    `wal-g backup-fetch "$PGDATA" LATEST`,
+    '',
+    '# Remove any recovery settings left in postgresql.auto.conf by a previous',
+    "# restore so they don't accumulate across cycles (postgres takes the last",
+    '# value, but stale duplicates are confusing).',
+    'if [ -f "$PGDATA/postgresql.auto.conf" ]; then',
+    '  sed -i "/^restore_command =/d; /^recovery_target_action =/d; /^recovery_target_time =/d; /^recovery_target_timeline =/d" "$PGDATA/postgresql.auto.conf"',
+    'fi',
+    '',
+    '# Write archive-recovery config so postgres replays WAL segments from S3',
+    '# and promotes to read-write on end-of-WAL (or at the PITR target).',
+    '# Uses >> so any existing postgresql.auto.conf settings are preserved.',
+    '{',
+    `  echo "restore_command = 'wal-g wal-fetch \\"%f\\" \\"%p\\"'"`,
+    `  echo "recovery_target_action = 'promote'"`,
+    // Pin recovery to the FETCHED base backup's own timeline. Default is
+    // 'latest', which makes postgres chase the newest timeline it can find a
+    // .history file for. In HA, repeated restore→promote cycles accumulate
+    // DIVERGENT timelines in the shared wal-g S3 prefix, and 'latest' picks one
+    // that forked off BEFORE this base backup's checkpoint — postgres then
+    // crash-loops "requested timeline N is not a child of this server's history"
+    // / wal-g "Archive '0000000N.history' does not exist". 'current' recovers
+    // along the base backup's timeline to end-of-WAL, then promotes fresh.
+    // (RCA 2026-06-01: compose-ha kept-rig restore.)
+    `  echo "recovery_target_timeline = 'current'"`,
+    pitrLine,
+    '} >> "$PGDATA/postgresql.auto.conf"',
+    '',
+    '# Signal file that tells postgres to enter archive recovery mode (PG 12+)',
+    'touch "$PGDATA/recovery.signal"',
+  ]
+    .filter((line) => line !== null)
+    .join('\n');
 }
 
 /**
- * Restore from a backup archive via Docker Compose.
- * Supports legacy .tar.gz (pg_dump) and new WAL-G point-in-time recovery.
+ * Restore from a wal-g base backup via Docker Compose with full archive
+ * recovery. Postgres replays WAL segments from S3 via restore_command and
+ * promotes to read-write on reaching end-of-WAL or the PITR target.
+ *
+ * S3 credentials are read from the db container's environment (sourced from
+ * the project .env file) — DO NOT pass them as -e overrides; the container
+ * already has WALG_S3_PREFIX + AWS_* set by carbon/docker-compose.yml.
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {string} projectName
+ * @param {'latest' | string} [target='latest']  'latest' or ISO-8601 PITR timestamp.
  */
-export async function restoreCompose(ip, sshKeyPath, projectName, backupFile, options = {}) {
+export async function restoreCompose(ip, sshKeyPath, projectName, target = 'latest') {
   const remoteDir = `/opt/${projectName}`;
-  const isWalg = backupFile === 'latest' || backupFile === 'walg';
 
-  // Stop the app container to prevent connections
+  // Build and validate the restore script early — fail before touching the
+  // running server if the target is malformed.
+  const scriptBody = composeRestoreScript(target);
+
+  // 1. Stop app to prevent connections during restore
   await sshRunAsync(
     ip,
     sshKeyPath,
     `cd ${remoteDir} && docker compose stop app 2>/dev/null || true`,
   );
 
-  if (isWalg) {
-    // --- BLAZING FAST WAL-G RESTORE ---
-    // 1. Stop DB
-    await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose stop db`);
-    // 2. Clear data (WAL-G restore-fetch requires empty dir)
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `cd ${remoteDir} && docker compose run --rm -v db_data:/data alpine sh -c "rm -rf /data/*"`,
-    );
-    // 3. Run WAL-G restore
-    const restoreCmd = `cd ${remoteDir} && docker compose run --rm \
-      -e WALG_S3_PREFIX="${options.walgS3Prefix}" \
-      -e AWS_ACCESS_KEY_ID="${options.s3AccessKey}" \
-      -e AWS_SECRET_ACCESS_KEY="${options.s3SecretKey}" \
-      -e AWS_ENDPOINT="${options.s3Endpoint}" \
-      -e AWS_REGION="${options.s3Region}" \
-      db wal-g backup-fetch /var/lib/postgresql/data LATEST`;
-    await sshRunAsync(ip, sshKeyPath, restoreCmd, { timeout: 900_000 });
-    // 4. Start DB
-    await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start db`);
-  } else {
-    // --- LEGACY PG_DUMP RESTORE ---
-    // Extract archive
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `mkdir -p /tmp/vb-restore && tar --no-xattrs --no-same-owner -xzf ${remoteDir}/backups/${backupFile} -C /tmp/vb-restore`,
-      { timeout: 120_000 },
-    );
+  // 2. Stop the DB. The restore container (step 4) clears PGDATA *itself*
+  //    from inside the db container — where the real ${project}_db_data volume
+  //    is mounted at $PGDATA — so there is no separate volume-clearing step.
+  //    (A `docker compose run --rm -v db_data:/data alpine` would clear a
+  //    literal `db_data` volume, NOT the project-prefixed one Compose uses.)
+  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose stop db`);
+
+  // 3. Write the restore script to the server (heredoc — no quoting risk) and
+  //    bind-mount it into the db container. The script body contains single
+  //    quotes (restore_command = 'wal-g wal-fetch ...') so it MUST go via a
+  //    file rather than a nested bash -c '...' invocation. The script clears
+  //    PGDATA, runs wal-g backup-fetch, and writes the archive-recovery config.
+  sshRunScript(
+    ip,
+    sshKeyPath,
+    [
+      'set -e',
+      `cat > ${remoteDir}/restore-walg.sh <<'VC_RESTORE_EOF'`,
+      scriptBody,
+      'VC_RESTORE_EOF',
+      `chmod +x ${remoteDir}/restore-walg.sh`,
+    ].join('\n'),
+    { timeout: 30_000 },
+  );
 
-    // Restore postgres database
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `cd ${remoteDir} && gunzip -c /tmp/vb-restore/postgres.sql.gz | docker compose exec -T db psql -U postgres -d postgres`,
-      { timeout: 300_000 },
-    );
+  await sshRunAsync(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose run --rm -v ${remoteDir}/restore-walg.sh:/restore.sh:ro db bash /restore.sh`,
+    { timeout: 900_000 },
+  );
+
+  // 4. Start the DB — postgres enters archive recovery, replays WAL, promotes
+  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start db`);
 
-    // Restore optional service databases if present
-    for (const db of ['n8n', 'metabase']) {
+  // 5. Poll until postgres has promoted (pg_isready + pg_is_in_recovery() = f).
+  //    WAL replay can take 1-2 minutes; deadline is 300s.
+  const deadline = Date.now() + 300_000;
+  let lastErr = '';
+  for (;;) {
+    try {
+      // First check: postgres accepting connections
       await sshRunAsync(
         ip,
         sshKeyPath,
-        `test -f /tmp/vb-restore/${db}.sql.gz && cd ${remoteDir} && gunzip -c /tmp/vb-restore/${db}.sql.gz | docker compose exec -T db psql -U postgres -d ${db} || true`,
-        { timeout: 300_000 },
+        `cd ${remoteDir} && docker compose exec -T db pg_isready -U postgres`,
+        { timeout: 10_000 },
       );
+      // Second check: promoted (no longer in recovery)
+      const recovering = await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db psql -U postgres -At -c 'SELECT pg_is_in_recovery()'`,
+        { timeout: 10_000 },
+      );
+      if (String(recovering).trim() === 'f') {
+        break; // promoted — restore complete
+      }
+      lastErr = 'pg_is_in_recovery() still true';
+    } catch (err) {
+      lastErr = err instanceof Error ? err.message : String(err);
     }
-
-    // Restore storage files if present
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `test -f /tmp/vb-restore/storage.tar.gz && cd ${remoteDir} && docker compose exec -T storage sh -c 'tar --no-xattrs --no-same-owner -xzf - -C /' < /tmp/vb-restore/storage.tar.gz || true`,
-      { timeout: 300_000 },
-    );
-
-    // Cleanup
-    await sshRunAsync(ip, sshKeyPath, 'rm -rf /tmp/vb-restore');
+    if (Date.now() >= deadline) {
+      throw new Error(
+        `restoreCompose: postgres did not promote within 300s after wal-g restore. ` +
+          `Last error: ${lastErr}`,
+      );
+    }
+    await new Promise((r) => setTimeout(r, 5_000));
   }
 
-  // Restart app
+  // 6. Bring the app back up
   await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start app`);
 }