vibecarbon 0.3.1 → 0.5.0

package/src/lib/deploy/compose/index.js

@@ -11,16 +11,16 @@
 
 import { spawn } from 'node:child_process';
 import { randomBytes } from 'node:crypto';
-import { existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'node:fs';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import * as p from '@clack/prompts';
-import { uploadS3Backup } from '../../backup-s3.js';
 import { runCommand, runCommandAsync } from '../../command.js';
 import { perfAsync, perfTimer } from '../../perf.js';
-import { shEscape } from '../../shell.js';
-import { scpDownload } from '../../ssh.js';
+import { parseDotenv, shEscape } from '../../shell.js';
+import { NO_PIN_SSH_OPTS, sshRunScript } from '../../ssh.js';
+import { postAdminUser, waitForGotrueHealth } from '../admin-user.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -41,18 +41,20 @@ export function loadCloudInitScript() {
   return readFileSync(CLOUD_INIT_PATH, 'utf-8');
 }
 
-// ServerAliveInterval+CountMax: force ssh to give up after 60s of no
-// SSH-protocol traffic, so a freshly-provisioned VPS that accepts TCP on 22
-// but never sends the SSH banner ("Connection timed out during banner
-// exchange") fails fast instead of hanging the caller indefinitely. Same
-// fix applied in lib/ssh.js — keep them in lockstep.
-const SSH_OPTS =
-  '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=4';
+// Single-sourced from lib/ssh.js (sshHostKeyOpts) so the string form used by
+// the runners below can't drift from the argv form — notably keeping the
+// ServerAlive keepalives that fail a banner-exchange hang fast instead of
+// stalling for ~600s. Re-exported as the canonical compose deploy SSH opts.
+// shell-safety-ignore: NO_PIN_SSH_OPTS is derived from lib/ssh.js sshHostKeyOpts(), which bakes in BatchMode=yes (validated there)
+const SSH_OPTS = NO_PIN_SSH_OPTS;
 
 /**
- * Run a command on a remote server via SSH
+ * Run a command on a remote server via SSH.
+ *
+ * Shared with compose/ha.js (imported, not re-copied) so both halves of the
+ * compose path use identical, single-sourced SSH options.
  */
-function sshRun(ip, sshKeyPath, command, options = {}) {
+export function sshRun(ip, sshKeyPath, command, options = {}) {
   const { timeout = 120_000 } = options;
   return runCommand(['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command], {
     stdio: 'pipe',
@@ -700,16 +702,6 @@ export async function verifyAppHealth(ip, sshKeyPath, projectName, options = {})
   return { healthy: false, status: lastStatus, details };
 }
 
-/**
- * Stop the Docker Compose stack on the remote server
- */
-export async function stopComposeStack(ip, sshKeyPath, projectName, options = {}) {
-  const remoteDir = `/opt/${projectName}`;
-  const flags = composeFileFlags(options);
-  const composeCmd = `cd ${remoteDir} && docker compose ${flags} down --timeout 30`;
-  await sshRunAsync(ip, sshKeyPath, composeCmd, { timeout: 120_000 });
-}
-
 /**
  * Pre-pull container images on the remote server.
  * Run this in parallel with the local Docker build to overlap I/O.
@@ -803,9 +795,8 @@ export async function runMigrations(ip, sshKeyPath, projectName) {
   // 2026-05-26. Use the canonical SQL NOTIFY on the pgrst channel instead
   // (db-channel-enabled defaults on); fall back to restarting rest if NOTIFY
   // doesn't land. Best-effort: a missed reload self-heals on the next deploy.
-  // Lives here (not just in deployCompose) so every caller — deployCompose,
-  // deployComposeHA, and the orchestrator's inline single-compose path — gets
-  // a queryable schema instead of PGRST205.
+  // Lives here so every caller — deployComposeHA and the orchestrator's inline
+  // single-compose path — gets a queryable schema instead of PGRST205.
   await sshRunAsync(
     ip,
     sshKeyPath,
@@ -815,153 +806,6 @@ export async function runMigrations(ip, sshKeyPath, projectName) {
   );
 }
 
-/**
- * Main Compose deployment orchestrator
- *
- * @param {object} options
- * @param {string} options.projectName - Project name
- * @param {string} options.environment - Environment name (prod, staging, etc.)
- * @param {string} options.ip - Server IP address
- * @param {string} options.sshKeyPath - Path to SSH private key
- * @param {string} [options.domain] - Custom domain
- * @param {boolean} [options.isNewServer] - Whether server was just provisioned
- * @param {object} [options.services] - Enabled services (observability, n8n, etc.)
- * @param {Function} [options.onProgress] - Progress callback
- * @returns {object} Deployment result
- */
-export async function deployCompose(options) {
-  const {
-    projectName,
-    ip,
-    sshKeyPath,
-    domain,
-    isNewServer = false,
-    services = {},
-    imageRef,
-    onProgress = () => {},
-  } = options;
-
-  if (!imageRef) {
-    throw new Error('deployCompose requires options.imageRef (ghcr.io/<owner>/<repo>:<tag>)');
-  }
-
-  // Step 1: Install Docker on new servers
-  if (isNewServer) {
-    onProgress('Configuring server...');
-    await setupServer(ip, sshKeyPath);
-  }
-
-  const serviceOpts = {
-    observability: services.observability,
-    n8n: services.n8n,
-    metabase: services.metabase,
-    redis: services.redis,
-  };
-
-  // Step 2: Copy project files + point APP_IMAGE at the CI-built ghcr.io tag
-  onProgress('Copying project files to server...');
-  await setupServerFiles(ip, sshKeyPath, projectName, {
-    domain,
-    image: imageRef,
-    ...serviceOpts,
-  });
-
-  // Step 3: Pull all images (app + Supabase) — docker compose pull handles it.
-  onProgress('Pulling images...');
-  await pullComposeImages(ip, sshKeyPath, projectName, serviceOpts);
-
-  // Step 4: Start Docker Compose stack
-  onProgress('Starting services...');
-  await startComposeStack(ip, sshKeyPath, projectName, serviceOpts);
-
-  // Step 5: Run migrations. A failure here means an empty/partial schema, which
-  // makes every DB-backed feature 500 — so let it abort the deploy rather than
-  // swallowing it (RCA prod-1 2026-05-26: the previous blanket try/catch hid a
-  // fully-failed migration and shipped a schema-less prod that reported success).
-  onProgress('Running database migrations...');
-  // runMigrations applies supabase/migrations/* AND reloads PostgREST's schema
-  // cache, so the new tables are immediately queryable (no PGRST205).
-  await runMigrations(ip, sshKeyPath, projectName);
-
-  return {
-    success: true,
-    ip,
-    domain,
-    deployMode: 'compose',
-  };
-}
-
-/**
- * Fast re-deploy: sync new config and reconcile.
- */
-export async function redeployCompose(options) {
-  const { projectName, ip, sshKeyPath, services = {}, imageRef, onProgress = () => {} } = options;
-
-  if (!imageRef) {
-    throw new Error('redeployCompose requires options.imageRef');
-  }
-
-  // Step 1: Sync new files (including .env with new image tag)
-  onProgress('Syncing configuration...');
-  await setupServerFiles(ip, sshKeyPath, projectName, {
-    image: imageRef,
-    observability: services.observability,
-    n8n: services.n8n,
-    metabase: services.metabase,
-    redis: services.redis,
-    domain: options.domain,
-  });
-
-  // Step 2: Reconcile (pulls images if needed and restarts)
-  onProgress('Reconciling services...');
-  await startComposeStack(ip, sshKeyPath, projectName, {
-    observability: services.observability,
-    n8n: services.n8n,
-    metabase: services.metabase,
-    redis: services.redis,
-  });
-
-  return { success: true };
-}
-
-/**
- * Get status of Docker Compose services on the remote server
- */
-export function getComposeStatus(ip, sshKeyPath, projectName) {
-  const remoteDir = `/opt/${projectName}`;
-
-  try {
-    const output = sshRun(ip, sshKeyPath, `cd ${remoteDir} && docker compose ps --format json`, {
-      timeout: 15_000,
-    });
-
-    if (!output?.trim()) {
-      return { running: false, services: [] };
-    }
-
-    // docker compose ps --format json outputs one JSON object per line
-    const services = output
-      .trim()
-      .split('\n')
-      .filter(Boolean)
-      .map((line) => {
-        try {
-          return JSON.parse(line);
-        } catch {
-          return null;
-        }
-      })
-      .filter(Boolean);
-
-    return {
-      running: services.some((s) => s.State === 'running'),
-      services,
-    };
-  } catch {
-    return { running: false, services: [] };
-  }
-}
-
 /**
  * Destroy a Docker Compose deployment
  */
@@ -980,248 +824,284 @@ export function destroyCompose(ip, sshKeyPath, projectName) {
 }
 
 /**
- * Create a full backup via Docker Compose.
- * Produces a .tar.gz archive containing:
- *   - postgres.sql.gz (main Supabase database, always)
- *   - n8n.sql.gz (if n8n database exists)
- *   - metabase.sql.gz (if metabase database exists)
- *   - storage.tar.gz (if file-backend storage files exist)
+ * Build the shell command that runs the wal-g base backup on a Compose VPS.
+ *
+ * `carbon/backup/compose-backup.sh` is the SINGLE source of truth for the
+ * backup logic (guard + PGUSER=supabase_admin + wal-g backup-push + delete
+ * retain, plus a flock and set -e). This builder is a thin, quote-safe
+ * wrapper so the on-demand path and the cron path run byte-identical commands.
+ *
+ * The `RETAIN=<n>` prefix binds to `bash` (a real command, so the assignment
+ * is exported into the script's environment) — NOT to `cd` (a builtin, where
+ * it would be discarded). The script reads it via `${RETAIN:-7}`.
+ *
+ * Invoking via `bash backup/compose-backup.sh` (relative to remoteDir, which
+ * we cd into) means the script's exec bit is irrelevant — same lesson as
+ * wal-archive.sh. The bundle ships it to `${remoteDir}/backup/`.
+ *
+ * @param {string} remoteDir  Absolute path to the project directory on the VPS
+ *                            (e.g. /opt/myapp)
+ * @param {number} [retain=7]  Number of full base backups to keep. Validated
+ *                             to a positive integer (injection-safe + sane).
+ * @returns {string}
  */
-export function backupCompose(ip, sshKeyPath, projectName, options = {}) {
-  const { clean = false } = options;
-  const remoteDir = `/opt/${projectName}`;
-  const timestamp = new Date().toISOString().replace(/T/, '_').replace(/[:.]/g, '').slice(0, 15);
-  const archiveName = `${projectName}_${timestamp}_full.tar.gz`;
-  const cleanFlags = clean ? '--clean --if-exists' : '';
-
-  // Step 1: Dump postgres database
-  const t1 = perfTimer('backup.pgDump.main');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir} && docker compose exec -T db pg_dump -U postgres ${cleanFlags} postgres | gzip > backups/_postgres.sql.gz`,
-    { timeout: 300_000 },
-  );
-  t1.end();
-
-  // Step 2: Dump optional service databases (n8n, metabase) if they exist
-  const t2 = perfTimer('backup.pgDump.features');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir} && for db in n8n metabase; do docker compose exec -T db psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='\${db}'" 2>/dev/null | grep -q 1 && docker compose exec -T db pg_dump -U postgres "\${db}" | gzip > "backups/_\${db}.sql.gz" || true; done`,
-    { timeout: 300_000 },
-  );
-  t2.end();
-
-  // Step 3: Backup storage files if using file backend
-  const t3 = perfTimer('backup.storageFiles');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir} && FCOUNT=$(docker compose exec -T storage sh -c 'find /var/lib/storage -type f 2>/dev/null | wc -l' 2>/dev/null || echo 0) && [ "$FCOUNT" -gt 0 ] 2>/dev/null && docker compose exec -T storage tar czf - -C / var/lib/storage > backups/_storage.tar.gz || true`,
-    { timeout: 300_000 },
-  );
-  t3.end();
-
-  // Step 4: Create combined archive from work files (prefixed with _)
-  const t4 = perfTimer('backup.combineArchive');
-  sshRun(
-    ip,
-    sshKeyPath,
-    `cd ${remoteDir}/backups && tar czf ${archiveName} $(ls _postgres.sql.gz _n8n.sql.gz _metabase.sql.gz _storage.tar.gz 2>/dev/null) && rm -f _postgres.sql.gz _n8n.sql.gz _metabase.sql.gz _storage.tar.gz`,
-    { timeout: 120_000 },
-  );
-  t4.end();
-
-  return archiveName;
+export function composeBackupCmd(remoteDir, retain = 7) {
+  const r = Number.isInteger(retain) && retain > 0 ? retain : 7;
+  return `cd ${remoteDir} && RETAIN=${r} bash backup/compose-backup.sh`;
 }
 
 /**
- * Upload a compose backup to S3.
+ * Trigger a wal-g base backup on a Compose VPS via compose-backup.sh.
+ *
+ * wal-g pushes the backup directly to S3 from inside the db container —
+ * there is no local archive to scp or upload. Success = exit 0.
  *
- * Downloads the archive from the VPS to a local temp file via SCP, then uploads
- * to S3 using the Node.js SDK. This replaces the previous approach of installing
- * awscli on the VPS (which failed silently on many server images).
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {string} projectName
+ * @param {object} [options]
+ * @param {number} [options.retain]  Full base backups to keep (default 7).
  */
-export async function uploadComposeBackupToS3(
-  ip,
-  sshKeyPath,
-  projectName,
-  archiveName,
-  backupS3Config,
-) {
-  const localTmpPath = join(tmpdir(), `vc-backup-${archiveName}`);
-  try {
-    await perfAsync('backup.scpFromServer', async () =>
-      scpDownload(ip, sshKeyPath, `/opt/${projectName}/backups/${archiveName}`, localTmpPath),
-    );
-    await perfAsync('backup.uploadS3', async () =>
-      uploadS3Backup(backupS3Config, localTmpPath, `backups/${archiveName}`),
-    );
-  } finally {
-    try {
-      unlinkSync(localTmpPath);
-    } catch {
-      // Ignore cleanup errors
-    }
-  }
+export function backupCompose(ip, sshKeyPath, projectName, options = {}) {
+  const remoteDir = `/opt/${projectName}`;
+  const t = perfTimer('backup.walgPush');
+  sshRun(ip, sshKeyPath, composeBackupCmd(remoteDir, options.retain ?? 7), { timeout: 900_000 });
+  t.end();
 }
 
 /**
- * Set up automated backup cron on the VPS.
+ * Set up the automated wal-g backup cron on the VPS.
+ *
+ * Installs a crontab entry that runs compose-backup.sh on the configured
+ * schedule — the SAME builder the on-demand path uses, so there is exactly
+ * one quoting/invocation path. No awscli required — wal-g already has S3
+ * credentials via env vars in docker-compose.yml.
+ *
+ * The cron line contains no single quotes (the builder is plain
+ * `cd … && RETAIN=N bash …`), so it is installed quote-safely by piping the
+ * new crontab through stdin (heredoc) rather than wrapping it in `echo '…'`.
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {string} projectName
+ * @param {object} [backupConfig]
+ * @param {string} [backupConfig.schedule]
+ * @param {number} [backupConfig.retentionDays]
+ * @param {object} [opts]
+ * @param {(ip: string, key: string, script: string, o: object) => unknown} [opts.runScript=sshRunScript]
+ *   Injectable script runner — defaults to the real SSH runner; overridden in unit tests.
  */
-export function setupComposeBackupCron(ip, sshKeyPath, projectName, backupS3Config, backupConfig) {
+export function setupComposeBackupCron(ip, sshKeyPath, projectName, backupConfig, opts = {}) {
+  const { runScript = sshRunScript } = opts;
   const remoteDir = `/opt/${projectName}`;
-  const schedule = backupConfig?.schedule || '0 */6 * * *';
-  const retentionDays = backupConfig?.retentionDays || 30;
-
-  // Build environment file for cron
-  const envLines = [`PROJECT_NAME=${projectName}`, `BACKUP_RETENTION_DAYS=${retentionDays}`];
-  if (backupS3Config) {
-    envLines.push(
-      `S3_BACKUP_BUCKET=${backupS3Config.bucket}`,
-      `S3_ACCESS_KEY=${backupS3Config.accessKey}`,
-      `S3_SECRET_KEY=${backupS3Config.secretKey}`,
-      `S3_ENDPOINT=${backupS3Config.endpoint}`,
-      `S3_REGION=${backupS3Config.region}`,
-    );
-  }
+  const schedule = backupConfig?.schedule || '0 2 * * *';
+  const retain =
+    Number.isInteger(backupConfig?.retentionDays) && backupConfig.retentionDays > 0
+      ? backupConfig.retentionDays
+      : 7;
+
+  const cronLine = `${schedule} ${composeBackupCmd(remoteDir, retain)} >> ${remoteDir}/backups/backup.log 2>&1`;
+
+  // Install quote-safely: build the next crontab on the server (existing minus
+  // any prior compose-backup.sh line, plus our line) and pipe via stdin. The
+  // cron line is fed to `cat` through a heredoc so no shell quoting can mangle
+  // it — there are no single quotes in it to collide with anyway.
+  const installScript = [
+    'set -e',
+    `mkdir -p ${remoteDir}/backups`,
+    'TMP_CRON=$(mktemp)',
+    'crontab -l 2>/dev/null | grep -v \'compose-backup.sh\' > "$TMP_CRON" || true',
+    'cat >> "$TMP_CRON" <<\'VC_CRON_EOF\'',
+    cronLine,
+    'VC_CRON_EOF',
+    'crontab "$TMP_CRON"',
+    'rm -f "$TMP_CRON"',
+  ].join('\n');
+
+  runScript(ip, sshKeyPath, installScript, { timeout: 60_000 });
+}
 
-  // Install awscli for automated S3 uploads (fail loudly if it can't be installed)
-  if (backupS3Config) {
-    sshRun(
-      ip,
-      sshKeyPath,
-      'command -v aws >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y awscli || pip3 install awscli)',
-      { timeout: 120_000 },
+// ISO-8601 datetime: YYYY-MM-DDTHH:MM:SS followed by Z or ±HH:MM
+const ISO_DATETIME_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|[+-]\d{2}:\d{2})$/;
+
+/**
+ * Build the shell script that fetches a wal-g base backup and writes the
+ * archive-recovery configuration so postgres replays WAL from S3 and
+ * promotes to read-write on completion (or at a PITR target).
+ *
+ * This is a pure function (no I/O) so it can be unit-tested without SSH.
+ * The script is SCP'd to the server and bind-mounted into the db container
+ * via `docker compose run --rm -v …:/restore.sh:ro db bash /restore.sh`,
+ * which avoids the quoting collision that would arise from embedding
+ * single-quoted postgres config values in a deeply-nested ssh command.
+ *
+ * Mirrors the k8s walg-restore init container in carbon/k8s/values/supabase.values.yaml.
+ *
+ * @param {'latest' | string} target  'latest' for the most-recent base backup,
+ *   or an ISO-8601 datetime string for point-in-time recovery.
+ * @returns {string} The bash script body.
+ */
+export function composeRestoreScript(target) {
+  if (target !== 'latest' && !ISO_DATETIME_RE.test(target)) {
+    throw new Error(
+      `Invalid target ${JSON.stringify(target)}: must be "latest" or an ISO-8601 datetime (e.g. 2026-05-31T12:00:00Z)`,
     );
   }
 
-  // Write env file and install cron
-  const envContent = envLines.join('\\n');
-  sshRun(
-    ip,
-    sshKeyPath,
-    [
-      `printf '${envContent}\\n' > ${remoteDir}/backups/.backup-env`,
-      `chmod 600 ${remoteDir}/backups/.backup-env`,
-      `chmod +x ${remoteDir}/backup/compose-backup.sh`,
-      `(crontab -l 2>/dev/null | grep -v 'compose-backup.sh' ; echo '${schedule} set -a && . ${remoteDir}/backups/.backup-env && set +a && ${remoteDir}/backup/compose-backup.sh >> ${remoteDir}/backups/backup.log 2>&1') | crontab -`,
-    ].join(' && '),
-    { timeout: 120_000 },
-  );
+  // pitrLine goes inside the { ... } block — no separate redirect needed
+  const pitrLine = target !== 'latest' ? `  echo "recovery_target_time = '${target}'"` : null;
+
+  return [
+    '#!/bin/bash',
+    'set -euo pipefail',
+    'PGDATA=/var/lib/postgresql/data',
+    '',
+    '# Empty PGDATA — wal-g backup-fetch requires an empty target dir. We are',
+    '# inside the db container, which mounts the real <project>_db_data volume',
+    '# at $PGDATA, so this clears the correct data (unlike a `docker compose run',
+    "# -v db_data:/data` flag, which bypasses Compose's project-volume naming).",
+    'if [ -f "$PGDATA/PG_VERSION" ]; then',
+    // biome-ignore lint/suspicious/noTemplateCurlyInString: ${PGDATA:?} is a deliberate bash "fail if unset" guard, not a JS template placeholder
+    '  rm -rf "${PGDATA:?}/"* "${PGDATA:?}/".* 2>/dev/null || true',
+    'fi',
+    '',
+    '# Fetch the base backup from S3 (wal-g reads S3 credentials from container env)',
+    `wal-g backup-fetch "$PGDATA" LATEST`,
+    '',
+    '# Remove any recovery settings left in postgresql.auto.conf by a previous',
+    "# restore so they don't accumulate across cycles (postgres takes the last",
+    '# value, but stale duplicates are confusing).',
+    'if [ -f "$PGDATA/postgresql.auto.conf" ]; then',
+    '  sed -i "/^restore_command =/d; /^recovery_target_action =/d; /^recovery_target_time =/d; /^recovery_target_timeline =/d" "$PGDATA/postgresql.auto.conf"',
+    'fi',
+    '',
+    '# Write archive-recovery config so postgres replays WAL segments from S3',
+    '# and promotes to read-write on end-of-WAL (or at the PITR target).',
+    '# Uses >> so any existing postgresql.auto.conf settings are preserved.',
+    '{',
+    `  echo "restore_command = 'wal-g wal-fetch \\"%f\\" \\"%p\\"'"`,
+    `  echo "recovery_target_action = 'promote'"`,
+    // Pin recovery to the FETCHED base backup's own timeline. Default is
+    // 'latest', which makes postgres chase the newest timeline it can find a
+    // .history file for. In HA, repeated restore→promote cycles accumulate
+    // DIVERGENT timelines in the shared wal-g S3 prefix, and 'latest' picks one
+    // that forked off BEFORE this base backup's checkpoint — postgres then
+    // crash-loops "requested timeline N is not a child of this server's history"
+    // / wal-g "Archive '0000000N.history' does not exist". 'current' recovers
+    // along the base backup's timeline to end-of-WAL, then promotes fresh.
+    // (RCA 2026-06-01: compose-ha kept-rig restore.)
+    `  echo "recovery_target_timeline = 'current'"`,
+    pitrLine,
+    '} >> "$PGDATA/postgresql.auto.conf"',
+    '',
+    '# Signal file that tells postgres to enter archive recovery mode (PG 12+)',
+    'touch "$PGDATA/recovery.signal"',
+  ]
+    .filter((line) => line !== null)
+    .join('\n');
 }
 
 /**
- * Restore from a backup archive via Docker Compose.
- * Supports legacy .tar.gz (pg_dump) and new WAL-G point-in-time recovery.
+ * Restore from a wal-g base backup via Docker Compose with full archive
+ * recovery. Postgres replays WAL segments from S3 via restore_command and
+ * promotes to read-write on reaching end-of-WAL or the PITR target.
+ *
+ * S3 credentials are read from the db container's environment (sourced from
+ * the project .env file) — DO NOT pass them as -e overrides; the container
+ * already has WALG_S3_PREFIX + AWS_* set by carbon/docker-compose.yml.
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {string} projectName
+ * @param {'latest' | string} [target='latest']  'latest' or ISO-8601 PITR timestamp.
  */
-export async function restoreCompose(ip, sshKeyPath, projectName, backupFile, options = {}) {
+export async function restoreCompose(ip, sshKeyPath, projectName, target = 'latest') {
   const remoteDir = `/opt/${projectName}`;
-  const isWalg = backupFile === 'latest' || backupFile === 'walg';
 
-  // Stop the app container to prevent connections
+  // Build and validate the restore script early — fail before touching the
+  // running server if the target is malformed.
+  const scriptBody = composeRestoreScript(target);
+
+  // 1. Stop app to prevent connections during restore
   await sshRunAsync(
     ip,
     sshKeyPath,
     `cd ${remoteDir} && docker compose stop app 2>/dev/null || true`,
   );
 
-  if (isWalg) {
-    // --- BLAZING FAST WAL-G RESTORE ---
-    // 1. Stop DB
-    await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose stop db`);
-    // 2. Clear data (WAL-G restore-fetch requires empty dir)
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `cd ${remoteDir} && docker compose run --rm -v db_data:/data alpine sh -c "rm -rf /data/*"`,
-    );
-    // 3. Run WAL-G restore
-    const restoreCmd = `cd ${remoteDir} && docker compose run --rm \
-      -e WALG_S3_PREFIX="${options.walgS3Prefix}" \
-      -e AWS_ACCESS_KEY_ID="${options.s3AccessKey}" \
-      -e AWS_SECRET_ACCESS_KEY="${options.s3SecretKey}" \
-      -e AWS_ENDPOINT="${options.s3Endpoint}" \
-      -e AWS_REGION="${options.s3Region}" \
-      db wal-g backup-fetch /var/lib/postgresql/data LATEST`;
-    await sshRunAsync(ip, sshKeyPath, restoreCmd, { timeout: 900_000 });
-    // 4. Start DB
-    await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start db`);
-  } else {
-    // --- LEGACY PG_DUMP RESTORE ---
-    // Extract archive
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `mkdir -p /tmp/vb-restore && tar --no-xattrs --no-same-owner -xzf ${remoteDir}/backups/${backupFile} -C /tmp/vb-restore`,
-      { timeout: 120_000 },
-    );
+  // 2. Stop the DB. The restore container (step 4) clears PGDATA *itself*
+  //    from inside the db container — where the real ${project}_db_data volume
+  //    is mounted at $PGDATA — so there is no separate volume-clearing step.
+  //    (A `docker compose run --rm -v db_data:/data alpine` would clear a
+  //    literal `db_data` volume, NOT the project-prefixed one Compose uses.)
+  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose stop db`);
+
+  // 3. Write the restore script to the server (heredoc — no quoting risk) and
+  //    bind-mount it into the db container. The script body contains single
+  //    quotes (restore_command = 'wal-g wal-fetch ...') so it MUST go via a
+  //    file rather than a nested bash -c '...' invocation. The script clears
+  //    PGDATA, runs wal-g backup-fetch, and writes the archive-recovery config.
+  sshRunScript(
+    ip,
+    sshKeyPath,
+    [
+      'set -e',
+      `cat > ${remoteDir}/restore-walg.sh <<'VC_RESTORE_EOF'`,
+      scriptBody,
+      'VC_RESTORE_EOF',
+      `chmod +x ${remoteDir}/restore-walg.sh`,
+    ].join('\n'),
+    { timeout: 30_000 },
+  );
 
-    // Restore postgres database
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `cd ${remoteDir} && gunzip -c /tmp/vb-restore/postgres.sql.gz | docker compose exec -T db psql -U postgres -d postgres`,
-      { timeout: 300_000 },
-    );
+  await sshRunAsync(
+    ip,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose run --rm -v ${remoteDir}/restore-walg.sh:/restore.sh:ro db bash /restore.sh`,
+    { timeout: 900_000 },
+  );
 
-    // Restore optional service databases if present
-    for (const db of ['n8n', 'metabase']) {
+  // 4. Start the DB — postgres enters archive recovery, replays WAL, promotes
+  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start db`);
+
+  // 5. Poll until postgres has promoted (pg_isready + pg_is_in_recovery() = f).
+  //    WAL replay can take 1-2 minutes; deadline is 300s.
+  const deadline = Date.now() + 300_000;
+  let lastErr = '';
+  for (;;) {
+    try {
+      // First check: postgres accepting connections
       await sshRunAsync(
         ip,
         sshKeyPath,
-        `test -f /tmp/vb-restore/${db}.sql.gz && cd ${remoteDir} && gunzip -c /tmp/vb-restore/${db}.sql.gz | docker compose exec -T db psql -U postgres -d ${db} || true`,
-        { timeout: 300_000 },
+        `cd ${remoteDir} && docker compose exec -T db pg_isready -U postgres`,
+        { timeout: 10_000 },
+      );
+      // Second check: promoted (no longer in recovery)
+      const recovering = await sshRunAsync(
+        ip,
+        sshKeyPath,
+        `cd ${remoteDir} && docker compose exec -T db psql -U postgres -At -c 'SELECT pg_is_in_recovery()'`,
+        { timeout: 10_000 },
       );
+      if (String(recovering).trim() === 'f') {
+        break; // promoted — restore complete
+      }
+      lastErr = 'pg_is_in_recovery() still true';
+    } catch (err) {
+      lastErr = err instanceof Error ? err.message : String(err);
     }
-
-    // Restore storage files if present
-    await sshRunAsync(
-      ip,
-      sshKeyPath,
-      `test -f /tmp/vb-restore/storage.tar.gz && cd ${remoteDir} && docker compose exec -T storage sh -c 'tar --no-xattrs --no-same-owner -xzf - -C /' < /tmp/vb-restore/storage.tar.gz || true`,
-      { timeout: 300_000 },
-    );
-
-    // Cleanup
-    await sshRunAsync(ip, sshKeyPath, 'rm -rf /tmp/vb-restore');
-  }
-
-  // Restart app
-  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start app`);
-}
-
-/**
- * Wait for an HTTPS endpoint to respond with a valid certificate.
- * Polls until a successful response (2xx/3xx) or timeout.
- *
- * @param {string} url - Full HTTPS URL to check
- * @param {object} [options]
- * @param {number} [options.timeout=90000] - Max wait time in ms
- * @param {number} [options.interval=5000] - Poll interval in ms
- * @returns {Promise<boolean>} true if SSL is ready, false if timed out
- */
-export async function waitForSSL(url, { timeout = 90_000, interval = 5_000 } = {}) {
-  const deadline = Date.now() + timeout;
-  while (Date.now() < deadline) {
-    try {
-      const result = runCommand(
-        ['curl', '-sf', '--max-time', '5', '-o', '/dev/null', '-w', '%{http_code}', url],
-        {
-          stdio: 'pipe',
-          encoding: 'utf-8',
-          timeout: 10_000,
-        },
+    if (Date.now() >= deadline) {
+      throw new Error(
+        `restoreCompose: postgres did not promote within 300s after wal-g restore. ` +
+          `Last error: ${lastErr}`,
       );
-      if (result?.trim().match(/^[23]\d\d$/)) return true;
-    } catch {
-      // Not ready yet
     }
-    await new Promise((r) => setTimeout(r, interval));
+    await new Promise((r) => setTimeout(r, 5_000));
   }
-  return false;
+
+  // 6. Bring the app back up
+  await sshRunAsync(ip, sshKeyPath, `cd ${remoteDir} && docker compose start app`);
 }
 
 /**
@@ -1236,6 +1116,28 @@ export async function waitForSSL(url, { timeout = 90_000, interval = 5_000 } = {
  * @param {string} projectName - Project name (for remote dir)
  * @returns {Promise<{success: boolean, message: string}>}
  */
+/**
+ * Extract the admin credentials a deploy needs to provision the super-admin
+ * out of a project `.env` file's text.
+ *
+ * `create` writes these via `escapeDotenv` (ADMIN_PASSWORD is POSIX
+ * single-quoted) and double-quotes for the rest, so the on-disk shapes are a
+ * mix of `'…'` and `"…"`. Decoding MUST go through `parseDotenv`/`unescapeDotenv`
+ * — the inverse of `escapeDotenv` — or single-quote wrappers leak into the
+ * value and GoTrue provisions a password the operator can never type.
+ *
+ * @param {string} envContent
+ * @returns {{adminEmail?: string, adminPassword?: string, serviceRoleKey?: string}}
+ */
+export function readAdminCredentials(envContent) {
+  const env = parseDotenv(envContent);
+  return {
+    adminEmail: env.ADMIN_EMAIL,
+    adminPassword: env.ADMIN_PASSWORD,
+    serviceRoleKey: env.SUPABASE_SERVICE_ROLE_KEY,
+  };
+}
+
 export async function createAdminUser(serverIp, sshKeyPath, projectName) {
   const envPath = join(process.cwd(), '.env');
   if (!existsSync(envPath)) {
@@ -1243,9 +1145,7 @@ export async function createAdminUser(serverIp, sshKeyPath, projectName) {
   }
 
   const envContent = readFileSync(envPath, 'utf-8');
-  const adminEmail = envContent.match(/^ADMIN_EMAIL="?([^"\n]+)"?/m)?.[1];
-  const adminPassword = envContent.match(/^ADMIN_PASSWORD="?([^"\n]+)"?/m)?.[1];
-  const serviceRoleKey = envContent.match(/^SUPABASE_SERVICE_ROLE_KEY="?([^"\n]+)"?/m)?.[1];
+  const { adminEmail, adminPassword, serviceRoleKey } = readAdminCredentials(envContent);
 
   if (!adminEmail || !adminPassword || !serviceRoleKey) {
     return { success: false, message: 'Admin credentials not found in .env' };
@@ -1299,54 +1199,22 @@ export async function createAdminUser(serverIp, sshKeyPath, projectName) {
   );
 
   try {
-    // Wait for port-forward to be ready. ssh -L is usually listening on
-    // localhost within 200-500ms; first few retries are at 200ms before
-    // falling back to 1s.
-    let connected = false;
-    for (let i = 0; i < 15; i++) {
-      try {
-        await fetch(`http://localhost:${localPort}/auth/v1/health`, {
-          signal: AbortSignal.timeout(2000),
-        });
-        connected = true;
-        break;
-      } catch {
-        const interval = i < 5 ? 200 : 1000;
-        await new Promise((r) => setTimeout(r, interval));
-      }
-    }
-
+    // Wait for the ssh -L tunnel to start forwarding (usually <500ms).
+    const connected = await waitForGotrueHealth(`http://localhost:${localPort}/auth/v1/health`, {
+      attempts: 15,
+      intervalMs: 500,
+    });
     if (!connected) {
       return { success: false, message: 'Could not reach auth service via SSH tunnel' };
     }
 
-    const response = await fetch(`http://localhost:${localPort}/auth/v1/admin/users`, {
-      method: 'POST',
-      headers: {
-        Authorization: `Bearer ${serviceRoleKey}`,
-        apikey: serviceRoleKey,
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        email: adminEmail,
-        password: adminPassword,
-        email_confirm: true,
-        app_metadata: { role: 'super_admin' },
-      }),
-      signal: AbortSignal.timeout(15000),
+    // POST through Kong, which rewrites /auth/v1 → GoTrue's /.
+    return await postAdminUser({
+      adminUsersUrl: `http://localhost:${localPort}/auth/v1/admin/users`,
+      serviceRoleKey,
+      adminEmail,
+      adminPassword,
     });
-
-    if (response.ok) {
-      return { success: true, message: `Admin user created: ${adminEmail}` };
-    }
-    if (response.status === 422) {
-      return { success: true, message: `Admin user already exists: ${adminEmail}` };
-    }
-    const body = await response.text().catch(() => '');
-    return {
-      success: false,
-      message: `Auth returned ${response.status}${body ? `: ${body}` : ''}`,
-    };
   } finally {
     pf.kill();
   }