vibecarbon 0.3.1 → 0.5.0

package/src/restore.js

@@ -16,10 +16,10 @@
  * memory:feedback_cli_single_dash_flags.
  */
 
-import { existsSync, unlinkSync } from 'node:fs';
-import { basename, join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { basename } from 'node:path';
 import * as p from '@clack/prompts';
-import { downloadS3Backup, listS3Backups } from './lib/backup-s3.js';
+import { listS3Backups } from './lib/backup-s3.js';
 import { renderHelp } from './lib/cli/help.js';
 import { parseFlags } from './lib/cli/parse-flags.js';
 import { selectEnvironment } from './lib/cli/select-environment.js';
@@ -31,19 +31,18 @@ import {
   loadProjectConfig,
   loadS3Config,
 } from './lib/config.js';
+// Static imports (no cycle: neither compose/index.js nor compose/ha.js imports
+// restore.js). Aliased so runComposeRestore can default to the real functions
+// while still accepting injected mocks for unit tests.
+import { configureStandbyReplication as realConfigureStandbyReplication } from './lib/deploy/compose/ha.js';
+import { restoreCompose as realRestoreCompose } from './lib/deploy/compose/index.js';
 import { getS3Credentials } from './lib/hetzner-guided-setup.js';
 import { requireLicense } from './lib/licensing/index.js';
 import { ensureOperatorIpAccess } from './lib/operator-ip.js';
 import { perfAsync } from './lib/perf.js';
+import { listPodBackups } from './lib/pod-backups.js';
 import { assertInProjectDir } from './lib/project-guard.js';
-import {
-  getPostgresPod,
-  getSSHKeyPath,
-  scpUpload,
-  sshKubectl,
-  sshRun,
-  sshRunScript,
-} from './lib/ssh.js';
+import { getPostgresPod, getSSHKeyPath, sshKubectl, sshRunScript } from './lib/ssh.js';
 import { createTracker } from './lib/tracker.js';
 import { validateBackupFilename } from './lib/validators.js';
 import { VERSION } from './lib/version.js';
@@ -97,45 +96,6 @@ const SPEC = {
   ],
 };
 
-// ============================================================================
-// KUBECTL HELPERS (k8s mode)
-// ============================================================================
-
-function listRemoteBackups(ip, sshKeyPath) {
-  const pod = getPostgresPod(ip, sshKeyPath);
-
-  try {
-    const output = sshKubectl(ip, sshKeyPath, [
-      'exec',
-      '-n',
-      'vibecarbon',
-      pod,
-      '--',
-      'sh',
-      '-c',
-      'ls -lt /backups/*_full.tar.gz /backups/*.sql.gz 2>/dev/null || echo NO_BACKUPS',
-    ]);
-
-    if (output === 'NO_BACKUPS' || !output) {
-      return [];
-    }
-
-    return output
-      .split('\n')
-      .filter((line) => line.includes('.tar.gz') || line.includes('.sql.gz'))
-      .map((line) => {
-        const parts = line.split(/\s+/);
-        return {
-          size: parts[4],
-          date: `${parts[5]} ${parts[6]} ${parts[7]}`,
-          name: parts[parts.length - 1].replace('/backups/', ''),
-        };
-      });
-  } catch {
-    return [];
-  }
-}
-
 // ============================================================================
 // RESTORE OPERATIONS (k8s mode)
 // ============================================================================
@@ -161,189 +121,54 @@ function scaleUpApp(ip, sshKeyPath) {
   sshKubectl(ip, sshKeyPath, ['scale', 'deployment', 'app', '-n', 'vibecarbon', '--replicas=2']);
 }
 
-function verifyPostgres(ip, sshKeyPath) {
-  const pod = getPostgresPod(ip, sshKeyPath);
-  sshKubectl(ip, sshKeyPath, [
-    'exec',
-    '-n',
-    'vibecarbon',
-    pod,
-    '--',
-    'pg_isready',
-    '-U',
-    'postgres',
-  ]);
-}
-
 /**
- * Restore from a full archive (.tar.gz containing postgres.sql.gz + optional storage.tar.gz)
+ * Wait until postgres on the db pod is actually accepting connections.
+ *
+ * After a wal-g restore, postgres replays archived WAL (restore_command) before
+ * promoting — minutes on a busy DB (observed 112s). The pod can be Ready while
+ * postgres is still "not accepting connections", so a single pg_isready races
+ * slow replay and reports a false restore failure. Poll until it accepts (or
+ * give up after `timeoutMs`).
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs=300000]
+ * @param {number} [opts.intervalMs=5000]
+ * @param {(ip: string, key: string, args: string[]) => unknown} [opts.exec=sshKubectl]
+ * @param {(ip: string, key: string) => string} [opts.getPod=getPostgresPod]
+ * @param {(ms: number) => Promise<void>} [opts.sleep]
  */
-function restoreFromArchive(ip, sshKeyPath, remotePath) {
-  const pod = getPostgresPod(ip, sshKeyPath);
-
-  // Extract the archive on the server — argv form; remotePath is never shell-interpolated.
-  sshRun(ip, sshKeyPath, ['mkdir', '-p', '/tmp/restore']);
-  // GNU tar strips leading `/` by default (no `-P` / `--absolute-names`),
-  // so absolute paths in a hostile archive never escape /tmp/restore.
-  sshRun(ip, sshKeyPath, [
-    'tar',
-    '--no-xattrs',
-    '--no-same-owner',
-    '-xzf',
-    remotePath,
-    '-C',
-    '/tmp/restore',
-  ]);
-
-  // Check archive contents
-  const contents = sshRun(ip, sshKeyPath, ['ls', '/tmp/restore/']);
-
-  // Restore database
-  if (contents.includes('postgres.sql.gz')) {
-    sshKubectl(ip, sshKeyPath, [
-      'cp',
-      '/tmp/restore/postgres.sql.gz',
-      `vibecarbon/${pod}:/tmp/postgres.sql.gz`,
-    ]);
-    sshKubectl(ip, sshKeyPath, [
-      'exec',
-      '-n',
-      'vibecarbon',
-      pod,
-      '--',
-      'sh',
-      '-c',
-      'gunzip -c /tmp/postgres.sql.gz | psql -U supabase_admin postgres',
-    ]);
-    sshKubectl(ip, sshKeyPath, [
-      'exec',
-      '-n',
-      'vibecarbon',
-      pod,
-      '--',
-      'rm',
-      '-f',
-      '/tmp/postgres.sql.gz',
-    ]);
-  }
-
-  // Restore storage files if present and using file backend
-  if (contents.includes('storage.tar.gz')) {
-    // kubectl get returns empty string if STORAGE_BACKEND env var is unset;
-    // default to 'file' so the legacy single-backend path still works.
-    let storageBackend;
+export async function verifyPostgres(ip, sshKeyPath, opts = {}) {
+  const {
+    timeoutMs = 300_000,
+    intervalMs = 5_000,
+    exec = sshKubectl,
+    getPod = getPostgresPod,
+    sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+  } = opts;
+  const pod = getPod(ip, sshKeyPath);
+  const deadline = Date.now() + timeoutMs;
+  let lastErr = '';
+  for (;;) {
     try {
-      storageBackend = sshKubectl(ip, sshKeyPath, [
-        'get',
-        'deployment',
-        'storage',
-        '-n',
-        'vibecarbon',
-        '-o',
-        'jsonpath={.spec.template.spec.containers[0].env[?(@.name=="STORAGE_BACKEND")].value}',
-      ]);
-    } catch {
-      storageBackend = 'file';
+      exec(ip, sshKeyPath, ['exec', '-n', 'vibecarbon', pod, '--', 'pg_isready', '-U', 'postgres']);
+      return; // exit 0 = accepting connections
+    } catch (err) {
+      lastErr = err instanceof Error ? err.message : String(err);
     }
-    if (!storageBackend) storageBackend = 'file';
-
-    if (storageBackend === 'file') {
-      const storagePod = sshKubectl(ip, sshKeyPath, [
-        'get',
-        'pods',
-        '-n',
-        'vibecarbon',
-        '-l',
-        'app.kubernetes.io/name=supabase-storage',
-        '-o',
-        'jsonpath={.items[0].metadata.name}',
-      ]);
-
-      sshKubectl(ip, sshKeyPath, [
-        'cp',
-        '/tmp/restore/storage.tar.gz',
-        `vibecarbon/${storagePod}:/tmp/storage.tar.gz`,
-      ]);
-      sshKubectl(ip, sshKeyPath, [
-        'exec',
-        '-n',
-        'vibecarbon',
-        storagePod,
-        '--',
-        'sh',
-        '-c',
-        'tar --no-xattrs --no-same-owner -xzf /tmp/storage.tar.gz -C / && rm -f /tmp/storage.tar.gz',
-      ]);
-    } else {
-      p.log.info('Storage backend is S3 — skipping file restore (files already in S3)');
+    if (Date.now() >= deadline) {
+      throw new Error(
+        `verifyPostgres: postgres did not accept connections within ${Math.round(timeoutMs / 1000)}s ` +
+          `(still replaying WAL after a wal-g restore?). Last pg_isready error: ${lastErr}`,
+      );
     }
+    await sleep(intervalMs);
   }
-
-  // Cleanup
-  sshRun(ip, sshKeyPath, ['rm', '-rf', '/tmp/restore']);
 }
 
-/**
- * Restore from a legacy .sql.gz file (database only)
- */
-function restoreFromSqlGz(ip, sshKeyPath, backupName) {
-  const pod = getPostgresPod(ip, sshKeyPath);
-
-  // backupName is validated via validateBackupFilename at entry; still pass it
-  // as a positional sh arg so the remote shell never expands it as code.
-  sshKubectl(ip, sshKeyPath, [
-    'exec',
-    '-n',
-    'vibecarbon',
-    pod,
-    '--',
-    'sh',
-    '-c',
-    'gunzip -c "/backups/$1" | psql -U supabase_admin postgres',
-    '_',
-    backupName,
-  ]);
-}
-
-async function restoreFromRemote(ip, sshKeyPath, backupName) {
-  scaleDownApp(ip, sshKeyPath);
-
-  if (backupName.endsWith('_full.tar.gz')) {
-    restoreFromArchive(ip, sshKeyPath, `/tmp/${backupName}`);
-  } else {
-    restoreFromSqlGz(ip, sshKeyPath, backupName);
-  }
-
-  scaleUpApp(ip, sshKeyPath);
-  verifyPostgres(ip, sshKeyPath);
-}
-
-async function uploadAndRestore(ip, sshKeyPath, localFile) {
-  const filename = basename(localFile);
-
-  // Upload local file to server
-  scpUpload(ip, sshKeyPath, localFile, `/tmp/${filename}`);
-
-  scaleDownApp(ip, sshKeyPath);
-
-  if (filename.endsWith('_full.tar.gz')) {
-    restoreFromArchive(ip, sshKeyPath, `/tmp/${filename}`);
-  } else if (filename.endsWith('.sql.gz')) {
-    const pod = getPostgresPod(ip, sshKeyPath);
-    sshKubectl(ip, sshKeyPath, [
-      'cp',
-      `/tmp/${filename}`,
-      `vibecarbon/${pod}:/backups/${filename}`,
-    ]);
-    restoreFromSqlGz(ip, sshKeyPath, filename);
-  }
-
-  // Clean up temp file on host
-  sshRun(ip, sshKeyPath, ['rm', '-f', `/tmp/${filename}`]);
-
-  scaleUpApp(ip, sshKeyPath);
-  verifyPostgres(ip, sshKeyPath);
-}
+// (k8s pg_dump restore helpers removed — k8s restore is now wal-g-native via
+// the db pod's init container; see runK8sRestore below.)
 
 // ============================================================================
 // SOURCE INTERPRETATION
@@ -434,7 +259,13 @@ export async function run(args) {
     seed: envSeed,
   });
 
-  const serverIp = envConfig.servers?.[0]?.ip;
+  // compose-ha: pick the CURRENT primary by role (a prior failover may have
+  // swapped roles while preserving array order, so servers[0] is not reliably
+  // the primary). Falls back to servers[0] for non-HA modes that have no role.
+  // This same serverIp feeds both the restore target and (in runComposeRestore)
+  // the re-seed source, so they stay consistent with the standby selection.
+  const primaryServer = envConfig.servers?.find((sv) => sv.role === 'primary');
+  const serverIp = primaryServer?.ip || envConfig.servers?.[0]?.ip;
   if (!serverIp) {
     p.log.error(`No server IP found for environment '${envName}'.`);
     process.exit(1);
@@ -511,19 +342,33 @@ export async function run(args) {
   // restores) ask for "the most recent" without first listing + parsing
   // names from `vibecarbon backup -l`.
   const wantsLatest = values.source === 'latest';
-  const chosenSource =
-    !wantsLatest && values.source
-      ? classifySource(/** @type {string} */ (values.source))
-      : await pickInteractiveSource({
-          isCompose,
-          envName,
-          projectName,
-          serverIp,
-          sshKeyPath,
-          s3,
-          useS3,
-          yes: wantsLatest || !!values.y,
-        });
+  let chosenSource;
+  if (!wantsLatest && values.source) {
+    chosenSource = classifySource(/** @type {string} */ (values.source));
+  } else if (wantsLatest) {
+    // BOTH compose and k8s restore are now wal-g-native: `latest` resolves to
+    // the LATEST base backup that wal-g pushed to S3 — k8s via the db pod's
+    // walg-restore init container, compose via `restoreCompose(..., 'latest')`
+    // → `wal-g backup-fetch LATEST` (see runK8sRestore / runComposeRestore).
+    // pickInteractiveSource lists legacy pg_dump `*_full.tar.gz` backups via
+    // listS3Backups, which CANNOT see wal-g `basebackups_005/base_*` objects, so
+    // routing `-source latest` through it wrongly exits "No backups found in S3"
+    // even though a wal-g base backup exists. Hand the `latest` sentinel
+    // straight to the restore runner and let wal-g resolve it. (RCA 2026-05-31:
+    // e2e k8s/k8s-ha restore step; compose found via kept-rig manual restore.)
+    chosenSource = { kind: 's3', name: 'latest' };
+  } else {
+    chosenSource = await pickInteractiveSource({
+      isCompose,
+      envName,
+      projectName,
+      serverIp,
+      sshKeyPath,
+      s3,
+      useS3,
+      yes: wantsLatest || !!values.y,
+    });
+  }
 
   if (!chosenSource) {
     p.log.error('No backup selected.');
@@ -570,7 +415,7 @@ export async function run(args) {
         projectName,
         serverIp,
         sshKeyPath,
-        s3WithCreds: s3,
+        envConfig,
       });
     } else {
       await runK8sRestore({
@@ -579,8 +424,6 @@ export async function run(args) {
         envName,
         serverIp,
         sshKeyPath,
-        s3,
-        useS3,
       });
     }
   } catch (error) {
@@ -662,7 +505,7 @@ async function runList({ isCompose, envName, projectName, serverIp, sshKeyPath }
   }
 
   // K8s pod fallback
-  const backups = listRemoteBackups(serverIp, sshKeyPath);
+  const backups = listPodBackups(serverIp, sshKeyPath);
   printBackupList(backups, envName);
 }
 
@@ -757,7 +600,7 @@ async function pickInteractiveSource({
 
   // K8s pod fallback
   s.start('Fetching available backups');
-  const backups = listRemoteBackups(serverIp, sshKeyPath);
+  const backups = listPodBackups(serverIp, sshKeyPath);
   s.stop('Backups retrieved');
   if (backups.length === 0) {
     p.log.error('No backups found on the server.');
@@ -780,84 +623,155 @@ async function pickInteractiveSource({
   return { kind: 's3', name: /** @type {string} */ (selected) };
 }
 
-async function runComposeRestore({
+// compose restore is wal-g-based (S3 pull, no file transfer). Mirrors
+// runK8sRestore's structure: reject local-file sources, resolve target,
+// call restoreCompose, verify postgres.
+//
+// COMPOSE-HA re-seed (Invariant 3): the wal-g restore rewinds the PRIMARY
+// (servers[0]) to an earlier LSN. For a compose-ha deployment the standby then
+// has WAL *ahead* of the restored primary (streamed from the old primary) and
+// cannot resume streaming — PostgreSQL would reject it with "requested timeline
+// does not contain minimum recovery point" (timeline divergence). So after the
+// primary promotes we re-seed the standby with a fresh pg_basebackup from the
+// restored primary via configureStandbyReplication, which is destructive-by-
+// design (it wipes the standby's PGDATA and re-basebackups — exactly what we
+// want here). Single-region compose has no standby, so the re-seed is skipped.
+//
+// restoreCompose + configureStandbyReplication are injectable so the dispatch
+// logic is unit-testable without SSH; production passes neither and they resolve
+// to the real module functions (static imports — no cycle: ha.js/compose-index
+// do not import restore.js).
+export async function runComposeRestore({
   s,
   chosenSource,
   envName,
   projectName,
   serverIp,
   sshKeyPath,
-  s3WithCreds,
+  envConfig = {},
+  restoreCompose = realRestoreCompose,
+  configureStandbyReplication = realConfigureStandbyReplication,
 }) {
-  const backupFile = chosenSource.name;
-
   if (chosenSource.kind === 'local') {
-    p.log.info(`Uploading local file: ${c.bold(chosenSource.path)}`);
-    scpUpload(serverIp, sshKeyPath, chosenSource.path, `/opt/${projectName}/backups/${backupFile}`);
-  } else if (s3WithCreds?.secretKey) {
-    s.start(`Downloading ${backupFile} from S3 to server`);
-    const tmpLocal = join(process.cwd(), `.restore-tmp-${backupFile}`);
-    try {
-      await downloadS3Backup(s3WithCreds, `backups/${backupFile}`, tmpLocal);
-      scpUpload(serverIp, sshKeyPath, tmpLocal, `/opt/${projectName}/backups/${backupFile}`);
+    throw new Error(
+      'compose restore is wal-g-based and pulls from S3 — local backup files are not supported. ' +
+        'Use `-source latest` or an ISO-8601 timestamp for PITR.',
+    );
+  }
+
+  // wal-g target: 'latest' → LATEST base backup; otherwise treat the source
+  // name as an ISO-8601 timestamp for point-in-time recovery.
+  const target =
+    chosenSource.name === 'latest' || !chosenSource.name ? 'latest' : chosenSource.name;
+
+  s.start('Restoring database via wal-g (app will be temporarily unavailable)');
+  await restoreCompose(serverIp, sshKeyPath, projectName, target);
+  s.stop('Database restored');
+
+  // compose-ha: the wal-g restore rewound the primary to an earlier LSN, so the
+  // standby now has WAL AHEAD of the primary and cannot resume streaming. Re-seed
+  // it with a fresh pg_basebackup from the restored primary so it rejoins the new
+  // timeline. (configureStandbyReplication is destructive-by-design: it wipes the
+  // standby's PGDATA and re-basebackups — exactly what we want here.)
+  //
+  // A re-seed failure is NOT a restore failure: the primary is restored and
+  // serving — only the standby is degraded. Warn (don't rethrow) so the restore
+  // still reports success; the operator can resync the standby via `deploy`.
+  if (envConfig.deployMode === 'compose-ha') {
+    const standby = envConfig.servers?.find((sv) => sv.role === 'standby');
+    if (standby?.ip) {
       try {
-        unlinkSync(tmpLocal);
-      } catch {
-        // ignore cleanup errors
+        s.start('Re-seeding standby from restored primary (pg_basebackup)');
+        await configureStandbyReplication(standby.ip, serverIp, sshKeyPath, projectName);
+        s.stop('Standby re-seeded from restored primary');
+      } catch (err) {
+        s.stop('Standby re-seed failed');
+        p.log.warn(
+          `Primary restored and serving, but standby re-seed failed: ${err.message}. ` +
+            `Run \`vibecarbon deploy ${envName}\` to resync the standby.`,
+        );
       }
-      s.stop('Backup transferred to server');
-    } catch (error) {
-      s.stop(`S3 download failed: ${error.message}`);
-      throw error;
+    } else {
+      p.log.warn(
+        'compose-ha restore: no standby server found to re-seed — skipping (verify replication manually).',
+      );
     }
   }
 
-  s.start('Restoring database...');
-  const { restoreCompose } = await import('./lib/deploy/compose/index.js');
-  restoreCompose(serverIp, sshKeyPath, projectName, backupFile);
-  s.stop('Database restored');
   p.log.success('Restore completed successfully');
-  // env name is unused by restoreCompose but kept in the closure for
-  // diagnostic logging consistency.
-  void envName;
+  p.log.info(`Verify with: ${c.info(`vibecarbon status ${envName}`)}`);
 }
 
-async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath, s3, useS3 }) {
-  s.start('Restoring database (app will be temporarily unavailable)');
+// k8s restore is wal-g-native: postgres + wal-g live in the supabase-db pod,
+// which continuously archives WAL + base backups to S3. Restore = set a
+// RESTORE_TARGET marker on vibecarbon-secrets → bounce the db StatefulSet so
+// its walg-restore init container runs `wal-g backup-fetch` into PGDATA before
+// postgres starts → clear the marker. No bytes flow through the operator and
+// no pg_dump/kubectl-cp; wal-g pulls from S3 inside the cluster. Mirrors the
+// compose wal-g restore, adapted to the StatefulSet/init-container shape.
+const DB_STATEFULSET = 'supabase-supabase-db';
+
+/** Set or clear the RESTORE_TARGET marker the init container reads. */
+function setRestoreMarker(ip, sshKeyPath, value) {
+  sshKubectl(ip, sshKeyPath, [
+    'patch',
+    'secret',
+    'vibecarbon-secrets',
+    '-n',
+    'vibecarbon',
+    '--type=merge',
+    '-p',
+    JSON.stringify({ stringData: { RESTORE_TARGET: value } }),
+  ]);
+}
 
+async function runK8sRestore({ s, chosenSource, envName, serverIp, sshKeyPath }) {
   if (chosenSource.kind === 'local') {
-    if (!chosenSource.path.endsWith('.sql.gz') && !chosenSource.path.endsWith('.tar.gz')) {
-      throw new Error('Backup file must be a .sql.gz or .tar.gz file');
-    }
-    await perfAsync('restore.uploadAndRestore', () =>
-      uploadAndRestore(serverIp, sshKeyPath, chosenSource.path),
-    );
-    s.stop('Database restored');
-  } else if (useS3) {
-    s.message('Downloading backup from S3');
-    const tempPath = join(process.cwd(), `.restore-tmp-${chosenSource.name}`);
-    await perfAsync('restore.downloadS3', () =>
-      downloadS3Backup(s3, `backups/${chosenSource.name}`, tempPath),
+    throw new Error(
+      'k8s restore is wal-g-based and pulls from S3 — local backup files are not supported. Use `-source latest` (or a timestamp for PITR).',
     );
-    s.message('Uploading backup to server');
-    await perfAsync('restore.scpUpload', async () =>
-      scpUpload(serverIp, sshKeyPath, tempPath, `/tmp/${chosenSource.name}`),
-    );
-    try {
-      unlinkSync(tempPath);
-    } catch {
-      // ignore cleanup errors
-    }
-    s.message('Restoring database');
-    await perfAsync('restore.restoreFromRemote', () =>
-      restoreFromRemote(serverIp, sshKeyPath, chosenSource.name),
-    );
-    s.stop('Database restored');
-  } else {
-    // Pod-only fallback: backup name refers to a file already in the pod.
-    await restoreFromRemote(serverIp, sshKeyPath, chosenSource.name);
-    s.stop('Database restored');
   }
+  // wal-g target: 'latest' → LATEST base backup; otherwise treat the source
+  // name as an ISO-8601 timestamp for point-in-time recovery.
+  const target =
+    chosenSource.name === 'latest' || !chosenSource.name ? 'latest' : chosenSource.name;
+
+  s.start('Restoring database via wal-g (app will be temporarily unavailable)');
+  // 1. Mark the restore so the db pod's init container fetches on next boot.
+  setRestoreMarker(serverIp, sshKeyPath, target);
+  // 2. Quiesce the app so it isn't querying mid-swap.
+  await perfAsync('restore.scaleDownApp', async () => scaleDownApp(serverIp, sshKeyPath));
+  try {
+    // 3. Bounce the db StatefulSet → walg-restore init container clears PGDATA
+    //    and runs `wal-g backup-fetch` before postgres starts.
+    s.message('Fetching from S3 + recovering (wal-g)');
+    await perfAsync('restore.walgFetch', async () => {
+      sshKubectl(serverIp, sshKeyPath, [
+        'rollout',
+        'restart',
+        `statefulset/${DB_STATEFULSET}`,
+        '-n',
+        'vibecarbon',
+      ]);
+      sshKubectl(serverIp, sshKeyPath, [
+        'rollout',
+        'status',
+        `statefulset/${DB_STATEFULSET}`,
+        '-n',
+        'vibecarbon',
+        '--timeout=600s',
+      ]);
+    });
+    // 4. Confirm postgres accepts connections post-recovery.
+    await perfAsync('restore.verifyPostgres', async () => verifyPostgres(serverIp, sshKeyPath));
+  } finally {
+    // 5. Always clear the marker so an unrelated future pod restart does NOT
+    //    re-fetch and wipe live data. Runs even if the restore failed.
+    setRestoreMarker(serverIp, sshKeyPath, '');
+  }
+  // 6. Bring the app back.
+  await perfAsync('restore.scaleUpApp', async () => scaleUpApp(serverIp, sshKeyPath));
+  s.stop('Database restored');
 
   p.log.success('Restore completed successfully');
   p.log.info('The application has been scaled back up.');