vibecarbon 0.4.0 → 0.5.0

package/src/lib/deploy/k8s/k3s.js

@@ -40,11 +40,17 @@ import * as p from '@clack/prompts';
 import { runCommandAsync } from '../../command.js';
 import { knownHostsPath } from '../../host-keys.js';
 import { loadCloudInit, renderScript } from '../../iac/cloud-init.js';
+import { dbImageRef } from '../../images.js';
 import { perfAsync } from '../../perf.js';
 import { postAdminUser, waitForGotrueHealth } from '../admin-user.js';
 
 export const K3S_VERSION = 'v1.31.5+k3s1';
 
+/** The db image is pre-published + multi-arch — pulled, never built/sideloaded. */
+export function resolveDbImageTag() {
+  return dbImageRef();
+}
+
 /**
  * Pattern of kube-apiserver transient signatures we've actually observed
  * mid-RPC during k3s control-plane warm-up. See runKubectlWithRetry —
@@ -53,7 +59,12 @@ export const K3S_VERSION = 'v1.31.5+k3s1';
  * obvious place to do it).
  */
 export const KUBECTL_TRANSIENT_PATTERN =
-  /http2: client connection lost|connection reset by peer|context deadline exceeded|EOF|i\/o timeout|TLS handshake|unexpected error when reading response body/i;
+  // `connection timed out` covers cross-cluster API blips like
+  // `read tcp <ip>:<port>->...:6443: read: connection timed out` — seen when a
+  // kubectl exec (e.g. enable-WAL-archiving) hits a momentarily unreachable
+  // standby control-plane in k8s-ha. Without it the call failed without
+  // retrying, aborting the deploy. (RCA 2026-06-01: k8s-ha standby deploy.)
+  /http2: client connection lost|connection reset by peer|context deadline exceeded|connection timed out|EOF|i\/o timeout|TLS handshake|unexpected error when reading response body/i;
 
 /**
  * Run a short-lived kubectl command with retry-on-transient-error.
@@ -209,6 +220,35 @@ export function pickIssuerName({ dnsProvider, acmeServer }) {
   return `letsencrypt-${stage}-${providerSuffix}`;
 }
 
+/**
+ * Build the `dnsNames` list for the `Certificate/vibecarbon-tls` patch.
+ *
+ * DNS-01 issuers (cloudflare, hetzner) support wildcard SANs — cert-manager
+ * can obtain `*.${domain}` in a single ACME order, covering every subdomain
+ * (api, studio, dashboard, grafana, n8n …) without separate per-subdomain
+ * certificates. One wildcard cert = no per-router cert-resolver needed =
+ * no per-domain ACME rate-limit risk.
+ *
+ * Manual issuers use HTTP-01 which cannot issue wildcards, so fall back to
+ * the apex domain only (the IngressRoutes already work; only non-apex
+ * subdomains are uncovered, but those are admin-only tools the operator
+ * configures manually for manual-DNS deploys).
+ *
+ * @param {string} domain - The apex deploy domain, e.g. `e1.carbonstack.dev`
+ * @param {string} issuerName - The ClusterIssuer name, e.g.
+ *   `letsencrypt-prod-hetzner`, produced by `pickIssuerName`
+ * @returns {string[]}
+ */
+export function certificateDnsNames(domain, issuerName) {
+  // Only the known DNS-01 providers (cloudflare, hetzner) support wildcard SANs.
+  // `manual` and any future/unknown issuer name falls back to apex-only so an
+  // unrecognised issuer never attempts a wildcard via HTTP-01 (which LE rejects).
+  const isDns01 = /-(cloudflare|hetzner)$/.test(issuerName);
+  if (!isDns01) return [domain];
+  // DNS-01 issuers (cloudflare, hetzner — prod or staging): issue one wildcard.
+  return [domain, `*.${domain}`];
+}
+
 /**
  * Render the per-DNS-provider Secret YAML that cert-manager's DNS-01
  * solvers read for API auth, or null when the provider doesn't need a
@@ -1287,12 +1327,14 @@ export async function installSupabase({
   }
   if (!dbImageTag) {
     throw new Error(
-      'installSupabase: dbImageTag is required (the wal-g-equipped db image, built by k3s-db-build).',
+      'installSupabase: dbImageTag is required (the pre-published wal-g db image, pulled from ghcr).',
     );
   }
   // Split the full image ref into repository + tag for the chart's
-  // image.db.{repository,tag}. dbImageTag is `10.0.1.1:5000/<project>-db:<ver>`
-  // — the registry host carries a port colon, so split on the LAST colon.
+  // image.db.{repository,tag} (and the walg-restore init's `{{DB_IMAGE}}:{{DB_IMAGE_TAG}}`).
+  // dbImageTag is now the pre-published `ghcr.io/<org>/postgres:<pg>-walg<ver>`
+  // ref — split on the LAST colon so the registry host (and any future port
+  // colon) stays in the repository.
   const lastColon = dbImageTag.lastIndexOf(':');
   const dbImageRepo = dbImageTag.slice(0, lastColon);
   const dbImageVer = dbImageTag.slice(lastColon + 1);
@@ -1797,42 +1839,73 @@ export async function applyK3sManifests({
   // Fan out the three deploys in parallel: kubectl wait blocks per-arg
   // sequentially, and the cainjector + webhook usually become Available
   // ~10-30s after the core deploy, so the serial form pays the longest
-  // arrival time three times in a row. Two-stage retry preserves the
-  // outer 180s+120s envelope of the previous --timeout=180s call.
+  // arrival time three times in a row.
+  //
+  // Each deploy POLLS to Available within a generous total budget rather than
+  // one or two fixed `kubectl wait` calls. On a fresh worker, cold image pulls
+  // can push cainjector/webhook readiness past a tight envelope even though the
+  // pod ultimately comes up healthy (observed 2026-06-01: k8s e2e failed
+  // `kubectl wait deploy/cert-manager-cainjector exited 1` while the pod was
+  // 1/1 Running moments later — the old 90s→120s=210s envelope just gave up
+  // first). Re-attempting `kubectl wait` in chunks until the budget also
+  // self-heals the fast-fail race where `wait` runs before the just-applied
+  // Deployment object has propagated (NotFound → exit 1 immediately). A
+  // genuinely stuck deploy still fails once the budget is exhausted.
   const certManagerDeploys = [
     'deploy/cert-manager',
     'deploy/cert-manager-webhook',
     'deploy/cert-manager-cainjector',
   ];
+  // 8-min budget: on a freshly provisioned cluster under load, cert-manager
+  // pods have been observed not even getting created until ~5 min after apply
+  // (control-plane/scheduler lag, not image pull), overrunning a tighter
+  // envelope even though the pods then come up healthy. This margin covers that
+  // cold-start tail; the poll below re-attempts within it rather than giving up.
+  const CERT_MANAGER_READY_BUDGET_MS = 480_000;
+  const waitDeployAvailableOnce = (deploy, timeoutSec) =>
+    new Promise((resolveFn, reject) => {
+      const child = spawn(
+        'kubectl',
+        [
+          '-n',
+          'cert-manager',
+          'wait',
+          '--for=condition=Available',
+          `--timeout=${timeoutSec}s`,
+          deploy,
+        ],
+        { stdio: 'inherit', env },
+      );
+      child.on('error', reject);
+      child.on('exit', (code) =>
+        code === 0 ? resolveFn() : reject(new Error(`kubectl wait ${deploy} exited ${code}`)),
+      );
+    });
+  const waitDeployAvailable = async (deploy) => {
+    const deadline = Date.now() + CERT_MANAGER_READY_BUDGET_MS;
+    let lastErr;
+    while (Date.now() < deadline) {
+      const remainingSec = Math.max(5, Math.ceil((deadline - Date.now()) / 1000));
+      try {
+        await waitDeployAvailableOnce(deploy, Math.min(60, remainingSec));
+        return;
+      } catch (err) {
+        lastErr = err;
+        if (Date.now() >= deadline) break;
+        // Brief backoff so a fast-failing wait (e.g. the Deployment object not
+        // yet propagated → immediate NotFound) doesn't busy-spin the budget.
+        await new Promise((r) => setTimeout(r, 3000));
+      }
+    }
+    throw (
+      lastErr ??
+      new Error(
+        `kubectl wait ${deploy} never became Available within ${CERT_MANAGER_READY_BUDGET_MS / 1000}s`,
+      )
+    );
+  };
   await perfAsync(`deploy.${perfPrefix}.certManager.wait`, () =>
-    Promise.all(
-      certManagerDeploys.map(async (deploy) => {
-        const waitOnce = (timeoutSec) =>
-          new Promise((resolveFn, reject) => {
-            const child = spawn(
-              'kubectl',
-              [
-                '-n',
-                'cert-manager',
-                'wait',
-                '--for=condition=Available',
-                `--timeout=${timeoutSec}s`,
-                deploy,
-              ],
-              { stdio: 'inherit', env },
-            );
-            child.on('error', reject);
-            child.on('exit', (code) =>
-              code === 0 ? resolveFn() : reject(new Error(`kubectl wait ${deploy} exited ${code}`)),
-            );
-          });
-        try {
-          await waitOnce(90);
-        } catch {
-          await waitOnce(120);
-        }
-      }),
-    ),
+    Promise.all(certManagerDeploys.map((deploy) => waitDeployAvailable(deploy))),
   );
   // 2b. Land the per-DNS-provider Secret in the cert-manager namespace
   //     BEFORE the kustomization that defines the issuers, so when the
@@ -2084,6 +2157,10 @@ export async function applyK3sManifests({
   // — DNS-01 vs HTTP-01 — cert-manager actually uses; see pickIssuerName.
   const acmeServer = envLocal?.ACME_CA_SERVER || process.env.ACME_CA_SERVER || '';
   const issuerName = pickIssuerName({ dnsProvider, acmeServer });
+  // DNS-01 issuers (cloudflare, hetzner) support wildcard SANs — one cert
+  // covers *.${domain} so every IngressRoute subdomain is included without
+  // separate per-router ACME orders. Manual (HTTP-01) falls back to apex-only.
+  const dnsNames = certificateDnsNames(domain, issuerName);
   await runKubectlWithRetry(
     [
       '-n',
@@ -2094,7 +2171,7 @@ export async function applyK3sManifests({
       '--type=merge',
       '-p',
       JSON.stringify({
-        spec: { dnsNames: [domain], issuerRef: { name: issuerName, kind: 'ClusterIssuer' } },
+        spec: { dnsNames, issuerRef: { name: issuerName, kind: 'ClusterIssuer' } },
       }),
     ],
     { env, description: 'applyK3sManifests: kubectl patch certificate/vibecarbon-tls' },
@@ -2652,26 +2729,11 @@ export async function deployK3s(options) {
   }
   const { tag: imageTag } = state.getStepResult('k3s-build');
 
-  // 7. Build the wal-g-equipped db image (carbon/db/Dockerfile: FROM
-  //    supabase/postgres + wal-g v3.0.5). This REPLACES the old separate
-  //    ~500MB backup image: the db image already carries wal-g + psql, and
-  //    co-locating wal-g with PGDATA is what enables physical base backups +
-  //    PITR (the old backup pod ran off-node and was stuck on pg_dump). The
-  //    supabase Helm chart's image.db is pointed at this build in
-  //    installSupabase. Built up-front so its sideload fans out with the app.
-  const dbBuildInputs = { projectName: `${options.projectName}-db` };
-  if (!state.shouldSkip('k3s-db-build', dbBuildInputs)) {
-    state.startStep('k3s-db-build', dbBuildInputs);
-    s.start('Building db image (wal-g)');
-    const built = await buildAppImage(join(projectDir, 'db'), `${options.projectName}-db`);
-    state.completeStep('k3s-db-build', {
-      tag: built.tag,
-      gitSha: built.gitSha,
-      isDirty: built.isDirty,
-    });
-    s.stop(`DB image built: ${built.tag}`);
-  }
-  const { tag: dbImageTag } = state.getStepResult('k3s-db-build');
+  // 7. The wal-g-equipped db image (supabase/postgres + wal-g) is now a
+  //    pre-published, multi-arch image pulled from ghcr — no per-deploy build
+  //    or sideload. The supabase Helm chart's image.db + the walg-restore init
+  //    container both reference it (see installSupabase + supabase.values.yaml).
+  const dbImageTag = resolveDbImageTag();
 
   // 7b. Sideload to every node (parallel SSH within sideloadK3s) AND fan out
   //     the app+db sideloads against each other. Both writes go through
@@ -2684,7 +2746,6 @@ export async function deployK3s(options) {
   if (supabaseIp) sshTargets.push(`root@${supabaseIp}`);
   for (const wIp of workerIps ?? []) sshTargets.push(`root@${wIp}`);
   const sideloadInputs = { imageTag, targets: sshTargets.join(',') };
-  const dbSideloadInputs = { dbImageTag, targets: sshTargets.join(',') };
   const sideloadTasks = [];
   if (!state.shouldSkip('k3s-sideload', sideloadInputs)) {
     state.startStep('k3s-sideload', sideloadInputs);
@@ -2694,18 +2755,10 @@ export async function deployK3s(options) {
       ).then(() => state.completeStep('k3s-sideload')),
     );
   }
-  if (!state.shouldSkip('k3s-db-sideload', dbSideloadInputs)) {
-    state.startStep('k3s-db-sideload', dbSideloadInputs);
-    sideloadTasks.push(
-      perfAsync(`deploy.${perfPrefix}.sideload.db`, () =>
-        sideloadK3s({ tag: dbImageTag, sshTargets, sshKey: privateKeyPath, khPath }),
-      ).then(() => state.completeStep('k3s-db-sideload')),
-    );
-  }
   if (sideloadTasks.length > 0) {
-    s.start(`Sideloading app + db to ${sshTargets.length} node(s) in parallel`);
+    s.start(`Sideloading app to ${sshTargets.length} node(s)`);
     await Promise.all(sideloadTasks);
-    s.stop('Sideload complete (app + db)');
+    s.stop('Sideload complete (app)');
   }
 
   // 8. Apply manifests + supabase Helm release + roll out the app.

package/src/lib/deploy/orchestrator.js

@@ -560,6 +560,7 @@ export async function executeDeployment(args, gatheredConfig) {
         startComposeStack,
         runMigrations,
         createAdminUser,
+        setupComposeBackupCron,
       } = await import('./compose/index.js');
       const { setupServerFiles, waitForSSH: waitForComposeSSH } = await import(
         './compose/index.js'
@@ -873,6 +874,27 @@ export async function executeDeployment(args, gatheredConfig) {
       }
       healthSpinner.stop(`App is serving requests (HTTP ${health.status})`);
 
+      // Install the scheduled wal-g backup cron on the server. Without this,
+      // a fresh compose deploy collects + persists backupConfig (envConfig.backup)
+      // but never schedules a backup — backups only ran after a `vibecarbon scale`
+      // event (scale.js was the lone caller). setupComposeBackupCron defaults the
+      // schedule + retain when backupConfig is absent, so it always installs a
+      // sensible cron. wal-g reads its S3 config from the db container env
+      // (rendered into .env at deploy) — the cron just runs compose-backup.sh,
+      // which self-skips (exit 0) when no S3 backup target is configured, so an
+      // always-installed cron never hard-fails on a no-S3 deploy.
+      //
+      // A cron-install failure must NOT fail an already-healthy deploy: the app
+      // is serving by this point. Downgrade to a warning so a transient ssh/cron
+      // hiccup doesn't roll back a successful deploy.
+      try {
+        await perfAsync('deploy.compose.backupCron', async () =>
+          setupComposeBackupCron(serverIp, sshKeyPath, projectConfig.projectName, backupConfig),
+        );
+      } catch (err) {
+        p.log.warn(`Scheduled backup cron install failed (deploy still succeeded): ${err.message}`);
+      }
+
       // No ACME self-heal needed here. Managed-DNS deploys (cloudflare/hetzner)
       // issue certs via DNS-01, which has no HTTP-01-vs-DNS-propagation race to
       // recover from; `manual` DNS was never self-healed (we don't manage its

package/src/lib/iac/index.js

@@ -135,6 +135,35 @@ export async function initBackend(options = {}) {
   }
 }
 
+// S3 (Hetzner Object Storage) throttles the Pulumi state backend with
+// `503 SlowDown` / `ServiceUnavailable` when many stack operations read/write
+// state concurrently — the documented blocker for running the e2e matrix in
+// parallel (multiple scenarios' `up`/`refresh`/`destroy` hit one S3 account at
+// once). It is also a transient any single deploy can hit. These ops are all
+// idempotent (Pulumi reconciles to desired state), so retrying on a throttle is
+// safe. Non-throttle errors propagate immediately.
+export const STATE_BACKEND_THROTTLE_PATTERN =
+  /SlowDown|ServiceUnavailable|RequestLimitExceeded|throttl|too many requests|\b503\b/i;
+
+export async function withStateBackendRetry(fn, desc, options = {}) {
+  const attempts = options.attempts ?? 5;
+  const baseMs = options.baseMs ?? 2000;
+  const sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      const msg = err?.message || String(err);
+      if (attempt >= attempts || !STATE_BACKEND_THROTTLE_PATTERN.test(msg)) throw err;
+      const backoff = Math.min(baseMs * 2 ** (attempt - 1), 30_000);
+      console.error(
+        `[pulumi] ${desc}: S3 state backend throttled (attempt ${attempt}/${attempts}), retrying in ${backoff}ms`,
+      );
+      await sleep(backoff);
+    }
+  }
+}
+
 /**
  * Run `pulumi up` and return the raw outputs. Streams Pulumi logs to an
  * optional `onOutput` sink so callers can forward progress to a spinner.
@@ -181,7 +210,10 @@ export async function upStack(stackName, program, options = {}) {
   }
   if (needsRefresh) {
     try {
-      await stack.refresh({ onOutput: options.onOutput, color: 'never' });
+      await withStateBackendRetry(
+        () => stack.refresh({ onOutput: options.onOutput, color: 'never' }),
+        'pre-up refresh',
+      );
     } catch (err) {
       // refresh can fail if the state file is completely corrupt or a
       // resource is unreachable. Log and proceed — `up` may still succeed
@@ -190,10 +222,10 @@ export async function upStack(stackName, program, options = {}) {
     }
   }
 
-  const result = await stack.up({
-    onOutput: options.onOutput,
-    color: 'never',
-  });
+  const result = await withStateBackendRetry(
+    () => stack.up({ onOutput: options.onOutput, color: 'never' }),
+    'up',
+  );
   // Flatten outputs: { foo: { value: 'bar', secret: false } } → { foo: 'bar' }
   const outputs = {};
   for (const [key, { value }] of Object.entries(result.outputs || {})) {
@@ -221,11 +253,17 @@ export async function destroyStack(stackName, program, options = {}) {
   // treats it as a hard error — surfaced in orphan-stack cleanup as
   // "server not found (not_found, ...)".
   try {
-    await stack.refresh({ onOutput: options.onOutput, color: 'never' });
+    await withStateBackendRetry(
+      () => stack.refresh({ onOutput: options.onOutput, color: 'never' }),
+      'pre-destroy refresh',
+    );
   } catch (err) {
     console.error(`[pulumi] pre-destroy refresh warning: ${err.message?.split('\n')[0] || err}`);
   }
-  await stack.destroy({ onOutput: options.onOutput, color: 'never' });
+  await withStateBackendRetry(
+    () => stack.destroy({ onOutput: options.onOutput, color: 'never' }),
+    'destroy',
+  );
   await stack.workspace.removeStack(stackName);
   return { destroyed: true };
 }

package/src/restore.js

@@ -16,10 +16,10 @@
  * memory:feedback_cli_single_dash_flags.
  */
 
-import { existsSync, unlinkSync } from 'node:fs';
-import { basename, join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { basename } from 'node:path';
 import * as p from '@clack/prompts';
-import { downloadS3Backup, listS3Backups } from './lib/backup-s3.js';
+import { listS3Backups } from './lib/backup-s3.js';
 import { renderHelp } from './lib/cli/help.js';
 import { parseFlags } from './lib/cli/parse-flags.js';
 import { selectEnvironment } from './lib/cli/select-environment.js';
@@ -31,13 +31,18 @@ import {
   loadProjectConfig,
   loadS3Config,
 } from './lib/config.js';
+// Static imports (no cycle: neither compose/index.js nor compose/ha.js imports
+// restore.js). Aliased so runComposeRestore can default to the real functions
+// while still accepting injected mocks for unit tests.
+import { configureStandbyReplication as realConfigureStandbyReplication } from './lib/deploy/compose/ha.js';
+import { restoreCompose as realRestoreCompose } from './lib/deploy/compose/index.js';
 import { getS3Credentials } from './lib/hetzner-guided-setup.js';
 import { requireLicense } from './lib/licensing/index.js';
 import { ensureOperatorIpAccess } from './lib/operator-ip.js';
 import { perfAsync } from './lib/perf.js';
 import { listPodBackups } from './lib/pod-backups.js';
 import { assertInProjectDir } from './lib/project-guard.js';
-import { getPostgresPod, getSSHKeyPath, scpUpload, sshKubectl, sshRunScript } from './lib/ssh.js';
+import { getPostgresPod, getSSHKeyPath, sshKubectl, sshRunScript } from './lib/ssh.js';
 import { createTracker } from './lib/tracker.js';
 import { validateBackupFilename } from './lib/validators.js';
 import { VERSION } from './lib/version.js';
@@ -116,18 +121,50 @@ function scaleUpApp(ip, sshKeyPath) {
   sshKubectl(ip, sshKeyPath, ['scale', 'deployment', 'app', '-n', 'vibecarbon', '--replicas=2']);
 }
 
-function verifyPostgres(ip, sshKeyPath) {
-  const pod = getPostgresPod(ip, sshKeyPath);
-  sshKubectl(ip, sshKeyPath, [
-    'exec',
-    '-n',
-    'vibecarbon',
-    pod,
-    '--',
-    'pg_isready',
-    '-U',
-    'postgres',
-  ]);
+/**
+ * Wait until postgres on the db pod is actually accepting connections.
+ *
+ * After a wal-g restore, postgres replays archived WAL (restore_command) before
+ * promoting — minutes on a busy DB (observed 112s). The pod can be Ready while
+ * postgres is still "not accepting connections", so a single pg_isready races
+ * slow replay and reports a false restore failure. Poll until it accepts (or
+ * give up after `timeoutMs`).
+ *
+ * @param {string} ip
+ * @param {string} sshKeyPath
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs=300000]
+ * @param {number} [opts.intervalMs=5000]
+ * @param {(ip: string, key: string, args: string[]) => unknown} [opts.exec=sshKubectl]
+ * @param {(ip: string, key: string) => string} [opts.getPod=getPostgresPod]
+ * @param {(ms: number) => Promise<void>} [opts.sleep]
+ */
+export async function verifyPostgres(ip, sshKeyPath, opts = {}) {
+  const {
+    timeoutMs = 300_000,
+    intervalMs = 5_000,
+    exec = sshKubectl,
+    getPod = getPostgresPod,
+    sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+  } = opts;
+  const pod = getPod(ip, sshKeyPath);
+  const deadline = Date.now() + timeoutMs;
+  let lastErr = '';
+  for (;;) {
+    try {
+      exec(ip, sshKeyPath, ['exec', '-n', 'vibecarbon', pod, '--', 'pg_isready', '-U', 'postgres']);
+      return; // exit 0 = accepting connections
+    } catch (err) {
+      lastErr = err instanceof Error ? err.message : String(err);
+    }
+    if (Date.now() >= deadline) {
+      throw new Error(
+        `verifyPostgres: postgres did not accept connections within ${Math.round(timeoutMs / 1000)}s ` +
+          `(still replaying WAL after a wal-g restore?). Last pg_isready error: ${lastErr}`,
+      );
+    }
+    await sleep(intervalMs);
+  }
 }
 
 // (k8s pg_dump restore helpers removed — k8s restore is now wal-g-native via
@@ -222,7 +259,13 @@ export async function run(args) {
     seed: envSeed,
   });
 
-  const serverIp = envConfig.servers?.[0]?.ip;
+  // compose-ha: pick the CURRENT primary by role (a prior failover may have
+  // swapped roles while preserving array order, so servers[0] is not reliably
+  // the primary). Falls back to servers[0] for non-HA modes that have no role.
+  // This same serverIp feeds both the restore target and (in runComposeRestore)
+  // the re-seed source, so they stay consistent with the standby selection.
+  const primaryServer = envConfig.servers?.find((sv) => sv.role === 'primary');
+  const serverIp = primaryServer?.ip || envConfig.servers?.[0]?.ip;
   if (!serverIp) {
     p.log.error(`No server IP found for environment '${envName}'.`);
     process.exit(1);
@@ -302,16 +345,17 @@ export async function run(args) {
   let chosenSource;
   if (!wantsLatest && values.source) {
     chosenSource = classifySource(/** @type {string} */ (values.source));
-  } else if (wantsLatest && !isCompose) {
-    // k8s restore is wal-g-native: `latest` resolves to the LATEST base backup
-    // INSIDE the cluster — the db pod's walg-restore init container runs
-    // `wal-g backup-fetch LATEST` (see runK8sRestore). pickInteractiveSource
-    // lists pg_dump `*_full.tar.gz` backups via listS3Backups, which cannot see
-    // wal-g `base_*` objects, so routing k8s `-source latest` through it wrongly
-    // exits "No backups found in S3" even though a wal-g base backup exists.
-    // Hand runK8sRestore the `latest` sentinel directly and let wal-g resolve
-    // it. (RCA 2026-05-31: e2e k8s/k8s-ha restore step — never reached before
-    // the backup chain (#4/#5/#6) was fixed.)
+  } else if (wantsLatest) {
+    // BOTH compose and k8s restore are now wal-g-native: `latest` resolves to
+    // the LATEST base backup that wal-g pushed to S3 — k8s via the db pod's
+    // walg-restore init container, compose via `restoreCompose(..., 'latest')`
+    // → `wal-g backup-fetch LATEST` (see runK8sRestore / runComposeRestore).
+    // pickInteractiveSource lists legacy pg_dump `*_full.tar.gz` backups via
+    // listS3Backups, which CANNOT see wal-g `basebackups_005/base_*` objects, so
+    // routing `-source latest` through it wrongly exits "No backups found in S3"
+    // even though a wal-g base backup exists. Hand the `latest` sentinel
+    // straight to the restore runner and let wal-g resolve it. (RCA 2026-05-31:
+    // e2e k8s/k8s-ha restore step; compose found via kept-rig manual restore.)
     chosenSource = { kind: 's3', name: 'latest' };
   } else {
     chosenSource = await pickInteractiveSource({
@@ -371,7 +415,7 @@ export async function run(args) {
         projectName,
         serverIp,
         sshKeyPath,
-        s3WithCreds: s3,
+        envConfig,
       });
     } else {
       await runK8sRestore({
@@ -579,46 +623,83 @@ async function pickInteractiveSource({
   return { kind: 's3', name: /** @type {string} */ (selected) };
 }
 
-async function runComposeRestore({
+// compose restore is wal-g-based (S3 pull, no file transfer). Mirrors
+// runK8sRestore's structure: reject local-file sources, resolve target,
+// call restoreCompose, verify postgres.
+//
+// COMPOSE-HA re-seed (Invariant 3): the wal-g restore rewinds the PRIMARY
+// (servers[0]) to an earlier LSN. For a compose-ha deployment the standby then
+// has WAL *ahead* of the restored primary (streamed from the old primary) and
+// cannot resume streaming — PostgreSQL would reject it with "requested timeline
+// does not contain minimum recovery point" (timeline divergence). So after the
+// primary promotes we re-seed the standby with a fresh pg_basebackup from the
+// restored primary via configureStandbyReplication, which is destructive-by-
+// design (it wipes the standby's PGDATA and re-basebackups — exactly what we
+// want here). Single-region compose has no standby, so the re-seed is skipped.
+//
+// restoreCompose + configureStandbyReplication are injectable so the dispatch
+// logic is unit-testable without SSH; production passes neither and they resolve
+// to the real module functions (static imports — no cycle: ha.js/compose-index
+// do not import restore.js).
+export async function runComposeRestore({
   s,
   chosenSource,
   envName,
   projectName,
   serverIp,
   sshKeyPath,
-  s3WithCreds,
+  envConfig = {},
+  restoreCompose = realRestoreCompose,
+  configureStandbyReplication = realConfigureStandbyReplication,
 }) {
-  const backupFile = chosenSource.name;
-
   if (chosenSource.kind === 'local') {
-    p.log.info(`Uploading local file: ${c.bold(chosenSource.path)}`);
-    scpUpload(serverIp, sshKeyPath, chosenSource.path, `/opt/${projectName}/backups/${backupFile}`);
-  } else if (s3WithCreds?.secretKey) {
-    s.start(`Downloading ${backupFile} from S3 to server`);
-    const tmpLocal = join(process.cwd(), `.restore-tmp-${backupFile}`);
-    try {
-      await downloadS3Backup(s3WithCreds, `backups/${backupFile}`, tmpLocal);
-      scpUpload(serverIp, sshKeyPath, tmpLocal, `/opt/${projectName}/backups/${backupFile}`);
+    throw new Error(
+      'compose restore is wal-g-based and pulls from S3 — local backup files are not supported. ' +
+        'Use `-source latest` or an ISO-8601 timestamp for PITR.',
+    );
+  }
+
+  // wal-g target: 'latest' → LATEST base backup; otherwise treat the source
+  // name as an ISO-8601 timestamp for point-in-time recovery.
+  const target =
+    chosenSource.name === 'latest' || !chosenSource.name ? 'latest' : chosenSource.name;
+
+  s.start('Restoring database via wal-g (app will be temporarily unavailable)');
+  await restoreCompose(serverIp, sshKeyPath, projectName, target);
+  s.stop('Database restored');
+
+  // compose-ha: the wal-g restore rewound the primary to an earlier LSN, so the
+  // standby now has WAL AHEAD of the primary and cannot resume streaming. Re-seed
+  // it with a fresh pg_basebackup from the restored primary so it rejoins the new
+  // timeline. (configureStandbyReplication is destructive-by-design: it wipes the
+  // standby's PGDATA and re-basebackups — exactly what we want here.)
+  //
+  // A re-seed failure is NOT a restore failure: the primary is restored and
+  // serving — only the standby is degraded. Warn (don't rethrow) so the restore
+  // still reports success; the operator can resync the standby via `deploy`.
+  if (envConfig.deployMode === 'compose-ha') {
+    const standby = envConfig.servers?.find((sv) => sv.role === 'standby');
+    if (standby?.ip) {
       try {
-        unlinkSync(tmpLocal);
-      } catch {
-        // ignore cleanup errors
+        s.start('Re-seeding standby from restored primary (pg_basebackup)');
+        await configureStandbyReplication(standby.ip, serverIp, sshKeyPath, projectName);
+        s.stop('Standby re-seeded from restored primary');
+      } catch (err) {
+        s.stop('Standby re-seed failed');
+        p.log.warn(
+          `Primary restored and serving, but standby re-seed failed: ${err.message}. ` +
+            `Run \`vibecarbon deploy ${envName}\` to resync the standby.`,
+        );
       }
-      s.stop('Backup transferred to server');
-    } catch (error) {
-      s.stop(`S3 download failed: ${error.message}`);
-      throw error;
+    } else {
+      p.log.warn(
+        'compose-ha restore: no standby server found to re-seed — skipping (verify replication manually).',
+      );
     }
   }
 
-  s.start('Restoring database...');
-  const { restoreCompose } = await import('./lib/deploy/compose/index.js');
-  restoreCompose(serverIp, sshKeyPath, projectName, backupFile);
-  s.stop('Database restored');
   p.log.success('Restore completed successfully');
-  // env name is unused by restoreCompose but kept in the closure for
-  // diagnostic logging consistency.
-  void envName;
+  p.log.info(`Verify with: ${c.info(`vibecarbon status ${envName}`)}`);
 }
 
 // k8s restore is wal-g-native: postgres + wal-g live in the supabase-db pod,