vibecarbon 0.9.0 → 0.11.0

package/src/lib/deploy/orchestrator.js

@@ -4,7 +4,6 @@
  */
 
 import dns from 'node:dns';
-import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import * as p from '@clack/prompts';
 import { c } from '../colors.js';
@@ -15,16 +14,13 @@ import {
   registerProject,
   saveProjectConfig,
 } from '../config.js';
-import { waitForDNSPropagation } from '../dns-propagation.js';
 import { ensureOperatorIpAccess } from '../operator-ip.js';
 import { perfAsync, perfTimer } from '../perf.js';
 import { createTracker } from '../tracker.js';
 import { useDnsChallenge } from './acme.js';
 import { renderBundle } from './bundle.js';
-import { sideloadCompose } from './image.js';
-import { buildRemote } from './remote-build.js';
 import { StateTracker } from './state.js';
-import { generateSSHKeyPair } from './utils.js';
+import { isComposeTier, isHATier, isK8sTier, resolveTier } from './tier-registry.js';
 
 /**
  * Probe the public /api/health endpoint to verify the app is reachable
@@ -194,8 +190,7 @@ export async function executeDeployment(args, gatheredConfig) {
   } = gatheredConfig;
 
   const { deployMode } = config;
-  const isComposeDeploy = deployMode === 'compose' || deployMode === 'compose-ha';
-  const isHA = deployMode === 'compose-ha' || !!config.ha;
+  const tier = resolveTier({ deployMode, ha: config.ha });
 
   // Operator-IP access: detect + persist + (when env already deployed) patch
   // the live Hetzner firewall. On a fresh first deploy this just appends to
@@ -205,7 +200,7 @@ export async function executeDeployment(args, gatheredConfig) {
   const accessResult = await ensureOperatorIpAccess({
     projectConfig,
     environment,
-    isHA,
+    isHA: isHATier(tier),
     apiToken,
     yes: !!args.yes,
     onMessage: (msg) => p.log.info(msg),
@@ -223,16 +218,45 @@ export async function executeDeployment(args, gatheredConfig) {
   }
 
   const s3Spinner = p.spinner();
-  const { HetznerS3Provider } = await import('../providers/hetzner-s3.js');
+  const { HetznerS3Provider, deriveStateBucketName } = await import('../providers/hetzner-s3.js');
   const s3Provider = new HetznerS3Provider(s3Config.accessKey, s3Config.secretKey, s3Config.region);
 
+  // Dedicated Pulumi-state bucket, derived once from the app storage bucket.
+  // Pulumi state used to live in the app storage bucket, which meant `destroy`
+  // deleting that bucket could yank the state backend mid-destroy. State now
+  // lives here instead. Both HA stacks share this ONE bucket.
+  const stateBucket = deriveStateBucketName(s3Config.bucket);
+
   // --- STEP 1: S3 Setup (Idempotent) ---
+  // `stateBucket` is part of the inputs hash so an upgraded CLI (whose prior
+  // deploy-state predates the dedicated state bucket) re-runs this step once to
+  // create + migrate the state bucket. Combined with the StateTracker version
+  // bump, no stale hash can skip the migration.
   const s3Inputs = {
     bucket: s3Config.bucket,
     backupBucket: backupS3Config.bucket,
+    stateBucket,
     region: s3Config.region,
   };
-  if (!state.shouldSkip('s3-setup', s3Inputs)) {
+  // Verify hook: even on a matching hash, re-run s3-setup if any of the buckets
+  // were deleted out-of-band (e.g. by a prior `destroy`). Without this, a
+  // resumed deploy skips bucket creation then fails downstream on NoSuchBucket
+  // (the exact resume hazard called out in Finding 2). Other steps that would
+  // benefit from a similar probe — `compose-setup-server` (server may be gone)
+  // and `dns-setup` — are left on the plain hash check to keep this change
+  // focused; s3-setup is the highest-value one because the state backend lives
+  // here.
+  const verifyBucketsExist = async () => {
+    const [app, backup, stateB] = await Promise.all([
+      s3Provider.bucketExists(s3Config.bucket),
+      backupS3Config?.bucket
+        ? s3Provider.bucketExists(backupS3Config.bucket)
+        : Promise.resolve(true),
+      s3Provider.bucketExists(stateBucket),
+    ]);
+    return app && backup && stateB;
+  };
+  if (!(await state.shouldSkipWithVerify('s3-setup', s3Inputs, verifyBucketsExist))) {
     state.startStep('s3-setup', s3Inputs);
     try {
       s3Spinner.start(`Creating S3 bucket: ${s3Config.bucket}`);
@@ -245,7 +269,22 @@ export async function executeDeployment(args, gatheredConfig) {
 
       try {
         s3Spinner.start('Configuring bucket CORS');
-        const corsOrigins = domain ? [`https://${domain}`] : ['*'];
+        // SECURITY: never fall back to a `*` wildcard — that would let any
+        // origin issue credentialed browser requests against the storage
+        // bucket. When a domain is configured we scope to its https origins
+        // (apex + api/app subdomains the app actually serves from); a
+        // domainless deploy has no browser origin to trust yet, so we scope
+        // to localhost dev origins and warn. The operator re-runs deploy once
+        // a domain is set to widen CORS to the real origin.
+        let corsOrigins;
+        if (domain) {
+          corsOrigins = [`https://${domain}`, `https://api.${domain}`, `https://app.${domain}`];
+        } else {
+          corsOrigins = ['http://localhost:3000', 'http://localhost:5173'];
+          p.log.warn(
+            'No domain configured — scoping bucket CORS to localhost dev origins instead of `*`. Re-run deploy with a domain to allow your production origin.',
+          );
+        }
         await s3Provider.configureCORS(s3Config.bucket, corsOrigins);
         s3Spinner.stop('CORS configured');
       } catch {
@@ -264,7 +303,98 @@ export async function executeDeployment(args, gatheredConfig) {
         s3Spinner.stop(`Backup bucket creation failed: ${backupBucketError.message}`);
       }
 
-      const result = { region: s3Provider.region, endpoint: s3Provider.getEndpoint() };
+      // Dedicated Pulumi-state bucket. Created imperatively (like the app +
+      // backup buckets) BEFORE any Pulumi stack op / initBackend. NOT
+      // web-facing: no CORS, not public.
+      s3Spinner.start(`Creating Pulumi state bucket: ${stateBucket}`);
+      const stateBucketResult = await s3Provider.createBucket(stateBucket);
+      s3Spinner.stop(
+        stateBucketResult.created
+          ? `Pulumi state bucket created: ${stateBucket}`
+          : `Pulumi state bucket exists: ${stateBucket}`,
+      );
+
+      // SECURITY/SAFETY: migrate legacy Pulumi state out of the app storage
+      // bucket. If the state bucket has no `.pulumi/` objects but the OLD app
+      // bucket does, this is an env deployed before the dedicated state bucket
+      // existed — copy the state prefix over so we resolve to real state, not
+      // empty state. Re-initializing an EMPTY state backend against live infra
+      // would make Pulumi believe nothing exists and either duplicate or orphan
+      // the running servers. So if the copy fails, we DO NOT proceed silently:
+      // abort loudly and tell the operator to migrate manually or
+      // destroy+redeploy. Fresh deploys hit neither branch (both buckets empty).
+      const PULUMI_STATE_PREFIX = '.pulumi/';
+      // Determine whether there is GENUINE legacy state to migrate. The check
+      // lists the just-created state bucket, which can race S3 list-after-create
+      // consistency and throw — that is NOT the orphan-risk case (a fresh deploy
+      // has no legacy state), so a check failure must NOT hard-abort. Only a
+      // CONFIRMED legacy-state copy failure aborts (below). Retry the checks a
+      // few times so a transient blip doesn't misfire either way.
+      let legacyState = false;
+      let checkOk = false;
+      for (let attempt = 1; attempt <= 3; attempt++) {
+        try {
+          const stateSeeded = await s3Provider.hasObjectsWithPrefix(
+            stateBucket,
+            PULUMI_STATE_PREFIX,
+          );
+          legacyState = stateSeeded
+            ? false
+            : await s3Provider.hasObjectsWithPrefix(s3Config.bucket, PULUMI_STATE_PREFIX);
+          checkOk = true;
+          break;
+        } catch (checkError) {
+          if (attempt === 3) {
+            // Couldn't determine migration need after retries (e.g. list-after-
+            // create consistency on the fresh state bucket). A fresh deploy — the
+            // overwhelming common case — has nothing to migrate, so proceed with
+            // a loud warning rather than blocking every deploy on a transient S3
+            // blip. (Pulumi then uses the dedicated state bucket normally.)
+            p.log.warn(
+              `Could not verify legacy Pulumi state after ${attempt} attempts — proceeding ` +
+                `(a fresh deploy has none to migrate): ${checkError.message}`,
+            );
+          } else {
+            await new Promise((r) => setTimeout(r, 1500 * attempt));
+          }
+        }
+      }
+      if (checkOk && legacyState) {
+        // CONFIRMED legacy state in the app bucket — migrate it. If THIS copy
+        // fails we DO NOT proceed silently: re-initializing an EMPTY state
+        // backend against live infra would orphan the running servers.
+        try {
+          s3Spinner.start(`Migrating Pulumi state ${s3Config.bucket} → ${stateBucket}`);
+          const { copied } = await s3Provider.copyPrefix(
+            s3Config.bucket,
+            stateBucket,
+            PULUMI_STATE_PREFIX,
+          );
+          s3Spinner.stop(`Migrated ${copied} Pulumi state object(s) to ${stateBucket}`);
+        } catch (copyError) {
+          s3Spinner.stop(`Pulumi state migration FAILED: ${copyError.message}`);
+          p.log.error(
+            [
+              'Could not migrate existing Pulumi state from the app storage bucket to the',
+              `dedicated state bucket (${stateBucket}). Proceeding would re-initialize an`,
+              'EMPTY state backend against your live infrastructure, which would orphan the',
+              'running servers (Pulumi would no longer track them).',
+              '',
+              'Aborting to protect your infrastructure. To recover, either:',
+              `  • Copy the ".pulumi/" prefix from ${s3Config.bucket} to ${stateBucket} manually`,
+              '    (e.g. via `aws s3 sync ... --endpoint-url <endpoint>`), then re-run deploy; or',
+              '  • Destroy this environment and redeploy from scratch.',
+            ].join('\n'),
+          );
+          process.exit(1);
+        }
+      }
+
+      const result = {
+        region: s3Provider.region,
+        endpoint: s3Provider.getEndpoint(),
+        stateBucket,
+      };
       state.completeStep('s3-setup', result);
     } catch (error) {
       const errMsg =
@@ -285,6 +415,11 @@ export async function executeDeployment(args, gatheredConfig) {
     backupS3Config.region = s3Config.region;
     backupS3Config.endpoint = s3Config.endpoint;
   }
+  // Always resolve stateBucket (deterministic derivation) so the Pulumi
+  // backend targets the dedicated bucket, whether s3-setup ran or was skipped.
+  // Every downstream iac call reads it via s3Config, and it's persisted into
+  // .vibecarbon.json so scale / failover / destroy resolve the same backend.
+  s3Config.stateBucket = s3Result?.stateBucket || stateBucket;
 
   // SKELETON SAVE: persist the minimum env entry that `vibecarbon destroy`
   // needs to recover, BEFORE any `pulumi up` runs. Without this, a crash
@@ -315,7 +450,12 @@ export async function executeDeployment(args, gatheredConfig) {
       ...(secondaryRegion ? { secondaryRegion } : {}),
       ...(domain ? { domain } : {}),
       ...(dnsProvider ? { dnsProvider } : {}),
-      s3: { bucket: s3Config.bucket, region: s3Config.region, endpoint: s3Config.endpoint },
+      s3: {
+        bucket: s3Config.bucket,
+        region: s3Config.region,
+        endpoint: s3Config.endpoint,
+        stateBucket: s3Config.stateBucket,
+      },
       backupS3: backupS3Config?.bucket
         ? {
             bucket: backupS3Config.bucket,
@@ -359,8 +499,7 @@ export async function executeDeployment(args, gatheredConfig) {
   const imageResolveTimer = perfTimer('deploy.image.resolve');
   const buildMode = resolveBuildMode(args, process.cwd(), deployMode);
   const isDirectDeploy = buildMode === 'direct';
-  const isComposeMode = deployMode === 'compose' || deployMode === 'compose-ha';
-  const isComposeLocal = buildMode === 'local' && isComposeMode;
+  const isComposeLocal = buildMode === 'local' && isComposeTier(tier);
   let imageReadyPromise;
   // For compose-local: kicked off here in parallel with iac.upStack below.
   // The build is awaited later (just before sideload). Other modes leave
@@ -474,7 +613,7 @@ export async function executeDeployment(args, gatheredConfig) {
   // DNS-propagation poll or the Traefik cert self-heal below. `manual` keeps
   // HTTP-01. Scoped to compose: k8s issues certs via cert-manager and ignores
   // this Traefik override entirely.
-  const dnsChallenge = isComposeDeploy && useDnsChallenge(dnsProvider);
+  const dnsChallenge = isComposeTier(tier) && useDnsChallenge(dnsProvider);
   const bundlePath = renderBundle(projectConfig.projectName, {
     domain,
     image: imageRef,
@@ -524,9 +663,79 @@ export async function executeDeployment(args, gatheredConfig) {
       : Promise.resolve(null);
 
   // --- STEP 5: Architecture Dispatch ---
-  let deployResult;
-  if (isComposeDeploy) {
-    if (deployMode === 'compose-ha') {
+  // One entry per tier, keyed by the registry. deploymentConfig is a cheap,
+  // pure object literal built unconditionally above the table; its k8s-only
+  // fields are simply unused on the compose paths.
+  const deploymentConfig = {
+    projectName: projectConfig.projectName,
+    environment: config.environment,
+    provider: config.provider,
+    region,
+    secondaryRegion,
+    masterServerType,
+    supabaseServerType,
+    workerServerType,
+    serverType,
+    minWorkers,
+    maxWorkers,
+    domain,
+    ha: config.ha,
+    observability: config.observability,
+    services,
+    apiToken,
+    cloudflareApiToken,
+    cloudflareZoneId,
+    hetznerDnsZoneId,
+    dnsProvider,
+    s3Config,
+    backupConfig,
+    backupBucketName: backupS3Config?.bucket || null,
+    // DR: when set (`deploy -restore latest|<ts>`), the db pod's init
+    // container seeds PGDATA from S3 via wal-g and applyMigrations is
+    // skipped (the restored DB is authoritative).
+    restore: args.restore || null,
+    // Finding #1: hard-gate k8s-HA replication unless the operator opts into a
+    // warm/degraded standby via `deploy -allow-degraded`.
+    allowDegraded: !!args.allowDegraded,
+    operatorCidrs: projectConfig.operatorCidrs ?? [],
+    imageReadyPromise,
+    tracker,
+    state,
+  };
+
+  const deployCtx = {
+    projectConfig,
+    environment,
+    envConfig,
+    region,
+    serverType,
+    services,
+    domain,
+    dnsProvider,
+    apiToken,
+    cloudflareApiToken,
+    cloudflareZoneId,
+    hetznerDnsZoneId,
+    backupConfig,
+    imageRef,
+    bundlePath,
+    ciReady,
+    s3Config,
+    state,
+    dnsChallenge,
+    dnsWarmupPromise,
+    isDirectDeploy,
+    isComposeLocal,
+    localImageTag,
+    composeLocalBuildPromise,
+  };
+
+  const TIER_DEPLOYERS = {
+    compose: async () => {
+      const { deployComposeSingle } = await import('./compose/single.js');
+      return deployComposeSingle(deployCtx);
+    },
+    'compose-ha': async () => {
       const { deployComposeHA } = await import('./compose/ha.js');
       const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
       const s = tracker.spinner();
@@ -534,7 +743,7 @@ export async function executeDeployment(args, gatheredConfig) {
       // Wrap the entire compose-HA pipeline so the perf trace shows the
       // headline cost (which contains compose-ha.deploy.* sub-stages from
       // compose/ha.js).
-      deployResult = await perfAsync('deploy.ha.compose.full', () =>
+      const result = await perfAsync('deploy.ha.compose.full', () =>
         deployComposeHA({
           projectName: projectConfig.projectName,
           environment,
@@ -558,426 +767,29 @@ export async function executeDeployment(args, gatheredConfig) {
           backupS3Config,
           backupConfig,
           bundlePath,
+          // Finding #1: hard-gate replication unless the operator opts into a
+          // warm/degraded standby via `deploy -allow-degraded`.
+          allowDegraded: !!args.allowDegraded,
           onProgress: (msg) => s.message(msg),
         }),
       );
       s.stop('Compose HA deployment complete');
-    } else {
-      const {
-        setupServer,
-        dockerLoginOnServer,
-        startComposeStack,
-        runMigrations,
-        createAdminUser,
-        setupComposeBackupCron,
-      } = await import('./compose/index.js');
-      const { setupServerFiles, waitForSSH: waitForComposeSSH } = await import(
-        './compose/index.js'
-      );
-      const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
-
-      let serverIp = envConfig.servers?.[0]?.ip || null;
-      // Track the Hetzner server's API id + literal name so destroy can find
-      // the VPS without falling back to the literal string "master" (which
-      // never matches the Pulumi-assigned name `${projectName}-${env}`).
-      // Without this, destroy → re-deploy hits 'server name is already used
-      // (uniqueness_error)' because the previous VPS was never deleted
-      // (caught in compose restore matrix runs).
-      let serverHetznerId = envConfig.servers?.[0]?.id || null;
-      let serverHetznerName = envConfig.servers?.[0]?.hetznerName || null;
-      if (!serverIp) {
-        generateSSHKeyPair(sshKeyPath);
-        const { upStack } = await import('../iac/index.js');
-        const { buildHetznerComposeProgram } = await import('../iac/programs/hetzner-compose.js');
-        const program = buildHetznerComposeProgram({
-          projectName: projectConfig.projectName,
-          environment,
-          sshPublicKey: readFileSync(`${sshKeyPath}.pub`, 'utf-8').trim(),
-          location: region,
-          serverType: serverType || 'cx23',
-          labels: { 'managed-by': 'vibecarbon' },
-          allowedSshIps: (projectConfig.operatorCidrs ?? []).map((e) => e.cidr),
-        });
-        // VM provisioning via Pulumi — typical 30-90s on Hetzner cx23.
-        const result = await perfAsync('deploy.iac.upStack', () =>
-          upStack(environment, program, { hcloudToken: apiToken, s3Config }),
-        );
-        serverIp = result.outputs.serverIp;
-        serverHetznerId = result.outputs.serverId || null;
-        serverHetznerName = `${projectConfig.projectName}-${environment}`;
-        await perfAsync('deploy.iac.waitForSSH', () => waitForComposeSSH(serverIp, sshKeyPath, 30));
-      }
-
-      // --- Warm path (StateTracker) ---
-      // On a second deploy against an already-provisioned server, skip the
-      // idempotent-but-slow steps (cloud-init probe, GHCR login, bundle rsync)
-      // when their inputs haven't changed. `docker compose up -d` is cheap
-      // and always runs — it applies diffs to the live stack.
-      const setupInputs = { serverIp };
-      if (!state.shouldSkip('compose-setup-server', setupInputs)) {
-        state.startStep('compose-setup-server', setupInputs);
-        // Cloud-init readiness probe — first-deploy spends 20-40s here while
-        // ufw + unattended-upgrades land. Warm deploys hit the ready marker
-        // immediately, so this is mostly cold-deploy noise.
-        await perfAsync('deploy.compose.cloudInitReady', async () => {
-          if (isDirectDeploy || isComposeLocal) {
-            // Both direct (build over DOCKER_HOST=ssh://) and local
-            // (sideload via docker save | ssh docker load) need dockerd
-            // up on the server before they can proceed. Push mode doesn't
-            // because the server pulls from GHCR after compose-up starts.
-            const { waitForDockerReady } = await import('./compose/index.js');
-            await setupServer(serverIp, sshKeyPath);
-            await waitForDockerReady(serverIp, sshKeyPath);
-          } else {
-            await setupServer(serverIp, sshKeyPath);
-          }
-        });
-        state.completeStep('compose-setup-server', { serverIp });
-      }
-
-      // --- Image transfer to server (mode-dependent) ---
-      if (isComposeLocal) {
-        // Local-build path: the build was kicked off in parallel with
-        // iac.upStack at the top of this function. By the time we reach
-        // here (post-cloudInitReady), the build has almost always finished.
-        // Await the promise (cheap if already settled) then sideload the
-        // image tarball over SSH. Sideload uses docker save | gzip | ssh
-        // | docker load — gzip drops ~600MB → ~150-200MB on the wire.
-        //
-        // Both run silently (build is silent:true; sideload streams no stdout),
-        // so without a spinner this is a 15-30s dead pause right after "Proceed"
-        // on warm redeploys (where the upStack spinner that normally overlaps
-        // the build was skipped). Drive a spinner so the operator sees progress.
-        const imageSpinner = p.spinner();
-        imageSpinner.start('Building app image');
-        await composeLocalBuildPromise;
-        imageSpinner.message('Sideloading app image to the server');
-        await perfAsync('deploy.image.sideload', async () =>
-          sideloadCompose({
-            tag: localImageTag,
-            sshTarget: `root@${serverIp}`,
-            sshKey: sshKeyPath,
-          }),
-        );
-        imageSpinner.stop('App image built and sideloaded');
-      } else if (isDirectDeploy) {
-        // Direct path: build the image on the server via DOCKER_HOST=ssh://.
-        // BuildKit caches aggressively, so unchanged source rebuilds in 1-3s.
-        // Slower than local+sideload (no upStack overlap) but works when the
-        // operator has no local Docker — selected by --direct or by
-        // resolveBuildMode's docker-not-found fallback. VITE_* args plumbed
-        // through here for the same reason as the local-build path above.
-        const { collectComposeBuildArgs } = await import('./compose/build-args.js');
-        const directBuildArgs = collectComposeBuildArgs(process.cwd(), {
-          projectName: projectConfig.projectName,
-          domain,
-        });
-        const success = await perfAsync('deploy.image.directBuild', () =>
-          buildRemote(serverIp, sshKeyPath, localImageTag, process.cwd(), directBuildArgs),
-        );
-        if (!success) process.exit(1);
-      }
-
-      // --- Docker Hub login (skip if creds unchanged) ---
-      // Without this, the reconcile script's `docker compose pull` for
-      // Supabase service images (rest, auth, storage, kong, imgproxy, etc.)
-      // hits Docker Hub anonymously. A fresh Hetzner VPS shares a NAT range
-      // with many other deploys, so the per-IP unauthenticated pull quota
-      // is routinely exhausted — observed 2026-04-26 matrix run #3
-      // ("toomanyrequests: You have reached your unauthenticated pull rate
-      // limit"). Compose-HA + scale already log in; this brings the
-      // single-compose deploy path in line.
-      const dockerHubCreds = loadCredentials().dockerHub || null;
-      if (dockerHubCreds) {
-        const dhLoginInputs = {
-          serverIp,
-          registry: 'docker.io',
-          user: dockerHubCreds.username,
-          tokenFp: dockerHubCreds.token ? dockerHubCreds.token.slice(0, 8) : '',
-        };
-        if (!state.shouldSkip('compose-dockerhub-login', dhLoginInputs)) {
-          state.startStep('compose-dockerhub-login', dhLoginInputs);
-          await dockerLoginOnServer(serverIp, sshKeyPath, dockerHubCreds);
-          state.completeStep('compose-dockerhub-login', { serverIp });
-        }
-      }
-
-      // --- GHCR login (skip if creds unchanged) ---
-      if (ciReady.ghcrPullCreds) {
-        const loginInputs = {
-          serverIp,
-          registry: 'ghcr.io',
-          user: ciReady.ghcrPullCreds.owner,
-          // Hash the token's fingerprint, not the token itself — we don't
-          // want the token written to .vibecarbon/deploy-state-*.json.
-          tokenFp: ciReady.ghcrPullCreds.token ? ciReady.ghcrPullCreds.token.slice(0, 8) : '',
-        };
-        if (!state.shouldSkip('compose-ghcr-login', loginInputs)) {
-          state.startStep('compose-ghcr-login', loginInputs);
-          await dockerLoginOnServer(serverIp, sshKeyPath, {
-            username: ciReady.ghcrPullCreds.owner,
-            token: ciReady.ghcrPullCreds.token,
-            registry: 'ghcr.io',
-          });
-          state.completeStep('compose-ghcr-login', { serverIp });
-        }
-      }
-
-      // --- Bundle rsync (skip when bundle inputs unchanged) ---
-      const bundleInputs = {
-        serverIp,
-        projectName: projectConfig.projectName,
-        imageRef,
-        domain,
-        services,
-      };
-      if (!state.shouldSkip('compose-setup-files', bundleInputs)) {
-        state.startStep('compose-setup-files', bundleInputs);
-        // setupServerFiles tars the bundle locally + streams over ssh +
-        // extracts on the server in one pipeline. Bucket the whole upload
-        // here; the inner tar/upload split is captured in compose/index.js.
-        await perfAsync('deploy.bundle.upload', () =>
-          setupServerFiles(serverIp, sshKeyPath, projectConfig.projectName, {
-            ...services,
-            domain,
-            image: imageRef,
-            bundlePath,
-          }),
-        );
-        state.completeStep('compose-setup-files', { serverIp });
-      }
-
-      // --- DNS update + propagation wait BEFORE compose-up ---
-      //
-      // Traefik attempts ACME HTTP-01 challenges immediately on container
-      // start and bursts ~7 attempts in ~30s before giving up. If LE's
-      // DNS resolver sees stale or absent records on those attempts,
-      // every challenge fails with "no valid A records found" and
-      // acme.json ends the deploy with zero certs issued — browser
-      // sees TRAEFIK DEFAULT CERT and NET::ERR_CERT_AUTHORITY_INVALID.
-      //
-      // The warm-up that fires in Step 4 writes a 0.0.0.0 placeholder;
-      // LE specifically rejects 0.0.0.0 as not-a-valid-A-record. So even
-      // when warm-up "succeeds," it doesn't help ACME — it actively
-      // poisons the resolver state until the real IP overwrites it.
-      //
-      // Fix (RCA from vibecarbon.com cold-deploy 2026-05-19): swap the
-      // ordering. Write the real IP now, wait for it to propagate to
-      // public resolvers, then start compose. Traefik's first ACME
-      // attempt sees real DNS, succeeds on the first try.
-      //
-      // The compose-HA path already writes DNS before compose-up at
-      // src/lib/deploy/compose/ha.js — single-region compose was the
-      // outlier that ran startComposeStack first then updated DNS.
-      if (domain && dnsProvider && dnsProvider !== 'manual') {
-        await perfAsync('deploy.dns.warm', async () => {
-          await dnsWarmupPromise;
-        });
-        const dnsUpdateInputs = { domain, dnsProvider, serverIp };
-        if (!state.shouldSkip('compose-dns-update', dnsUpdateInputs)) {
-          state.startStep('compose-dns-update', dnsUpdateInputs);
-          const { setupSimple: updateDnsIp } =
-            dnsProvider === 'cloudflare'
-              ? await import('../cloudflare.js')
-              : await import('../hetzner-dns.js');
-          const token = dnsProvider === 'cloudflare' ? cloudflareApiToken : apiToken;
-          const zoneId = dnsProvider === 'cloudflare' ? cloudflareZoneId : hetznerDnsZoneId;
-          const dnsSpinner = p.spinner();
-          dnsSpinner.start(`Pointing ${domain} → ${serverIp}...`);
-          await perfAsync('deploy.dns.create', () => updateDnsIp(token, zoneId, domain, serverIp));
-          dnsSpinner.stop(`DNS A record set: ${domain} → ${serverIp}`);
-          state.completeStep('compose-dns-update', { serverIp });
-        }
-        // HTTP-01 only: block compose-up until public resolvers see the real
-        // IP. 120s budget covers Cloudflare's typical edge propagation (under
-        // 60s) with margin. Returns false on timeout; we proceed anyway —
-        // Traefik can still acquire certs on its own retries, the customer
-        // just sees a brief self-signed-cert window. Hard-failing here would
-        // mask deploys that succeed eventually.
-        //
-        // DNS-01 (managed providers) skips this entirely: lego validates via a
-        // TXT record it writes through the provider API, so cert issuance does
-        // not wait on the A record propagating. The A record we just wrote
-        // propagates in the background; the post-deploy health probe uses
-        // localhost + Host header and never depends on it.
-        //
-        // Visible spinner so the operator isn't staring at a frozen
-        // "Yes" screen for up to 120s on a cold deploy where the DNS
-        // edge is slow. `timer` indicator forces the rendered string to
-        // change every second so progress remains visibly alive.
-        if (!dnsChallenge) {
-          const dnsSpinner = p.spinner({ indicator: 'timer' });
-          dnsSpinner.start(`Waiting for ${domain} to resolve to ${serverIp}`);
-          const dnsPropagated = await perfAsync('deploy.dns.waitForPropagation', () =>
-            waitForDNSPropagation(domain, serverIp, 120_000),
-          );
-          dnsSpinner.stop(
-            dnsPropagated
-              ? `DNS resolves: ${domain} → ${serverIp}`
-              : `DNS propagation timed out — proceeding anyway`,
-          );
-        }
-      }
-
-      // `docker compose up -d` is idempotent and cheap — always run so a
-      // config tweak that didn't change the bundle hash (e.g. env var)
-      // still takes effect. Reconcile.sh on the server runs `docker compose
-      // pull` (when not local image) + `docker compose up -d` — see
-      // compose/index.js for the inner perf instrumentation.
-      await perfAsync('deploy.reconcile.run', () =>
-        startComposeStack(serverIp, sshKeyPath, projectConfig.projectName, services),
-      );
-
-      // Apply app migrations + reload PostgREST. The orchestrator's inline
-      // single-compose path previously skipped this (only deployComposeHA ran
-      // runMigrations), so compose-single shipped an EMPTY app schema — 0 public
-      // tables, every DB-backed feature 500'd, and db_schema verify failed with
-      // PGRST205. runMigrations waits for supabase_admin, applies each
-      // supabase/migrations/* with ON_ERROR_STOP=1 (a real failure aborts the
-      // deploy rather than silently shipping a schema-less prod), then reloads
-      // PostgREST's schema cache. Mirrors the HA path.
-      await perfAsync('deploy.compose.migrations', () =>
-        runMigrations(serverIp, sshKeyPath, projectConfig.projectName),
-      );
-
-      // Create the production app super-admin in auth.users via GoTrue's admin
-      // API. Same class of trap as runMigrations above: this inline single-
-      // compose path previously skipped it (only deployComposeHA called
-      // createAdminUser), so a single-server deploy shipped a prod app the
-      // operator couldn't actually log into — the only admin user lived in
-      // their local Docker. Idempotent (422 = already exists); a failure here
-      // is a warning, not a hard abort, so a transient GoTrue blip doesn't
-      // fail an otherwise-healthy deploy — re-running `vibecarbon deploy`
-      // retries it.
-      const adminResult = await perfAsync('deploy.compose.createAdminUser', () =>
-        createAdminUser(serverIp, sshKeyPath, projectConfig.projectName),
-      );
-      if (adminResult.success) {
-        p.log.success(adminResult.message);
-      } else {
-        p.log.warn(`${adminResult.message}. Run \`vibecarbon deploy\` again to retry.`);
-      }
-
-      // --- Verifiable success gate ---
-      // Probe the app's own /api/health endpoint on the server itself (via
-      // localhost + Host header), bypassing DNS/TLS. A successful deploy
-      // must yield an HTTP 2xx here; otherwise fail the deploy loudly with
-      // container state + log tail.
-      //
-      // verifyAppHealth polls every 10s for up to 3 minutes while the app
-      // container restarts and binds port 3000. Without a visible spinner
-      // the operator just sees a frozen "Stack reconciled" line for that
-      // entire window — observed 2026-05-19 vibecarbon.com re-deploy
-      // where compose-up finished in ~5s but the app took ~3min to come
-      // back, leaving the screen looking hung. `timer` indicator so the
-      // elapsed-seconds counter ticks visibly even on terminals where
-      // the spinner frame-character animation gets throttled under load.
-      const { verifyAppHealth } = await import('./compose/index.js');
-      const healthSpinner = p.spinner({ indicator: 'timer' });
-      healthSpinner.start('Waiting for app to start serving requests');
-      let health;
-      try {
-        health = await perfAsync('deploy.health.probe', () =>
-          verifyAppHealth(serverIp, sshKeyPath, projectConfig.projectName, {
-            domain,
-          }),
-        );
-      } catch (err) {
-        healthSpinner.stop('App health probe errored', 1);
-        throw err;
-      }
-      if (!health.healthy) {
-        healthSpinner.stop(`App not serving requests (status: ${health.status})`, 1);
-        p.log.error(
-          `Deploy produced an unhealthy app (probe status: ${health.status}):\n${health.details}`,
-        );
-        process.exit(1);
-      }
-      healthSpinner.stop(`App is serving requests (HTTP ${health.status})`);
-
-      // Install the scheduled wal-g backup cron on the server. Without this,
-      // a fresh compose deploy collects + persists backupConfig (envConfig.backup)
-      // but never schedules a backup — backups only ran after a `vibecarbon scale`
-      // event (scale.js was the lone caller). setupComposeBackupCron defaults the
-      // schedule + retain when backupConfig is absent, so it always installs a
-      // sensible cron. wal-g reads its S3 config from the db container env
-      // (rendered into .env at deploy) — the cron just runs compose-backup.sh,
-      // which self-skips (exit 0) when no S3 backup target is configured, so an
-      // always-installed cron never hard-fails on a no-S3 deploy.
-      //
-      // A cron-install failure must NOT fail an already-healthy deploy: the app
-      // is serving by this point. Downgrade to a warning so a transient ssh/cron
-      // hiccup doesn't roll back a successful deploy.
-      try {
-        await perfAsync('deploy.compose.backupCron', async () =>
-          setupComposeBackupCron(serverIp, sshKeyPath, projectConfig.projectName, backupConfig),
-        );
-      } catch (err) {
-        p.log.warn(`Scheduled backup cron install failed (deploy still succeeded): ${err.message}`);
-      }
-
-      // No ACME self-heal needed here. Managed-DNS deploys (cloudflare/hetzner)
-      // issue certs via DNS-01, which has no HTTP-01-vs-DNS-propagation race to
-      // recover from; `manual` DNS was never self-healed (we don't manage its
-      // records). The old acme.json poll + Traefik restart (acme-verify.js)
-      // existed only for the HTTP-01 race the DNS-01 switch eliminates.
-
-      deployResult = {
-        masterIp: serverIp,
-        serverId: serverHetznerId || 'manual',
-        serverName: serverHetznerName,
-      };
-    }
-  } else {
-    // Kubernetes (k3s on Ubuntu)
-    const { deployK3s } = await import('./k8s/k3s.js');
-    const { deployK8sHA } = await import('./k8s/ha/index.js');
-
-    const deploymentConfig = {
-      projectName: projectConfig.projectName,
-      environment: config.environment,
-      provider: config.provider,
-      region,
-      secondaryRegion,
-      masterServerType,
-      supabaseServerType,
-      workerServerType,
-      serverType,
-      minWorkers,
-      maxWorkers,
-      domain,
-      ha: config.ha,
-      observability: config.observability,
-      services,
-      apiToken,
-      cloudflareApiToken,
-      cloudflareZoneId,
-      hetznerDnsZoneId,
-      dnsProvider,
-      s3Config,
-      backupConfig,
-      backupBucketName: backupS3Config?.bucket || null,
-      // DR: when set (`deploy -restore latest|<ts>`), the db pod's init
-      // container seeds PGDATA from S3 via wal-g and applyMigrations is
-      // skipped (the restored DB is authoritative).
-      restore: args.restore || null,
-      operatorCidrs: projectConfig.operatorCidrs ?? [],
-      imageReadyPromise,
-      tracker,
-      state,
-    };
-
-    if (config.ha) {
+      return result;
+    },
+    k8s: async () => {
+      const { deployK3s } = await import('./k8s/k3s.js');
+      return perfAsync('deploy.k3s.full', () => deployK3s(deploymentConfig));
+    },
+    'k8s-ha': async () => {
       // K8s HA = 2 clusters (primary + standby) provisioned in parallel,
       // each running k3s + Helm + sideload. Cold HA observed ~25-40 min.
-      deployResult = await perfAsync('deploy.ha.k8s.full', () => deployK8sHA(deploymentConfig));
-    } else {
-      deployResult = await perfAsync('deploy.k3s.full', () => deployK3s(deploymentConfig));
-    }
+      const { deployK8sHA } = await import('./k8s/ha/index.js');
+      return perfAsync('deploy.ha.k8s.full', () => deployK8sHA(deploymentConfig));
+    },
+  };
+  const deployResult = await TIER_DEPLOYERS[tier]();
 
-    // K3s is local-first: deployK3s already built the image, sideloaded it
-    // to every node, and applied k8s manifests inline. Layer in Flux + GHA
-    // via `vibecarbon configure cicd <env>` (PR 7) after the cluster is up.
+  if (isK8sTier(tier)) {
     {
       const kubeconfigPath = join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}`);
       const masterIp = deployResult.primary?.masterIp || deployResult.masterIp;
@@ -1086,15 +898,14 @@ export async function executeDeployment(args, gatheredConfig) {
         // root cause without requiring a follow-up `vibecarbon diagnose`.
         // The cluster is about to be destroyed by the test runner; once
         // it's gone, kubectl probes return nothing. We spawn each tool via
-        // execFileSync (array argv, no shell) — domain and environment come
+        // runCommandAsync (array argv, no shell) — domain and environment come
         // from validated config, but we still avoid shell-string interp.
         try {
-          const { execFileSync } = await import('node:child_process');
           // HA deploys split into kubeconfig-<env>-primary / -standby; standalone
           // uses just kubeconfig-<env>. Prefer primary (the user-facing cluster
           // — Traefik/cert-manager state we want to inspect lives there) and
           // fall back to the standalone path. fs.existsSync check would race
-          // against the FS, so prefer the file-not-found error in execFileSync.
+          // against the FS, so prefer the file-not-found error in runCommandAsync.
           const { existsSync } = await import('node:fs');
           const kubeconfigCandidates = [
             join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}-primary`),
@@ -1102,18 +913,18 @@ export async function executeDeployment(args, gatheredConfig) {
           ];
           const kubeconfig =
             kubeconfigCandidates.find((c) => existsSync(c)) ?? kubeconfigCandidates[1];
-          // Run each tool via execFileSync. Capture stderr alongside stdout
+          // Run each tool via runCommandAsync. Capture stderr alongside stdout
           // (kubectl prints errors to stderr; without merging we get the
           // useless first line of err.message — observed 2026-04-26 run #5
           // where every kubectl probe surfaced as "(kubectl failed: Command
           // failed: kubectl ...)" with no actual diagnostic).
-          const safeRun = (cmd, argv) => {
+          const safeRun = async (cmd, argv) => {
             try {
-              return execFileSync(cmd, argv, {
-                encoding: 'utf8',
+              const output = await runCommandAsync([cmd, ...argv], {
+                silent: true,
                 timeout: 30_000,
-                stdio: ['ignore', 'pipe', 'pipe'],
-              }).trim();
+              });
+              return output.trim();
             } catch (err) {
               const stdout = err?.stdout?.toString?.()?.trim() ?? '';
               const stderr = err?.stderr?.toString?.()?.trim() ?? '';
@@ -1121,10 +932,10 @@ export async function executeDeployment(args, gatheredConfig) {
               return `(${cmd} failed exit=${err?.status ?? '?'}):\n${tail || (err instanceof Error ? err.message : String(err))}`;
             }
           };
-          const dig = safeRun('dig', ['+short', domain, '@1.1.1.1']);
+          const dig = await safeRun('dig', ['+short', domain, '@1.1.1.1']);
           const kcExists = existsSync(kubeconfig);
           const certs = kcExists
-            ? safeRun('kubectl', [
+            ? await safeRun('kubectl', [
                 '--kubeconfig',
                 kubeconfig,
                 'get',
@@ -1136,7 +947,7 @@ export async function executeDeployment(args, gatheredConfig) {
               ])
             : `(kubeconfig not present at ${kubeconfig} — checked ${kubeconfigCandidates.join(', ')})`;
           const events = kcExists
-            ? safeRun('kubectl', [
+            ? await safeRun('kubectl', [
                 '--kubeconfig',
                 kubeconfig,
                 'get',
@@ -1153,13 +964,13 @@ export async function executeDeployment(args, gatheredConfig) {
           // kong 503, etc.). curl exits non-zero on connect/TLS errors so we
           // route through safeRun. -k skips cert validation — we want the
           // body even if origin's cert is staging or self-signed.
-          const curlVerbose = safeRun('curl', [
+          const curlVerbose = await safeRun('curl', [
             '-skvI',
             '--max-time',
             '10',
             `https://${domain}/api/health`,
           ]);
-          const curlPodIp = safeRun('curl', [
+          const curlPodIp = await safeRun('curl', [
             '-skv',
             '--max-time',
             '10',
@@ -1175,7 +986,7 @@ export async function executeDeployment(args, gatheredConfig) {
           // turns "deploy probe got 404" from a riddle into a directly
           // actionable error.
           const pods = kcExists
-            ? safeRun('kubectl', [
+            ? await safeRun('kubectl', [
                 '--kubeconfig',
                 kubeconfig,
                 'get',
@@ -1187,7 +998,7 @@ export async function executeDeployment(args, gatheredConfig) {
               ])
             : '(skipped — no kubeconfig)';
           const endpoints = kcExists
-            ? safeRun('kubectl', [
+            ? await safeRun('kubectl', [
                 '--kubeconfig',
                 kubeconfig,
                 'get',
@@ -1199,7 +1010,7 @@ export async function executeDeployment(args, gatheredConfig) {
           // App pod logs (last 100 lines). If multiple replicas, this picks
           // by deployment selector — same as `kubectl logs deploy/app`.
           const appLogs = kcExists
-            ? safeRun('kubectl', [
+            ? await safeRun('kubectl', [
                 '--kubeconfig',
                 kubeconfig,
                 'logs',
@@ -1321,11 +1132,11 @@ export async function executeDeployment(args, gatheredConfig) {
       supabaseIp: deployResult.supabaseIp,
       region,
       serverType,
-      ...(isComposeDeploy ? {} : { role: 'master' }),
+      ...(isComposeTier(tier) ? {} : { role: 'master' }),
     });
     // K8s only — compose has neither a separate supabase node nor worker
     // pool. Compose deploys leave servers as a single-element array.
-    if (!isComposeDeploy) {
+    if (isK8sTier(tier)) {
       if (deployResult.supabaseIp) {
         servers.push({
           name: 'supabase',
@@ -1405,6 +1216,11 @@ export async function executeDeployment(args, gatheredConfig) {
         region,
         ...(isHADeploy && {
           secondaryRegion,
+          // Finding #1: persist DR posture. A default (gated) HA deploy that
+          // reaches this point is streaming; only `-allow-degraded` finalizes a
+          // warm/degraded standby, which the inner deploy flags on deployResult.
+          replication: deployResult?.degraded ? 'degraded' : 'streaming',
+          degraded: !!deployResult?.degraded,
           // Persist per-cluster floatingIp + supabaseIp under ha.primary /
           // ha.standby so failover.identifyServers takes the structured
           // path that surfaces floatingIp. Without these, identifyServers
@@ -1475,7 +1291,12 @@ export async function executeDeployment(args, gatheredConfig) {
         }),
         deployedAt: new Date().toISOString(),
         deployedCommit,
-        s3: { bucket: s3Config.bucket, region: s3Config.region, endpoint: s3Config.endpoint },
+        s3: {
+          bucket: s3Config.bucket,
+          region: s3Config.region,
+          endpoint: s3Config.endpoint,
+          stateBucket: s3Config.stateBucket,
+        },
         ...(backupS3Persist && { backupS3: backupS3Persist }),
         backup: backupConfig,
         services,