vibecarbon 0.1.0

package/src/lib/deploy/orchestrator.js

@@ -0,0 +1,1401 @@
+/**
+ * Deployment Orchestrator
+ * Logic for executing the deployment after configuration is gathered
+ */
+
+import dns from 'node:dns';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import * as p from '@clack/prompts';
+import { c } from '../colors.js';
+import { runCommand, runCommandAsync } from '../command.js';
+import {
+  loadCredentials,
+  loadProjectConfig,
+  registerProject,
+  saveProjectConfig,
+} from '../config.js';
+import { waitForDNSPropagation } from '../dns-propagation.js';
+import { ensureOperatorIpAccess } from '../operator-ip.js';
+import { perfAsync, perfTimer } from '../perf.js';
+import { createTracker } from '../tracker.js';
+import { renderBundle } from './bundle.js';
+import { sideloadCompose } from './image.js';
+import { buildRemote } from './remote-build.js';
+import { StateTracker } from './state.js';
+import { generateSSHKeyPair } from './utils.js';
+
+/**
+ * Probe the public /api/health endpoint to verify the app is reachable
+ * over the internet.
+ *
+ * K8s cold deploys need a generous budget: after the GHA workflow returns,
+ * cert-manager may still be finishing cert issuance, DNS may be
+ * propagating, and the ingress may be picking up new routes. 10 minutes
+ * handles first-deploy cert+DNS tail without false-negatives. Compose
+ * already has a faster on-server health gate before we ever get here, so
+ * 60s is enough to catch DNS edge cases.
+ *
+ * When `ACME_CA_SERVER` is set to a Let's Encrypt staging URL (e2e
+ * tests, dev envs avoiding LE rate limits), cert-manager issues a cert
+ * signed by the LE Staging CA which Node's default trust store does not
+ * include — fetch() would reject every probe with "self-signed cert"
+ * even though the app is healthy. Detect that case and use an undici
+ * dispatcher that skips TLS verification. Real-world prod runs are
+ * unaffected.
+ *
+ * Returns `{ ok: true, status }` on first 2xx, or a structured failure
+ * descriptor on timeout — never null. Capturing last-status and last-error
+ * lets the caller throw with actionable context (e.g. "lastStatus=503 over
+ * 120 attempts" tells you ingress is up but app is failing) instead of the
+ * old opaque null which made the error message default to whatever stderr
+ * tail the runner happened to scrape (the 2026-04-27 morning matrix's k8s
+ * timeout surfaced as a CSP header dump because of this exact gap).
+ */
+async function probePublicHealth(domain, { timeoutMs = 60_000, intervalMs = 10_000 } = {}) {
+  const url = `https://${domain}/api/health`;
+  const deadline = Date.now() + timeoutMs;
+  let firstAttempt = true;
+  let attempts = 0;
+  let lastStatus = null;
+  let lastErrorClass = null;
+  let lastErrorMessage = null;
+  let lastErrorCause = null;
+  // The probe runs IMMEDIATELY after we (re)write the DNS record, so the
+  // system resolver may still have a cached NXDOMAIN or a prior placeholder
+  // IP. Fetch then fails for the entire probe budget while curl (run once
+  // in the diagnostic dump much later) gets a fresh getaddrinfo and
+  // succeeds — the symptom that surfaced as "TypeError: fetch failed (121
+  // attempts)" while the diagnostic curl returned `{"status":"ok"}` in
+  // matrix #3 (k8s + k8s-ha). Pin the resolver to public DNS for the
+  // probe so we always see the freshest record. setServers is per-process
+  // and doesn't affect anything else in this orchestrator instance.
+  try {
+    dns.setDefaultResultOrder('verbatim');
+    dns.setServers(['1.1.1.1', '8.8.8.8']);
+  } catch {
+    // setServers can throw if called with an empty list or invalid IPs;
+    // both are caller errors, not runtime concerns. Fall through and use
+    // whatever resolver is currently configured.
+  }
+  // Lazy-load undici only when we need an insecure dispatcher — keeps the
+  // happy path zero-cost and avoids a hard dep on undici internals.
+  //
+  // CRITICAL: when we use undici's Agent we MUST also use undici's `fetch`,
+  // not Node 24's `globalThis.fetch`. Node 24 ships its own bundled undici;
+  // its built-in fetch's dispatcher protocol is incompatible with the npm
+  // undici package's Agent (different `onRequestStart` signature). Mixing
+  // them produces `InvalidArgumentError: invalid onRequestStart method` on
+  // every probe call, with the misleading wrapper "TypeError: fetch failed"
+  // — which is what dominated every k8s + k8s-ha matrix failure for weeks.
+  // Always pair Agent + fetch from the same undici instance.
+  let dispatcher = null;
+  let fetchFn = fetch;
+  if ((process.env.ACME_CA_SERVER || '').includes('staging')) {
+    const undici = await import('undici');
+    dispatcher = new undici.Agent({ connect: { rejectUnauthorized: false } });
+    fetchFn = undici.fetch;
+  }
+  // Periodic progress logging: every 5 attempts, write a one-line summary
+  // of the most recent failure class + lastStatus to stderr. Without this,
+  // the only signal we get from a 5-min probe is the FINAL outcome — which
+  // tells us nothing about whether the early failures were DNS-related,
+  // TCP-refused, or app-side 5xx. iter-perfwave2 analysis: k8s-ha probe
+  // hits exactly 300s = 30 × 10s (DNS TTL match suspicious), but with no
+  // per-attempt visibility we can't distinguish "DNS cached for 300s" from
+  // "ACME issuance pending for 300s" from "app slow to converge".
+  const startTime = Date.now();
+  const logProgress = () => {
+    const elapsedSec = Math.round((Date.now() - startTime) / 1000);
+    const lastSig = lastStatus
+      ? `lastStatus=${lastStatus}`
+      : lastErrorCause
+        ? `lastError=${lastErrorClass}/${lastErrorCause}`
+        : lastErrorClass
+          ? `lastError=${lastErrorClass}`
+          : 'no-response-yet';
+    console.error(`[probe] ${url} attempt=${attempts} elapsed=${elapsedSec}s ${lastSig}`);
+  };
+  while (Date.now() < deadline) {
+    if (!firstAttempt) await new Promise((r) => setTimeout(r, intervalMs));
+    firstAttempt = false;
+    attempts++;
+    try {
+      const res = await fetchFn(url, {
+        method: 'GET',
+        signal: AbortSignal.timeout(8_000),
+        ...(dispatcher && { dispatcher }),
+      });
+      lastStatus = res.status;
+      if (res.ok) return { ok: true, status: String(res.status), attempts };
+    } catch (err) {
+      // Capture failure class + short message so the eventual throw can
+      // say *why* the probe never reached 2xx. Common patterns:
+      //   - TimeoutError → ingress not routing or pod hung
+      //   - TypeError "fetch failed" → DNS or TCP refused
+      //   - Error "self-signed certificate" → LE staging cert + dispatcher
+      //     misconfig (we should have set the insecure dispatcher above)
+      // Undici nests the actual reason in err.cause (e.g. ENOTFOUND,
+      // ECONNREFUSED, EAI_AGAIN, UND_ERR_SOCKET). The wrapper "fetch
+      // failed" is useless without it — capture the cause's name+message
+      // so we can distinguish DNS failure from TCP refused from TLS
+      // handshake failure without re-running the matrix.
+      lastErrorClass = err?.name ?? 'Unknown';
+      lastErrorMessage = (err instanceof Error ? err.message : String(err)).slice(0, 200);
+      const cause = err && typeof err === 'object' && 'cause' in err ? err.cause : null;
+      if (cause) {
+        const causeName = cause?.name ?? cause?.code ?? 'UnknownCause';
+        const causeMsg = (cause instanceof Error ? cause.message : String(cause)).slice(0, 200);
+        lastErrorCause = `${causeName}: ${causeMsg}`;
+      }
+    }
+    if (attempts % 5 === 0) logProgress();
+  }
+  return {
+    ok: false,
+    attempts,
+    timeoutMs,
+    lastStatus,
+    lastErrorClass,
+    lastErrorMessage,
+    lastErrorCause,
+    url,
+  };
+}
+
+/**
+ * Execute the deployment based on gathered configuration
+ */
+export async function executeDeployment(args, gatheredConfig) {
+  const {
+    projectConfig,
+    envConfig,
+    environment,
+    config,
+    apiToken,
+    region,
+    secondaryRegion,
+    serverType,
+    masterServerType,
+    supabaseServerType,
+    workerServerType,
+    minWorkers,
+    maxWorkers,
+    domain,
+    dnsProvider,
+    cloudflareApiToken,
+    cloudflareZoneId,
+    hetznerDnsZoneId,
+    s3Config,
+    backupS3Config,
+    backupConfig,
+    services,
+  } = gatheredConfig;
+
+  const { deployMode } = config;
+  const isComposeDeploy = deployMode === 'compose' || deployMode === 'compose-ha';
+  const isHA = deployMode === 'compose-ha' || !!config.ha;
+
+  // Operator-IP access: detect + persist + (when env already deployed) patch
+  // the live Hetzner firewall. On a fresh first deploy this just appends to
+  // projectConfig.operatorCidrs in memory and on disk — the firewall itself
+  // is created later in this function with the correct source_ips because
+  // every program-builder call site reads projectConfig.operatorCidrs.
+  const accessResult = await ensureOperatorIpAccess({
+    projectConfig,
+    environment,
+    isHA,
+    apiToken,
+    yes: !!args.yes,
+    onMessage: (msg) => p.log.info(msg),
+  });
+  if (accessResult.added && accessResult.fromEnv) {
+    p.log.info(`Bootstrapped operator CIDRs from ALLOWED_SSH_IPS env var: ${accessResult.cidr}`);
+  }
+
+  // Initialize state tracker for granular resume
+  const state = new StateTracker(projectConfig.projectName, environment);
+
+  // Clear state if --full is passed
+  if (args.full) {
+    state.clear();
+  }
+
+  const s3Spinner = p.spinner();
+  const { HetznerS3Provider } = await import('../providers/hetzner-s3.js');
+  const s3Provider = new HetznerS3Provider(s3Config.accessKey, s3Config.secretKey, s3Config.region);
+
+  // --- STEP 1: S3 Setup (Idempotent) ---
+  const s3Inputs = {
+    bucket: s3Config.bucket,
+    backupBucket: backupS3Config.bucket,
+    region: s3Config.region,
+  };
+  if (!state.shouldSkip('s3-setup', s3Inputs)) {
+    state.startStep('s3-setup', s3Inputs);
+    try {
+      s3Spinner.start(`Creating S3 bucket: ${s3Config.bucket}`);
+      const bucket = await s3Provider.createBucket(s3Config.bucket);
+      s3Spinner.stop(
+        bucket.created
+          ? `S3 bucket created: ${s3Config.bucket}`
+          : `S3 bucket exists: ${s3Config.bucket}`,
+      );
+
+      try {
+        s3Spinner.start('Configuring bucket CORS');
+        const corsOrigins = domain ? [`https://${domain}`] : ['*'];
+        await s3Provider.configureCORS(s3Config.bucket, corsOrigins);
+        s3Spinner.stop('CORS configured');
+      } catch {
+        s3Spinner.stop('CORS configuration skipped (not supported by provider)');
+      }
+
+      s3Spinner.start(`Creating backup bucket: ${backupS3Config.bucket}`);
+      try {
+        const backupBucket = await s3Provider.createBucket(backupS3Config.bucket);
+        s3Spinner.stop(
+          backupBucket.created
+            ? `Backup bucket created: ${backupS3Config.bucket}`
+            : `Backup bucket exists: ${backupS3Config.bucket}`,
+        );
+      } catch (backupBucketError) {
+        s3Spinner.stop(`Backup bucket creation failed: ${backupBucketError.message}`);
+      }
+
+      const result = { region: s3Provider.region, endpoint: s3Provider.getEndpoint() };
+      state.completeStep('s3-setup', result);
+    } catch (error) {
+      const errMsg =
+        error.message ||
+        error.Code ||
+        error.name ||
+        `HTTP ${error.$metadata?.httpStatusCode || 'unknown'}`;
+      s3Spinner.stop(`S3 setup failed: ${errMsg}`);
+      process.exit(1);
+    }
+  }
+
+  // Apply resolved S3 values from state
+  const s3Result = state.getStepResult('s3-setup');
+  if (s3Result) {
+    s3Config.region = s3Result.region;
+    s3Config.endpoint = s3Result.endpoint;
+    backupS3Config.region = s3Config.region;
+    backupS3Config.endpoint = s3Config.endpoint;
+  }
+
+  // SKELETON SAVE: persist the minimum env entry that `vibecarbon destroy`
+  // needs to recover, BEFORE any `pulumi up` runs. Without this, a crash
+  // during k3s install / cloud-init / health probe leaves Pulumi state in
+  // S3 + real Hetzner infra alive while .vibecarbon.json has no entry —
+  // destroy reads the empty config, finds nothing, exits cleanly, and the
+  // PG/network/firewall/FIP outlive the test until the sweep catches them
+  // (or, worse, blocks the next deploy with a quota error). Observed in
+  // matrix #3 (3 PGs + 3 networks + 1 firewall leaked from k8s/k8s-ha
+  // health-probe failures). Fields: deployMode + region + s3 give k8s
+  // destroy the Pulumi backend; secondaryRegion + ha.enabled give k8s-ha
+  // destroy the `-primary` / `-standby` stack enumeration; the compose
+  // destroy path already falls back to the deterministic Pulumi name
+  // `${projectName}-${envName}` when servers[] is empty (see
+  // src/destroy.js:1067-1093), so the skeleton is sufficient there too.
+  // Subsequent saves (mid-flight from compose/ha.js + k8s/ha, plus the
+  // final save at the end of executeDeployment) merge with on-disk state,
+  // so the skeleton is non-destructive — they layer on top of it.
+  {
+    const existingEnv = projectConfig.environments?.[environment] ?? {};
+    const skeletonEnv = {
+      ...existingEnv,
+      envName: environment,
+      status: 'deploying',
+      deployMode,
+      provider: config.provider,
+      region,
+      ...(secondaryRegion ? { secondaryRegion } : {}),
+      ...(domain ? { domain } : {}),
+      ...(dnsProvider ? { dnsProvider } : {}),
+      s3: { bucket: s3Config.bucket, region: s3Config.region, endpoint: s3Config.endpoint },
+      backupS3: backupS3Config?.bucket
+        ? {
+            bucket: backupS3Config.bucket,
+            region: backupS3Config.region,
+            endpoint: backupS3Config.endpoint,
+          }
+        : existingEnv.backupS3,
+      ...(deployMode === 'compose-ha' || (deployMode === 'kubernetes' && secondaryRegion)
+        ? {
+            ha: {
+              ...(existingEnv.ha ?? {}),
+              enabled: true,
+              ...(secondaryRegion ? { failoverRegion: secondaryRegion } : {}),
+            },
+          }
+        : {}),
+      lastAttempt: new Date().toISOString(),
+    };
+    projectConfig.environments = projectConfig.environments ?? {};
+    projectConfig.environments[environment] = skeletonEnv;
+    saveProjectConfig(projectConfig);
+  }
+
+  const tracker = createTracker('deploy', {
+    environment: config.environment,
+    provider: config.provider,
+    region,
+  });
+
+  // --- STEP 2: Image Resolve ---
+  // resolveBuildMode returns one of:
+  //   'local'  — k3s default: deployK3s builds + sideloads + applies
+  //              manifests inline. No CI image dependency.
+  //   'direct' — compose: build locally on dev, push image directly
+  //              to server (auto-detected when CI/CD is not configured).
+  //   'push'   — compose: git push triggers GHA build + Flux reconcile
+  //              (auto-detected when CI/CD is configured).
+  const { ensureCIImageReady, buildImageRef, resolveBuildMode, waitForGhcrManifest } = await import(
+    '../ci-setup.js'
+  );
+  const imageResolveTimer = perfTimer('deploy.image.resolve');
+  const buildMode = resolveBuildMode(args, process.cwd(), deployMode);
+  const isDirectDeploy = buildMode === 'direct';
+  const isComposeMode = deployMode === 'compose' || deployMode === 'compose-ha';
+  const isComposeLocal = buildMode === 'local' && isComposeMode;
+  let imageReadyPromise;
+  // For compose-local: kicked off here in parallel with iac.upStack below.
+  // The build is awaited later (just before sideload). Other modes leave
+  // this null and use the existing remote-build / push paths.
+  let composeLocalBuildPromise = null;
+
+  const localImageTag = `${projectConfig.projectName}-app:local`;
+
+  if (isComposeLocal) {
+    // Build the app image locally NOW, in parallel with iac.upStack which
+    // starts ~immediately below. The tag is fixed (no serverIp dependency)
+    // and matches what bundle.js stamps into APP_IMAGE in the rendered
+    // .env, so docker-compose.prod.yml's `image: ${APP_IMAGE}` resolves to
+    // the sideloaded image. iac.upStack typically runs ~109s; a warm
+    // operator-side build runs ~15-30s, so the build finishes well inside
+    // the upStack window — net saving is the full build duration off the
+    // critical path. State-tracker stepId 'compose-localbuild' is distinct
+    // from 'compose-upstack' so concurrent state writes don't collide
+    // (state.js uses last-writer-wins per stepId).
+    // silent: true so BuildKit progress doesn't interleave with the
+    // spinner output from upStack (which is running concurrently). On
+    // failure runCommandAsync includes captured stdout/stderr in the
+    // thrown error — operator still sees what went wrong.
+    // Plumb VITE_* build args through to docker build so the browser bundle
+    // ships with real values for VITE_SUPABASE_URL etc. Without these the
+    // Dockerfile's ARG VITE_* defaults to empty strings and Vite inlines
+    // empty values, causing the client-side supabase init to throw at
+    // page load. Mirrors what docker-compose.yml's build.args block does
+    // for local-dev `docker compose up --build`. See
+    // src/lib/deploy/compose/build-args.js for the rationale.
+    const { collectComposeBuildArgs, buildArgFlags } = await import('./compose/build-args.js');
+    const localBuildArgs = collectComposeBuildArgs(process.cwd(), {
+      projectName: projectConfig.projectName,
+      domain,
+    });
+    composeLocalBuildPromise = perfAsync('deploy.image.localBuild', () =>
+      runCommandAsync(
+        ['docker', 'build', ...buildArgFlags(localBuildArgs), '-t', localImageTag, process.cwd()],
+        { silent: true },
+      ),
+    );
+    imageReadyPromise = Promise.resolve({
+      imageTag: localImageTag,
+      githubOwner: 'noop',
+      repoName: projectConfig.projectName,
+      ghcrPullCreds: null,
+      isLocal: true,
+    });
+  } else if (buildMode === 'local') {
+    // K8s 'local' build mode: deployK3s handles the build + sideload +
+    // manifest apply inline. No CI image to wait for; stub the
+    // image-ready contract so downstream code sees a deterministic shape.
+    imageReadyPromise = Promise.resolve({
+      imageTag: localImageTag,
+      githubOwner: 'noop',
+      repoName: projectConfig.projectName,
+      ghcrPullCreds: null,
+      isLocal: true,
+    });
+  } else if (buildMode === 'direct') {
+    imageReadyPromise = Promise.resolve({ imageTag: localImageTag, isLocal: true });
+  } else {
+    // CI path: ensureCIImageReady performs the workflow dispatch + GHA-build
+    // wait. On a cold push deploy this is the dominant cost (~10 min).
+    imageReadyPromise = perfAsync('deploy.image.ciWait', () =>
+      ensureCIImageReady({ yes: args.yes, tracker }),
+    );
+  }
+
+  const ciReady = await imageReadyPromise;
+  if (!ciReady.isLocal) {
+    p.log.info(
+      `Deploying image: ${c.bold(buildImageRef(ciReady.githubOwner, ciReady.repoName, ciReady.imageTag))}`,
+    );
+    // Bridge the race between the GitHub packages API (ghcrTagExists) and
+    // the OCI manifests endpoint that kubelet/docker actually pull from.
+    // HEAD-probe the manifest URL with short polling instead of a blanket
+    // 30s sleep — typical hit is <5s, worst observed ~15s.
+    const s = p.spinner();
+    s.start('Waiting for GHCR manifest to propagate...');
+    const manifestReady = await perfAsync('deploy.image.manifestPropagate', () =>
+      waitForGhcrManifest(ciReady.githubOwner, ciReady.repoName, ciReady.imageTag),
+    );
+    s.stop(manifestReady ? 'Manifest propagated' : 'Manifest probe timed out (proceeding anyway)');
+  }
+  imageResolveTimer.end();
+  const imageRef = ciReady.isLocal
+    ? ciReady.imageTag
+    : buildImageRef(ciReady.githubOwner, ciReady.repoName, ciReady.imageTag);
+
+  // --- STEP 3: Local Bundle Rendering (Unified) ---
+  // renderBundle is sync (cp + writeFile fan-out); perfTimer captures total
+  // wall-clock time for the staging-dir build. Typical ~50-300ms.
+  const bundleRenderTimer = perfTimer('deploy.bundle.pack');
+  // verbose=false: per-file `[bundle] Copying X` lines bypass the clack
+  // gutter and print flush-left, breaking the visual flow. Bundle content
+  // is reproducible from the project tree; debug it by re-running with
+  // VIBECARBON_PERF=1 + VIBECARBON_BUNDLE_VERBOSE=1 if needed.
+  const bundlePath = renderBundle(projectConfig.projectName, {
+    domain,
+    image: imageRef,
+    observability: services.observability,
+    n8n: services.n8n,
+    metabase: services.metabase,
+    redis: services.redis,
+    s3: s3Config,
+    backupS3: backupS3Config,
+    verbose: process.env.VIBECARBON_BUNDLE_VERBOSE === '1',
+  });
+  bundleRenderTimer.end();
+
+  // --- STEP 4: DNS Warm-up (parallel with VM provisioning) ---
+  // Kick off the placeholder-DNS write in the background so TTL starts
+  // draining while we spin up the VM. The real-IP update below awaits the
+  // warm-up handle to avoid racing two writes on the same record.
+  const dnsInputs = { domain, dnsProvider, cloudflareZoneId, hetznerDnsZoneId };
+  const dnsWarmupPromise =
+    domain && dnsProvider && dnsProvider !== 'manual' && !state.shouldSkip('dns-setup', dnsInputs)
+      ? (async () => {
+          state.startStep('dns-setup', dnsInputs);
+          try {
+            const { setupSimple: updateDnsIp } =
+              dnsProvider === 'cloudflare'
+                ? await import('../cloudflare.js')
+                : await import('../hetzner-dns.js');
+            const token = dnsProvider === 'cloudflare' ? cloudflareApiToken : apiToken;
+            const zoneId = dnsProvider === 'cloudflare' ? cloudflareZoneId : hetznerDnsZoneId;
+            if (!zoneId) {
+              p.log.info('DNS warm-up skipped (Zone ID not found)');
+              return null;
+            }
+            const result = await updateDnsIp(token, zoneId, domain, '0.0.0.0');
+            state.completeStep('dns-setup', result);
+            p.log.info('DNS records warmed up');
+            return result;
+          } catch (error) {
+            p.log.warn(`Initial DNS setup failed: ${error.message}`);
+            return null;
+          }
+        })()
+      : Promise.resolve(null);
+
+  // --- STEP 5: Architecture Dispatch ---
+  let deployResult;
+  if (isComposeDeploy) {
+    if (deployMode === 'compose-ha') {
+      const { deployComposeHA } = await import('./compose/ha.js');
+      const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
+      const s = tracker.spinner();
+      s.start('Starting HA deployment...');
+      // Wrap the entire compose-HA pipeline so the perf trace shows the
+      // headline cost (which contains compose-ha.deploy.* sub-stages from
+      // compose/ha.js).
+      deployResult = await perfAsync('deploy.ha.compose.full', () =>
+        deployComposeHA({
+          projectName: projectConfig.projectName,
+          environment,
+          sshKeyPath,
+          domain,
+          region,
+          secondaryRegion: secondaryRegion || undefined,
+          serverType: serverType || 'cx23',
+          apiToken,
+          operatorCidrs: projectConfig.operatorCidrs ?? [],
+          cloudflareApiToken,
+          cloudflareZoneId,
+          hetznerDnsZoneId,
+          dnsProvider,
+          services,
+          dockerHubCreds: loadCredentials().dockerHub || null,
+          ghcrPullCreds: ciReady.ghcrPullCreds,
+          imageRef,
+          envConfig,
+          s3Config,
+          backupS3Config,
+          backupConfig,
+          bundlePath,
+          onProgress: (msg) => s.message(msg),
+        }),
+      );
+      s.stop('Compose HA deployment complete');
+    } else {
+      const { setupServer, dockerLoginOnServer, startComposeStack } = await import(
+        './compose/index.js'
+      );
+      const { setupServerFiles, waitForSSH: waitForComposeSSH } = await import(
+        './compose/index.js'
+      );
+      const sshKeyPath = join(process.cwd(), '.vibecarbon', `deploy_key_${environment}`);
+
+      let serverIp = envConfig.servers?.[0]?.ip || null;
+      // Track the Hetzner server's API id + literal name so destroy can find
+      // the VPS without falling back to the literal string "master" (which
+      // never matches the Pulumi-assigned name `${projectName}-${env}`).
+      // Without this, destroy → re-deploy hits 'server name is already used
+      // (uniqueness_error)' because the previous VPS was never deleted
+      // (caught in compose restore matrix runs).
+      let serverHetznerId = envConfig.servers?.[0]?.id || null;
+      let serverHetznerName = envConfig.servers?.[0]?.hetznerName || null;
+      if (!serverIp) {
+        generateSSHKeyPair(sshKeyPath);
+        const { upStack } = await import('../iac/index.js');
+        const { buildHetznerComposeProgram } = await import('../iac/programs/hetzner-compose.js');
+        const program = buildHetznerComposeProgram({
+          projectName: projectConfig.projectName,
+          environment,
+          sshPublicKey: readFileSync(`${sshKeyPath}.pub`, 'utf-8').trim(),
+          location: region,
+          serverType: serverType || 'cx23',
+          labels: { 'managed-by': 'vibecarbon' },
+          allowedSshIps: (projectConfig.operatorCidrs ?? []).map((e) => e.cidr),
+        });
+        // VM provisioning via Pulumi — typical 30-90s on Hetzner cx23.
+        const result = await perfAsync('deploy.iac.upStack', () =>
+          upStack(environment, program, { hcloudToken: apiToken, s3Config }),
+        );
+        serverIp = result.outputs.serverIp;
+        serverHetznerId = result.outputs.serverId || null;
+        serverHetznerName = `${projectConfig.projectName}-${environment}`;
+        await perfAsync('deploy.iac.waitForSSH', () => waitForComposeSSH(serverIp, sshKeyPath, 30));
+      }
+
+      // --- Warm path (StateTracker) ---
+      // On a second deploy against an already-provisioned server, skip the
+      // idempotent-but-slow steps (cloud-init probe, GHCR login, bundle rsync)
+      // when their inputs haven't changed. `docker compose up -d` is cheap
+      // and always runs — it applies diffs to the live stack.
+      const setupInputs = { serverIp };
+      if (!state.shouldSkip('compose-setup-server', setupInputs)) {
+        state.startStep('compose-setup-server', setupInputs);
+        // Cloud-init readiness probe — first-deploy spends 20-40s here while
+        // ufw + unattended-upgrades land. Warm deploys hit the ready marker
+        // immediately, so this is mostly cold-deploy noise.
+        await perfAsync('deploy.compose.cloudInitReady', async () => {
+          if (isDirectDeploy || isComposeLocal) {
+            // Both direct (build over DOCKER_HOST=ssh://) and local
+            // (sideload via docker save | ssh docker load) need dockerd
+            // up on the server before they can proceed. Push mode doesn't
+            // because the server pulls from GHCR after compose-up starts.
+            const { waitForDockerReady } = await import('./compose/index.js');
+            await setupServer(serverIp, sshKeyPath);
+            await waitForDockerReady(serverIp, sshKeyPath);
+          } else {
+            await setupServer(serverIp, sshKeyPath);
+          }
+        });
+        state.completeStep('compose-setup-server', { serverIp });
+      }
+
+      // --- Image transfer to server (mode-dependent) ---
+      if (isComposeLocal) {
+        // Local-build path: the build was kicked off in parallel with
+        // iac.upStack at the top of this function. By the time we reach
+        // here (post-cloudInitReady), the build has almost always finished.
+        // Await the promise (cheap if already settled) then sideload the
+        // image tarball over SSH. Sideload uses docker save | gzip | ssh
+        // | docker load — gzip drops ~600MB → ~150-200MB on the wire.
+        await composeLocalBuildPromise;
+        await perfAsync('deploy.image.sideload', async () =>
+          sideloadCompose({
+            tag: localImageTag,
+            sshTarget: `root@${serverIp}`,
+            sshKey: sshKeyPath,
+          }),
+        );
+      } else if (isDirectDeploy) {
+        // Direct path: build the image on the server via DOCKER_HOST=ssh://.
+        // BuildKit caches aggressively, so unchanged source rebuilds in 1-3s.
+        // Slower than local+sideload (no upStack overlap) but works when the
+        // operator has no local Docker — selected by --direct or by
+        // resolveBuildMode's docker-not-found fallback. VITE_* args plumbed
+        // through here for the same reason as the local-build path above.
+        const { collectComposeBuildArgs } = await import('./compose/build-args.js');
+        const directBuildArgs = collectComposeBuildArgs(process.cwd(), {
+          projectName: projectConfig.projectName,
+          domain,
+        });
+        const success = await perfAsync('deploy.image.directBuild', () =>
+          buildRemote(serverIp, sshKeyPath, localImageTag, process.cwd(), directBuildArgs),
+        );
+        if (!success) process.exit(1);
+      }
+
+      // --- Docker Hub login (skip if creds unchanged) ---
+      // Without this, the reconcile script's `docker compose pull` for
+      // Supabase service images (rest, auth, storage, kong, imgproxy, etc.)
+      // hits Docker Hub anonymously. A fresh Hetzner VPS shares a NAT range
+      // with many other deploys, so the per-IP unauthenticated pull quota
+      // is routinely exhausted — observed 2026-04-26 matrix run #3
+      // ("toomanyrequests: You have reached your unauthenticated pull rate
+      // limit"). Compose-HA + scale already log in; this brings the
+      // single-compose deploy path in line.
+      const dockerHubCreds = loadCredentials().dockerHub || null;
+      if (dockerHubCreds) {
+        const dhLoginInputs = {
+          serverIp,
+          registry: 'docker.io',
+          user: dockerHubCreds.username,
+          tokenFp: dockerHubCreds.token ? dockerHubCreds.token.slice(0, 8) : '',
+        };
+        if (!state.shouldSkip('compose-dockerhub-login', dhLoginInputs)) {
+          state.startStep('compose-dockerhub-login', dhLoginInputs);
+          await dockerLoginOnServer(serverIp, sshKeyPath, dockerHubCreds);
+          state.completeStep('compose-dockerhub-login', { serverIp });
+        }
+      }
+
+      // --- GHCR login (skip if creds unchanged) ---
+      if (ciReady.ghcrPullCreds) {
+        const loginInputs = {
+          serverIp,
+          registry: 'ghcr.io',
+          user: ciReady.ghcrPullCreds.owner,
+          // Hash the token's fingerprint, not the token itself — we don't
+          // want the token written to .vibecarbon/deploy-state-*.json.
+          tokenFp: ciReady.ghcrPullCreds.token ? ciReady.ghcrPullCreds.token.slice(0, 8) : '',
+        };
+        if (!state.shouldSkip('compose-ghcr-login', loginInputs)) {
+          state.startStep('compose-ghcr-login', loginInputs);
+          await dockerLoginOnServer(serverIp, sshKeyPath, {
+            username: ciReady.ghcrPullCreds.owner,
+            token: ciReady.ghcrPullCreds.token,
+            registry: 'ghcr.io',
+          });
+          state.completeStep('compose-ghcr-login', { serverIp });
+        }
+      }
+
+      // --- Bundle rsync (skip when bundle inputs unchanged) ---
+      const bundleInputs = {
+        serverIp,
+        projectName: projectConfig.projectName,
+        imageRef,
+        domain,
+        services,
+      };
+      if (!state.shouldSkip('compose-setup-files', bundleInputs)) {
+        state.startStep('compose-setup-files', bundleInputs);
+        // setupServerFiles tars the bundle locally + streams over ssh +
+        // extracts on the server in one pipeline. Bucket the whole upload
+        // here; the inner tar/upload split is captured in compose/index.js.
+        await perfAsync('deploy.bundle.upload', () =>
+          setupServerFiles(serverIp, sshKeyPath, projectConfig.projectName, {
+            ...services,
+            domain,
+            image: imageRef,
+            bundlePath,
+          }),
+        );
+        state.completeStep('compose-setup-files', { serverIp });
+      }
+
+      // --- DNS update + propagation wait BEFORE compose-up ---
+      //
+      // Traefik attempts ACME HTTP-01 challenges immediately on container
+      // start and bursts ~7 attempts in ~30s before giving up. If LE's
+      // DNS resolver sees stale or absent records on those attempts,
+      // every challenge fails with "no valid A records found" and
+      // acme.json ends the deploy with zero certs issued — browser
+      // sees TRAEFIK DEFAULT CERT and NET::ERR_CERT_AUTHORITY_INVALID.
+      //
+      // The warm-up that fires in Step 4 writes a 0.0.0.0 placeholder;
+      // LE specifically rejects 0.0.0.0 as not-a-valid-A-record. So even
+      // when warm-up "succeeds," it doesn't help ACME — it actively
+      // poisons the resolver state until the real IP overwrites it.
+      //
+      // Fix (RCA from vibecarbon.com cold-deploy 2026-05-19): swap the
+      // ordering. Write the real IP now, wait for it to propagate to
+      // public resolvers, then start compose. Traefik's first ACME
+      // attempt sees real DNS, succeeds on the first try.
+      //
+      // The compose-HA path already writes DNS before compose-up at
+      // src/lib/deploy/compose/ha.js — single-region compose was the
+      // outlier that ran startComposeStack first then updated DNS.
+      if (domain && dnsProvider && dnsProvider !== 'manual') {
+        await perfAsync('deploy.dns.warm', async () => {
+          await dnsWarmupPromise;
+        });
+        const dnsUpdateInputs = { domain, dnsProvider, serverIp };
+        if (!state.shouldSkip('compose-dns-update', dnsUpdateInputs)) {
+          state.startStep('compose-dns-update', dnsUpdateInputs);
+          const { setupSimple: updateDnsIp } =
+            dnsProvider === 'cloudflare'
+              ? await import('../cloudflare.js')
+              : await import('../hetzner-dns.js');
+          const token = dnsProvider === 'cloudflare' ? cloudflareApiToken : apiToken;
+          const zoneId = dnsProvider === 'cloudflare' ? cloudflareZoneId : hetznerDnsZoneId;
+          await perfAsync('deploy.dns.create', () => updateDnsIp(token, zoneId, domain, serverIp));
+          state.completeStep('compose-dns-update', { serverIp });
+        }
+        // Block compose-up until public resolvers see the real IP. 120s
+        // budget covers Cloudflare's typical edge propagation (under 60s)
+        // with margin. Returns false on timeout; we proceed anyway —
+        // Traefik can still acquire certs on its own retries, the
+        // customer just sees a brief self-signed-cert window. Hard-
+        // failing here would mask deploys that succeed eventually.
+        //
+        // Visible spinner so the operator isn't staring at a frozen
+        // "Yes" screen for up to 120s on a cold deploy where the DNS
+        // edge is slow. `timer` indicator forces the rendered string to
+        // change every second so progress remains visibly alive.
+        const dnsSpinner = p.spinner({ indicator: 'timer' });
+        dnsSpinner.start(`Waiting for ${domain} to resolve to ${serverIp}`);
+        const dnsPropagated = await perfAsync('deploy.dns.waitForPropagation', () =>
+          waitForDNSPropagation(domain, serverIp, 120_000),
+        );
+        dnsSpinner.stop(
+          dnsPropagated
+            ? `DNS resolves: ${domain} → ${serverIp}`
+            : `DNS propagation timed out — proceeding anyway`,
+        );
+      }
+
+      // `docker compose up -d` is idempotent and cheap — always run so a
+      // config tweak that didn't change the bundle hash (e.g. env var)
+      // still takes effect. Reconcile.sh on the server runs `docker compose
+      // pull` (when not local image) + `docker compose up -d` — see
+      // compose/index.js for the inner perf instrumentation.
+      await perfAsync('deploy.reconcile.run', () =>
+        startComposeStack(serverIp, sshKeyPath, projectConfig.projectName, services),
+      );
+
+      // --- Verifiable success gate ---
+      // Probe the app's own /api/health endpoint on the server itself (via
+      // localhost + Host header), bypassing DNS/TLS. A successful deploy
+      // must yield an HTTP 2xx here; otherwise fail the deploy loudly with
+      // container state + log tail.
+      //
+      // verifyAppHealth polls every 10s for up to 3 minutes while the app
+      // container restarts and binds port 3000. Without a visible spinner
+      // the operator just sees a frozen "Stack reconciled" line for that
+      // entire window — observed 2026-05-19 vibecarbon.com re-deploy
+      // where compose-up finished in ~5s but the app took ~3min to come
+      // back, leaving the screen looking hung. `timer` indicator so the
+      // elapsed-seconds counter ticks visibly even on terminals where
+      // the spinner frame-character animation gets throttled under load.
+      const { verifyAppHealth } = await import('./compose/index.js');
+      const healthSpinner = p.spinner({ indicator: 'timer' });
+      healthSpinner.start('Waiting for app to start serving requests');
+      let health;
+      try {
+        health = await perfAsync('deploy.health.probe', () =>
+          verifyAppHealth(serverIp, sshKeyPath, projectConfig.projectName, {
+            domain,
+          }),
+        );
+      } catch (err) {
+        healthSpinner.stop('App health probe errored', 1);
+        throw err;
+      }
+      if (!health.healthy) {
+        healthSpinner.stop(`App not serving requests (status: ${health.status})`, 1);
+        p.log.error(
+          `Deploy produced an unhealthy app (probe status: ${health.status}):\n${health.details}`,
+        );
+        process.exit(1);
+      }
+      healthSpinner.stop(`App is serving requests (HTTP ${health.status})`);
+
+      // --- ACME / TLS self-heal (compose only) ---
+      // The internal health probe above bypasses DNS/TLS and confirms the
+      // app container is up. It does NOT confirm Traefik has issued a real
+      // Let's Encrypt cert. Verify by reading acme.json; if Traefik is
+      // stuck on a stale failed-challenge state (typically because a
+      // previous deploy on v3.6.1 raced ahead of DNS propagation), restart
+      // Traefik in-line to retry ACME. RCA: vibecarbon.com customer hit
+      // this on 2026-05-19 — cold deploy "succeeded" with HTTP 200 from
+      // internal probe, but browsers saw NET::ERR_CERT_AUTHORITY_INVALID
+      // because Traefik served its self-signed default cert.
+      if (domain && dnsProvider && dnsProvider !== 'manual') {
+        const { ensureTraefikCert } = await import('./compose/acme-verify.js');
+        await perfAsync('deploy.acme.verify', () =>
+          ensureTraefikCert(serverIp, sshKeyPath, projectConfig.projectName, domain, p.log),
+        );
+      }
+
+      deployResult = {
+        masterIp: serverIp,
+        serverId: serverHetznerId || 'manual',
+        serverName: serverHetznerName,
+      };
+    }
+  } else {
+    // Kubernetes (k3s on Ubuntu)
+    const { deployK3s } = await import('./k8s/k3s.js');
+    const { deployK8sHA } = await import('./k8s/ha/index.js');
+
+    const deploymentConfig = {
+      projectName: projectConfig.projectName,
+      environment: config.environment,
+      provider: config.provider,
+      region,
+      secondaryRegion,
+      masterServerType,
+      supabaseServerType,
+      workerServerType,
+      serverType,
+      minWorkers,
+      maxWorkers,
+      domain,
+      ha: config.ha,
+      observability: config.observability,
+      services,
+      apiToken,
+      cloudflareApiToken,
+      cloudflareZoneId,
+      hetznerDnsZoneId,
+      dnsProvider,
+      s3Config,
+      backupConfig,
+      backupBucketName: backupS3Config?.bucket || null,
+      operatorCidrs: projectConfig.operatorCidrs ?? [],
+      imageReadyPromise,
+      tracker,
+      state,
+    };
+
+    if (config.ha) {
+      // K8s HA = 2 clusters (primary + standby) provisioned in parallel,
+      // each running k3s + Helm + sideload. Cold HA observed ~25-40 min.
+      deployResult = await perfAsync('deploy.ha.k8s.full', () => deployK8sHA(deploymentConfig));
+    } else {
+      deployResult = await perfAsync('deploy.k3s.full', () => deployK3s(deploymentConfig));
+    }
+
+    // K3s is local-first: deployK3s already built the image, sideloaded it
+    // to every node, and applied k8s manifests inline. Layer in Flux + GHA
+    // via `vibecarbon configure cicd <env>` (PR 7) after the cluster is up.
+    {
+      const kubeconfigPath = join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}`);
+      const masterIp = deployResult.primary?.masterIp || deployResult.masterIp;
+      p.log.success('Local-first deploy complete. App is sideloaded + running on the cluster.');
+      p.note(
+        [
+          `Kubeconfig:  ${kubeconfigPath}`,
+          `Master IP:   ${masterIp ?? '(unknown — check Pulumi output)'}`,
+          '',
+          'Next steps:',
+          `  vibecarbon shell ${environment}    # bash with KUBECONFIG set`,
+          `  vibecarbon diagnose ${environment} # full cluster state dump`,
+          `  vibecarbon destroy ${environment}  # tear down when done`,
+          '',
+          `To layer in Flux + GitHub Actions: vibecarbon configure cicd ${environment}`,
+        ].join('\n'),
+        'Cluster + app ready',
+      );
+      // Fall through to Step 6 so .vibecarbon.json persists the serverlist —
+      // without it, `vibecarbon destroy` can't find the master/supabase IPs
+      // and the cluster leaks.
+    }
+
+    // --- Post-deploy DNS update ---
+    // The dnsWarmupPromise (started in Step 4) created records pointing at
+    // 0.0.0.0 so Cloudflare's edge has the zone record cached before traffic
+    // hits. Now we know the cluster's actual floating IP and need to update
+    // the records to point at it — without this, the verify-app-health probe
+    // below resolves to 0.0.0.0 and times out.
+    //
+    // Compose does the equivalent in its own branch above (line ~436).
+    // Local-first k3s manages DNS records inline here rather than via
+    // Flux annotations + external-dns.
+    {
+      const k8sFloatingIp =
+        deployResult.primary?.floatingIp ||
+        deployResult.floatingIp ||
+        deployResult.primary?.masterIp ||
+        deployResult.masterIp;
+      if (domain && dnsProvider && dnsProvider !== 'manual' && k8sFloatingIp) {
+        await perfAsync('deploy.dns.warm', async () => {
+          await dnsWarmupPromise;
+        });
+        const dnsUpdateInputs = { domain, dnsProvider, serverIp: k8sFloatingIp };
+        if (!state.shouldSkip('k8s-dns-update', dnsUpdateInputs)) {
+          state.startStep('k8s-dns-update', dnsUpdateInputs);
+          const { setupSimple: updateDnsIp } =
+            dnsProvider === 'cloudflare'
+              ? await import('../cloudflare.js')
+              : await import('../hetzner-dns.js');
+          const token = dnsProvider === 'cloudflare' ? cloudflareApiToken : apiToken;
+          const zoneId = dnsProvider === 'cloudflare' ? cloudflareZoneId : hetznerDnsZoneId;
+          await perfAsync('deploy.dns.create', () =>
+            updateDnsIp(token, zoneId, domain, k8sFloatingIp),
+          );
+          state.completeStep('k8s-dns-update', { serverIp: k8sFloatingIp });
+        }
+      }
+    }
+
+    // --- Verifiable success gate for k8s ---
+    // The GHA deploy workflow already waits for Flux reconciliation; this
+    // extra probe confirms the app is reachable from the public internet
+    // (not just healthy inside the cluster). DNS propagation, cert issuance,
+    // and ingress routing all need to land before a user can actually load
+    // the URL — any of those failing mid-deploy would leave us exiting 0
+    // while the URL returns 5xx.
+    //
+    // K8s first-deploy timing on a cold cluster:
+    //   - DNS warmup record (0.0.0.0) → real IP swap propagates through
+    //     Cloudflare's edge in 1-3 min (TTL=auto = 5 min worst case).
+    //   - cert-manager HTTP-01 challenge: needs DNS to resolve to origin,
+    //     then LE issues. Typical 1-3 min once DNS is correct, but
+    //     orange-cloud + CF redirect quirks can push it to 8-12 min.
+    //   - Traefik picks up the new cert: ~30s.
+    // Old 10-min budget tripped on the cold tail (observed 2026-04-25 batch
+    // run #1 — k8s standalone, k8s-ha both failed at the probe). 20 min
+    // covers the p99 cold path; warm rollouts still finish in <60s.
+    if (domain) {
+      const s = p.spinner();
+      s.start(`Probing https://${domain}/api/health ...`);
+      // Public health probe — DNS propagation + cert issuance + ingress
+      // routing all roll up here. p99 cold k8s-ha = 8-12 min; warm <60s.
+      const probeResult = await perfAsync('deploy.health.probe', () =>
+        probePublicHealth(domain, { timeoutMs: 1_200_000 }),
+      );
+      if (probeResult?.ok) {
+        s.stop(`App is serving requests (HTTP ${probeResult.status})`);
+      } else {
+        // Build a concise one-line failure summary as the spinner stop text
+        // AND as the first line of the diagnostic dump below. The runner
+        // captures the stderr tail as the step's error message; if this
+        // line is the most recent error-level emission, the runner's
+        // failure attribution will say WHAT went wrong (last status, last
+        // network error class) instead of just dumping curl HEAD response
+        // headers — which was the 2026-04-27 morning matrix's "[unknown]
+        // CSP header dump" failure mode.
+        const r = probeResult ?? {};
+        const reason = r.lastStatus
+          ? `last HTTP status ${r.lastStatus} after ${r.attempts} attempts over ${Math.round((r.timeoutMs ?? 0) / 1000)}s`
+          : r.lastErrorClass
+            ? `${r.lastErrorClass}: ${r.lastErrorMessage}${r.lastErrorCause ? ` (cause: ${r.lastErrorCause})` : ''} (${r.attempts} attempts)`
+            : 'never connected';
+        s.stop(`Public health probe failed against https://${domain}/api/health — ${reason}`);
+        // Capture diagnostics inline so a CI run failure surfaces the
+        // root cause without requiring a follow-up `vibecarbon diagnose`.
+        // The cluster is about to be destroyed by the test runner; once
+        // it's gone, kubectl probes return nothing. We spawn each tool via
+        // execFileSync (array argv, no shell) — domain and environment come
+        // from validated config, but we still avoid shell-string interp.
+        try {
+          const { execFileSync } = await import('node:child_process');
+          // HA deploys split into kubeconfig-<env>-primary / -standby; standalone
+          // uses just kubeconfig-<env>. Prefer primary (the user-facing cluster
+          // — Traefik/cert-manager state we want to inspect lives there) and
+          // fall back to the standalone path. fs.existsSync check would race
+          // against the FS, so prefer the file-not-found error in execFileSync.
+          const { existsSync } = await import('node:fs');
+          const kubeconfigCandidates = [
+            join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}-primary`),
+            join(process.cwd(), '.vibecarbon', `kubeconfig-${environment}`),
+          ];
+          const kubeconfig =
+            kubeconfigCandidates.find((c) => existsSync(c)) ?? kubeconfigCandidates[1];
+          // Run each tool via execFileSync. Capture stderr alongside stdout
+          // (kubectl prints errors to stderr; without merging we get the
+          // useless first line of err.message — observed 2026-04-26 run #5
+          // where every kubectl probe surfaced as "(kubectl failed: Command
+          // failed: kubectl ...)" with no actual diagnostic).
+          const safeRun = (cmd, argv) => {
+            try {
+              return execFileSync(cmd, argv, {
+                encoding: 'utf8',
+                timeout: 30_000,
+                stdio: ['ignore', 'pipe', 'pipe'],
+              }).trim();
+            } catch (err) {
+              const stdout = err?.stdout?.toString?.()?.trim() ?? '';
+              const stderr = err?.stderr?.toString?.()?.trim() ?? '';
+              const tail = [stdout, stderr].filter(Boolean).join('\n').slice(0, 2000);
+              return `(${cmd} failed exit=${err?.status ?? '?'}):\n${tail || (err instanceof Error ? err.message : String(err))}`;
+            }
+          };
+          const dig = safeRun('dig', ['+short', domain, '@1.1.1.1']);
+          const kcExists = existsSync(kubeconfig);
+          const certs = kcExists
+            ? safeRun('kubectl', [
+                '--kubeconfig',
+                kubeconfig,
+                'get',
+                'certificate,certificaterequest,order,challenge',
+                '-n',
+                'vibecarbon',
+                '-o',
+                'wide',
+              ])
+            : `(kubeconfig not present at ${kubeconfig} — checked ${kubeconfigCandidates.join(', ')})`;
+          const events = kcExists
+            ? safeRun('kubectl', [
+                '--kubeconfig',
+                kubeconfig,
+                'get',
+                'events',
+                '-n',
+                'vibecarbon',
+                '--sort-by=.lastTimestamp',
+              ])
+            : '(skipped — no kubeconfig)';
+          // Trim events to the most recent 30 lines.
+          const recentEvents = events.split('\n').slice(-30).join('\n');
+          // Direct curl probes — surface the actual TLS/HTTP error from the
+          // public side (cert-manager not yet issued, traefik not routing,
+          // kong 503, etc.). curl exits non-zero on connect/TLS errors so we
+          // route through safeRun. -k skips cert validation — we want the
+          // body even if origin's cert is staging or self-signed.
+          const curlVerbose = safeRun('curl', [
+            '-skvI',
+            '--max-time',
+            '10',
+            `https://${domain}/api/health`,
+          ]);
+          const curlPodIp = safeRun('curl', [
+            '-skv',
+            '--max-time',
+            '10',
+            '--http1.1',
+            `https://${domain}/api/health`,
+          ]);
+          // Pods + endpoints + app logs. When traefik returns a 19-byte
+          // "404 page not found" but everything in cert-manager is green,
+          // the cause is almost always "service `app` has zero endpoints
+          // because all app pods are NotReady". Capturing pod state +
+          // endpoints + the app pod's stdout (where the readiness probe
+          // failure logs live — `Readiness check failed: { error: ... }`)
+          // turns "deploy probe got 404" from a riddle into a directly
+          // actionable error.
+          const pods = kcExists
+            ? safeRun('kubectl', [
+                '--kubeconfig',
+                kubeconfig,
+                'get',
+                'pods',
+                '-n',
+                'vibecarbon',
+                '-o',
+                'wide',
+              ])
+            : '(skipped — no kubeconfig)';
+          const endpoints = kcExists
+            ? safeRun('kubectl', [
+                '--kubeconfig',
+                kubeconfig,
+                'get',
+                'endpoints',
+                '-n',
+                'vibecarbon',
+              ])
+            : '(skipped — no kubeconfig)';
+          // App pod logs (last 100 lines). If multiple replicas, this picks
+          // by deployment selector — same as `kubectl logs deploy/app`.
+          const appLogs = kcExists
+            ? safeRun('kubectl', [
+                '--kubeconfig',
+                kubeconfig,
+                'logs',
+                'deployment/app',
+                '-n',
+                'vibecarbon',
+                '--tail=100',
+                '--all-containers',
+              ])
+            : '(skipped — no kubeconfig)';
+          p.log.error(
+            `Deploy completed but the app is not serving traffic.\n` +
+              `\n--- DNS (dig +short ${domain}) ---\n${dig || '(empty)'}\n` +
+              `\n--- cert-manager state (vibecarbon ns) ---\n${certs}\n` +
+              `\n--- Pods (vibecarbon ns) ---\n${pods}\n` +
+              `\n--- Service endpoints (vibecarbon ns) ---\n${endpoints}\n` +
+              `\n--- App pod logs (last 100) ---\n${appLogs}\n` +
+              `\n--- Recent events (last 30) ---\n${recentEvents}\n` +
+              `\n--- curl -kvI https://${domain}/api/health (HEAD) ---\n${curlVerbose}\n` +
+              `\n--- curl -kv https://${domain}/api/health (GET) ---\n${curlPodIp}\n`,
+          );
+        } catch (diagErr) {
+          p.log.error(
+            `Deploy completed but the app is not serving traffic. Check:\n` +
+              `  - DNS: dig +short ${domain}\n` +
+              `  - Pods: kubectl --kubeconfig ~/.vibecarbon/kubeconfig-${environment} get pods -A\n` +
+              `  - Certs: kubectl --kubeconfig ~/.vibecarbon/kubeconfig-${environment} get certificate -n vibecarbon\n` +
+              `(Auto-diagnostic capture failed: ${diagErr instanceof Error ? diagErr.message : String(diagErr)})`,
+          );
+        }
+        // Emit the structured probe reason as the VERY LAST line on BOTH
+        // stdout and stderr so the e2e runner's failure-tail capture
+        // grabs it regardless of which stream wins. An earlier version
+        // wrote it to stderr only, but the runner's `detail = stdoutClean
+        // || stderrClean` prefers stdout — where @clack's p.log.error
+        // wrote a CSP header dump from the curl -kvI diagnostic, masking
+        // the structured reason in matrix attributions. Writing to both
+        // streams costs nothing and guarantees the runner sees it as the
+        // last line.
+        const failLine = `\nFAIL: public health probe never reached 2xx — ${reason}\n`;
+        console.error(failLine);
+        console.log(failLine);
+        // Throw rather than process.exit(1): process.exit aborts before
+        // the deploy-logger's stream flushes, so the entire diagnostic
+        // dump above is silently lost from the .vibecarbon/logs/ file.
+        // (RCA from k8s-ha probe failure 2026-04-30: detail log ended
+        // mid-spinner with no diagnostic preserved.) withDeployLog's
+        // catch path writes the error trace, awaits the file close, then
+        // re-throws — so the deploy still exits non-zero, but with a
+        // self-contained log.
+        throw new Error(`Public health probe failed: ${reason}`);
+      }
+    }
+  }
+
+  // --- STEP 6: Finalize Config (Consolidated) ---
+  // Persist supabaseIp alongside masterIp for k8s deploys. The failover flow
+  // in src/failover.js needs the supabase node IP (where the postgres pod
+  // runs) to pg_basebackup-reseed standby from primary — without it, the
+  // failover uses whatever (potentially stale) state standby has. Compose
+  // deploys only have a single masterIp so supabaseIp is omitted there.
+  const servers = [];
+  if (deployResult.primary) {
+    // k8s-HA shape: { primary: { masterIp, supabaseIp }, standby: {...} }
+    servers.push(
+      {
+        name: 'primary',
+        ip: deployResult.primary.masterIp,
+        supabaseIp: deployResult.primary.supabaseIp,
+        region,
+      },
+      {
+        name: 'standby',
+        ip: deployResult.standby.masterIp,
+        supabaseIp: deployResult.standby.supabaseIp,
+        region: secondaryRegion,
+      },
+    );
+  } else if (deployResult.primaryIp && deployResult.standbyIp) {
+    // compose-HA shape: { primaryIp, standbyIp } — no supabaseIp because
+    // compose-HA runs Supabase on the same VM as the app.
+    servers.push(
+      { name: 'primary', ip: deployResult.primaryIp, region, serverType },
+      { name: 'standby', ip: deployResult.standbyIp, region: secondaryRegion, serverType },
+    );
+  } else if (deployResult.masterIp) {
+    // Single-server fall-through: hit by compose (one VPS) and k8s
+    // (master + supabase + worker[s]) deploys.
+    //
+    // Compose: persist the Hetzner-assigned `id` + literal name (e.g.
+    // `${projectName}-${environment}`) so `vibecarbon destroy` can locate
+    // the VPS via the API and `vibecarbon scale` can run its blue-green
+    // migration through the Hetzner provider. The role-style
+    // `name: 'master'` is kept because existing call-sites filter on it
+    // (failover, etc.). Without `hetznerName`, destroy's name-fallback
+    // searches Hetzner for the literal "master" — never matches — and the
+    // next deploy collides on the unique server name. Without `id`, scale
+    // treats the server as "deployed via --ssh, can't resize through the
+    // API" and silently no-ops. scaleCompose replays the old server's full
+    // `.env` into the new bundle as `envOverrides`, so the migration path
+    // works end-to-end across both fields.
+    //
+    // K8s: in addition to the master entry, persist one entry per node
+    // role (supabase + worker-N) so post-deploy commands (scale, destroy,
+    // e2e verify-scale) can enumerate every IP without having to
+    // re-derive worker IPs from Pulumi outputs. Master stays at index 0 —
+    // a handful of compose-only paths read `envConfig.servers?.[0]`, and
+    // preserving that contract is cheap. `supabaseIp` is also kept on the
+    // master entry so failover.js can read `servers[0].supabaseIp` without
+    // first iterating to find the supabase entry — redundant with the
+    // dedicated supabase entry below, but keeps the existing failover
+    // path working.
+    servers.push({
+      name: 'master',
+      ...(deployResult.serverId &&
+        deployResult.serverId !== 'manual' && { id: deployResult.serverId }),
+      ...(deployResult.serverName && { hetznerName: deployResult.serverName }),
+      ip: deployResult.masterIp,
+      supabaseIp: deployResult.supabaseIp,
+      region,
+      serverType,
+      ...(isComposeDeploy ? {} : { role: 'master' }),
+    });
+    // K8s only — compose has neither a separate supabase node nor worker
+    // pool. Compose deploys leave servers as a single-element array.
+    if (!isComposeDeploy) {
+      if (deployResult.supabaseIp) {
+        servers.push({
+          name: 'supabase',
+          ip: deployResult.supabaseIp,
+          region,
+          serverType,
+          role: 'supabase',
+        });
+      }
+      const workerIps = Array.isArray(deployResult.workerIps) ? deployResult.workerIps : [];
+      workerIps.forEach((wIp, idx) => {
+        if (!wIp) return;
+        servers.push({
+          name: `worker-${idx + 1}`,
+          ip: wIp,
+          region,
+          serverType,
+          role: 'worker',
+        });
+      });
+    }
+  }
+
+  let deployedCommit = null;
+  try {
+    deployedCommit = runCommand(['git', 'rev-parse', 'HEAD'], {
+      encoding: 'utf-8',
+      silent: true,
+    }).trim();
+  } catch {}
+
+  // Persist backupS3 alongside the storage-s3 block. The backup + restore
+  // commands both read it via loadBackupS3Config() from .vibecarbon.json
+  // — without it here, `vibecarbon restore` against a destroyed-then-redeployed
+  // environment falls through to server-side `ls /opt/*/backups/*` on a
+  // fresh VM with no files, so DR from a populated backup bucket is broken.
+  // Keep the shape in sync with loadBackupS3Config (bucket/region/endpoint,
+  // no credentials — those live in ~/.vibecarbon/credentials.json).
+  const backupS3Persist = backupS3Config
+    ? {
+        bucket: backupS3Config.bucket,
+        region: backupS3Config.region,
+        endpoint: backupS3Config.endpoint,
+      }
+    : null;
+
+  // Persist HA flags + secondaryRegion so post-deploy commands can tell
+  // a k8s-HA env (`deployMode='kubernetes' + ha.enabled + secondaryRegion`)
+  // apart from a single-region k8s env (`deployMode='kubernetes'` only).
+  // scale.js relies on this distinction to route Pulumi up against the
+  // per-cluster `${env}-primary` / `${env}-standby` stacks instead of a
+  // bare `${env}` stack that has no Pulumi state. Without these fields,
+  // k8s-HA scale would try to create a fresh stack and Hetzner would
+  // reject the duplicate SshKey post.
+  const isHADeploy = !!config.ha && !!secondaryRegion;
+
+  // Re-read the on-disk project config before merging. Inner deploy paths
+  // (compose/ha.js, k8s/ha/index.js) call saveProjectConfig themselves
+  // mid-flight to persist server IDs / per-cluster details that this
+  // outer save cannot reconstruct from `deployResult` alone (the
+  // orchestrator's `servers` array drops `id`, for example). If we spread
+  // the in-memory `projectConfig` (captured at function entry), we'd
+  // silently overwrite those mid-flight saves — observed in compose-HA
+  // matrix runs where destroyComposeHA found servers with no `id` and
+  // skipped the VPS delete entirely, leaving Hetzner state to collide
+  // on restore re-deploy.
+  const persistedConfig = loadProjectConfig() || projectConfig;
+  const persistedEnvConfig = persistedConfig.environments?.[environment] || envConfig;
+  const finalConfig = {
+    ...persistedConfig,
+    environments: {
+      ...persistedConfig.environments,
+      [environment]: {
+        ...persistedEnvConfig,
+        status: 'deployed',
+        deployMode,
+        region,
+        ...(isHADeploy && {
+          secondaryRegion,
+          // Persist per-cluster floatingIp + supabaseIp under ha.primary /
+          // ha.standby so failover.identifyServers takes the structured
+          // path that surfaces floatingIp. Without these, identifyServers
+          // falls through to the flat `servers[]` lookup which has only
+          // `ip` — failover then targets the standby's master IP for the
+          // DNS A record instead of the floating IP, bypassing the
+          // FIP-based design.
+          ha: {
+            enabled: true,
+            failoverRegion: secondaryRegion,
+            ...(deployResult.primary && {
+              primary: {
+                masterIp: deployResult.primary.masterIp,
+                floatingIp: deployResult.primary.floatingIp,
+                supabaseIp: deployResult.primary.supabaseIp,
+                region,
+              },
+              standby: {
+                masterIp: deployResult.standby.masterIp,
+                floatingIp: deployResult.standby.floatingIp,
+                supabaseIp: deployResult.standby.supabaseIp,
+                region: secondaryRegion,
+              },
+            }),
+          },
+        }),
+        domain,
+        dnsProvider,
+        // Persist the nested `dns: { provider, cloudflareZoneId | hetznerZoneId }`
+        // shape too. failover.detectScenario reads `envConfig.dns?.provider`
+        // (NOT the flat `dnsProvider`) — without this, k8s/k8s-ha failover
+        // falls through to ha_manual, prints "Update DNS yourself" instead of
+        // calling Cloudflare/Hetzner, and the public health probe in
+        // verify-failover never points at the standby. Mirror the shape
+        // compose-ha already persists in src/lib/deploy/compose/ha.js.
+        dns:
+          dnsProvider === 'cloudflare' && cloudflareZoneId
+            ? { provider: 'cloudflare', cloudflareZoneId }
+            : dnsProvider === 'hetzner' && hetznerDnsZoneId
+              ? { provider: 'hetzner', hetznerZoneId: hetznerDnsZoneId }
+              : null,
+        // Prefer the inner deploy's richer `servers` (with `id`) when it
+        // already wrote one. Compose-HA persists
+        //   [{ name, id, ip, region, serverType, role }, ...]
+        // The outer `servers` we'd build here drops `id` (deployResult
+        // for compose-HA only carries primaryIp/standbyIp). Falling back
+        // to the inner save preserves the IDs that destroyComposeHA needs.
+        servers:
+          Array.isArray(persistedEnvConfig.servers) &&
+          persistedEnvConfig.servers.length > 0 &&
+          persistedEnvConfig.servers.some((s) => s.id != null)
+            ? persistedEnvConfig.servers
+            : servers,
+        // Persist k8s sizing inputs so post-deploy `vibecarbon scale` can
+        // rebuild the same Pulumi inputs without falling back to defaults.
+        // Without `minWorkers/maxWorkers` here, scale.js would read
+        // `envConfig.minWorkers/maxWorkers` as `undefined` and fall back to
+        // built-in defaults — which on a clean type-bump made Pulumi plan a
+        // delete of `worker-1`, replacing master+supabase along the way
+        // (etcd loss). Master/supabase/worker types are also persisted so
+        // the per-role split survives across deploy → scale.
+        ...(deployMode === 'kubernetes' && {
+          masterServerType,
+          supabaseServerType,
+          workerServerType,
+          minWorkers,
+          maxWorkers,
+        }),
+        deployedAt: new Date().toISOString(),
+        deployedCommit,
+        s3: { bucket: s3Config.bucket, region: s3Config.region, endpoint: s3Config.endpoint },
+        ...(backupS3Persist && { backupS3: backupS3Persist }),
+        backup: backupConfig,
+        services,
+      },
+    },
+  };
+  saveProjectConfig(finalConfig);
+  registerProject(projectConfig.projectName, process.cwd());
+
+  const { formatted: timeStr } = tracker.finish();
+  p.log.success(`Deployment to ${c.bold(environment)} complete! ${c.dim(`(${timeStr})`)}`);
+  if (domain) p.log.info(`URL: ${c.info(`https://${domain}`)}`);
+}