vibecarbon 0.8.0 → 0.10.0

package/src/lib/deploy/k8s/k3s.js

@@ -37,9 +37,9 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { setTimeout as delay } from 'node:timers/promises';
 import * as p from '@clack/prompts';
-import { runCommandAsync } from '../../command.js';
+import { runCommandAsync, writeSecretFile } from '../../command.js';
 import { featureConfigKeys, featureSecretKeys } from '../../config-registry.js';
-import { knownHostsPath } from '../../host-keys.js';
+import { knownHostsPath, seedKnownHosts } from '../../host-keys.js';
 import { loadCloudInit, renderScript } from '../../iac/cloud-init.js';
 import { dbImageRef } from '../../images.js';
 import { perfAsync } from '../../perf.js';
@@ -1101,21 +1101,28 @@ export async function pushImageToLocalRegistry({
       );
     });
 
-  // Three attempts with progressive backoff. Failure modes we're
-  // covering, in priority order:
+  // Progressive backoff across attempts. Failure modes we're covering, in
+  // priority order:
   //   1. Stale tunnel from a parallel run holds :5000 → reopen tunnel.
   //      Recovers in seconds (handled by the per-attempt teardown).
   //   2. Registry pod restarted between back-to-back pushes (e.g.
   //      probe-killed mid-prior-push) → manifest PUT returns EOF or
   //      `manifest blob unknown to registry`. Pod recovery is ~10–20s,
-  //      so the second/third attempt has the longer backoff to let the
+  //      so the later attempts have the longer backoff to let the
   //      replacement pod become Ready before re-pushing.
-  //   3. S3 transient (Hetzner Object Storage 5xx during blob upload)
-  //      → registry returns the same `blob unknown` after the layer is
-  //      "uploaded" but not yet readable. Same recovery shape as (2).
-  // The 1s/15s/30s schedule keeps the happy path fast (single attempt
-  // ~6s end-to-end) while giving genuine restarts time to recover.
-  const settleDelays = settleDelaysMs ?? [1_000, 15_000, 30_000];
+  //   3. S3 throttle (Hetzner Object Storage 503 SlowDown during blob
+  //      upload) → the registry propagates a `503 Service Unavailable` to
+  //      the docker client. This is LOAD-induced: k8s-HA deploys the primary
+  //      and standby clusters in parallel, and BOTH registries write layers
+  //      to the same Hetzner S3 account at once, so the account throttles.
+  //      3 short attempts (~46s of settle) couldn't ride it out. The longer
+  //      tail (60s/120s) extends the window past the throttle window AND past
+  //      the sibling cluster's push draining, which halves the S3 write load.
+  // The 1s/15s head keeps the happy path fast (single attempt ~6s end-to-end)
+  // while the 30s/60s/120s tail rides out genuine restarts + S3 throttling.
+  // Note docker push is content-addressable: layers whose S3 write already
+  // succeeded are skipped on retry, so later attempts get progressively cheaper.
+  const settleDelays = settleDelaysMs ?? [1_000, 15_000, 30_000, 60_000, 120_000];
   const maxAttempts = settleDelays.length;
   const attemptErrors = [];
   const taggedAliases = new Set();
@@ -1325,6 +1332,7 @@ export async function installSupabase({
   envLocal,
   dbImageTag,
   backupBucketName,
+  walgRole = 'primary',
 }) {
   const env = { ...process.env, KUBECONFIG: kubeconfig };
   // Render the values template — substitute every {{TOKEN}} the template
@@ -1351,6 +1359,15 @@ export async function installSupabase({
   // WAL-G S3 prefix — dedicated backup bucket if provided, else the storage
   // bucket (mirrors compose's S3_BACKUP_BUCKET:-S3_BUCKET fallback). Never
   // leave it as s3:/// (wal-g would error).
+  //
+  // SINGLE canonical prefix (NO role segment). Reads (backup-fetch/restore/
+  // reseed) and writes (backup-push/wal-push) must all agree on ONE prefix — a
+  // role-segmented prefix made the standby read an empty `…/walg/standby` and
+  // fail restore/reseed with "No backups found" (caught by compose-ha scale
+  // e2e). Anti-collision (finding #3: a standby / bring-up-phase independent
+  // primary must never WRITE here) is enforced by the WALG_ROLE WRITE-GUARD:
+  // wal-archive.sh and the backup CronJob both no-op when WALG_ROLE=standby.
+  // walgRole is rendered into the db env below so those write guards can read it.
   const walgBucket = backupBucketName || s3Config?.bucket || '';
   const walgS3Prefix = walgBucket
     ? `s3://${walgBucket}/backups/${projectName || 'vibecarbon'}/walg`
@@ -1360,6 +1377,7 @@ export async function installSupabase({
     .replace(/\{\{PROJECT_NAME\}\}/g, projectName || 'vibecarbon')
     .replace(/\{\{S3_BACKUP_BUCKET\}\}/g, walgBucket)
     .replace(/\{\{WALG_S3_PREFIX\}\}/g, walgS3Prefix)
+    .replace(/\{\{WALG_ROLE\}\}/g, walgRole)
     .replace(/\{\{S3_ENDPOINT\}\}/g, s3Config?.endpoint ?? '')
     .replace(/\{\{S3_REGION\}\}/g, s3Config?.region ?? '')
     .replace(/\{\{DB_IMAGE\}\}/g, dbImageRepo)
@@ -1370,7 +1388,9 @@ export async function installSupabase({
     .replace(/\{\{ADMIN_EMAIL\}\}/g, envLocal?.ADMIN_EMAIL ?? '')
     .replace(/\{\{ADMIN_PASSWORD\}\}/g, envLocal?.ADMIN_PASSWORD ?? '');
   const tmpValues = join(tmpdir(), `vibecarbon-supabase-values-${process.pid}-${Date.now()}.yaml`);
-  writeFileSync(tmpValues, rendered);
+  // SECURITY: the rendered values contain ADMIN_EMAIL/ADMIN_PASSWORD and live
+  // for the whole `helm upgrade --wait` window — write 0o600, never 0o644.
+  writeSecretFile(tmpValues, rendered);
   try {
     // Add the supabase-community helm repo (idempotent — non-zero if exists).
     try {
@@ -2344,6 +2364,12 @@ export async function applyK3sManifests({
   //    timeout in chart). image.db points at the wal-g-equipped db image
   //    (sideloaded in step 7b), and the chart mounts wal-archive.sh + the
   //    non-secret WALG_* env from supabase.values.yaml.
+  // Derive the wal-g WRITE-GUARD role (NOT a path segment). deployK8sHA calls
+  // deployK3s with environment `${env}-primary` / `${env}-standby`; single-
+  // cluster deploys pass the plain env name → 'primary'. Rendered into the db
+  // container env as WALG_ROLE so wal-archive.sh + the backup CronJob no-op on
+  // the standby. The S3 prefix itself stays canonical (no role segment).
+  const walgRole = environment?.endsWith('-standby') ? 'standby' : 'primary';
   await perfAsync(`deploy.${perfPrefix}.installSupabase`, async () =>
     installSupabase({
       kubeconfig,
@@ -2354,6 +2380,7 @@ export async function applyK3sManifests({
       envLocal,
       dbImageTag,
       backupBucketName,
+      walgRole,
     }),
   );
   // 7b. Enable continuous WAL archiving on the supabase-db. The chart can't
@@ -2593,7 +2620,7 @@ export async function applyK3sManifests({
  * App install + sideload land in a follow-up commit (Task 7).
  *
  * @param {K3sDeployOptions} options
- * @returns {Promise<{masterIp: string, floatingIp: string, supabaseIp: string, workerIps: string[], networkId: string|number, kubeconfig: string, sshKeyPath: string}>}
+ * @returns {Promise<{masterIp: string, floatingIp: string, supabaseIp: string, supabasePrivateIp: string, workerIps: string[], networkId: string|number, kubeconfig: string, sshKeyPath: string}>}
  */
 export async function deployK3s(options) {
   const { state } = options;
@@ -2689,7 +2716,14 @@ export async function deployK3s(options) {
     s.stop(`Cluster provisioned — master=${outputs.masterIp} floating=${outputs.floatingIp}`);
   }
   const infraOutputs = state.getStepResult('k3s-infra');
-  const { masterIp, floatingIp, supabaseIp, workerIps, networkId } = infraOutputs;
+  const {
+    masterIp,
+    floatingIp,
+    supabaseIp,
+    supabasePrivateIp = '10.0.1.2',
+    workerIps,
+    networkId,
+  } = infraOutputs;
 
   // 4. Wait for cloud-init to finish + k3s to be Ready
   if (!state.shouldSkip('k3s-ready', { masterIp })) {
@@ -2709,6 +2743,18 @@ export async function deployK3s(options) {
     s.stop('k3s reachable');
   }
 
+  // Trusted host-key seed. The SSH opts above pin per-env with accept-new
+  // (TOFU on first connect); seeding the known_hosts from a real ssh-keyscan
+  // now — on (re)provision, once SSH is reachable — turns accept-new into a
+  // strict pin for the rest of this deploy AND every later command, and
+  // re-pins a Hetzner-recycled IP cleanly (seedKnownHosts drops the stale
+  // per-IP line). Best-effort: if ssh-keyscan can't reach a node we fall back
+  // to accept-new TOFU. See host-keys.js seedKnownHosts for the full security
+  // rationale (why only provisioning re-seeds).
+  for (const seedIp of [masterIp, supabaseIp, ...(workerIps ?? [])].filter(Boolean)) {
+    await seedKnownHosts(khPath, seedIp);
+  }
+
   // 5. Fetch + patch kubeconfig
   let kubeconfig;
   if (!state.shouldSkip('k3s-kubeconfig', { masterIp })) {
@@ -2827,6 +2873,11 @@ export async function deployK3s(options) {
     masterIp,
     floatingIp,
     supabaseIp,
+    // The supabase node's Hetzner private-network IP (deterministic in the IaC
+    // program, but threaded through the output rather than hardcoded at the use
+    // site). setupReplication renders it into the repl-gateway egress
+    // NetworkPolicy + socat bind for this cluster.
+    supabasePrivateIp,
     workerIps,
     networkId,
     kubeconfig,

package/src/lib/deploy/orchestrator.js

@@ -16,6 +16,7 @@ import {
   saveProjectConfig,
 } from '../config.js';
 import { waitForDNSPropagation } from '../dns-propagation.js';
+import { knownHostsPathForKey, seedKnownHosts } from '../host-keys.js';
 import { ensureOperatorIpAccess } from '../operator-ip.js';
 import { perfAsync, perfTimer } from '../perf.js';
 import { createTracker } from '../tracker.js';
@@ -223,16 +224,45 @@ export async function executeDeployment(args, gatheredConfig) {
   }
 
   const s3Spinner = p.spinner();
-  const { HetznerS3Provider } = await import('../providers/hetzner-s3.js');
+  const { HetznerS3Provider, deriveStateBucketName } = await import('../providers/hetzner-s3.js');
   const s3Provider = new HetznerS3Provider(s3Config.accessKey, s3Config.secretKey, s3Config.region);
 
+  // Dedicated Pulumi-state bucket, derived once from the app storage bucket.
+  // Pulumi state used to live in the app storage bucket, which meant `destroy`
+  // deleting that bucket could yank the state backend mid-destroy. State now
+  // lives here instead. Both HA stacks share this ONE bucket.
+  const stateBucket = deriveStateBucketName(s3Config.bucket);
+
   // --- STEP 1: S3 Setup (Idempotent) ---
+  // `stateBucket` is part of the inputs hash so an upgraded CLI (whose prior
+  // deploy-state predates the dedicated state bucket) re-runs this step once to
+  // create + migrate the state bucket. Combined with the StateTracker version
+  // bump, no stale hash can skip the migration.
   const s3Inputs = {
     bucket: s3Config.bucket,
     backupBucket: backupS3Config.bucket,
+    stateBucket,
     region: s3Config.region,
   };
-  if (!state.shouldSkip('s3-setup', s3Inputs)) {
+  // Verify hook: even on a matching hash, re-run s3-setup if any of the buckets
+  // were deleted out-of-band (e.g. by a prior `destroy`). Without this, a
+  // resumed deploy skips bucket creation then fails downstream on NoSuchBucket
+  // (the exact resume hazard called out in Finding 2). Other steps that would
+  // benefit from a similar probe — `compose-setup-server` (server may be gone)
+  // and `dns-setup` — are left on the plain hash check to keep this change
+  // focused; s3-setup is the highest-value one because the state backend lives
+  // here.
+  const verifyBucketsExist = async () => {
+    const [app, backup, stateB] = await Promise.all([
+      s3Provider.bucketExists(s3Config.bucket),
+      backupS3Config?.bucket
+        ? s3Provider.bucketExists(backupS3Config.bucket)
+        : Promise.resolve(true),
+      s3Provider.bucketExists(stateBucket),
+    ]);
+    return app && backup && stateB;
+  };
+  if (!(await state.shouldSkipWithVerify('s3-setup', s3Inputs, verifyBucketsExist))) {
     state.startStep('s3-setup', s3Inputs);
     try {
       s3Spinner.start(`Creating S3 bucket: ${s3Config.bucket}`);
@@ -245,7 +275,22 @@ export async function executeDeployment(args, gatheredConfig) {
 
       try {
         s3Spinner.start('Configuring bucket CORS');
-        const corsOrigins = domain ? [`https://${domain}`] : ['*'];
+        // SECURITY: never fall back to a `*` wildcard — that would let any
+        // origin issue credentialed browser requests against the storage
+        // bucket. When a domain is configured we scope to its https origins
+        // (apex + api/app subdomains the app actually serves from); a
+        // domainless deploy has no browser origin to trust yet, so we scope
+        // to localhost dev origins and warn. The operator re-runs deploy once
+        // a domain is set to widen CORS to the real origin.
+        let corsOrigins;
+        if (domain) {
+          corsOrigins = [`https://${domain}`, `https://api.${domain}`, `https://app.${domain}`];
+        } else {
+          corsOrigins = ['http://localhost:3000', 'http://localhost:5173'];
+          p.log.warn(
+            'No domain configured — scoping bucket CORS to localhost dev origins instead of `*`. Re-run deploy with a domain to allow your production origin.',
+          );
+        }
         await s3Provider.configureCORS(s3Config.bucket, corsOrigins);
         s3Spinner.stop('CORS configured');
       } catch {
@@ -264,7 +309,98 @@ export async function executeDeployment(args, gatheredConfig) {
         s3Spinner.stop(`Backup bucket creation failed: ${backupBucketError.message}`);
       }
 
-      const result = { region: s3Provider.region, endpoint: s3Provider.getEndpoint() };
+      // Dedicated Pulumi-state bucket. Created imperatively (like the app +
+      // backup buckets) BEFORE any Pulumi stack op / initBackend. NOT
+      // web-facing: no CORS, not public.
+      s3Spinner.start(`Creating Pulumi state bucket: ${stateBucket}`);
+      const stateBucketResult = await s3Provider.createBucket(stateBucket);
+      s3Spinner.stop(
+        stateBucketResult.created
+          ? `Pulumi state bucket created: ${stateBucket}`
+          : `Pulumi state bucket exists: ${stateBucket}`,
+      );
+
+      // SECURITY/SAFETY: migrate legacy Pulumi state out of the app storage
+      // bucket. If the state bucket has no `.pulumi/` objects but the OLD app
+      // bucket does, this is an env deployed before the dedicated state bucket
+      // existed — copy the state prefix over so we resolve to real state, not
+      // empty state. Re-initializing an EMPTY state backend against live infra
+      // would make Pulumi believe nothing exists and either duplicate or orphan
+      // the running servers. So if the copy fails, we DO NOT proceed silently:
+      // abort loudly and tell the operator to migrate manually or
+      // destroy+redeploy. Fresh deploys hit neither branch (both buckets empty).
+      const PULUMI_STATE_PREFIX = '.pulumi/';
+      // Determine whether there is GENUINE legacy state to migrate. The check
+      // lists the just-created state bucket, which can race S3 list-after-create
+      // consistency and throw — that is NOT the orphan-risk case (a fresh deploy
+      // has no legacy state), so a check failure must NOT hard-abort. Only a
+      // CONFIRMED legacy-state copy failure aborts (below). Retry the checks a
+      // few times so a transient blip doesn't misfire either way.
+      let legacyState = false;
+      let checkOk = false;
+      for (let attempt = 1; attempt <= 3; attempt++) {
+        try {
+          const stateSeeded = await s3Provider.hasObjectsWithPrefix(
+            stateBucket,
+            PULUMI_STATE_PREFIX,
+          );
+          legacyState = stateSeeded
+            ? false
+            : await s3Provider.hasObjectsWithPrefix(s3Config.bucket, PULUMI_STATE_PREFIX);
+          checkOk = true;
+          break;
+        } catch (checkError) {
+          if (attempt === 3) {
+            // Couldn't determine migration need after retries (e.g. list-after-
+            // create consistency on the fresh state bucket). A fresh deploy — the
+            // overwhelming common case — has nothing to migrate, so proceed with
+            // a loud warning rather than blocking every deploy on a transient S3
+            // blip. (Pulumi then uses the dedicated state bucket normally.)
+            p.log.warn(
+              `Could not verify legacy Pulumi state after ${attempt} attempts — proceeding ` +
+                `(a fresh deploy has none to migrate): ${checkError.message}`,
+            );
+          } else {
+            await new Promise((r) => setTimeout(r, 1500 * attempt));
+          }
+        }
+      }
+      if (checkOk && legacyState) {
+        // CONFIRMED legacy state in the app bucket — migrate it. If THIS copy
+        // fails we DO NOT proceed silently: re-initializing an EMPTY state
+        // backend against live infra would orphan the running servers.
+        try {
+          s3Spinner.start(`Migrating Pulumi state ${s3Config.bucket} → ${stateBucket}`);
+          const { copied } = await s3Provider.copyPrefix(
+            s3Config.bucket,
+            stateBucket,
+            PULUMI_STATE_PREFIX,
+          );
+          s3Spinner.stop(`Migrated ${copied} Pulumi state object(s) to ${stateBucket}`);
+        } catch (copyError) {
+          s3Spinner.stop(`Pulumi state migration FAILED: ${copyError.message}`);
+          p.log.error(
+            [
+              'Could not migrate existing Pulumi state from the app storage bucket to the',
+              `dedicated state bucket (${stateBucket}). Proceeding would re-initialize an`,
+              'EMPTY state backend against your live infrastructure, which would orphan the',
+              'running servers (Pulumi would no longer track them).',
+              '',
+              'Aborting to protect your infrastructure. To recover, either:',
+              `  • Copy the ".pulumi/" prefix from ${s3Config.bucket} to ${stateBucket} manually`,
+              '    (e.g. via `aws s3 sync ... --endpoint-url <endpoint>`), then re-run deploy; or',
+              '  • Destroy this environment and redeploy from scratch.',
+            ].join('\n'),
+          );
+          process.exit(1);
+        }
+      }
+
+      const result = {
+        region: s3Provider.region,
+        endpoint: s3Provider.getEndpoint(),
+        stateBucket,
+      };
       state.completeStep('s3-setup', result);
     } catch (error) {
       const errMsg =
@@ -285,6 +421,11 @@ export async function executeDeployment(args, gatheredConfig) {
     backupS3Config.region = s3Config.region;
     backupS3Config.endpoint = s3Config.endpoint;
   }
+  // Always resolve stateBucket (deterministic derivation) so the Pulumi
+  // backend targets the dedicated bucket, whether s3-setup ran or was skipped.
+  // Every downstream iac call reads it via s3Config, and it's persisted into
+  // .vibecarbon.json so scale / failover / destroy resolve the same backend.
+  s3Config.stateBucket = s3Result?.stateBucket || stateBucket;
 
   // SKELETON SAVE: persist the minimum env entry that `vibecarbon destroy`
   // needs to recover, BEFORE any `pulumi up` runs. Without this, a crash
@@ -315,7 +456,12 @@ export async function executeDeployment(args, gatheredConfig) {
       ...(secondaryRegion ? { secondaryRegion } : {}),
       ...(domain ? { domain } : {}),
       ...(dnsProvider ? { dnsProvider } : {}),
-      s3: { bucket: s3Config.bucket, region: s3Config.region, endpoint: s3Config.endpoint },
+      s3: {
+        bucket: s3Config.bucket,
+        region: s3Config.region,
+        endpoint: s3Config.endpoint,
+        stateBucket: s3Config.stateBucket,
+      },
       backupS3: backupS3Config?.bucket
         ? {
             bucket: backupS3Config.bucket,
@@ -558,6 +704,9 @@ export async function executeDeployment(args, gatheredConfig) {
           backupS3Config,
           backupConfig,
           bundlePath,
+          // Finding #1: hard-gate replication unless the operator opts into a
+          // warm/degraded standby via `deploy -allow-degraded`.
+          allowDegraded: !!args.allowDegraded,
           onProgress: (msg) => s.message(msg),
         }),
       );
@@ -606,6 +755,14 @@ export async function executeDeployment(args, gatheredConfig) {
         serverHetznerId = result.outputs.serverId || null;
         serverHetznerName = `${projectConfig.projectName}-${environment}`;
         await perfAsync('deploy.iac.waitForSSH', () => waitForComposeSSH(serverIp, sshKeyPath, 30));
+        // Trusted host-key seed on fresh provision — captures the new VPS's
+        // host key via ssh-keyscan into the per-env known_hosts so the pinned
+        // SSH opts (accept-new) become a strict pin for this deploy and every
+        // later command. Re-seeding here also cleanly re-pins a Hetzner-
+        // recycled IP (destroy → redeploy). Best-effort: falls back to
+        // accept-new TOFU if the scan can't reach the server yet. See
+        // host-keys.js seedKnownHosts for the full security rationale.
+        await seedKnownHosts(knownHostsPathForKey(sshKeyPath), serverIp);
       }
 
       // --- Warm path (StateTracker) ---
@@ -961,6 +1118,9 @@ export async function executeDeployment(args, gatheredConfig) {
       // container seeds PGDATA from S3 via wal-g and applyMigrations is
       // skipped (the restored DB is authoritative).
       restore: args.restore || null,
+      // Finding #1: hard-gate k8s-HA replication unless the operator opts into a
+      // warm/degraded standby via `deploy -allow-degraded`.
+      allowDegraded: !!args.allowDegraded,
       operatorCidrs: projectConfig.operatorCidrs ?? [],
       imageReadyPromise,
       tracker,
@@ -1405,6 +1565,11 @@ export async function executeDeployment(args, gatheredConfig) {
         region,
         ...(isHADeploy && {
           secondaryRegion,
+          // Finding #1: persist DR posture. A default (gated) HA deploy that
+          // reaches this point is streaming; only `-allow-degraded` finalizes a
+          // warm/degraded standby, which the inner deploy flags on deployResult.
+          replication: deployResult?.degraded ? 'degraded' : 'streaming',
+          degraded: !!deployResult?.degraded,
           // Persist per-cluster floatingIp + supabaseIp under ha.primary /
           // ha.standby so failover.identifyServers takes the structured
           // path that surfaces floatingIp. Without these, identifyServers
@@ -1475,7 +1640,12 @@ export async function executeDeployment(args, gatheredConfig) {
         }),
         deployedAt: new Date().toISOString(),
         deployedCommit,
-        s3: { bucket: s3Config.bucket, region: s3Config.region, endpoint: s3Config.endpoint },
+        s3: {
+          bucket: s3Config.bucket,
+          region: s3Config.region,
+          endpoint: s3Config.endpoint,
+          stateBucket: s3Config.stateBucket,
+        },
         ...(backupS3Persist && { backupS3: backupS3Persist }),
         backup: backupConfig,
         services,

package/src/lib/deploy/remote-build.js

@@ -9,6 +9,7 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import * as p from '@clack/prompts';
 import { runCommand } from '../command.js';
+import { knownHostsPathForKey } from '../host-keys.js';
 import { perfTimer } from '../perf.js';
 
 /**
@@ -41,8 +42,13 @@ export async function buildRemote(ip, sshKeyPath, imageTag, cwd, buildArgs = {})
   // session helper drops with "error reading preface ... file already closed".
   // Keepalives let ssh detect a dead peer (~60s) and fail the attempt cleanly
   // so the retry below can re-establish, instead of hanging to the build timeout.
+  // Host-key pinned against the per-env known_hosts (passed via env var so no
+  // shell-quoting concern), NOT /dev/null + no. accept-new TOFU's an
+  // ephemeral/recycled Hetzner IP but rejects a changed key for an already-
+  // pinned host (MITM on an established env fails); GlobalKnownHostsFile=
+  // /dev/null ignores the system file and never touches ~/.ssh/known_hosts.
   const sshWrapper = `#!/bin/bash
-exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 "$@"
+exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$VIBECARBON_KNOWN_HOSTS" -o GlobalKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 "$@"
 `;
   writeFileSync(sshWrapperPath, sshWrapper, { mode: 0o755 });
 
@@ -52,6 +58,7 @@ exec /usr/bin/ssh -i "$VIBECARBON_SSH_KEY" -o StrictHostKeyChecking=no -o UserKn
     DOCKER_BUILDKIT: '1',
     PATH: `${tempBinDir}:${process.env.PATH}`, // Prepend our wrapper
     VIBECARBON_SSH_KEY: sshKeyPath,
+    VIBECARBON_KNOWN_HOSTS: knownHostsPathForKey(sshKeyPath),
   };
 
   try {