vibecarbon 0.9.0 → 0.10.0

package/src/lib/deploy/compose/ha.js

@@ -21,10 +21,24 @@ import {
   saveProjectConfig,
 } from '../../config.js';
 import { fetchWithRetry } from '../../fetch-retry.js';
+import { knownHostsPathForKey, seedKnownHosts } from '../../host-keys.js';
 import { perfAsync } from '../../perf.js';
 import { getProvider } from '../../providers/index.js';
 import { useDnsChallenge } from '../acme.js';
-import { generateSSHKeyPair, readReplPassword } from '../utils.js';
+import {
+  assertReplicationStreamingOrDegraded,
+  buildPrimaryConninfo,
+  buildReplicationHbaLine,
+  buildStagedBasebackupScript,
+  verifyStreaming,
+} from '../replication.js';
+import {
+  generateSSHKeyPair,
+  mergeRemoteDotenv,
+  pinnedSshOptsString,
+  readReplPassword,
+} from '../utils.js';
+import { WG_SUBNET_CIDR } from '../wireguard.js';
 import {
   createAdminUser,
   dockerLoginOnServer,
@@ -211,11 +225,12 @@ async function deleteFirewallByName(name, hetznerHeaders, deadlineMs = 30_000) {
  * Configure the primary PostgreSQL server for streaming replication.
  *
  * 1. Create a dedicated replication user
- * 2. Allow the standby IP in pg_hba.conf
+ * 2. Allow replication from the WireGuard tunnel subnet in pg_hba.conf (plain
+ *    `host` — WireGuard encrypts the wire, so Postgres doesn't also need TLS)
  * 3. Set wal_level=replica and max_wal_senders
  * 4. Restart PostgreSQL to apply changes
  */
-async function configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, projectName) {
+async function configurePrimaryReplication(primaryIp, sshKeyPath, projectName) {
   const remoteDir = `/opt/${projectName}`;
   // replPassword is generated at create time from crypto.randomBytes and is
   // restricted to base64url characters — no shell escaping needed.
@@ -239,8 +254,10 @@ async function configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, pro
   // hold WAL until the standby confirms receipt, closing the drift
   // window.
   //
-  // The SQL is base64-encoded so the password never appears in argv —
-  // the remote shell only sees the base64 blob.
+  // The SQL (which embeds the replication password) is piped to psql via STDIN
+  // (`input:`), so it never appears in argv / the ssh command string / the
+  // remote shell's /proc/cmdline. `docker compose exec -T` forwards our stdin
+  // straight into psql.
   const createRoleSql = `
 DO $$
 BEGIN
@@ -263,27 +280,29 @@ BEGIN
   END IF;
 END $$;
 `;
-  const createRoleB64 = Buffer.from(createRoleSql).toString('base64');
   await sshRun(
     primaryIp,
     sshKeyPath,
-    `cd ${remoteDir} && echo ${createRoleB64} | base64 -d | docker compose exec -T db psql -U supabase_admin -d postgres`,
-    { timeout: 30_000 },
+    `cd ${remoteDir} && docker compose exec -T db psql -U supabase_admin -d postgres`,
+    { timeout: 30_000, input: createRoleSql },
   );
 
-  // Update pg_hba.conf to allow replication from standby. The supabase/postgres
-  // image overrides hba_file to /etc/postgresql/pg_hba.conf (NOT the default
+  // Update pg_hba.conf to allow replication from the WireGuard tunnel subnet
+  // (WG_SUBNET_CIDR) — one line covers both peers regardless of which side is
+  // currently primary/standby. The supabase/postgres image overrides hba_file
+  // to /etc/postgresql/pg_hba.conf (NOT the default
   // /var/lib/postgresql/data/pg_hba.conf). RCA: compose-ha 2026-05-01 fanout12
   // — appended to the wrong file, postgres never saw the rule, every
   // replication connection was rejected with "no pg_hba.conf entry" while
   // pg_hba_file_rules reflected the actual /etc loaded contents (no replicator
   // entry). Resolve the path at runtime via SHOW hba_file so we don't break
   // again if the image moves the file.
+  const hbaLine = buildReplicationHbaLine(WG_SUBNET_CIDR);
   await sshRun(
     primaryIp,
     sshKeyPath,
     `cd ${remoteDir} && HBA_FILE=$(docker compose exec -T db psql -U supabase_admin -d postgres -tAc 'SHOW hba_file;' | tr -d '[:space:]') && ` +
-      `docker compose exec -T db sh -c "echo 'host replication replicator ${standbyIp}/32 md5' >> $HBA_FILE && ` +
+      `docker compose exec -T db sh -c "echo '${hbaLine}' >> $HBA_FILE && ` +
       `psql -U supabase_admin -d postgres -tAc 'SELECT pg_reload_conf();'"`,
     { timeout: 15_000 },
   );
@@ -296,6 +315,15 @@ END $$;
     { timeout: 30_000 },
   );
 
+  // TODO(wireguard-transport): tunnel bring-up — Task 6.
+  //
+  // This used to enable server-side TLS (verify-ca) here: copy the per-deploy
+  // CA/cert/key from the bind-mounted Secret into PGDATA and ALTER SYSTEM SET
+  // ssl = 'on' + ssl_cert_file/ssl_key_file/ssl_ca_file. That's public-IP-era
+  // plumbing for the TLS transport being replaced — WireGuard encrypts the
+  // wire instead, so there's no cert material to install here. Task 6 wires
+  // the actual wg0 tunnel bring-up for compose-ha.
+
   // Restart PostgreSQL to apply WAL changes
   await sshRun(primaryIp, sshKeyPath, `cd ${remoteDir} && docker compose restart db`, {
     timeout: 60_000,
@@ -371,64 +399,34 @@ END $$;`;
     timeout: 60_000,
   });
 
-  // Run pg_basebackup from primary into standby's data directory.
-  //
-  // -Fp: plain format (writes directly to PGDATA)
-  // -Xs: stream WAL during backup
-  // -R:  auto-create standby.signal and write primary_conninfo to postgresql.auto.conf
-  // -C --slot: use the physical replication slot created by
-  //            configurePrimaryReplication. This makes the primary hold
-  //            WAL for this standby specifically — so streaming stays
-  //            caught up indefinitely instead of silently falling behind
-  //            when wal_keep_size runs out.
-  //
-  // After pg_basebackup, append `primary_slot_name` to postgresql.auto.conf
-  // so the continuous-streaming connection (after initial backup) also uses
-  // the slot. `-R` writes primary_conninfo but not primary_slot_name.
+  // Run pg_basebackup from primary into standby's data directory, using the
+  // SHARED staged-basebackup script — the SAME hardened script the k8s path +
+  // failover/restore re-seed use, so the deploy-time seed and the
+  // recovery-time re-seed can't drift apart. It stages into a tmp dir +
+  // atomically swaps in only after verifying PG_VERSION, so a failed/partial
+  // basebackup never leaves PGDATA half-formed (the `initdb: directory exists
+  // but is not empty` crash-loop). probeFirst aborts cleanly (exit 0, PGDATA
+  // untouched) when the primary's REPL_PORT is unreachable. Plaintext (no
+  // PGSSLMODE/PGSSLROOTCERT) — see TODO(wireguard-transport) above; the
+  // connection is unencrypted until Task 6 lands the tunnel.
   //
-  // `set -e -o pipefail` is load-bearing: without it, a failed
-  // pg_basebackup leaves the data dir with only `postgresql.auto.conf`
-  // (because the trailing `>>` succeeds creating it from scratch), no
-  // `PG_VERSION`. The next `docker compose start db` then loops forever
-  // on `initdb: directory exists but is not empty` (RCA from compose-ha
-  // failover failure 2026-04-30). With `set -e`, basebackup failure
-  // exits the script non-zero, sshRun throws, the caller can react.
-  const basebackupScript = `
-set -e -o pipefail
-rm -rf /var/lib/postgresql/data/*
-PGPASSWORD='${replPassword}' pg_basebackup -h ${primaryIp} -p 5433 -U replicator -D /var/lib/postgresql/data -Fp -Xs -P -R -S vibecarbon_standby_slot -v
-# pg_basebackup -R is supposed to (a) create standby.signal and (b)
-# write primary_conninfo to postgresql.auto.conf. RCA from compose-ha
-# fanout9 2026-05-01: in supabase/postgres:15.8.1.085, neither happened
-# — postgresql.auto.conf came back at 88 bytes (just the ALTER SYSTEM
-# header) and standby.signal was missing. Postgres on next start saw
-# no standby.signal, came up as PRIMARY, started archiving WAL via
-# wal-g, and pg_is_in_recovery() returned 'f' for the entire 60s probe
-# window. Write everything ourselves — don't trust -R.
-touch /var/lib/postgresql/data/standby.signal
-# Replace postgresql.auto.conf wholesale (don't append — the upstream
-# image's first start can ALTER SYSTEM RESET ALL and wipe whatever -R
-# claimed to write). Both primary_conninfo and primary_slot_name are
-# load-bearing: missing primary_conninfo means recovery has nowhere
-# to read WAL from, missing primary_slot_name means primary won't
-# hold WAL for this standby and streaming silently falls behind.
-cat > /var/lib/postgresql/data/postgresql.auto.conf <<EOF
-# Do not edit this file manually!
-# It will be overwritten by the ALTER SYSTEM command.
-primary_conninfo = 'host=${primaryIp} port=5433 user=replicator password=${replPassword} application_name=standby'
-primary_slot_name = 'vibecarbon_standby_slot'
-EOF
-chown -R postgres:postgres /var/lib/postgresql/data
-# Final sanity checks. Each one is the postcondition for a different
-# downstream failure mode: PG_VERSION → docker compose start initdb
-# loop; standby.signal → postgres comes up as primary; primary_conninfo
-# in auto.conf → recovery has no source.
-test -f /var/lib/postgresql/data/PG_VERSION
-test -f /var/lib/postgresql/data/standby.signal
-grep -q '^primary_conninfo' /var/lib/postgresql/data/postgresql.auto.conf
-grep -q '^primary_slot_name' /var/lib/postgresql/data/postgresql.auto.conf
+  // After the swap we pin primary_conninfo explicitly (last entry in
+  // postgresql.auto.conf wins): pg_basebackup -R records a conninfo from the
+  // backup connection, but we write our own anyway so it's byte-identical to
+  // what the shared builder produces. When probeFirst skips, the staged
+  // script exits 0 before reaching this tail, so the independent standby is
+  // left untouched.
+  const basebackupScript = `${buildStagedBasebackupScript({
+    replPassword,
+    primaryHost: primaryIp,
+    probeFirst: true,
+    label: 'compose-ha-repl',
+  })}
+cat >> /var/lib/postgresql/data/postgresql.auto.conf <<'REPL_CONNINFO_EOF'
+primary_conninfo = '${buildPrimaryConninfo({ primaryHost: primaryIp, replPassword })}'
+REPL_CONNINFO_EOF
+chown postgres:postgres /var/lib/postgresql/data/postgresql.auto.conf
 `;
-  const basebackupB64 = Buffer.from(basebackupScript).toString('base64');
   try {
     await sshRun(
       standbyIp,
@@ -442,14 +440,22 @@ grep -q '^primary_slot_name' /var/lib/postgresql/data/postgresql.auto.conf
       // missing standby.signal / wiped postgresql.auto.conf goes
       // undetected until pg_is_in_recovery() returns 'f' an entire
       // basebackup later. RCA: compose-ha 2026-05-01 fanout10 failover.
-      `cd ${remoteDir} && echo ${basebackupB64} | base64 -d | docker compose run --rm -T --entrypoint bash db 2>&1`,
+      //
+      // The script (which embeds the replication password in PGPASSWORD +
+      // primary_conninfo) is piped to bash via STDIN (`input:` below), so it
+      // never appears in argv / the ssh command string. `-T` forwards stdin.
+      //
+      // Include the replication overlay for consistency with the other
+      // compose invocations against this server (docker-compose.replication.yml
+      // carries the REPL_PORT mapping used elsewhere in the HA flow).
+      `cd ${remoteDir} && docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.replication.yml run --rm -T --entrypoint bash db 2>&1`,
       // silent:true makes sshRun THROW on non-zero exit instead of returning
       // false. Without it, a failed basebackup (e.g. pg_hba.conf rejecting
       // the connection) is silently swallowed: try-catch never enters, code
       // proceeds to docker compose start db, probe loop times out, and the
       // operator sees "did not enter recovery mode within 60s" with no clue
       // that pg_basebackup never even ran. RCA: fanout12 2026-05-01.
-      { timeout: 300_000, silent: true },
+      { timeout: 300_000, silent: true, input: basebackupScript },
     );
   } catch (err) {
     // Surface the actual basebackup output. sshRun on failure attaches
@@ -686,6 +692,7 @@ export async function deployComposeHA(options) {
     backupConfig = null,
     bundlePath = null,
     operatorCidrs = [],
+    allowDegraded = false,
     onProgress = () => {},
   } = options;
 
@@ -834,6 +841,9 @@ export async function deployComposeHA(options) {
         bucket: s3Config.bucket,
         region: s3Config.region,
         endpoint: s3Config.endpoint,
+        // Dedicated Pulumi-state bucket set by the orchestrator s3-setup step.
+        // Persisted so destroy / scale / failover resolve the same backend.
+        stateBucket: s3Config.stateBucket,
       },
     }),
     ...(backupS3Config && { backupS3: backupS3Config }),
@@ -860,6 +870,15 @@ export async function deployComposeHA(options) {
     throw new Error('Timed out waiting for SSH on one or both servers');
   }
 
+  // Seed the per-env known_hosts pin with BOTH servers' real host keys now that
+  // they answer SSH. Every subsequent ssh/scp (setupServerFiles, replication,
+  // failover) strict-checks against this file (accept-new), so pre-seeding via
+  // ssh-keyscan means we pin the real keys instead of blindly trusting whatever
+  // the first connection presents.
+  const haKnownHostsPath = knownHostsPathForKey(sshKeyPath);
+  await seedKnownHosts(haKnownHostsPath, primaryIp);
+  await seedKnownHosts(haKnownHostsPath, standbyIp);
+
   // Ensure Docker is running, configure firewall, enable auto-updates
   onProgress('Configuring servers...');
   await perfAsync('deploy.ha.compose.cloudInitReady.both', () =>
@@ -993,6 +1012,21 @@ export async function deployComposeHA(options) {
   }
   await perfAsync('deploy.ha.compose.setupFan', () => Promise.all(setupTasks));
 
+  // wal-g WRITE-GUARD (finding #3): write WALG_ROLE into each node's .env. It is
+  // NOT a path segment — the S3 prefix is a single canonical
+  // `…/walg` on both nodes so restore/reseed on the standby can READ the
+  // primary's base backups. WALG_ROLE=standby instead makes wal-archive.sh +
+  // compose-backup.sh no-op on the standby, so a standby never WRITES into the
+  // canonical prefix (prevents WAL-timeline collision / split-brain corruption).
+  // Runs after setupServerFiles copied the base .env; before compose-up so the
+  // db container reads the role. mergeRemoteDotenv is idempotent across redeploys.
+  const haSshOpts = pinnedSshOptsString(sshKeyPath);
+  const haRemoteDir = `/opt/${projectName}`;
+  await Promise.all([
+    mergeRemoteDotenv(primaryIp, haSshOpts, haRemoteDir, { WALG_ROLE: 'primary' }),
+    mergeRemoteDotenv(standbyIp, haSshOpts, haRemoteDir, { WALG_ROLE: 'standby' }),
+  ]);
+
   // Pull app + base images on both servers in parallel. No remote build —
   // image comes from ghcr.io (CI-built, multi-arch manifest; each node
   // pulls its arch). Each server is independent; no cross-cluster dep.
@@ -1153,14 +1187,16 @@ export async function deployComposeHA(options) {
   }
 
   // Expose raw PostgreSQL on port 5433 for replication (supavisor owns 5432).
-  // Add a docker-compose.replication.yml overlay that maps db:5432 → host:5433.
-  // Fan out across primary + standby — the overlay write + db-only `compose up`
+  // The docker-compose.replication.yml overlay maps db:5432 → host:5433 on
+  // both nodes so pg_basebackup + streaming can reach the peer. Fan out
+  // across primary + standby — the overlay write + db-only `compose up`
   // touch disjoint VPS hosts and need to land on both before
-  // configurePrimaryReplication grants pub access. Previously serial via a
-  // for-loop using sync sshRun (~4 ssh calls × 2-3s each = ~8-12s blocking the
-  // event loop); Promise.all halves wall-clock and unblocks any concurrent
-  // perfAsync timers from being charged for the dead time.
-  const replOverlay = `version: "3.8"\nservices:\n  db:\n    ports:\n      - "5433:5432"\n`;
+  // configurePrimaryReplication grants pub access.
+  //
+  // TODO(wireguard-transport): tunnel bring-up — Task 6. This overlay (direct
+  // port-mapping, no TLS cert bind-mount) is the interim pre-WireGuard
+  // transport; Task 6 replaces it with the wg0 tunnel.
+  const replOverlay = 'version: "3.8"\nservices:\n  db:\n    ports:\n      - "5433:5432"\n';
   await Promise.all(
     [primaryIp, standbyIp].map(async (ip) => {
       await sshRunAsync(
@@ -1185,7 +1221,7 @@ export async function deployComposeHA(options) {
   // dominated by pg_basebackup for the standby seed.
   await perfAsync('deploy.ha.replication.setup', async () => {
     onProgress('Configuring PostgreSQL replication on primary...');
-    await configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, projectName);
+    await configurePrimaryReplication(primaryIp, sshKeyPath, projectName);
 
     onProgress('Configuring standby as hot replica...');
     let replicationOk = false;
@@ -1197,13 +1233,15 @@ export async function deployComposeHA(options) {
         projectName,
       );
     } catch (err) {
-      // Soft-fail in deploy: replication has been chronically broken
-      // (see project_replication_broken.md) and hard-failing every
-      // compose-ha deploy here would block iteration on the rest of
-      // the scenario. Surface the captured pg_basebackup output so
-      // we can root-cause it, then continue with a clear warning.
-      // Failover-path still hard-fails (see line ~1185) so we don't
-      // cascade into a data-wipe + CrashLoopBackOff there.
+      // DECISION (open, for the maintainer): replication failure is downgraded
+      // to a WARNING and the deploy still finalizes as `deployed`. Replication
+      // has been chronically fragile (memory: project_replication_broken) and
+      // hard-failing every compose-ha deploy would block iteration on the rest
+      // of the scenario. We keep the deploy green but report the REAL streaming
+      // state below (never a hardcoded "streaming"). Promoting replication
+      // failure to a hard deploy gate is a product decision NOT made here.
+      // Failover-path still hard-fails (reseed abort) so a data-wipe +
+      // CrashLoopBackOff can't cascade there.
       p.log.warn(`Replication setup error (continuing with warning): ${err.message}`);
     }
     if (!replicationOk) {
@@ -1217,30 +1255,53 @@ export async function deployComposeHA(options) {
   // hardcoded 1s/3s shell-out: fast happy path (probe at 250ms, 750ms,
   // 1.75s, …) without giving up too early under load. Total budget ~25s.
   onProgress('Verifying replication...');
-  const REPL_DELAYS_MS = [250, 500, 1000, 2000, 3000, 3000, 3000, 3000, 3000, 3000];
-  let replActive = false;
-  for (let i = 0; i < REPL_DELAYS_MS.length; i++) {
-    try {
-      const count = await sshRun(
+  // Shared verify poller (finding #2) — the k8s path uses the same primitive.
+  // readState reports pg_stat_replication.state on the primary ('streaming' when
+  // the standby is caught up, else another state / '' when no replica connected);
+  // lastState feeds the finding-#1 hard-gate error message below.
+  const { streaming: replActive, lastState: replLastState } = await verifyStreaming({
+    readState: async () => {
+      const state = await sshRun(
         primaryIp,
         sshKeyPath,
-        `cd /opt/${projectName} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "SELECT count(*) FROM pg_stat_replication WHERE state = 'streaming'"`,
+        `cd /opt/${projectName} && docker compose exec -T db psql -U supabase_admin -d postgres -tAc "SELECT state FROM pg_stat_replication ORDER BY (state = 'streaming') DESC LIMIT 1"`,
         { timeout: 10_000 },
       );
-      if (Number.parseInt(count?.trim(), 10) > 0) {
-        replActive = true;
-        break;
-      }
-    } catch {
-      // Retry
-    }
-    if (i === REPL_DELAYS_MS.length - 1) break;
-    await new Promise((r) => setTimeout(r, REPL_DELAYS_MS[i]));
-  }
+      return typeof state === 'string' ? state.trim() : '';
+    },
+    attempts: 10,
+    delaysMs: [250, 500, 1000, 2000, 3000],
+  });
+
+  // DECISION (finding #1, hard-gate-by-default): a compose-HA deploy whose
+  // standby is NOT verifiably streaming now FAILS by default rather than
+  // finalizing as `deployed` with a warning. Warm-standby / degraded DR is only
+  // accepted when the operator explicitly opts in with `-allow-degraded`
+  // (options.allowDegraded). The failover path already hard-fails a bad re-seed;
+  // this brings the deploy path to the same posture so a green `deployed` always
+  // means a real replica exists. Reuses the honest pg_stat_replication verify
+  // above (never a hardcoded "streaming").
+  // Shared hard-gate (finding #1): throws by default when not streaming; with
+  // -allow-degraded it returns { degraded: true } so we finalize a warm standby.
+  const { degraded } = assertReplicationStreamingOrDegraded({
+    streaming: replActive,
+    lastState: replLastState,
+    allowDegraded,
+    fixHint:
+      'Verify the replication port (5433) is open between the two regions and the standby ' +
+      'completed pg_basebackup.',
+  });
   if (replActive) {
     p.log.success('Streaming replication active');
   } else {
-    p.log.warn('Streaming replication not detected — standby may fall behind');
+    p.log.warn(
+      'DEGRADED HA (-allow-degraded): streaming replication NOT detected ' +
+        `(last pg_stat_replication.state=${
+          replLastState ? JSON.stringify(replLastState) : 'no replica connected'
+        }). The standby is a warm/cold spare and may be missing recent writes — ` +
+        'disaster recovery is NOT guaranteed. Resync with `vibecarbon deploy` (no ' +
+        '-allow-degraded) once the replication issue is resolved.',
+    );
   }
 
   // Install the scheduled wal-g backup cron on BOTH nodes. Invariant 2: the
@@ -1285,6 +1346,11 @@ export async function deployComposeHA(options) {
       [environment]: {
         ...pendingEnvConfig,
         status: 'deployed',
+        // Record DR posture so `status` and downstream commands can see the env
+        // is running with a warm-standby (only reachable via -allow-degraded;
+        // the default gate throws before this point when not streaming).
+        replication: degraded ? 'degraded' : 'streaming',
+        degraded,
         deployedAt: new Date().toISOString(),
         lastAttempt: undefined,
       },
@@ -1300,7 +1366,17 @@ export async function deployComposeHA(options) {
   p.log.message(`  ${c.dim('Primary')}        ${c.info(primaryIp)} (${region})`);
   p.log.message(`  ${c.dim('Standby')}        ${c.info(standbyIp)} (${standbyRegion})`);
   p.log.message(`  ${c.dim('Deploy method')}  Docker Compose HA`);
-  p.log.message(`  ${c.dim('Replication')}    PostgreSQL streaming`);
+  // Report the REAL replication state (verified via pg_stat_replication above),
+  // not a hardcoded "streaming". A standby that isn't streaming means DR is not
+  // guaranteed — say so loudly rather than implying a healthy replica.
+  if (replActive) {
+    p.log.message(`  ${c.dim('Replication')}    ${c.success('PostgreSQL streaming')}`);
+  } else {
+    // Only reachable via -allow-degraded (the default gate threw otherwise).
+    p.log.message(
+      `  ${c.dim('Replication')}    ${c.warning('DEGRADED (warm standby) — NOT streaming; DR is not guaranteed')}`,
+    );
+  }
   p.log.message('');
 
   return {
@@ -1309,6 +1385,8 @@ export async function deployComposeHA(options) {
     standbyIp,
     domain,
     deployMode: 'compose-ha',
+    degraded,
+    replication: degraded ? 'degraded' : 'streaming',
   };
 }
 

package/src/lib/deploy/compose/index.js

@@ -17,9 +17,10 @@ import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import * as p from '@clack/prompts';
 import { runCommandAsync } from '../../command.js';
+import { knownHostsPathForKey } from '../../host-keys.js';
 import { perfAsync, perfTimer } from '../../perf.js';
 import { parseDotenv, shEscape } from '../../shell.js';
-import { NO_PIN_SSH_OPTS, sshRunScript } from '../../ssh.js';
+import { sshRunScript } from '../../ssh.js';
 import { postAdminUser, waitForGotrueHealth } from '../admin-user.js';
 
 const __filename = fileURLToPath(import.meta.url);
@@ -41,12 +42,52 @@ export function loadCloudInitScript() {
   return readFileSync(CLOUD_INIT_PATH, 'utf-8');
 }
 
-// Single-sourced from lib/ssh.js (sshHostKeyOpts) so the string form used by
-// the runners below can't drift from the argv form — notably keeping the
-// ServerAlive keepalives that fail a banner-exchange hang fast instead of
-// stalling for ~600s. Re-exported as the canonical compose deploy SSH opts.
-// shell-safety-ignore: NO_PIN_SSH_OPTS is derived from lib/ssh.js sshHostKeyOpts(), which bakes in BatchMode=yes (validated there)
-const SSH_OPTS = NO_PIN_SSH_OPTS;
+/**
+ * Build the compose deploy path's SSH `-o` option tokens, pinning host keys
+ * against the per-env known_hosts file DERIVED from the key path
+ * (knownHostsPathForKey) — so this path no longer uses UserKnownHostsFile=
+ * /dev/null + StrictHostKeyChecking=no. accept-new bootstraps an empty pin and
+ * TOFU's a fresh/recycled Hetzner IP, but rejects a changed key for an
+ * already-pinned host (MITM on an established env fails). GlobalKnownHostsFile=
+ * /dev/null ignores the system file; the operator's ~/.ssh/known_hosts is never
+ * touched. Keeps the ServerAlive keepalives that fail a banner-exchange hang
+ * fast (~60s) instead of stalling ~600s.
+ *
+ * @param {string} sshKeyPath
+ * @returns {string[]} argv `-o` tokens
+ */
+function composeSshOpts(sshKeyPath) {
+  return [
+    '-o',
+    `UserKnownHostsFile=${knownHostsPathForKey(sshKeyPath)}`,
+    '-o',
+    'GlobalKnownHostsFile=/dev/null',
+    '-o',
+    'StrictHostKeyChecking=accept-new',
+    '-o',
+    'BatchMode=yes',
+    '-o',
+    'ConnectTimeout=10',
+    '-o',
+    'ServerAliveInterval=15',
+    '-o',
+    'ServerAliveCountMax=4',
+  ];
+}
+
+/**
+ * The same opts as a shell-safe space-joined string for the string-command
+ * runners / raw `ssh ...` interpolations below. The known_hosts path is
+ * shEscaped so a project dir with spaces can't split the argument.
+ *
+ * @param {string} sshKeyPath
+ * @returns {string}
+ */
+function composeSshOptsString(sshKeyPath) {
+  return composeSshOpts(sshKeyPath)
+    .map((tok) => (tok.startsWith('UserKnownHostsFile=') ? shEscape(tok) : tok))
+    .join(' ');
+}
 
 /**
  * Run a command on a remote server via SSH.
@@ -58,7 +99,8 @@ export async function sshRun(ip, sshKeyPath, command, options = {}) {
   const { timeout = 120_000, ...rest } = options;
   try {
     return await runCommandAsync(
-      ['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command],
+      // shell-safety-ignore: composeSshOpts() bakes in BatchMode=yes (validated in its definition)
+      ['ssh', ...composeSshOpts(sshKeyPath), '-i', sshKeyPath, `root@${ip}`, command],
       { silent: true, timeout, ...rest },
     );
   } catch {
@@ -76,7 +118,7 @@ export async function sshRun(ip, sshKeyPath, command, options = {}) {
  */
 export async function sshRunAsync(ip, sshKeyPath, command, options = {}) {
   const { timeout = 120_000, retries = 3, ignoreError, ...rest } = options;
-  const sshArgs = ['ssh', ...SSH_OPTS.split(' '), '-i', sshKeyPath, `root@${ip}`, command];
+  const sshArgs = ['ssh', ...composeSshOpts(sshKeyPath), '-i', sshKeyPath, `root@${ip}`, command];
   // Strip ignoreError from inner call so connection errors always throw and
   // the retry loop can catch them. We apply ignoreError ourselves after retries.
   const runOpts = { silent: true, timeout, ...rest };
@@ -140,8 +182,9 @@ export async function waitForSSH(host, sshKeyPath, maxAttempts = 30) {
       // this returned true on the very first probe regardless of SSH state.)
       await runCommandAsync(
         [
+          // shell-safety-ignore: composeSshOpts() bakes in BatchMode=yes (validated in its definition)
           'ssh',
-          ...SSH_OPTS.split(' '),
+          ...composeSshOpts(sshKeyPath),
           '-i',
           sshKeyPath,
           '-o',
@@ -367,13 +410,17 @@ export async function setupServerFiles(ip, sshKeyPath, projectName, options = {}
           [
             'bash',
             '-c',
-            'cat "$1" | ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -i "$2" "root@$3" "mkdir -p $4/backups && tar --no-xattrs --no-same-owner -xzf - -C $4 && cp $4/$5.service /etc/systemd/system/ && systemctl daemon-reload"',
+            // Host-key pinned: accept-new against the per-env known_hosts
+            // ($6, a positional so the path is safely quoted), never
+            // /dev/null + StrictHostKeyChecking=no.
+            'cat "$1" | ssh -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$6" -o GlobalKnownHostsFile=/dev/null -o BatchMode=yes -i "$2" "root@$3" "mkdir -p $4/backups && tar --no-xattrs --no-same-owner -xzf - -C $4 && cp $4/$5.service /etc/systemd/system/ && systemctl daemon-reload"',
             '--',
             tarPath,
             sshKeyPath,
             ip,
             remoteDir,
             projectName,
+            knownHostsPathForKey(sshKeyPath),
           ],
           { silent: true, timeout: 300_000 },
         ),
@@ -459,8 +506,9 @@ export async function transferImageBetweenServers(sourceIp, destIp, sshKeyPath,
   // images compress 3-5x; without gzip a ~600MB save streams uncompressed
   // across two regions on top of the operator's residential upload, which
   // historically tripped the 600s timeout under load.
-  const saveCmd = `ssh ${SSH_OPTS} -i ${shEscape(sshKeyPath)} root@${sourceIp} docker save ${shEscape(imageRef)}`;
-  const loadCmd = `ssh ${SSH_OPTS} -i ${shEscape(sshKeyPath)} root@${destIp} 'gunzip | docker load'`;
+  const sshOpts = composeSshOptsString(sshKeyPath);
+  const saveCmd = `ssh ${sshOpts} -i ${shEscape(sshKeyPath)} root@${sourceIp} docker save ${shEscape(imageRef)}`;
+  const loadCmd = `ssh ${sshOpts} -i ${shEscape(sshKeyPath)} root@${destIp} 'gunzip | docker load'`;
   await runCommandAsync(['bash', '-c', `set -o pipefail && ${saveCmd} | gzip -1 | ${loadCmd}`], {
     silent: true,
     timeout: 600_000,
@@ -534,7 +582,7 @@ export async function startComposeStack(ip, sshKeyPath, projectName, _options =
     // after the wrapper timed out) but the timer ate the run. 30 min covers
     // a slow-pull cold start on any of the three S3 regions plus margin
     // without masking a true regression (a real hang would still surface
-    // via the 60s ServerAlive keepalives in lib/ssh.js + compose SSH_OPTS).
+    // via the 60s ServerAlive keepalives in lib/ssh.js + composeSshOpts).
     await perfAsync('deploy.compose.up', () =>
       sshRunAsync(ip, sshKeyPath, `/bin/bash ${remoteDir}/reconcile.sh`, { timeout: 1_800_000 }),
     );
@@ -1193,12 +1241,7 @@ export async function createAdminUser(serverIp, sshKeyPath, projectName) {
   const pf = spawn(
     'ssh',
     [
-      '-o',
-      'StrictHostKeyChecking=no',
-      '-o',
-      'UserKnownHostsFile=/dev/null',
-      '-o',
-      'BatchMode=yes',
+      ...composeSshOpts(sshKeyPath),
       '-i',
       sshKeyPath,
       '-N',