vibecarbon 0.8.0 → 0.10.0

package/src/lib/deploy/k8s/ha/index.js

@@ -2,7 +2,7 @@
  * High Availability Kubernetes Deployment Module (Diamond & Fullerene)
  *
  * Handles multi-region Kubernetes deployment with HA capabilities.
- * Licensed under AGPL v3.
+ * Licensed under FSL-1.1-MIT.
  *
  * Features:
  * - Multi-region Kubernetes clusters
@@ -29,12 +29,28 @@ import { c } from '../../../colors.js';
 import { runCommandAsync } from '../../../command.js';
 import { fetchWithRetry } from '../../../fetch-retry.js';
 import { setupHA as setupHetznerDnsHA } from '../../../hetzner-dns.js';
+import { knownHostsPathForKey, seedKnownHosts } from '../../../host-keys.js';
 import { initBackend } from '../../../iac/index.js';
 import { perfAsync } from '../../../perf.js';
 import { HetznerProvider } from '../../../providers/hetzner.js';
 import { createPrefixedTracker } from '../../../tracker.js';
+import {
+  assertReplicationStreamingOrDegraded,
+  buildPrimaryConninfo,
+  buildReplicationFirewallRules,
+  buildReplicationHbaLine,
+  buildStagedBasebackupScript,
+  REPL_PORT,
+  verifyStreaming,
+} from '../../replication.js';
 import { StateTracker } from '../../state.js';
 import { readReplPassword } from '../../utils.js';
+import {
+  exchangeAndBringUpTunnel,
+  REPL_GATEWAY_PORT,
+  WG_PRIMARY_IP,
+  WG_SUBNET_CIDR,
+} from '../../wireguard.js';
 import { deployK3s } from '../k3s.js';
 
 /**
@@ -104,6 +120,99 @@ const __dirname = dirname(__filename);
 // Path to HA scripts in the template
 const HA_SCRIPTS_PATH = join(__dirname, '../../../../../carbon/ha');
 
+// The repl-gateway manifest (Pod + additive egress NetworkPolicy). HA-only, so
+// it lives outside the base kustomization; setupReplication renders its
+// placeholders per-cluster and applies it via `kubectl apply -f -`.
+const REPL_GATEWAY_MANIFEST = join(
+  __dirname,
+  '../../../../../carbon/k8s/base/repl-gateway/repl-gateway.yaml',
+);
+
+/**
+ * The two socat address specs for the repl-gateway relay, keyed by cluster
+ * role. Pure — unit-tested; rendered into the repl-gateway Pod manifest.
+ *
+ *   - primary: bind the tunnel IP (10.99.0.1:15433) and forward to the local
+ *     postgres hostPort (127.0.0.1:5433). Reachable only via wg0.
+ *   - standby: bind the node's private IP (<priv>:15433) and forward INTO the
+ *     tunnel to the primary gateway (10.99.0.1:15433). This is the address the
+ *     standby db pod dials.
+ *
+ * @param {'primary'|'standby'} role
+ * @param {string} nodePrivateIp - this cluster's supabase node private IP
+ * @returns {{relayListen: string, relayTarget: string}}
+ */
+export function buildReplGatewaySocatArgs(role, nodePrivateIp) {
+  if (role === 'primary') {
+    return {
+      relayListen: `TCP-LISTEN:${REPL_GATEWAY_PORT},bind=${WG_PRIMARY_IP},fork,reuseaddr`,
+      relayTarget: `TCP:127.0.0.1:${REPL_PORT}`,
+    };
+  }
+  if (role === 'standby') {
+    return {
+      relayListen: `TCP-LISTEN:${REPL_GATEWAY_PORT},bind=${nodePrivateIp},fork,reuseaddr`,
+      relayTarget: `TCP:${WG_PRIMARY_IP}:${REPL_GATEWAY_PORT}`,
+    };
+  }
+  throw new Error(`buildReplGatewaySocatArgs: unknown role '${role}'`);
+}
+
+/**
+ * Render the repl-gateway manifest for one cluster: substitute the socat relay
+ * direction + the local node private IP (egress NetworkPolicy scope). The
+ * result is applied via `kubectl apply -f -` — placeholders are resolved at
+ * apply time, never left for runtime.
+ *
+ * @param {object} o
+ * @param {string} o.template - the raw repl-gateway.yaml
+ * @param {'primary'|'standby'} o.role
+ * @param {string} o.nodePrivateIp - this cluster's supabase node private IP
+ * @returns {string}
+ */
+export function renderReplGatewayManifest({ template, role, nodePrivateIp }) {
+  const { relayListen, relayTarget } = buildReplGatewaySocatArgs(role, nodePrivateIp);
+  return template
+    .replaceAll('__RELAY_LISTEN__', relayListen)
+    .replaceAll('__RELAY_TARGET__', relayTarget)
+    .replaceAll('__SUPABASE_PRIVATE_IP__', nodePrivateIp);
+}
+
+/**
+ * Strategic-merge patch that binds the primary's postgres to REPL_PORT (hostPort)
+ * → containerPort 5432 on the node. The primary repl-gateway (hostNetwork socat)
+ * relays tunnel traffic to `127.0.0.1:REPL_PORT` — this hostPort IS that local
+ * listener. NOTE: the port is NOT opened in the Hetzner firewall (the firewall
+ * now admits only WG udp/51821), so postgres is reachable only via localhost on
+ * the node, never cross-cluster over the public internet. Without this hostPort
+ * the db is reachable only inside the cluster and the gateway's local relay
+ * target would have nothing to connect to. Mirrors the traefik/registry hostPort
+ * pattern. One db pod per node (StatefulSet), so the node's :REPL_PORT is
+ * uncontended.
+ *
+ * The container name in the supabase chart's db StatefulSet is `supabase-db`.
+ * A strategic-merge patch keyed by container `name` + port `containerPort`
+ * updates the existing port entry in place (adds hostPort) rather than
+ * appending a duplicate.
+ * @returns {object}
+ */
+export function buildDbHostPortPatch() {
+  return {
+    spec: {
+      template: {
+        spec: {
+          containers: [
+            {
+              name: 'supabase-db',
+              ports: [{ containerPort: 5432, hostPort: REPL_PORT }],
+            },
+          ],
+        },
+      },
+    },
+  };
+}
+
 /**
  * Deploy with High Availability
  * @param {object} options - Deployment options
@@ -308,84 +417,129 @@ export async function deployK8sHA(options) {
   const standbyResult = standbySettled.value;
   p.log.success('Both clusters deployed');
 
-  // 3. Open port 5432 in the primary's Hetzner Cloud firewall so the standby's supabase node
-  // can reach the primary's postgres directly via hostPort. The base firewall (declared in
-  // src/lib/iac/programs/hetzner-k8s.js) only allows SSH, HTTP, HTTPS, K8s API, and
-  // private-network traffic.
-  if (options.apiToken && standbyResult.supabaseIp) {
-    const primaryClusterName = `${options.projectName}-${options.environment}-primary`;
-    const firewallName = `${primaryClusterName}-firewall`;
-    try {
-      const headers = {
-        Authorization: `Bearer ${options.apiToken}`,
-        'Content-Type': 'application/json',
-      };
-      const fwListRes = await fetchWithRetry(
-        `https://api.hetzner.cloud/v1/firewalls?name=${encodeURIComponent(firewallName)}`,
-        { headers },
-      );
-      const fwListData = await fwListRes.json();
-      const firewall = fwListData.firewalls?.find((fw) => fw.name === firewallName);
-      if (firewall) {
-        const existingRules = firewall.rules || [];
-        const hasReplRule = existingRules.some(
-          (r) => r.port === '5432' && r.source_ips?.includes(`${standbyResult.supabaseIp}/32`),
+  // 3. Open the WireGuard tunnel port (UDP 51821) in BOTH clusters' Hetzner
+  // Cloud firewalls, each admitting its PEER supabase node's PUBLIC IP — the
+  // WG endpoints are the supabase nodes' public IPs, and the firewall's default
+  // udp rule only admits the private range. Symmetric on both clusters: the
+  // standby streams from the primary at deploy time, and the post-failover
+  // reverse re-seed dials the promoted standby (neither failover.js nor
+  // restore.js touches firewalls, so deploy must set up both directions). The
+  // per-cluster firewall (`<project>-<env>-<role>-firewall`) is attached to all
+  // that cluster's nodes incl. the supabase/WG node. buildReplicationFirewallRules
+  // emits the udp/51821 rule (and scrubs stale TCP 5432/5433/30432 rules from the
+  // retired public-IP TLS transport).
+  if (options.apiToken) {
+    const headers = {
+      Authorization: `Bearer ${options.apiToken}`,
+      'Content-Type': 'application/json',
+    };
+    for (const [clusterRole, peerSupabaseIp] of [
+      ['primary', standbyResult.supabaseIp],
+      ['standby', primaryResult.supabaseIp],
+    ]) {
+      if (!peerSupabaseIp) continue;
+      const firewallName = `${options.projectName}-${options.environment}-${clusterRole}-firewall`;
+      try {
+        const fwListRes = await fetchWithRetry(
+          `https://api.hetzner.cloud/v1/firewalls?name=${encodeURIComponent(firewallName)}`,
+          { headers },
         );
-        if (!hasReplRule) {
-          // Remove any stale NodePort 30432 rules from previous deploys
-          const cleanedRules = existingRules.filter((r) => r.port !== '30432');
-          const updatedRules = [
-            ...cleanedRules,
-            {
-              direction: 'in',
-              protocol: 'tcp',
-              port: '5432',
-              source_ips: [`${standbyResult.supabaseIp}/32`],
-              description: 'PostgreSQL replication from standby supabase node',
-            },
-          ];
-          await fetchWithRetry(
-            `https://api.hetzner.cloud/v1/firewalls/${firewall.id}/actions/set_rules`,
-            {
-              method: 'POST',
-              headers,
-              body: JSON.stringify({ rules: updatedRules }),
-            },
-          );
+        const fwListData = await fwListRes.json();
+        const firewall = fwListData.firewalls?.find((fw) => fw.name === firewallName);
+        if (firewall) {
+          const updatedRules = buildReplicationFirewallRules(firewall.rules || [], peerSupabaseIp);
+          if (updatedRules) {
+            await fetchWithRetry(
+              `https://api.hetzner.cloud/v1/firewalls/${firewall.id}/actions/set_rules`,
+              {
+                method: 'POST',
+                headers,
+                body: JSON.stringify({ rules: updatedRules }),
+              },
+            );
+          }
         }
+      } catch {
+        // Non-fatal: manual firewall config may suffice
       }
-    } catch {
-      // Non-fatal: manual firewall config may suffice
     }
   }
 
   // 4. Configure PostgreSQL replication (must run after both clusters finish —
   // replication restarts postgres pods, which would break concurrent tail work).
+  //
+  // DECISION (finding #1, hard-gate-by-default): an HA deploy whose standby is
+  // NOT verifiably streaming now FAILS by default. Warm-standby / degraded DR is
+  // no longer silently accepted — a deploy that reports `deployed` must have a
+  // real replica. The operator can opt back into the old warn-and-continue
+  // behavior with `vibecarbon deploy -allow-degraded` (options.allowDegraded),
+  // which finalizes the env in a DEGRADED (warm-standby) state. k8s-HA
+  // replication has a documented history of fragility (memory:
+  // project_replication_broken), so this gate MAY turn previously green deploys
+  // red — that is the intended safety behavior; the failure message below is
+  // actionable (why + last observed state + the -allow-degraded escape hatch).
+  const allowDegraded = !!options.allowDegraded;
   s.start('Configuring PostgreSQL replication');
   let replicationStatus = 'not configured';
+  let replStreaming = false;
+  let replLastState = '';
+  let replError = null;
   try {
     const replResult = await setupReplication({
       primaryIp: primaryResult.masterIp,
       standbyIp: standbyResult.masterIp,
       primarySupabaseIp: primaryResult.supabaseIp,
       standbySupabaseIp: standbyResult.supabaseIp,
+      primarySupabasePrivateIp: primaryResult.supabasePrivateIp,
+      standbySupabasePrivateIp: standbyResult.supabasePrivateIp,
       sshKeyPath: sharedSshKeyPath,
     });
-    if (replResult.streaming) {
-      s.stop('PostgreSQL replication configured — streaming');
-      replicationStatus = 'streaming';
-    } else {
-      s.stop('PostgreSQL replication configured — verify manually');
-      replicationStatus = 'configured (verify streaming)';
-    }
+    replStreaming = replResult.streaming;
+    replLastState = replResult.lastState || '';
   } catch (error) {
-    const msg = error.message || String(error);
+    replError = error;
+  }
+
+  // If setupReplication threw, fold its message into the last-state hint so the
+  // shared gate surfaces the real cause (e.g. a cross-cluster connectivity error)
+  // rather than a generic "no replica connected".
+  let lastStateHint = replLastState;
+  if (replError && !lastStateHint) {
+    const msg = replError.message || String(replError);
     const lines = msg.split('\n').filter((l) => l.trim() && !l.includes('Warning: Permanently'));
     const lastLine = lines[lines.length - 1]?.trim() || msg;
-    const shortError = lastLine.length > 120 ? `${lastLine.slice(0, 120)}…` : lastLine;
-    s.stop(`Replication setup failed: ${shortError}`);
-    replicationStatus = `failed: ${shortError}`;
-    p.log.warn('You may need to configure replication manually');
+    lastStateHint = lastLine.length > 160 ? `${lastLine.slice(0, 160)}…` : lastLine;
+  }
+
+  // Shared hard-gate (finding #1): throws by default when not streaming; with
+  // -allow-degraded it returns { degraded: true }. k8s-HA replication has a
+  // documented history of fragility (memory: project_replication_broken), so
+  // this gate MAY turn previously green deploys red — that is the intended
+  // safety behavior.
+  try {
+    assertReplicationStreamingOrDegraded({
+      streaming: replStreaming,
+      lastState: lastStateHint,
+      allowDegraded,
+      fixHint: 'Fix the replication issue (see the k8s-ha replication notes).',
+    });
+  } catch (gateErr) {
+    s.stop('Replication verification failed — aborting HA deploy');
+    throw gateErr;
+  }
+
+  if (replStreaming) {
+    s.stop('PostgreSQL replication configured — streaming');
+    replicationStatus = 'streaming';
+  } else {
+    s.stop('Replication DEGRADED — continuing (-allow-degraded)');
+    replicationStatus = 'DEGRADED (warm standby — NOT streaming)';
+    p.log.warn(
+      'DEGRADED HA: the standby is not streaming from the primary. Disaster recovery ' +
+        'is NOT guaranteed — the standby is a warm/cold spare that may be missing recent ' +
+        'writes. You proceeded with -allow-degraded; resync with `vibecarbon deploy` (no ' +
+        '-allow-degraded) once the replication issue is resolved.',
+    );
   }
 
   // 5. Configure Cloudflare DNS for failover
@@ -474,6 +628,11 @@ export async function deployK8sHA(options) {
     primary: primaryResult,
     standby: standbyResult,
     domain: options.domain,
+    // Surface DR posture so the orchestrator can persist a degraded marker. When
+    // the gate passed, replicationStatus === 'streaming'; with -allow-degraded it
+    // reflects the warm-standby state.
+    replicationStatus,
+    degraded: replicationStatus !== 'streaming',
   };
 }
 
@@ -483,16 +642,24 @@ export async function deployK8sHA(options) {
  * @param {string} options.primaryIp - Primary master node IP (for SSH)
  * @param {string} options.standbyIp - Standby master node IP (for SSH)
  * @param {string} options.sshKeyPath - Path to shared SSH key
- * @param {string} options.primarySupabaseIp - Primary supabase node IP (where postgres runs)
- * @param {string} options.standbySupabaseIp - Standby supabase node IP (where postgres runs)
+ * @param {string} options.primarySupabaseIp - Primary supabase node PUBLIC IP (WG endpoint + SSH)
+ * @param {string} options.standbySupabaseIp - Standby supabase node PUBLIC IP (WG endpoint + SSH)
+ * @param {string} [options.primarySupabasePrivateIp] - Primary supabase node private IP (gateway bind + egress netpol)
+ * @param {string} [options.standbySupabasePrivateIp] - Standby supabase node private IP (gateway bind + egress netpol)
  * @returns {Promise<{streaming: boolean}>} Replication status
  */
 export async function setupReplication(options) {
   const { primaryIp, standbyIp, sshKeyPath } = options;
-  // pg_basebackup connects to the supabase node IP (where postgres pod runs),
-  // not the master node. Fall back to master IP if supabase IP not provided.
+  // The supabase node PUBLIC IPs are BOTH the WireGuard tunnel endpoints and the
+  // SSH targets for the gateway bring-up. Fall back to master IP if not provided.
   const _primarySupabaseIp = options.primarySupabaseIp || primaryIp;
   const _standbySupabaseIp = options.standbySupabaseIp || standbyIp;
+  // The supabase nodes' PRIVATE IPs. Deterministic in the IaC program
+  // (10.0.1.2 on both clusters' 10.0.1.0/24 subnets), but threaded through the
+  // deploy outputs rather than hardcoded at the use site. Used for the
+  // repl-gateway socat bind + the egress NetworkPolicy scope on each cluster.
+  const _primarySupabasePrivateIp = options.primarySupabasePrivateIp || '10.0.1.2';
+  const _standbySupabasePrivateIp = options.standbySupabasePrivateIp || '10.0.1.2';
 
   // replPassword is generated at create time from crypto.randomBytes and is
   // restricted to base64url characters — no shell escaping needed. Read from
@@ -506,15 +673,26 @@ export async function setupReplication(options) {
     );
   }
 
-  // SSH opts as argv — keeps all command builders below in argv form
-  // (no local-shell parsing). Host-key pinning migration tracked as H-1.
+  // SSH opts as argv — keeps all command builders below in argv form (no
+  // local-shell parsing). Host keys are pinned to the per-env known_hosts file
+  // derived from the shared HA deploy key. Seed both cluster IPs first (the
+  // clusters just finished deploying, so they're reachable) so accept-new pins
+  // the real host keys via ssh-keyscan instead of blindly trusting the first
+  // connection. GlobalKnownHostsFile=/dev/null ignores the system-wide file;
+  // accept-new TOFUs a fresh/recycled Hetzner IP but REJECTS a changed key for
+  // an already-pinned host.
+  const khPath = knownHostsPathForKey(sshKeyPath);
+  await seedKnownHosts(khPath, primaryIp);
+  await seedKnownHosts(khPath, standbyIp);
   const sshOpts = [
     '-i',
     sshKeyPath,
     '-o',
-    'StrictHostKeyChecking=no',
+    `UserKnownHostsFile=${khPath}`,
+    '-o',
+    'GlobalKnownHostsFile=/dev/null',
     '-o',
-    'UserKnownHostsFile=/dev/null',
+    'StrictHostKeyChecking=accept-new',
     '-o',
     'BatchMode=yes',
   ];
@@ -531,14 +709,13 @@ export async function setupReplication(options) {
   // separate tokens and the final remote-command string is the only place
   // a shell parses anything. No local template-literal interpolation.
 
-  // Render primary-init.sql with the real REPL_PASSWORD, SCP to remote, then clean up.
-  // The SQL template contains the legacy placeholder password; replace it with the
-  // per-deploy random password. We avoid writing the placeholder literal here by
-  // building it from parts so this source file itself passes the no-legacy-password check.
+  // Render primary-init.sql with the real REPL_PASSWORD, SCP to remote, then
+  // clean up. The SQL template ships the `{{REPL_PASSWORD}}` placeholder (a
+  // token, not a credential); substitute the per-deploy random password. These
+  // two MUST stay in sync — if the template's placeholder is renamed, k8s-HA
+  // would SCP an unrendered SQL file and replication auth would break.
   const sqlTemplate = readFileSync(primaryInitSql, 'utf-8');
-  // Legacy placeholder in the SQL file: 'repl' + '_password'  (split to avoid literal match)
-  const legacyPlaceholder = "'repl" + "_password'";
-  const renderedSql = sqlTemplate.replaceAll(legacyPlaceholder, `'${replPassword}'`);
+  const renderedSql = sqlTemplate.replaceAll('{{REPL_PASSWORD}}', replPassword);
   const tmpDir = mkdtempSync(join(tmpdir(), 'vibecarbon-repl-'));
   const renderedSqlPath = join(tmpDir, 'primary-init.sql');
   try {
@@ -568,22 +745,100 @@ export async function setupReplication(options) {
     });
   }
 
-  // 2. Add pg_hba.conf entry allowing replication from any IP (password-protected).
-  // NodePort traffic is SNAT'd by kube-proxy, so the source IP seen by postgres is
-  // an internal cluster IP, not the standby's public IP. We cannot tighten the CIDR
-  // without moving replication off NodePort (follow-up). The defense in depth is:
-  //   1. Per-deploy random REPL_PASSWORD (generated in create.js via crypto.randomBytes).
-  //   2. Hetzner firewall restricts external source IPs reaching the NodePort.
-  //   3. md5 auth on pg_hba.conf.
+  // 1a. Stand up the WireGuard replication transport.
+  //
+  // Replication runs over a point-to-point WireGuard tunnel (wg0, UDP
+  // WG_PORT=51821) between the two clusters' supabase nodes, relayed by a
+  // hostNetwork `repl-gateway` socat pod on each cluster:
+  //
+  //   standby db pod ─tcp <priv>:15433─► standby repl-gateway ─wg0→10.99.0.1:15433─►
+  //     primary repl-gateway ─tcp 127.0.0.1:5433─► primary postgres (hostPort)
+  //
+  // The old public-IP TLS transport (per-deploy CA/cert Secret + public-IP
+  // NetworkPolicy allowances) is gone: WireGuard encrypts the wire, so there is
+  // no cert Secret, and the ONLY netpol change is the single additive
+  // `allow-db-repl-gateway-egress` shipped INSIDE the gateway manifest (db pod →
+  // local node private IP tcp 15433) — default-deny-all and rp_filter are
+  // untouched.
+  //
+  // Render + apply the gateway manifest to BOTH clusters (each with its own
+  // relay direction + node private IP), then generate keys + bring up wg0.
+  const gatewayTemplate = readFileSync(REPL_GATEWAY_MANIFEST, 'utf-8');
+  await runCommandAsync(['ssh', ...sshOpts, `root@${primaryIp}`, 'kubectl apply -f -'], {
+    silent: true,
+    input: renderReplGatewayManifest({
+      template: gatewayTemplate,
+      role: 'primary',
+      nodePrivateIp: _primarySupabasePrivateIp,
+    }),
+  });
+  await runCommandAsync(['ssh', ...sshOpts, `root@${standbyIp}`, 'kubectl apply -f -'], {
+    silent: true,
+    input: renderReplGatewayManifest({
+      template: gatewayTemplate,
+      role: 'standby',
+      nodePrivateIp: _standbySupabasePrivateIp,
+    }),
+  });
+
+  // Generate a keypair on each supabase node, cross-distribute PUBLIC keys, and
+  // bring up wg0. Private keys are generated on-node and never leave it (see
+  // exchangeAndBringUpTunnel). The WG endpoints/SSH targets are the supabase
+  // nodes' PUBLIC IPs.
+  await exchangeAndBringUpTunnel({
+    primaryIp: _primarySupabaseIp,
+    standbyIp: _standbySupabaseIp,
+    sshKeyPath,
+  });
+
+  // 1b. Bind the primary's postgres to a node hostPort (REPL_PORT → containerPort
+  // 5432) so the primary repl-gateway's socat can relay tunnel traffic to it at
+  // 127.0.0.1:REPL_PORT. Not opened in the firewall — localhost-only on the node
+  // (the gateway is hostNetwork, so it shares the node netns). Without the
+  // hostPort the db is cluster-internal only and the gateway relay target has
+  // nothing to connect to. The strategic-merge patch travels via stdin
+  // (--patch-file /dev/stdin) so the JSON never hits argv / the remote shell's
+  // /proc/cmdline. The patch triggers a pod recreate; wait for the StatefulSet
+  // rollout before opening replication.
   await runCommandAsync(
     [
       'ssh',
       ...sshOpts,
       `root@${primaryIp}`,
-      `kubectl exec -n vibecarbon supabase-supabase-db-0 -- bash -c "echo 'host replication replicator 0.0.0.0/0 md5' >> /var/lib/postgresql/data/pg_hba.conf"`,
+      'kubectl patch statefulset supabase-supabase-db -n vibecarbon --type strategic --patch-file /dev/stdin',
+    ],
+    { silent: true, input: JSON.stringify(buildDbHostPortPatch()) },
+  );
+  await runCommandAsync(
+    [
+      'ssh',
+      ...sshOpts,
+      `root@${primaryIp}`,
+      'kubectl rollout status statefulset/supabase-supabase-db -n vibecarbon --timeout=180s',
     ],
     { silent: true },
   );
+
+  // 2. Add a pg_hba.conf entry allowing replication from the WireGuard tunnel
+  // subnet (WG_SUBNET_CIDR). Plain `host` (buildReplicationHbaLine), not
+  // `hostssl` — WireGuard encrypts the wire, so Postgres doesn't also require
+  // TLS. One line (not per-peer /32s) because the tunnel subnet covers BOTH
+  // peers regardless of which side is currently primary/standby: after a
+  // failover the roles swap, and pg_basebackup copies this line byte-for-byte
+  // into the standby, so it stays valid for the reverse re-seed with no
+  // per-peer bookkeeping. Defense in depth: per-deploy random REPL_PASSWORD +
+  // the WireGuard tunnel + scram-sha-256 auth + this subnet-scoped pg_hba
+  // entry.
+  const hbaLine = buildReplicationHbaLine(WG_SUBNET_CIDR);
+  await runCommandAsync(
+    [
+      'ssh',
+      ...sshOpts,
+      `root@${primaryIp}`,
+      'kubectl exec -i -n vibecarbon supabase-supabase-db-0 -- bash -c "cat >> /var/lib/postgresql/data/pg_hba.conf"',
+    ],
+    { silent: true, input: `${hbaLine}\n` },
+  );
   await runCommandAsync(
     [
       'ssh',
@@ -727,12 +982,14 @@ export async function setupReplication(options) {
   );
 
   // 4. Initialize standby as a streaming replica of the primary.
-  // Modeled after standby-init.sh — runs pg_basebackup directly inside the standby pod
-  // connecting to the primary's supabase node via hostPort 5432 (the Hetzner Cloud firewall
-  // was opened in step 3 of the HA orchestrator). Uses -Fp (plain format) and -R to auto-
-  // create standby.signal and primary_conninfo.
-  const primaryHost = _primarySupabaseIp;
-  const primaryPort = '5432';
+  // Runs pg_basebackup inside the standby pod, dialing the standby's OWN node
+  // private IP on the repl-gateway relay port (REPL_GATEWAY_PORT=15433). That
+  // intra-node hop is admitted by the additive allow-db-repl-gateway-egress
+  // netpol; the standby gateway relays it through the WireGuard tunnel to the
+  // primary gateway, which forwards to the primary postgres hostPort. Plaintext
+  // (no PGSSLMODE/PGSSLROOTCERT) — WireGuard is the encryption layer.
+  const primaryHost = _standbySupabasePrivateIp;
+  const primaryPort = String(REPL_GATEWAY_PORT);
 
   // 4a. Stop standby postgres gracefully
   await runCommandAsync(
@@ -762,51 +1019,17 @@ export async function setupReplication(options) {
   //      PGDATA in a half-formed state.
   //   4. Sanity check PG_VERSION is present after basebackup before swapping in.
   //
-  // -Fp/-Xs/-R/-S: see prior comment — unchanged semantics.
-  // Static staging path (single-instance pod, no PID collision concern).
-  // Literal strings everywhere — avoids the JS-template / bash-escape
-  // confusion that would silently produce `$STAGE` literals if backslashes
-  // are off by one.
-  const pgBasebackupScript = `
-set -e -o pipefail
-
-# Probe primary's hostPort 5432 from inside the standby pod. If unreachable
-# (cross-cluster network gap is the historical case), abort cleanly so the
-# standby keeps its self-bootstrapped postgres rather than entering a
-# permanent CrashLoopBackOff via half-wiped PGDATA.
-if ! pg_isready -h ${primaryHost} -p ${primaryPort} -t 5 -U replicator -d postgres > /dev/null 2>&1; then
-  echo "[ha-replication] primary postgres at ${primaryHost}:${primaryPort} not reachable from standby pod — skipping pg_basebackup. Standby will run as an independent postgres; failover serves the app but data does not sync from primary." >&2
-  exit 0
-fi
-
-# Stage into a tmp dir; atomic swap only after basebackup verifies good.
-rm -rf /var/lib/postgresql/data.staging
-mkdir -p /var/lib/postgresql/data.staging
-chown postgres:postgres /var/lib/postgresql/data.staging
-
-PGPASSWORD='${replPassword}' pg_basebackup \\
-  -h ${primaryHost} -p ${primaryPort} \\
-  -U replicator -D /var/lib/postgresql/data.staging \\
-  -Fp -Xs -R -c fast -S vibecarbon_standby_slot
-
-# Sanity: a successful pg_basebackup always writes PG_VERSION + standby.signal (with -R).
-if [ ! -f /var/lib/postgresql/data.staging/PG_VERSION ]; then
-  echo "[ha-replication] pg_basebackup completed without PG_VERSION — refusing to swap" >&2
-  rm -rf /var/lib/postgresql/data.staging
-  exit 1
-fi
-# -R should already have created standby.signal; touch belt-and-suspenders.
-touch /var/lib/postgresql/data.staging/standby.signal
-echo "primary_slot_name = 'vibecarbon_standby_slot'" >> /var/lib/postgresql/data.staging/postgresql.auto.conf
-chown -R postgres:postgres /var/lib/postgresql/data.staging
-
-# Atomic-ish swap. Old PGDATA is moved aside first; on success it's removed.
-# A failure during swap leaves the original at data.prev for manual recovery.
-rm -rf /var/lib/postgresql/data.prev
-mv /var/lib/postgresql/data /var/lib/postgresql/data.prev
-mv /var/lib/postgresql/data.staging /var/lib/postgresql/data
-rm -rf /var/lib/postgresql/data.prev
-`.trim();
+  // -Fp/-Xs/-R/-S: see prior comment — unchanged semantics. The staged-swap
+  // script + probe-first guard are built by the shared buildStagedBasebackupScript
+  // (finding #2): the SAME hardened script the failover/restore re-seed uses, so
+  // the deploy-time seed and the recovery-time re-seed can't drift apart.
+  const pgBasebackupScript = buildStagedBasebackupScript({
+    replPassword,
+    primaryHost,
+    primaryPort,
+    probeFirst: true,
+    label: 'ha-replication',
+  });
   await runCommandAsync(
     [
       'ssh',
@@ -814,7 +1037,35 @@ rm -rf /var/lib/postgresql/data.prev
       `root@${standbyIp}`,
       'kubectl exec -i -n vibecarbon supabase-supabase-db-0 -- bash',
     ],
-    { silent: false, input: pgBasebackupScript, ignoreError: true },
+    // input: pipes the basebackup script over stdin — requires silent:true
+    // (the stdin pipe is unavailable with inherited stdio). Without this the
+    // call throws before the standby is ever seeded, so it never streams.
+    { silent: true, input: pgBasebackupScript, ignoreError: true },
+  );
+
+  // 4b2. Pin the standby's streaming primary_conninfo (plaintext — sslmode=disable;
+  // WireGuard is the encryption layer). The standby streams through its LOCAL
+  // repl-gateway (own node private IP : REPL_GATEWAY_PORT), same path as the
+  // basebackup above — NOT the primary's public IP. The staged basebackup's
+  // `-R` writes a primary_conninfo from the backup connection, but we append our
+  // explicit conninfo anyway (last entry in postgresql.auto.conf wins) so it's
+  // byte-identical to what the shared builder produces. The password is
+  // delivered over stdin (never argv). No-op-safe when basebackup was skipped:
+  // without standby.signal, postgres ignores primary_conninfo.
+  const standbyConninfo = buildPrimaryConninfo({
+    primaryHost: _standbySupabasePrivateIp,
+    replPassword,
+    port: REPL_GATEWAY_PORT,
+  });
+  await runCommandAsync(
+    [
+      'ssh',
+      ...sshOpts,
+      `root@${standbyIp}`,
+      'kubectl exec -i -n vibecarbon supabase-supabase-db-0 -- bash -c ' +
+        '"cat >> /var/lib/postgresql/data/postgresql.auto.conf"',
+    ],
+    { silent: true, input: `primary_conninfo = '${standbyConninfo}'\n`, ignoreError: true },
   );
 
   // 4c. Start standby postgres in recovery mode (no-op when basebackup was
@@ -853,30 +1104,26 @@ rm -rf /var/lib/postgresql/data.prev
   }
 
   // 5. Verify replication is streaming on the primary. Once standby catches
-  // up, pg_stat_replication.state flips to 'streaming' on the next read.
-  let streaming = false;
-  for (let i = 0; i < 15; i++) {
-    try {
+  // up, pg_stat_replication.state flips to 'streaming' on the next read. Uses
+  // the shared verifyStreaming poller (finding #2) with a k8s readState closure
+  // so compose + k8s share ONE verify implementation; lastState feeds the
+  // finding-#1 hard-gate error message.
+  const { streaming, lastState } = await verifyStreaming({
+    readState: async () => {
       const result = await runCommandAsync(
         [
           'ssh',
           ...sshOpts,
           `root@${primaryIp}`,
-          `kubectl exec -n vibecarbon supabase-supabase-db-0 -- psql -U supabase_admin -tAc 'SELECT state FROM pg_stat_replication LIMIT 1'`,
+          `kubectl exec -n vibecarbon supabase-supabase-db-0 -- psql -U supabase_admin -tAc "SELECT state FROM pg_stat_replication ORDER BY (state = 'streaming') DESC LIMIT 1"`,
         ],
         { silent: true, ignoreError: true },
       );
-      if (result?.trim() === 'streaming') {
-        streaming = true;
-        break;
-      }
-    } catch {
-      // Retry
-    }
-    if (i === 14) break;
-    const interval = i < 3 ? 500 : 2000;
-    await new Promise((r) => setTimeout(r, interval));
-  }
+      return typeof result === 'string' ? result.trim() : '';
+    },
+    attempts: 15,
+    delaysMs: [500, 500, 2000],
+  });
 
-  return { streaming };
+  return { streaming, lastState };
 }