vibecarbon 0.11.0 → 0.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vibecarbon",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Create and manage production-ready Vibecarbon applications",
   "type": "module",
   "bin": "./src/cli.js",

package/src/lib/deploy/compose/ha.js

@@ -28,8 +28,10 @@ import { useDnsChallenge } from '../acme.js';
 import {
   assertReplicationStreamingOrDegraded,
   buildPrimaryConninfo,
+  buildReplicationFirewallRules,
   buildReplicationHbaLine,
   buildStagedBasebackupScript,
+  REPL_PORT,
   verifyStreaming,
 } from '../replication.js';
 import {
@@ -38,7 +40,13 @@ import {
   pinnedSshOptsString,
   readReplPassword,
 } from '../utils.js';
-import { WG_SUBNET_CIDR } from '../wireguard.js';
+import {
+  exchangeAndBringUpTunnel,
+  REPL_GATEWAY_PORT,
+  WG_PRIMARY_IP,
+  WG_STANDBY_IP,
+  WG_SUBNET_CIDR,
+} from '../wireguard.js';
 import {
   createAdminUser,
   dockerLoginOnServer,
@@ -221,17 +229,83 @@ async function deleteFirewallByName(name, hetznerHeaders, deadlineMs = 30_000) {
 //   deployMode === 'compose-ha' (skipped for single-region compose, which has no
 //   standby). configureStandbyReplication is exported from this file for that.
 
+/**
+ * Build the compose replication overlay (`docker-compose.replication.yml`) for
+ * ONE node, with that node's own WireGuard tunnel IP baked into the relay.
+ *
+ * Two pieces:
+ *   1. `db` publishes the raw replication port (REPL_PORT=5433 → container 5432)
+ *      so the local socat relay has a plaintext postgres to forward to
+ *      (supavisor owns 5432 and can't speak the replication protocol).
+ *   2. `repl-gateway` — a socat relay on the HOST network namespace (so it sees
+ *      the host's `wg0`, brought up by exchangeAndBringUpTunnel) that binds this
+ *      node's OWN tunnel IP (`selfWgIp:REPL_GATEWAY_PORT`) and forwards to the
+ *      local db (`127.0.0.1:REPL_PORT`). Binding the tunnel IP (not 0.0.0.0)
+ *      means the relay is reachable ONLY over wg0 — never on the public IP — so
+ *      it needs no firewall rule of its own. NO `NET_ADMIN`/in-container
+ *      WireGuard: the tunnel lives on the host, the container is a pure relay.
+ *      `restart: unless-stopped` so a transient socat exit self-heals.
+ *
+ * Each node self-exposes its LOCAL db at `<self-tunnel-ip>:REPL_GATEWAY_PORT`,
+ * which makes the transport symmetric for failover: whichever node is currently
+ * standby dials the current primary's tunnel IP over wg0 (docker bridge → host
+ * wg0), and the promoted node already exposes its db on its own tunnel IP.
+ *
+ * @param {string} selfWgIp - this node's WireGuard tunnel IP (WG_PRIMARY_IP on
+ *   the primary, WG_STANDBY_IP on the standby)
+ * @returns {string} the YAML overlay content
+ */
+export function buildReplicationOverlay(selfWgIp) {
+  return `version: "3.8"
+services:
+  db:
+    ports:
+      - "${REPL_PORT}:5432"
+  repl-gateway:
+    image: alpine/socat
+    network_mode: host
+    restart: unless-stopped
+    command:
+      - "TCP-LISTEN:${REPL_GATEWAY_PORT},bind=${selfWgIp},fork,reuseaddr"
+      - "TCP:127.0.0.1:${REPL_PORT}"
+`;
+}
+
+// The compose flags that include the replication overlay so `up`/`restart`
+// against the repl-gateway service resolves it.
+const REPL_COMPOSE_FLAGS =
+  '-f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.replication.yml';
+
 /**
  * Configure the primary PostgreSQL server for streaming replication.
  *
- * 1. Create a dedicated replication user
- * 2. Allow replication from the WireGuard tunnel subnet in pg_hba.conf (plain
+ * 1. Bring up the point-to-point WireGuard tunnel (host wg0, UDP 51821) between
+ *    the two nodes and start the primary's `repl-gateway` socat relay
+ * 2. Create a dedicated replication user
+ * 3. Allow replication from the WireGuard tunnel subnet in pg_hba.conf (plain
  *    `host` — WireGuard encrypts the wire, so Postgres doesn't also need TLS)
- * 3. Set wal_level=replica and max_wal_senders
- * 4. Restart PostgreSQL to apply changes
+ * 4. Set wal_level=replica and max_wal_senders
+ * 5. Restart PostgreSQL to apply changes
  */
-async function configurePrimaryReplication(primaryIp, sshKeyPath, projectName) {
+export async function configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, projectName) {
   const remoteDir = `/opt/${projectName}`;
+
+  // Establish the WireGuard replication transport. Keys are generated on-node
+  // (private key never leaves the node); only the derived public keys transit
+  // the orchestrator. Endpoints/SSH targets are the two VPS PUBLIC IPs. This
+  // must run BEFORE starting the socat relay, since the relay binds the tunnel
+  // IP (10.99.0.x) which only exists once `wg0` is up.
+  await exchangeAndBringUpTunnel({ primaryIp, standbyIp, sshKeyPath });
+
+  // Start the primary's repl-gateway (socat, host netns). The overlay file was
+  // written to the node earlier in the deploy flow; `up -d repl-gateway` binds
+  // WG_PRIMARY_IP:REPL_GATEWAY_PORT → 127.0.0.1:REPL_PORT now that wg0 exists.
+  await sshRun(
+    primaryIp,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose ${REPL_COMPOSE_FLAGS} up -d repl-gateway 2>&1`,
+    { timeout: 60_000 },
+  );
   // replPassword is generated at create time from crypto.randomBytes and is
   // restricted to base64url characters — no shell escaping needed.
   // Read from process.env first (CI), then .env.local (the `vibecarbon create`
@@ -315,14 +389,10 @@ END $$;
     { timeout: 30_000 },
   );
 
-  // TODO(wireguard-transport): tunnel bring-up — Task 6.
-  //
-  // This used to enable server-side TLS (verify-ca) here: copy the per-deploy
-  // CA/cert/key from the bind-mounted Secret into PGDATA and ALTER SYSTEM SET
-  // ssl = 'on' + ssl_cert_file/ssl_key_file/ssl_ca_file. That's public-IP-era
-  // plumbing for the TLS transport being replaced — WireGuard encrypts the
-  // wire instead, so there's no cert material to install here. Task 6 wires
-  // the actual wg0 tunnel bring-up for compose-ha.
+  // No app-layer TLS. Replication runs over the WireGuard tunnel (brought up
+  // above), which encrypts the wire, so Postgres itself speaks plaintext — the
+  // retired verify-ca transport's cert install + `ALTER SYSTEM SET ssl='on'`
+  // are gone (public-IP-era plumbing).
 
   // Restart PostgreSQL to apply WAL changes
   await sshRun(primaryIp, sshKeyPath, `cd ${remoteDir} && docker compose restart db`, {
@@ -394,6 +464,19 @@ END $$;`;
     { timeout: 30_000 },
   );
 
+  // Start the standby's repl-gateway (socat, host netns). The tunnel (host wg0)
+  // is already up — configurePrimaryReplication brought it up on both nodes at
+  // deploy time, and it persists across the restore/re-seed re-entry. Binding
+  // WG_STANDBY_IP:REPL_GATEWAY_PORT self-exposes the standby's LOCAL db over the
+  // tunnel so the transport is symmetric after a failover (the promoted standby
+  // is then the primary, and the old primary streams from WG_STANDBY_IP).
+  await sshRun(
+    standbyIp,
+    sshKeyPath,
+    `cd ${remoteDir} && docker compose ${REPL_COMPOSE_FLAGS} up -d repl-gateway 2>&1`,
+    { timeout: 60_000 },
+  );
+
   // Stop the standby database
   await sshRun(standbyIp, sshKeyPath, `cd ${remoteDir} && docker compose stop db`, {
     timeout: 60_000,
@@ -406,9 +489,17 @@ END $$;`;
   // atomically swaps in only after verifying PG_VERSION, so a failed/partial
   // basebackup never leaves PGDATA half-formed (the `initdb: directory exists
   // but is not empty` crash-loop). probeFirst aborts cleanly (exit 0, PGDATA
-  // untouched) when the primary's REPL_PORT is unreachable. Plaintext (no
-  // PGSSLMODE/PGSSLROOTCERT) — see TODO(wireguard-transport) above; the
-  // connection is unencrypted until Task 6 lands the tunnel.
+  // untouched) when the primary's gateway is unreachable. Plaintext (no
+  // PGSSLMODE/PGSSLROOTCERT) — WireGuard is the encryption layer.
+  //
+  // The connection targets the PRIMARY's WireGuard tunnel IP (WG_PRIMARY_IP) on
+  // the repl-gateway relay port (REPL_GATEWAY_PORT), NOT the primary's public
+  // IP: the standby db container reaches 10.99.0.1:15433 via the host's wg0
+  // (docker bridge → host route → wg0), where the primary's socat relay
+  // forwards to the primary db (127.0.0.1:REPL_PORT). WG_PRIMARY_IP is fixed to
+  // whichever node is currently primary (exchangeAndBringUpTunnel assigns it),
+  // so this stays correct across a role swap once failover re-establishes the
+  // tunnel with swapped endpoints.
   //
   // After the swap we pin primary_conninfo explicitly (last entry in
   // postgresql.auto.conf wins): pg_basebackup -R records a conninfo from the
@@ -418,12 +509,13 @@ END $$;`;
   // left untouched.
   const basebackupScript = `${buildStagedBasebackupScript({
     replPassword,
-    primaryHost: primaryIp,
+    primaryHost: WG_PRIMARY_IP,
+    primaryPort: String(REPL_GATEWAY_PORT),
     probeFirst: true,
     label: 'compose-ha-repl',
   })}
 cat >> /var/lib/postgresql/data/postgresql.auto.conf <<'REPL_CONNINFO_EOF'
-primary_conninfo = '${buildPrimaryConninfo({ primaryHost: primaryIp, replPassword })}'
+primary_conninfo = '${buildPrimaryConninfo({ primaryHost: WG_PRIMARY_IP, port: REPL_GATEWAY_PORT, replPassword })}'
 REPL_CONNINFO_EOF
 chown postgres:postgres /var/lib/postgresql/data/postgresql.auto.conf
 `;
@@ -549,44 +641,41 @@ chown postgres:postgres /var/lib/postgresql/data/postgresql.auto.conf
 // ============================================================================
 
 /**
- * Open PostgreSQL port (5432) between the two servers via UFW.
- * Only allows traffic from the peer's IP — not from the public internet.
- */
-/**
- * Open the dedicated replication port between HA peers.
+ * Open the WireGuard tunnel port (UDP 51821) from the HA peer in UFW.
  *
- * Port 5433 maps directly to the db container's PostgreSQL (bypassing supavisor
- * on port 5432). This is required because supavisor doesn't support PostgreSQL
- * replication protocol, and pg_basebackup / streaming replication need a raw
- * PostgreSQL connection.
+ * Replication no longer traverses a public TCP port — it runs over a
+ * point-to-point WireGuard tunnel between the two nodes' public IPs. The only
+ * inbound the peer needs is the WG handshake/data on UDP 51821 (51820 is
+ * flannel-wg's in k3s; compose uses 51821 for parity). Scoped to the peer's IP,
+ * never the public internet. The raw replication port (5433) is no longer
+ * exposed to the peer at the host firewall — the socat relay binds the tunnel
+ * IP and is reachable only over wg0.
  */
-async function openReplicationPort(ip, peerIp, sshKeyPath) {
-  await sshRunAsync(ip, sshKeyPath, `ufw allow from ${peerIp} to any port 5433 proto tcp`, {
+async function openWireguardPortUfw(ip, peerIp, sshKeyPath) {
+  await sshRunAsync(ip, sshKeyPath, `ufw allow from ${peerIp} to any port 51821 proto udp`, {
     timeout: 15_000,
   });
 }
 
 /**
- * Add an `in tcp 5433` rule scoped to the peer's IP on a server's Hetzner Cloud
- * firewall. UFW alone isn't enough — Hetzner Cloud Firewall sits in front of
- * the host and blocks SYN before it ever reaches UFW. Without this, every
- * pg_basebackup attempt from the peer hits a TCP `Connection timed out`,
- * configureStandbyReplication's bash script (with set -e + bash) fails on
- * pg_basebackup, the catch path throws "pg_basebackup failed", but historically
- * the script ran under dash (no `set -e -o pipefail` support) and silently
- * proceeded — so the warning surfaced two stages later as "did not enter
- * recovery mode within 60s" with the standby actually running its first-init
- * primary-mode state. RCA: compose-ha 2026-05-01 fanout11.
+ * Open the WireGuard tunnel port (UDP 51821) scoped to the peer's IP on a
+ * server's Hetzner Cloud firewall. UFW alone isn't enough — the Cloud Firewall
+ * sits in front of the host and drops the packet before it ever reaches UFW.
+ * Without this, the WG handshake from the peer never lands, the tunnel never
+ * comes up, and pg_basebackup over the relay times out.
  *
  * The Pulumi-managed firewall (src/lib/iac/programs/hetzner-compose.js) only
- * declares 22/80/443 because the standby's IP isn't known at primary stack-up
- * time. Patch in 5433 here, after both stacks are up.
+ * declares 22/80/443 because the peer's IP isn't known at stack-up time. Patch
+ * in the UDP 51821 rule here, after both stacks are up. Uses the shared
+ * buildReplicationFirewallRules builder so compose-ha and k8s-ha compute the
+ * exact same rule set (peer-scoped udp/51821 + scrubs any stale tcp
+ * 5432/5433/30432 rules from the retired public-IP TLS transport).
  *
- * Idempotent: skips if a matching rule already exists. Non-fatal on error —
- * the deploy continues with a warning so the operator can iterate the rest of
- * the scenario; HA replication just won't actually work until 5433 is open.
+ * Idempotent: buildReplicationFirewallRules returns null when the rule already
+ * exists and no stale rules remain. Non-fatal on error — the deploy continues
+ * with a warning; HA replication just won't work until 51821 is open.
  */
-async function openReplicationPortHetznerFirewall(serverName, peerIp, hetznerToken) {
+async function openWireguardPortHetznerFirewall(serverName, peerIp, hetznerToken) {
   if (!hetznerToken) return;
   const firewallName = `${serverName}-firewall`;
   const headers = {
@@ -600,22 +689,8 @@ async function openReplicationPortHetznerFirewall(serverName, peerIp, hetznerTok
   const fwListData = await fwListRes.json();
   const firewall = fwListData.firewalls?.find((fw) => fw.name === firewallName);
   if (!firewall) return;
-  const existingRules = firewall.rules || [];
-  const peerCidr = `${peerIp}/32`;
-  const hasReplRule = existingRules.some(
-    (r) => r.port === '5433' && r.source_ips?.includes(peerCidr),
-  );
-  if (hasReplRule) return;
-  const updatedRules = [
-    ...existingRules,
-    {
-      direction: 'in',
-      protocol: 'tcp',
-      port: '5433',
-      source_ips: [peerCidr],
-      description: 'PostgreSQL replication from HA peer',
-    },
-  ];
+  const updatedRules = buildReplicationFirewallRules(firewall.rules || [], peerIp);
+  if (!updatedRules) return; // already correct
   await fetchWithRetry(`https://api.hetzner.cloud/v1/firewalls/${firewall.id}/actions/set_rules`, {
     method: 'POST',
     headers,
@@ -958,27 +1033,28 @@ export async function deployComposeHA(options) {
     perfAsync('deploy.ha.compose.setupFiles.standby', () =>
       setupServerFiles(standbyIp, sshKeyPath, projectName, serviceOpts),
     ),
-    perfAsync('deploy.ha.compose.openReplPort.primary', () =>
-      openReplicationPort(primaryIp, standbyIp, sshKeyPath),
+    perfAsync('deploy.ha.compose.openWgPortUfw.primary', () =>
+      openWireguardPortUfw(primaryIp, standbyIp, sshKeyPath),
     ),
-    perfAsync('deploy.ha.compose.openReplPort.standby', () =>
-      openReplicationPort(standbyIp, primaryIp, sshKeyPath),
+    perfAsync('deploy.ha.compose.openWgPortUfw.standby', () =>
+      openWireguardPortUfw(standbyIp, primaryIp, sshKeyPath),
     ),
-    // Open 5433 in the Hetzner Cloud firewall for both peers. UFW is the
-    // host-level layer and was already configured above; without these two
-    // calls the Cloud Firewall (Pulumi-declared with only 22/80/443) silently
-    // drops every replication packet at the edge. Both directions because
-    // post-failover the standby becomes primary and needs its own 5433 open
-    // for the new standby (now the old primary) to reconnect.
-    perfAsync('deploy.ha.compose.cloudFirewall5433.primary', () =>
-      openReplicationPortHetznerFirewall(
+    // Open the WireGuard tunnel port (UDP 51821) in the Hetzner Cloud firewall
+    // for both peers. UFW is the host-level layer configured above; without
+    // these two calls the Cloud Firewall (Pulumi-declared with only 22/80/443)
+    // silently drops the WG handshake at the edge and the tunnel never comes
+    // up. Both directions because post-failover the standby becomes primary and
+    // the WG endpoints reverse — the promoted node must already admit the old
+    // primary's WG traffic.
+    perfAsync('deploy.ha.compose.cloudFirewallWg.primary', () =>
+      openWireguardPortHetznerFirewall(
         `${projectName}-${environment}-primary`,
         standbyIp,
         apiToken,
       ),
     ),
-    perfAsync('deploy.ha.compose.cloudFirewall5433.standby', () =>
-      openReplicationPortHetznerFirewall(
+    perfAsync('deploy.ha.compose.cloudFirewallWg.standby', () =>
+      openWireguardPortHetznerFirewall(
         `${projectName}-${environment}-standby`,
         primaryIp,
         apiToken,
@@ -1186,42 +1262,45 @@ export async function deployComposeHA(options) {
     );
   }
 
-  // Expose raw PostgreSQL on port 5433 for replication (supavisor owns 5432).
-  // The docker-compose.replication.yml overlay maps db:5432 → host:5433 on
-  // both nodes so pg_basebackup + streaming can reach the peer. Fan out
-  // across primary + standby — the overlay write + db-only `compose up`
-  // touch disjoint VPS hosts and need to land on both before
-  // configurePrimaryReplication grants pub access.
-  //
-  // TODO(wireguard-transport): tunnel bring-up — Task 6. This overlay (direct
-  // port-mapping, no TLS cert bind-mount) is the interim pre-WireGuard
-  // transport; Task 6 replaces it with the wg0 tunnel.
-  const replOverlay = 'version: "3.8"\nservices:\n  db:\n    ports:\n      - "5433:5432"\n';
+  // Write the WireGuard replication overlay (docker-compose.replication.yml) to
+  // both nodes. It (a) publishes db:5432 → host:REPL_PORT so the socat relay has
+  // a plaintext postgres to forward to, and (b) declares the `repl-gateway`
+  // socat relay bound to THIS node's own tunnel IP (WG_PRIMARY_IP on the
+  // primary, WG_STANDBY_IP on the standby). The relay is NOT started here —
+  // socat binds the tunnel IP, which only exists after wg0 is up; the tunnel is
+  // brought up (and the gateways started) inside configure{Primary,Standby}
+  // Replication below. We DO recreate the db service now so its REPL_PORT
+  // mapping is live before the relay attaches. Fan out — both nodes are
+  // disjoint hosts.
   await Promise.all(
-    [primaryIp, standbyIp].map(async (ip) => {
+    [
+      [primaryIp, WG_PRIMARY_IP],
+      [standbyIp, WG_STANDBY_IP],
+    ].map(async ([ip, selfWgIp]) => {
       await sshRunAsync(
         ip,
         sshKeyPath,
-        `cat > /opt/${projectName}/docker-compose.replication.yml << 'REPL'\n${replOverlay}REPL`,
+        `cat > /opt/${projectName}/docker-compose.replication.yml << 'REPL'\n${buildReplicationOverlay(selfWgIp)}REPL`,
         { timeout: 10_000 },
       );
-      // Recreate only the db service with the new port mapping
+      // Recreate only the db service with the new port mapping (the relay is
+      // started later, once wg0 exists).
       await sshRunAsync(
         ip,
         sshKeyPath,
-        `cd /opt/${projectName} && docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.replication.yml up -d db 2>&1`,
+        `cd /opt/${projectName} && docker compose ${REPL_COMPOSE_FLAGS} up -d db 2>&1`,
         { timeout: 60_000 },
       );
     }),
   );
 
   // Configure PostgreSQL streaming replication.
-  // Wrap the entire setup (primary + standby + slot creation + pg_basebackup)
-  // so the perf trace shows the headline replication cost — typical 30-90s,
-  // dominated by pg_basebackup for the standby seed.
+  // Wrap the entire setup (tunnel + gateways + primary + standby + slot +
+  // pg_basebackup) so the perf trace shows the headline replication cost —
+  // typical 30-90s, dominated by pg_basebackup for the standby seed.
   await perfAsync('deploy.ha.replication.setup', async () => {
     onProgress('Configuring PostgreSQL replication on primary...');
-    await configurePrimaryReplication(primaryIp, sshKeyPath, projectName);
+    await configurePrimaryReplication(primaryIp, standbyIp, sshKeyPath, projectName);
 
     onProgress('Configuring standby as hot replica...');
     let replicationOk = false;
@@ -1288,8 +1367,8 @@ export async function deployComposeHA(options) {
     lastState: replLastState,
     allowDegraded,
     fixHint:
-      'Verify the replication port (5433) is open between the two regions and the standby ' +
-      'completed pg_basebackup.',
+      'Verify the WireGuard tunnel port (UDP 51821) is open between the two regions, wg0 ' +
+      'is up on both nodes, and the standby completed pg_basebackup.',
   });
   if (replActive) {
     p.log.success('Streaming replication active');