underpost 3.2.9 → 3.2.11

package/src/cli/lxd.js

@@ -2,6 +2,7 @@
  * LXD module for managing LXD virtual machines as K3s nodes.
  * @module src/cli/lxd.js
  * @namespace UnderpostLxd
+ *
  */
 
 import { getNpmRootPath } from '../server/conf.js';
@@ -13,46 +14,92 @@ import Underpost from '../index.js';
 
 const logger = loggerFactory(import.meta);
 
-/**
- * @class UnderpostLxd
- * @description Provides a set of static methods to interact with LXD,
- * encapsulating common LXD operations for VM management and network testing.
- * @memberof UnderpostLxd
- */
+const ENGINE_ROOT_IN_VM = '/home/dd/engine';
+const ENGINE_ROOT_ON_HOST = '/home/dd/engine';
+const ADMIN_PROFILE = 'admin-profile';
+const BRIDGE_NETWORK = 'lxdbr0';
+const BRIDGE_SUBNET_PREFIX = '10.250.250';
+
 class UnderpostLxd {
+  static _project = '';
+
+  static _lxcCmd() {
+    return UnderpostLxd._project ? `lxc --project ${UnderpostLxd._project}` : 'lxc';
+  }
+
   static API = {
     /**
      * @method callback
-     * @description Main entry point for LXD VM operations. Each VM is a K3s node (control or worker).
+     * @description Main entry point for all LXD CLI operations.
+     * @param {string} [vmId=''] - Positional VM identifier for boolean
+     *   vm lifecycle flags.
      * @param {object} options
      * @param {boolean} [options.init=false] - Initialize LXD via preseed.
-     * @param {boolean} [options.reset=false] - Remove LXD snap and purge data.
+     * @param {boolean} [options.reset=false] - Host-safe teardown of VMs, proxy
+     *   devices, admin-profile, and lxdbr0. Does NOT touch the LXD snap or
+     *   storage pools.
+     * @param {boolean} [options.purge=false] - Gracefully shut the LXD daemon
+     *   down (60s timeout) and remove the LXD snap. Combine with `--reset` to
+     *   wipe per-VM state first. Without `--reset`, snap removal alone wipes
+     *   everything.
+     * @param {boolean} [options.shutdown=false] - Pre-host-reboot procedure:
+     *   gracefully stop every VM and the LXD daemon. Run before `reboot` /
+     *   `poweroff` to keep the host bootable.
+     * @param {boolean} [options.restore=false] - Symmetric to `--shutdown`:
+     *   starts the LXD daemon (`snap start lxd`), waits for it to become
+     *   responsive, then starts every VM that exists. VMs created with
+     *   `admin-profile` have `boot.autostart=false`, so this is the explicit
+     *   "bring the lab back online" command.
      * @param {boolean} [options.dev=false] - Use local paths instead of npm global.
      * @param {boolean} [options.install=false] - Install LXD snap.
-     * @param {boolean} [options.createVirtualNetwork=false] - Create lxdbr0 bridge network.
-     * @param {string} [options.ipv4Address='10.250.250.1/24'] - IPv4 address/CIDR for the lxdbr0 bridge network.
+     * @param {boolean} [options.createVirtualNetwork=false] - Create lxdbr0 as a LXD-managed
+     *   bridge with NAT, but with DHCP/DNS off and dnsmasq neutralized (raw.dnsmasq=port=0) so
+     *   it coexists with MAAS. The managed subnet enables static NIC IPs for `--expose` proxies.
+     * @param {string} [options.ipv4Address='10.250.250.1/24'] - Managed gateway address/CIDR
+     *   for lxdbr0 (LXD assigns this to the bridge and masquerades VM egress).
      * @param {boolean} [options.createAdminProfile=false] - Create admin-profile for VMs.
-     * @param {boolean} [options.control=false] - Initialize VM as K3s control plane node.
-     * @param {boolean} [options.worker=false] - Initialize VM as K3s worker node.
-     * @param {string} [options.initVm=''] - VM name to initialize as a K3s node.
-     * @param {string} [options.deleteVm=''] - VM name to stop and delete.
-     * @param {string} [options.createVm=''] - VM name to create (copies launch command to clipboard).
-     * @param {string} [options.infoVm=''] - VM name to inspect.
-     * @param {string} [options.rootSize=''] - Root disk size in GiB for new VMs (e.g. '32').
-     * @param {string} [options.joinNode=''] - Join format: 'workerName,controlName' (standalone join). When used with --init-vm --worker, just the control node name.
+     * @param {boolean} [options.control=false] - Initialize VM as K3s control plane.
+     * @param {boolean} [options.worker=false] - Initialize VM as K3s worker.
+     * @param {boolean} [options.vmInit=false] - Bring the VM identified by
+     *   `vmId` up as a K3s node end-to-end.
+     * @param {boolean} [options.vmDelete=false] - Safely stop and delete the
+     *   VM identified by `vmId`.
+     * @param {boolean} [options.vmCreate=false] - Surface the launch command
+     *   for the VM identified by `vmId`.
+     * @param {boolean} [options.vmInfo=false] - Inspect the VM identified by
+     *   `vmId`.
+     * @param {string} [options.rootSize=''] - Root disk size in GiB for new VMs.
+     * @param {string} [options.joinNode=''] - Join format: 'workerName,controlName'.
      * @param {string} [options.expose=''] - Expose VM ports to host: 'vmName:port1,port2'.
+     * @param {string} [options.nodePort=''] - Custom VM-side (connect) port for `--expose`. When set, the
+     *   host listens on each requested port but the proxy connects to this port inside the VM (e.g. expose
+     *   host 27017 → VM NodePort 32017). Defaults to the same port on both sides.
      * @param {string} [options.deleteExpose=''] - Remove exposed ports: 'vmName:port1,port2'.
-     * @param {string} [options.test=''] - VM name to run connectivity and health checks on.
-     * @param {string} [options.workflowId=''] - Workflow ID for runWorkflow.
-     * @param {string} [options.vmId=''] - VM name for workflow execution.
-     * @param {string} [options.deployId=''] - Deployment ID for workflow context.
-     * @param {string} [options.namespace=''] - Kubernetes namespace context.
+     * @param {boolean} [options.vmTest=false] - Run connectivity and health
+     *   checks on the VM identified by `vmId`.
+     * @param {boolean} [options.vmSyncEngine=false] - Re-copy the host engine
+     *   source into the VM identified by `vmId`, overriding whatever is
+     *   currently there. Equivalent to step 2 of `--vm-init` in isolation.
+     * @param {boolean} [options.copy=false] - For two-phase flows that surface a
+     *   command for the user to execute (e.g. `--create-admin-profile` phase 1):
+     *   when set, copy the command to the clipboard. When unset, print it to
+     *   the terminal so the user can read it directly.
+     * @param {string} [options.maasProject=''] - LXD project managed by MAAS
+     *   (e.g. 'k3s-cluster'). When set, all lxc commands target this project so
+     *   MAAS can enumerate the VMs in its machines UI.
+     * @param {boolean} [options.moveToProject=false] - Stop the VM identified
+     *   by `vmId`, move it from the default project to `maasProject`, then start
+     *   it so MAAS picks it up. Requires `--maas-project`.
      * @memberof UnderpostLxd
      */
     async callback(
+      vmId = '',
       options = {
         init: false,
         reset: false,
+        purge: false,
+        shutdown: false,
+        restore: false,
         dev: false,
         install: false,
         createVirtualNetwork: false,
@@ -60,297 +107,1126 @@ class UnderpostLxd {
         createAdminProfile: false,
         control: false,
         worker: false,
-        initVm: '',
-        deleteVm: '',
-        createVm: '',
-        infoVm: '',
+        vmInit: false,
+        vmDelete: false,
+        vmCreate: false,
+        vmInfo: false,
         rootSize: '',
         joinNode: '',
         expose: '',
+        nodePort: '',
         deleteExpose: '',
-        test: '',
-        workflowId: '',
-        vmId: '',
-        deployId: '',
-        namespace: '',
+        vmTest: false,
+        vmSyncEngine: false,
+        copy: false,
+        maasProject: '',
+        moveToProject: false,
       },
     ) {
       const npmRoot = getNpmRootPath();
       const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      const currentVmId = vmId ? String(vmId).trim() : '';
+      const vmCreate = options.vmCreate === true;
+      const vmDelete = options.vmDelete === true;
+      const vmInfo = options.vmInfo === true;
+      const vmInit = options.vmInit === true;
+      const vmTest = options.vmTest === true;
+      const vmSyncEngine = options.vmSyncEngine === true;
+      UnderpostLxd._project = options.maasProject ? String(options.maasProject).trim() : '';
 
+      // =====================================================================
+      // SHUTDOWN: graceful pre-host-reboot procedure
+      // =====================================================================
+      if (options.shutdown === true) {
+        UnderpostLxd._gracefulShutdownAll();
+        return;
+      }
+
+      // =====================================================================
+      // RESTORE: symmetric counterpart to --shutdown
+      // =====================================================================
+      if (options.restore === true) {
+        UnderpostLxd._restoreAll();
+        return;
+      }
+
+      // =====================================================================
+      // RESET / PURGE: host-safe teardown variants
+      //   --reset       wipes VMs, proxy devices, admin-profile, lxdbr0
+      //   --purge       gracefully stops the daemon, then snap remove --purge
+      //   --reset --purge   both, in order
+      // =====================================================================
       if (options.reset === true) {
-        shellExec(`sudo systemctl stop snap.lxd.daemon`);
-        shellExec(`sudo snap remove lxd --purge`);
+        UnderpostLxd._safeReset();
+      }
+      if (options.purge === true) {
+        UnderpostLxd._safePurge();
       }
+      if (options.reset === true || options.purge === true) return;
 
-      if (options.install === true) shellExec(`sudo snap install lxd`);
+      // =====================================================================
+      // INSTALL (idempotent: skip if already installed)
+      // =====================================================================
+      if (options.install === true) {
+        if (UnderpostLxd._snapInstalled('lxd')) {
+          logger.info('LXD snap is already installed; skipping.');
+        } else {
+          shellExec(`sudo snap install lxd`);
+        }
+      }
 
+      // =====================================================================
+      // INIT (LXD preseed)
+      // =====================================================================
       if (options.init === true) {
         shellExec(`sudo systemctl start snap.lxd.daemon`);
         shellExec(`sudo systemctl status snap.lxd.daemon`);
-        const lxdPreseedContent = fs
-          .readFileSync(`${underpostRoot}/manifests/lxd/lxd-preseed.yaml`, 'utf8')
-          .replaceAll(`127.0.0.1`, Underpost.dns.getLocalIPv4Address());
-        shellExec(`echo "${lxdPreseedContent}" | lxd init --preseed`);
-        shellExec(`lxc cluster list`);
+        if (UnderpostLxd._lxdInitialized()) {
+          logger.info('LXD is already initialized (storage pool present); skipping preseed.');
+        } else {
+          const lxdPreseedContent = fs
+            .readFileSync(`${underpostRoot}/manifests/lxd/lxd-preseed.yaml`, 'utf8')
+            .replaceAll(`127.0.0.1`, Underpost.dns.getLocalIPv4Address());
+          shellExec(`echo "${lxdPreseedContent}" | lxd init --preseed`);
+        }
+        shellExec(`${UnderpostLxd._lxcCmd()} cluster list`);
       }
 
+      // =====================================================================
+      // CREATE VIRTUAL NETWORK
+      //
+      // LXD-managed bridge so its native features work — the host is the
+      // gateway (10.250.250.1), LXD masquerades VM egress (`ipv4.nat`), and the
+      // managed subnet lets instance NICs carry a static `ipv4.address`, which
+      // NAT-mode proxy devices (`--expose`) require.
+      //
+      // MAAS harmony: LXD spawns a dnsmasq for any managed subnet, and on this
+      // host that dnsmasq cannot bind :53/:67 (MAAS's named/dhcpd already own
+      // them), so a default managed bridge dies with "Address already in use".
+      // We neutralize dnsmasq instead of dropping the subnet:
+      //   - ipv4.dhcp=false  → no DHCP, no :67 bind (MAAS owns DHCP).
+      //   - dns.mode=none + raw.dnsmasq=port=0 → no DNS, no :53 bind. dns.mode
+      //     alone does not force port=0 in dnsmasq, hence the explicit raw line.
+      // dnsmasq then starts but opens no listening sockets — no collision.
+      //
+      // Settings are applied inline at create time so dnsmasq is neutralized on
+      // its first spawn (not after a racing default-config start). Idempotent:
+      // reconcile when the network already exists.
+      // =====================================================================
       if (options.createVirtualNetwork === true) {
-        const ipv4Address = options.ipv4Address ? options.ipv4Address : '10.250.250.1/24';
-        shellExec(`lxc network create lxdbr0 \
-ipv4.address=${ipv4Address} \
-ipv4.nat=true \
-ipv4.dhcp=true \
-ipv6.address=none`);
+        const gatewayCidr = options.ipv4Address ? options.ipv4Address : '10.250.250.1/24';
+        // Order matters for the reconcile path: neutralize dnsmasq (no DHCP, no
+        // DNS listener) BEFORE assigning ipv4.address, or setting the subnet on
+        // an existing bridge spawns a default dnsmasq that collides with MAAS
+        // before raw.dnsmasq lands. On a fresh inline create this is atomic.
+        const bridgeSettings = {
+          'ipv4.dhcp': 'false',
+          'dns.mode': 'none',
+          'raw.dnsmasq': 'port=0',
+          'ipv6.address': 'none',
+          'ipv4.address': gatewayCidr,
+          'ipv4.nat': 'true',
+          'ipv4.firewall': 'true',
+        };
+
+        if (UnderpostLxd._networkExists(BRIDGE_NETWORK)) {
+          logger.info(`Network '${BRIDGE_NETWORK}' already exists; reconciling managed bridge settings.`);
+          for (const [key, value] of Object.entries(bridgeSettings)) {
+            shellExec(`${UnderpostLxd._lxcCmd()} network set ${BRIDGE_NETWORK} ${key} "${value}"`);
+          }
+        } else {
+          const inlineConfig = Object.entries(bridgeSettings)
+            .map(([key, value]) => `${key}="${value}"`)
+            .join(' ');
+          shellExec(`${UnderpostLxd._lxcCmd()} network create ${BRIDGE_NETWORK} ${inlineConfig}`);
+        }
+
+        UnderpostLxd._ensureBridgeInTrustedZone(BRIDGE_NETWORK);
+        UnderpostLxd._ensureBridgeForwardingAccept(BRIDGE_NETWORK);
       }
 
+      // =====================================================================
+      // CREATE ADMIN PROFILE (two-phase to sidestep `lxc profile create` hangs)
+      //
+      // Phase 1 (profile absent): copy `lxc profile create admin-profile` to
+      //   the clipboard and exit. The user runs it themselves in their shell.
+      // Phase 2 (profile present): load the YAML into the existing profile.
+      //
+      // Driven by an explicit pre-condition check; no shell command runs that
+      // could hang waiting on stdin/tty.
+      // =====================================================================
       if (options.createAdminProfile === true) {
-        const existingProfiles = await new Promise((resolve) => {
-          shellExec(`lxc profile show admin-profile`, {
-            silent: true,
-            callback: (...args) => resolve(JSON.stringify(args)),
-          });
-        });
-        if (existingProfiles.toLowerCase().match('error')) {
-          logger.warn('Profile does not exist. Use the following command to create it:');
-          pbcopy(`lxc profile create admin-profile`);
+        if (!UnderpostLxd._profileExists(ADMIN_PROFILE)) {
+          const createCmd = `lxc profile create ${ADMIN_PROFILE}`;
+          if (options.copy === true) {
+            logger.warn(
+              `Profile '${ADMIN_PROFILE}' does not exist. The create command has been copied to your clipboard — run it, then re-run this command to load the YAML.`,
+            );
+            pbcopy(createCmd);
+          } else {
+            logger.warn(
+              `Profile '${ADMIN_PROFILE}' does not exist. Run the command below in your shell, then re-run this command to load the YAML. (Pass --copy to put it on the clipboard instead.)`,
+            );
+            console.log(`\n  ${createCmd}\n`);
+          }
         } else {
-          shellExec(`cat ${underpostRoot}/manifests/lxd/lxd-admin-profile.yaml | lxc profile edit admin-profile`);
-          shellExec(`lxc profile show admin-profile`);
+          shellExec(
+            `cat ${underpostRoot}/manifests/lxd/lxd-admin-profile.yaml | ${UnderpostLxd._lxcCmd()} profile edit ${ADMIN_PROFILE}`,
+          );
+          shellExec(`${UnderpostLxd._lxcCmd()} profile show ${ADMIN_PROFILE}`);
         }
       }
 
-      if (options.deleteVm) {
-        const vmName = options.deleteVm;
-        logger.info(`Stopping VM: ${vmName}`);
-        shellExec(`lxc stop ${vmName}`);
-        logger.info(`Deleting VM: ${vmName}`);
-        shellExec(`lxc delete ${vmName}`);
-        logger.info(`VM ${vmName} deleted.`);
+      // =====================================================================
+      // DELETE VM (idempotent via pre-condition checks; no silent errors)
+      // =====================================================================
+      if (vmDelete) {
+        if (!currentVmId) {
+          throw new Error(`--vm-delete requires the [vm-id] command argument.`);
+        }
+        UnderpostLxd._safeDeleteVm(currentVmId);
       }
 
-      if (options.createVm) {
-        pbcopy(
-          `lxc launch images:rockylinux/9 ${
-            options.createVm
-          } --vm --target lxd-node1 -c limits.cpu=2 -c limits.memory=4GB --profile admin-profile -d root,size=${
-            options.rootSize ? options.rootSize + 'GiB' : '32GiB'
-          }`,
-        );
+      // =====================================================================
+      // MOVE VM TO PROJECT (stop + cross-project move + start for MAAS)
+      // =====================================================================
+      if (options.moveToProject === true) {
+        if (!currentVmId) {
+          throw new Error(`--move-to-project requires the [vm-id] command argument.`);
+        }
+        if (!UnderpostLxd._project) {
+          throw new Error(`--move-to-project requires --maas-project <projectName>.`);
+        }
+        const rawState = shellExec(`lxc list ${currentVmId} --format json`, { stdout: true }).trim();
+        const arr = JSON.parse(rawState || '[]');
+        const inst = Array.isArray(arr) ? arr.find((i) => i?.name === currentVmId) : null;
+        if (!inst) throw new Error(`VM '${currentVmId}' not found in the default project.`);
+
+        // Ensure every profile the VM references exists in the target project.
+        // `lxc move` across projects fails with "Profile not found" if the
+        // target project does not have the same profiles as the source.
+        //
+        // Two-phase pattern (mirrors --create-admin-profile): `lxc profile create`
+        // can hang waiting on stdin/tty, so we NEVER run it programmatically.
+        //   Phase 1 (profile absent in target): surface the create command and exit.
+        //   Phase 2 (profile present in target): sync the YAML from the default project.
+        const vmProfiles = Array.isArray(inst.profiles) ? inst.profiles : [];
+        const targetProfilesRaw = shellExec(`${UnderpostLxd._lxcCmd()} profile list --format json`, {
+          stdout: true,
+        }).trim();
+        const targetProfiles = JSON.parse(targetProfilesRaw || '[]');
+        const targetProfileNames = Array.isArray(targetProfiles) ? targetProfiles.map((p) => p?.name) : [];
+        for (const profileName of vmProfiles) {
+          if (profileName === 'default') continue; // every project already has 'default'
+          if (!targetProfileNames.includes(profileName)) {
+            const createCmd = `lxc --project ${UnderpostLxd._project} profile create ${profileName}`;
+            if (options.copy === true) {
+              logger.warn(
+                `Profile '${profileName}' not found in project '${UnderpostLxd._project}'. The create command has been copied to your clipboard — run it, then re-run --move-to-project.`,
+              );
+              pbcopy(createCmd);
+            } else {
+              logger.warn(
+                `Profile '${profileName}' not found in project '${UnderpostLxd._project}'. Run the command below in your shell, then re-run --move-to-project. (Pass --copy to put it on the clipboard instead.)`,
+              );
+              console.log(`\n  ${createCmd}\n`);
+            }
+            return;
+          }
+          // Phase 2: profile exists in target — sync YAML from default project.
+          // Explicitly use --project default on the source side so the read is
+          // unambiguous regardless of any active project context.
+          logger.info(`Syncing profile '${profileName}' YAML into project '${UnderpostLxd._project}'...`);
+          shellExec(
+            `lxc --project default profile show ${profileName} | ${UnderpostLxd._lxcCmd()} profile edit ${profileName}`,
+          );
+          logger.info(`  Profile '${profileName}' synced.`);
+        }
+
+        if (inst.status === 'Running' || inst.status === 'Frozen') {
+          logger.info(`Stopping VM '${currentVmId}' before cross-project move...`);
+          shellExec(`lxc stop ${currentVmId} --timeout 60`);
+        }
+        logger.info(`Moving VM '${currentVmId}' to project '${UnderpostLxd._project}'...`);
+        shellExec(`lxc move ${currentVmId} ${currentVmId} --target-project ${UnderpostLxd._project}`);
+        logger.info(`VM '${currentVmId}' is now in project '${UnderpostLxd._project}'. Starting...`);
+        shellExec(`${UnderpostLxd._lxcCmd()} start ${currentVmId}`);
+        logger.info(`VM '${currentVmId}' started in project '${UnderpostLxd._project}'.`);
+        return;
       }
 
-      if (options.initVm) {
-        const vmName = options.initVm;
+      // =====================================================================
+      // CREATE VM (surface the launch command for the user to run)
+      //
+      // Default: print to terminal. With `--copy`: copy to clipboard.
+      // Same two-phase pattern as `--create-admin-profile`: the CLI never runs
+      // `lxc launch` itself (it can hang on first image fetch or AppArmor
+      // negotiation in some snap setups), so the user always invokes it.
+      // =====================================================================
+      if (vmCreate) {
+        if (!currentVmId) {
+          throw new Error(`--vm-create requires the [vm-id] command argument.`);
+        }
+        const vmName = currentVmId;
+        const launchCmd = `${UnderpostLxd._lxcCmd()} launch images:rockylinux/9 ${
+          vmName
+        } --vm --target lxd-node1 -c limits.cpu=2 -c limits.memory=4GB --profile ${ADMIN_PROFILE} -d root,size=${
+          options.rootSize ? options.rootSize + 'GiB' : '32GiB'
+        }`;
+        if (options.copy === true) {
+          logger.info(`Launch command for VM '${vmName}' copied to clipboard. Run it in your shell.`);
+          pbcopy(launchCmd);
+        } else {
+          logger.info(
+            `Run the launch command below in your shell to create VM '${vmName}'. (Pass --copy to put it on the clipboard instead.)`,
+          );
+          console.log(`\n  ${launchCmd}\n`);
+        }
+      }
+
+      // =====================================================================
+      // INIT VM (OS setup + engine bootstrap + K3s role)
+      // =====================================================================
+      if (vmInit) {
+        if (!currentVmId) {
+          throw new Error(`--vm-init requires the [vm-id] command argument.`);
+        }
+        const vmName = currentVmId;
+        if (!UnderpostLxd._vmExists(vmName)) {
+          throw new Error(`VM '${vmName}' does not exist. Create it first with 'underpost lxd ${vmName} --vm-create'.`);
+        }
         const lxdSetupPath = `${underpostRoot}/scripts/lxd-vm-setup.sh`;
         const k3sSetupPath = `${underpostRoot}/scripts/k3s-node-setup.sh`;
 
-        // Step 1: OS base setup (disk, packages, kernel modules)
-        shellExec(`cat ${lxdSetupPath} | lxc exec ${vmName} -- bash`);
+        const fallbackIp = UnderpostLxd._allocateFallbackIp(vmName);
+        logger.info(`[${vmName}] Step 1/3: OS base setup (DHCP fallback IP: ${fallbackIp}/24)...`);
+        shellExec(
+          `cat ${lxdSetupPath} | ${UnderpostLxd._lxcCmd()} exec ${vmName} --env LXD_FALLBACK_IPV4_CIDR=${fallbackIp}/24 --env LXD_NODE_NAME=${vmName} -- bash`,
+        );
 
-        // Step 2: Push engine source from host to VM
-        await Underpost.lxd.runWorkflow({ workflowId: 'engine', vmName, dev: options.dev });
+        logger.info(`[${vmName}] Step 2/3: Bootstrapping engine source into VM...`);
+        await UnderpostLxd._bootstrapEngineSource(vmName);
 
-        // Step 3: K3s role setup (installs Node, npm deps, then k3s via node bin --dev)
+        // Step 3: K3s role setup, driven by the local engine source.
+        logger.info(`[${vmName}] Step 3/3: K3s role setup...`);
+        const baseArgs = `--engine-root=${ENGINE_ROOT_IN_VM}`;
         if (options.worker === true) {
-          if (options.joinNode) {
-            const controlNode = options.joinNode.includes(',') ? options.joinNode.split(',').pop() : options.joinNode;
-            const k3sToken = shellExec(
-              `lxc exec ${controlNode} -- bash -c 'sudo cat /var/lib/rancher/k3s/server/node-token'`,
-              { stdout: true },
-            ).trim();
-            const controlPlaneIp = shellExec(
-              `lxc list ${controlNode} --format json | jq -r '.[0].state.network.enp5s0.addresses[] | select(.family=="inet") | .address'`,
-              { stdout: true },
-            ).trim();
-            logger.info(`Initializing worker ${vmName} and joining control plane ${controlNode} (${controlPlaneIp})`);
-            shellExec(
-              `cat ${k3sSetupPath} | lxc exec ${vmName} -- bash -s -- --worker --control-ip=${controlPlaneIp} --token=${k3sToken}`,
+          if (!options.joinNode) {
+            throw new Error(
+              `--vm-init --worker requires --join-node <controlVmName>. A worker is meaningless without a control plane to join; the script would only fail after npm install completes.`,
             );
-          } else {
-            shellExec(`cat ${k3sSetupPath} | lxc exec ${vmName} -- bash -s -- --worker`);
           }
+          const controlNode = options.joinNode.includes(',') ? options.joinNode.split(',').pop() : options.joinNode;
+          const { ip: controlPlaneIp, token: k3sToken } = UnderpostLxd._readControlPlaneJoinInfo(controlNode);
+          logger.info(`[${vmName}] Joining control plane ${controlNode} (${controlPlaneIp})`);
+          shellExec(
+            `cat ${k3sSetupPath} | ${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -s -- ${baseArgs} --worker --control-ip=${controlPlaneIp} --token=${k3sToken}`,
+          );
+          UnderpostLxd._labelWorkerNodeRole(controlNode, vmName);
         } else {
-          shellExec(`cat ${k3sSetupPath} | lxc exec ${vmName} -- bash -s -- --control`);
+          shellExec(
+            `cat ${k3sSetupPath} | ${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -s -- ${baseArgs} --control`,
+          );
         }
+        logger.info(`[${vmName}] Init complete. Engine mirrored at ${ENGINE_ROOT_IN_VM}.`);
       }
 
-      if (options.workflowId) {
-        await Underpost.lxd.runWorkflow({
-          workflowId: options.workflowId,
-          vmName: options.vmId,
-          deployId: options.deployId,
-          dev: options.dev,
-        });
-      }
-
-      // Standalone join: --join-node workerName,controlName (without --init-vm)
-      if (options.joinNode && !options.initVm) {
+      // =====================================================================
+      // STANDALONE JOIN
+      // =====================================================================
+      if (options.joinNode && !vmInit) {
         const [workerNode, controlNode] = options.joinNode.split(',');
-        const k3sToken = shellExec(
-          `lxc exec ${controlNode} -- bash -c 'sudo cat /var/lib/rancher/k3s/server/node-token'`,
-          { stdout: true },
-        ).trim();
-        const controlPlaneIp = shellExec(
-          `lxc list ${controlNode} --format json | jq -r '.[0].state.network.enp5s0.addresses[] | select(.family=="inet") | .address'`,
-          { stdout: true },
-        ).trim();
+        if (!workerNode || !controlNode) {
+          throw new Error(`--join-node standalone requires 'workerName,controlName' format.`);
+        }
+        if (!UnderpostLxd._vmExists(workerNode)) {
+          throw new Error(`Worker VM '${workerNode}' does not exist.`);
+        }
+        const { ip: controlPlaneIp, token: k3sToken } = UnderpostLxd._readControlPlaneJoinInfo(controlNode);
+        const k3sSetupPath = `${underpostRoot}/scripts/k3s-node-setup.sh`;
         logger.info(`Joining K3s worker ${workerNode} to control plane ${controlNode} (${controlPlaneIp})`);
         shellExec(
-          `lxc exec ${workerNode} -- bash -c 'K3S_URL=https://${controlPlaneIp}:6443 K3S_TOKEN=${k3sToken} curl -sfL https://get.k3s.io | sh -s - agent'`,
+          `cat ${k3sSetupPath} | ${UnderpostLxd._lxcCmd()} exec ${workerNode} -- bash -s -- --engine-root=${ENGINE_ROOT_IN_VM} --worker --control-ip=${controlPlaneIp} --token=${k3sToken}`,
         );
+        UnderpostLxd._labelWorkerNodeRole(controlNode, workerNode);
         logger.info(`Worker ${workerNode} joined successfully.`);
       }
 
-      if (options.infoVm) {
-        shellExec(`lxc config show ${options.infoVm}`);
-        shellExec(`lxc info --show-log ${options.infoVm}`);
-        shellExec(`lxc info ${options.infoVm}`);
-        shellExec(`lxc list ${options.infoVm}`);
+      // =====================================================================
+      // INFO VM
+      // =====================================================================
+      if (vmInfo) {
+        if (!currentVmId) {
+          throw new Error(`--vm-info requires the [vm-id] command argument.`);
+        }
+        const vmName = currentVmId;
+        shellExec(`${UnderpostLxd._lxcCmd()} config show ${vmName}`);
+        shellExec(`${UnderpostLxd._lxcCmd()} info --show-log ${vmName}`);
+        shellExec(`${UnderpostLxd._lxcCmd()} info ${vmName}`);
+        shellExec(`${UnderpostLxd._lxcCmd()} list ${vmName}`);
       }
 
+      // =====================================================================
+      // EXPOSE (host LAN port -> VM NodePort via LXD NAT-mode proxy device)
+      //
+      // NAT-mode proxy on a VM requires the host to be the gateway (it is) and a
+      // static ipv4.address on the instance NIC. _ensureNicStaticIpv4 pins that
+      // (with security.ipv4_filtering so it's accepted on the DHCP-less bridge),
+      // then the proxy device forwards listen=host -> connect=VM, preserving the
+      // client address via NAT. When exposing known service ports (MongoDB 27017,
+      // Valkey 6379), also persist the host-side runtime env so `node src/server`
+      // on the physical host dials the LXD proxy instead of localhost defaults.
+      // =====================================================================
       if (options.expose) {
         const [vmName, ports] = options.expose.split(':');
         const protocols = ['tcp'];
         const hostIp = Underpost.dns.getLocalIPv4Address();
-        const vmIp = shellExec(
-          `lxc list ${vmName} --format json | jq -r '.[0].state.network.enp5s0.addresses[] | select(.family=="inet") | .address'`,
-          { stdout: true },
-        ).trim();
+        const exposedHostPorts = ports
+          .split(',')
+          .map((port) => port.trim())
+          .filter((port) => port.length > 0);
+        const vmIp = UnderpostLxd._vmIpv4(vmName);
         if (!vmIp) {
-          logger.error(`Could not get VM IP for ${vmName}. Cannot expose ports.`);
-          return;
+          throw new Error(`Could not resolve VM IP for ${vmName}. Cannot expose ports.`);
         }
-        for (const port of ports.split(',')) {
+        UnderpostLxd._ensureNicStaticIpv4(vmName, vmIp);
+        for (const port of exposedHostPorts) {
+          const connectPort = options.nodePort ? options.nodePort : port;
           for (const protocol of protocols) {
             const deviceName = `${vmName}-${protocol}-port-${port}`;
-            shellExec(`lxc config device remove ${vmName} ${deviceName}`);
+            if (UnderpostLxd._vmHasDevice(vmName, deviceName)) {
+              shellExec(`${UnderpostLxd._lxcCmd()} config device remove ${vmName} ${deviceName}`);
+            }
             shellExec(
-              `lxc config device add ${vmName} ${deviceName} proxy listen=${protocol}:${hostIp}:${port} connect=${protocol}:${vmIp}:${port} nat=true`,
+              `${UnderpostLxd._lxcCmd()} config device add ${vmName} ${deviceName} proxy listen=${protocol}:${hostIp}:${port} connect=${protocol}:${vmIp}:${connectPort} nat=true`,
             );
-            logger.info(`Exposed ${protocol}:${hostIp}:${port} -> ${vmIp}:${port} on ${vmName}`);
+            logger.info(`Exposed ${protocol}:${hostIp}:${port} -> ${vmIp}:${connectPort} on ${vmName}`);
           }
         }
+        if (exposedHostPorts.includes('27017') || exposedHostPorts.includes('6379')) {
+          Underpost.cluster.syncServiceConnectionEnv({
+            serviceHost: hostIp,
+            mongodb: exposedHostPorts.includes('27017'),
+            valkey: exposedHostPorts.includes('6379'),
+            options,
+          });
+        }
       }
 
+      // =====================================================================
+      // DELETE EXPOSE
+      // =====================================================================
       if (options.deleteExpose) {
         const [vmName, ports] = options.deleteExpose.split(':');
         const protocols = ['tcp'];
         for (const port of ports.split(',')) {
           for (const protocol of protocols) {
-            shellExec(`lxc config device remove ${vmName} ${vmName}-${protocol}-port-${port}`);
+            const deviceName = `${vmName}-${protocol}-port-${port}`;
+            if (UnderpostLxd._vmHasDevice(vmName, deviceName)) {
+              shellExec(`${UnderpostLxd._lxcCmd()} config device remove ${vmName} ${deviceName}`);
+            } else {
+              logger.info(`Device ${deviceName} not present on ${vmName}; skipping.`);
+            }
           }
         }
       }
 
-      if (options.test) {
-        const vmName = options.test;
-        const vmIp = shellExec(
-          `lxc list ${vmName} --format json | jq -r '.[0].state.network.enp5s0.addresses[] | select(.family=="inet") | .address'`,
-          { stdout: true },
-        ).trim();
+      // =====================================================================
+      // SYNC ENGINE (re-copy host engine source into VM)
+      // =====================================================================
+      if (vmSyncEngine) {
+        if (!currentVmId) {
+          throw new Error(`--vm-sync-engine requires the [vm-id] command argument.`);
+        }
+        const vmName = currentVmId;
+        if (!UnderpostLxd._vmExists(vmName)) {
+          throw new Error(`VM '${vmName}' does not exist.`);
+        }
+        logger.info(`[${vmName}] Syncing engine source from host...`);
+        await UnderpostLxd._bootstrapEngineSource(vmName);
+        UnderpostLxd._execVmNodeCommand(vmName, `cd ${ENGINE_ROOT_IN_VM} && npm install`, { requireNpm: true });
+        logger.info(`[${vmName}] Engine source sync complete.`);
+        return;
+      }
+
+      // =====================================================================
+      // TEST (connectivity and health checks)
+      // =====================================================================
+      if (vmTest) {
+        if (!currentVmId) {
+          throw new Error(`--vm-test requires the [vm-id] command argument.`);
+        }
+        const vmName = currentVmId;
+        const vmIp = UnderpostLxd._vmIpv4(vmName);
         logger.info(`VM ${vmName} IPv4: ${vmIp || 'none'}`);
         const httpStatus = shellExec(
-          `lxc exec ${vmName} -- curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://google.com`,
+          `${UnderpostLxd._lxcCmd()} exec ${vmName} -- curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://google.com`,
           { stdout: true },
         ).trim();
         logger.info(`VM ${vmName} HTTP connectivity: ${httpStatus}`);
         logger.info(`Health report for VM: ${vmName}`);
-        shellExec(`lxc list ${vmName} --format json`);
-        shellExec(`lxc exec ${vmName} -- bash -c 'top -bn1 | grep "Cpu(s)"'`);
-        shellExec(`lxc exec ${vmName} -- bash -c 'free -m'`);
-        shellExec(`lxc exec ${vmName} -- bash -c 'df -h /'`);
-        shellExec(`lxc exec ${vmName} -- bash -c 'ip a'`);
-        shellExec(`lxc exec ${vmName} -- bash -c 'cat /etc/resolv.conf'`);
-        shellExec(`lxc exec ${vmName} -- bash -c 'sudo k3s kubectl get nodes'`);
+        shellExec(`${UnderpostLxd._lxcCmd()} list ${vmName} --format json`);
+        shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'top -bn1 | grep "Cpu(s)"'`);
+        shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'free -m'`);
+        shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'df -h /'`);
+        shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'ip a'`);
+        shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'cat /etc/resolv.conf'`);
+        shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'sudo k3s kubectl get nodes'`);
       }
     },
+  };
 
-    /**
-     * @method pushDirectory
-     * @description Pushes a host directory into a VM using ignore-walk (respecting .gitignore)
-     * and a tar pipe. Skips gitignored paths (e.g. node_modules, .git, build artifacts).
-     * @param {object} params
-     * @param {string} params.srcPath - Absolute path of the source directory on the host.
-     * @param {string} params.vmName - Target LXD VM name.
-     * @param {string} params.destPath - Absolute path of the destination directory inside the VM.
-     * @param {string[]} [params.ignoreFiles=['.gitignore']] - Ignore-file names to respect during walk.
-     * @returns {Promise<void>}
-     * @memberof UnderpostLxd
-     */
-    async pushDirectory({ srcPath, vmName, destPath, ignoreFiles }) {
-      const includesFile = `/tmp/lxd-push-${vmName}-${Date.now()}.txt`;
-      if (!ignoreFiles) ignoreFiles = ['.gitignore'];
-      // Collect non-ignored files via ignore-walk
-      const files = await new Promise((resolve) =>
-        walk(
-          {
-            path: srcPath,
-            ignoreFiles,
-            includeEmpty: false,
-            follow: false,
-          },
-          (_, result) => resolve(result),
-        ),
-      );
+  // =====================================================================
+  // PRIVATE HELPERS — lookups that legitimately tolerate "absent" return
+  // values do so via list-style commands that always exit 0, not by
+  // suppressing error signals from destructive commands.
+  // =====================================================================
+
+  /**
+   * Lists all LXD VM (virtual-machine) instance names. Returns [] when no VMs.
+   * `lxc list --format json` always exits 0; an empty cluster yields `[]`.
+   * @returns {string[]}
+   * @private
+   */
+  static _listVms() {
+    const raw = shellExec(
+      `${UnderpostLxd._lxcCmd()} list --format json | jq -r '.[] | select(.type=="virtual-machine") | .name // empty'`,
+      {
+        stdout: true,
+      },
+    ).trim();
+    if (!raw) return [];
+    return raw.split('\n').filter((n) => n.length > 0);
+  }
+
+  /**
+   * Returns the named VM's status string (e.g. 'Running', 'Stopped', 'Frozen')
+   * or `null` if the VM does not exist. Never throws on absence.
+   * @param {string} vmName
+   * @returns {string|null}
+   * @private
+   */
+  static _vmState(vmName) {
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} list ${vmName} --format json`, { stdout: true }).trim();
+    if (!raw) return null;
+    const arr = JSON.parse(raw);
+    const inst = Array.isArray(arr) ? arr.find((i) => i?.name === vmName) : null;
+    return inst ? inst.status || 'Unknown' : null;
+  }
 
-      // Write relative paths (one per line) to a temp includes file
-      fs.writeFileSync(includesFile, files.join('\n'));
-      logger.info(`lxd pushDirectory: ${files.length} files collected`, { srcPath, vmName, destPath, includesFile });
+  /**
+   * @param {string} vmName
+   * @returns {boolean}
+   * @private
+   */
+  static _vmExists(vmName) {
+    return UnderpostLxd._vmState(vmName) !== null;
+  }
 
-      // Reset destination directory inside the VM
-      shellExec(`lxc exec ${vmName} -- bash -c 'rm -rf ${destPath} && mkdir -p ${destPath}'`);
+  /**
+   * Resolves the VM's primary IPv4, preferring the guest interface that owns
+   * the default route. This avoids selecting K3s bridge/CNI addresses like
+   * 10.42.0.1 after the control plane comes up.
+   * @param {string} vmName
+   * @returns {string}
+   * @private
+   */
+  static _vmIpv4(vmName) {
+    const defaultRoute = shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- ip -4 -o route show default`, {
+      stdout: true,
+    }).trim();
+    const defaultRouteTokens = defaultRoute ? defaultRoute.split(/\s+/) : [];
+    const devIndex = defaultRouteTokens.indexOf('dev');
+    const defaultIface = devIndex >= 0 ? defaultRouteTokens[devIndex + 1] || '' : '';
 
-      // Stream tar archive from host into VM
+    if (defaultIface) {
+      const defaultIfaceAddr = shellExec(
+        `${UnderpostLxd._lxcCmd()} exec ${vmName} -- ip -4 -o addr show dev ${defaultIface} scope global`,
+        {
+          stdout: true,
+        },
+      ).trim();
+      const routeScopedIp = defaultIfaceAddr.match(/\binet\s+([0-9.]+)\//)?.[1] || '';
+      if (routeScopedIp) return routeScopedIp;
+    }
+
+    return shellExec(
+      `${UnderpostLxd._lxcCmd()} list ${vmName} --format json | jq -r '[.[0].state.network | to_entries[] | select(.key!="lo") | .value.addresses[]? | select(.family=="inet" and .scope=="global") | .address | select(test("^10\\.42\\.|^10\\.43\\.|^169\\.254\\.") | not)] | .[0] // empty'`,
+      { stdout: true },
+    ).trim();
+  }
+
+  /**
+   * Pins the VM's lxdbr0 NIC to a static `ipv4.address` equal to `vmIp`, which
+   * NAT-mode proxy devices require. `security.ipv4_filtering=true` is set in the
+   * same call: lxdbr0 runs no DHCP, and LXD only permits a static NIC address
+   * on a DHCP-less managed bridge when filtering is enabled (it also anti-spoofs
+   * the VM to that IP). The eth0 NIC comes from admin-profile, so override it on
+   * first touch and set thereafter. Same IP as the current lease -> no disruption.
+   * @param {string} vmName
+   * @param {string} vmIp
+   * @private
+   */
+  static _ensureNicStaticIpv4(vmName, vmIp) {
+    const nic = 'eth0';
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} list ${vmName} --format json`, { stdout: true }).trim();
+    const arr = JSON.parse(raw || '[]');
+    const inst = Array.isArray(arr) ? arr.find((i) => i?.name === vmName) : null;
+    const instanceDevices = inst?.devices || {};
+    const currentStatic = instanceDevices[nic]?.['ipv4.address'] || '';
+    const hasLocalNic = !!instanceDevices[nic];
+    const verb = hasLocalNic ? 'set' : 'override';
+    if (currentStatic !== vmIp) {
       shellExec(
-        `tar -C ${srcPath} -cf - --files-from=${includesFile} | lxc exec ${vmName} -- tar -C ${destPath} -xf -`,
+        `${UnderpostLxd._lxcCmd()} config device ${verb} ${vmName} ${nic} ipv4.address=${vmIp} security.ipv4_filtering=true`,
       );
+      logger.info(`  Pinned ${vmName} NIC ${nic} to static ${vmIp} (required for NAT proxy on VMs).`);
+    } else {
+      shellExec(`${UnderpostLxd._lxcCmd()} config device set ${vmName} ${nic} security.ipv4_filtering=true`, {
+        silentOnError: true,
+      });
+      logger.info(`  NIC ${nic} on ${vmName} already pinned to static ${vmIp}.`);
+    }
+  }
 
-      // Clean up temp includes file
-      fs.removeSync(includesFile);
-    },
+  /**
+   * Returns true if a named device is currently attached (expanded) to the VM.
+   * @param {string} vmName
+   * @param {string} deviceName
+   * @returns {boolean}
+   * @private
+   */
+  static _vmHasDevice(vmName, deviceName) {
+    if (!UnderpostLxd._vmExists(vmName)) return false;
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} list ${vmName} --format json`, { stdout: true }).trim();
+    const arr = JSON.parse(raw || '[]');
+    const inst = Array.isArray(arr) ? arr.find((i) => i?.name === vmName) : null;
+    if (!inst) return false;
+    return Object.prototype.hasOwnProperty.call(inst.expanded_devices || {}, deviceName);
+  }
 
-    /**
-     * @method runWorkflow
-     * @description Executes predefined workflows on LXD VMs.
-     * @param {object} params
-     * @param {string} params.workflowId - Workflow ID to execute.
-     * @param {string} params.vmName - Target VM name.
-     * @param {string} [params.deployId] - Deployment identifier.
-     * @param {boolean} [params.dev=false] - Use local paths.
-     * @memberof UnderpostLxd
-     */
-    async runWorkflow({ workflowId, vmName, deployId, dev }) {
-      switch (workflowId) {
-        case 'engine': {
-          await Underpost.lxd.pushDirectory({
-            srcPath: `/home/dd/engine`,
-            vmName,
-            destPath: `/home/dd/engine`,
-          });
-          await Underpost.lxd.pushDirectory({
-            srcPath: `/home/dd/engine/engine-private`,
-            vmName,
-            destPath: `/home/dd/engine/engine-private`,
-            ignoreFiles: ['/home/dd/engine/.gitignore', '.gitignore'],
-          });
-          break;
-        }
-        case 'engine-recursive-push': {
-          const enginePath = '/home/dd/engine';
-          shellExec(`lxc exec ${vmName} -- bash -c 'rm -rf ${enginePath}'`);
-          shellExec(`lxc exec ${vmName} -- bash -c 'mkdir -p /home/dd'`);
-          shellExec(`lxc file push ${enginePath} ${vmName}/home/dd --recursive`);
-          break;
-        }
-        case 'dev-reset': {
-          shellExec(
-            `lxc exec ${vmName} -- bash -lc 'cd /home/dd/engine && node bin cluster --dev --reset --k3s && node bin cluster --dev --k3s'`,
-          );
-          break;
-        }
+  /**
+   * @param {string} name
+   * @returns {boolean}
+   * @private
+   */
+  static _profileExists(name) {
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} profile list --format json`, { stdout: true }).trim();
+    const arr = JSON.parse(raw || '[]');
+    return Array.isArray(arr) && arr.some((p) => p?.name === name);
+  }
+
+  /**
+   * @param {string} name
+   * @returns {boolean}
+   * @private
+   */
+  static _networkExists(name) {
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} network list --format json`, { stdout: true }).trim();
+    const arr = JSON.parse(raw || '[]');
+    return Array.isArray(arr) && arr.some((n) => n?.name === name);
+  }
+
+  /**
+   * True once `lxd init --preseed` has bootstrapped this daemon. Detected by the
+   * presence of any storage pool: the preseed creates `local`, and a fresh
+   * daemon has none. `lxc storage list` exits 0 with `[]` before init, so this
+   * never throws on a not-yet-initialized host.
+   * @returns {boolean}
+   * @private
+   */
+  static _lxdInitialized() {
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} storage list --format json`, {
+      stdout: true,
+      silentOnError: true,
+    }).trim();
+    if (!raw) return false;
+    let arr;
+    try {
+      arr = JSON.parse(raw);
+    } catch {
+      return false;
+    }
+    return Array.isArray(arr) && arr.length > 0;
+  }
+
+  /**
+   * Adds the bridge to the firewalld `trusted` zone so VM<->host and VM
+   * outbound traffic isn't dropped by the host firewall. Idempotent and
+   * resilient: re-adding an already-trusted interface is a no-op, and hosts
+   * without firewalld are skipped rather than aborting bridge creation.
+   * @param {string} bridge
+   * @private
+   */
+  static _ensureBridgeInTrustedZone(bridge) {
+    const hasFirewalld = shellExec(`command -v firewall-cmd`, { stdout: true, silentOnError: true }).trim();
+    if (!hasFirewalld) {
+      logger.info(`firewall-cmd not found; skipping trusted-zone binding for ${bridge}.`);
+      return;
+    }
+    shellExec(`sudo firewall-cmd --permanent --zone=trusted --add-interface=${bridge}`, { silentOnError: true });
+    shellExec(`sudo firewall-cmd --reload`, { silentOnError: true });
+  }
+
+  /**
+   * Explicitly accepts forwarded traffic to/from the plain bridge in the
+   * iptables FORWARD chain. A LXD-managed bridge inserts these itself
+   * (`ipv4.firewall=true`); a plain bridge does not, so on a host where Docker
+   * has set the FORWARD policy to DROP, VM<->VM traffic on lxdbr0 (e.g. a k3s
+   * worker dialing the control plane API) is silently dropped once br_netfilter
+   * routes bridged frames through netfilter. Rules are prepended (position 1) so
+   * they win over Docker's DROP, and guarded by `-C` so re-runs don't duplicate.
+   * Not persisted across reboots/iptables flush by design — re-run
+   * `--create-virtual-network`, consistent with the rest of this lab flow.
+   * @param {string} bridge
+   * @private
+   */
+  static _ensureBridgeForwardingAccept(bridge) {
+    const hasIptables = shellExec(`command -v iptables`, { stdout: true, silentOnError: true }).trim();
+    if (!hasIptables) {
+      logger.info(`iptables not found; skipping FORWARD accept rules for ${bridge}.`);
+      return;
+    }
+    for (const dir of ['-i', '-o']) {
+      const present = shellExec(`sudo iptables -C FORWARD ${dir} ${bridge} -j ACCEPT`, { silentOnError: true });
+      if (present.code !== 0) {
+        shellExec(`sudo iptables -I FORWARD 1 ${dir} ${bridge} -j ACCEPT`);
       }
-    },
-  };
+    }
+    logger.info(`Ensured FORWARD ACCEPT for ${bridge} (counters Docker/default DROP for VM<->VM traffic).`);
+  }
+
+  /**
+   * Returns true if a snap with the given name is installed. `snap list` exits
+   * 0 with the full installed-snap table; we grep for an exact-name row.
+   * @param {string} name
+   * @returns {boolean}
+   * @private
+   */
+  static _snapInstalled(name) {
+    const raw = shellExec(`snap list`, { stdout: true });
+    return raw.split('\n').some((line) => new RegExp(`^${name}\\s`).test(line));
+  }
+
+  /**
+   * Single-quotes a shell argument for safe `bash -lc '...'` usage.
+   * @param {string} value
+   * @returns {string}
+   * @private
+   */
+  static _shellSingleQuote(value) {
+    return `'${`${value}`.replace(/'/g, `'\\''`)}'`;
+  }
+
+  /**
+   * Runs a command inside a VM with Node/NPM restored from the NVM install
+   * that k3s-node-setup.sh lays down. Non-login `bash -c` shells do not keep
+   * that PATH, so resolve it explicitly here.
+   * @param {string} vmName
+   * @param {string} command
+   * @param {object} [options]
+   * @param {boolean} [options.requireNpm=false]
+   * @param {number} [options.timeoutSeconds=0]
+   * @private
+   */
+  static _execVmNodeCommand(vmName, command, options = { requireNpm: false, timeoutSeconds: 0 }) {
+    const { requireNpm = false, timeoutSeconds = 0 } = options;
+    const runtimeBootstrap = [
+      'export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"',
+      '[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"',
+      `if ! command -v node >/dev/null 2>&1${requireNpm ? ' || ! command -v npm >/dev/null 2>&1' : ''}; then latest_nvm_bin="$(ls -d "$NVM_DIR"/versions/node/*/bin 2>/dev/null | sort -V | tail -n 1)"; if [ -n "$latest_nvm_bin" ]; then export PATH="$latest_nvm_bin:$PATH"; fi; fi`,
+      'command -v node >/dev/null 2>&1 || { echo "ERROR: node not found in PATH or NVM_DIR=$NVM_DIR" >&2; exit 127; }',
+      requireNpm
+        ? 'command -v npm >/dev/null 2>&1 || { echo "ERROR: npm not found in PATH or NVM_DIR=$NVM_DIR" >&2; exit 127; }'
+        : '',
+      command,
+    ]
+      .filter(Boolean)
+      .join(' && ');
+    const lxcExecCmd = `${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -lc ${UnderpostLxd._shellSingleQuote(runtimeBootstrap)}`;
+    return shellExec(timeoutSeconds > 0 ? `timeout ${timeoutSeconds} ${lxcExecCmd}` : lxcExecCmd);
+  }
+
+  /**
+   * Deterministic per-VM IPv4 in the lxdbr0 /24, used as the static fallback
+   * inside lxd-vm-setup.sh when DHCP is unavailable. The previous shared
+   * `10.250.250.100/24` fallback caused IP collisions across VMs and broke
+   * worker→control K3s joins (the worker dialed its own NIC). Offset is the
+   * sum of vmName char codes mod 253, +2 — avoids .0, .1 (gateway), .255.
+   * @param {string} vmName
+   * @returns {string}
+   * @private
+   */
+  static _allocateFallbackIp(vmName) {
+    let sum = 0;
+    for (let i = 0; i < vmName.length; i++) sum += vmName.charCodeAt(i);
+    return `${BRIDGE_SUBNET_PREFIX}.${(sum % 253) + 2}`;
+  }
+
+  /**
+   * Reads the K3s join info (control plane IPv4 + node token) from the control
+   * VM. The control must already be running — VMs created with `admin-profile`
+   * have `boot.autostart=false`, so after a host reboot bring it up explicitly
+   * ('lxc start <control>' or 'node bin lxd --restore') before joining a worker.
+   * Throws if either value is missing — callers depend on both.
+   * @param {string} controlNode
+   * @returns {{ip: string, token: string}}
+   * @private
+   */
+  static _readControlPlaneJoinInfo(controlNode) {
+    const state = UnderpostLxd._vmState(controlNode);
+    if (state === null) {
+      throw new Error(`Control node VM '${controlNode}' does not exist.`);
+    }
+    if (state !== 'Running') {
+      throw new Error(
+        `Control node VM '${controlNode}' is ${state}. Start it first ('lxc start ${controlNode}' or 'node bin lxd --restore'), then re-run the worker join.`,
+      );
+    }
+    const token = shellExec(
+      `${UnderpostLxd._lxcCmd()} exec ${controlNode} -- bash -c 'sudo cat /var/lib/rancher/k3s/server/node-token'`,
+      {
+        stdout: true,
+      },
+    ).trim();
+    const ip = UnderpostLxd._vmIpv4(controlNode);
+    if (!ip || !token) {
+      throw new Error(`Could not read join info from control node '${controlNode}' (ip='${ip}', token='${token}').`);
+    }
+    return { ip, token };
+  }
+
+  /**
+   * Applies the `node-role.kubernetes.io/worker` label to a freshly joined
+   * worker. A K3s agent cannot self-apply `node-role.kubernetes.io/*` labels
+   * (the NodeRestriction admission plugin rejects them), so the label must be
+   * set from the control plane after the worker registers — otherwise the node
+   * shows ROLES `<none>`. The K3s node name defaults to the VM hostname, which
+   * LXD sets to the instance name, so `workerName` is the VM name. Waits up to
+   * 60s for the node to appear before labeling.
+   * @param {string} controlNode
+   * @param {string} workerName
+   * @private
+   */
+  static _labelWorkerNodeRole(controlNode, workerName) {
+    logger.info(`Labeling worker '${workerName}' as node-role.kubernetes.io/worker (from control '${controlNode}')...`);
+    shellExec(
+      `${UnderpostLxd._lxcCmd()} exec ${controlNode} -- bash -c 'for i in $(seq 1 30); do if sudo k3s kubectl get node ${workerName} >/dev/null 2>&1; then sudo k3s kubectl label node ${workerName} node-role.kubernetes.io/worker=worker --overwrite && exit 0; fi; sleep 2; done; echo "WARN: worker ${workerName} did not register within 60s; role label not applied." >&2'`,
+    );
+  }
+
+  /**
+   * Enumerates and removes every device of `type: proxy` attached to a VM
+   * (the `--expose` NAT proxy devices). Naming-agnostic. Skips if the VM is
+   * already gone; otherwise every `lxc config device remove` propagates errors.
+   * @param {string} vmName
+   * @private
+   */
+  static _removeProxyDevices(vmName) {
+    if (!UnderpostLxd._vmExists(vmName)) {
+      logger.info(`  Skipping proxy cleanup: VM '${vmName}' is already gone.`);
+      return;
+    }
+    logger.info(`  Removing proxy devices from ${vmName}...`);
+    const raw = shellExec(`${UnderpostLxd._lxcCmd()} list ${vmName} --format json`, { stdout: true }).trim();
+    const arr = JSON.parse(raw || '[]');
+    const inst = Array.isArray(arr) ? arr.find((i) => i?.name === vmName) : null;
+    const expandedDevices = inst?.expanded_devices || {};
+    const proxyNames = Object.entries(expandedDevices)
+      .filter(([, dev]) => dev?.type === 'proxy')
+      .map(([name]) => name);
+    if (proxyNames.length === 0) {
+      logger.info(`  No proxy devices found on ${vmName}.`);
+      return;
+    }
+    for (const name of proxyNames) {
+      logger.info(`    Removing device: ${name}`);
+      shellExec(`${UnderpostLxd._lxcCmd()} config device remove ${vmName} ${name}`);
+    }
+  }
+
+  /**
+   * Delegates K3s teardown inside a running VM to the centralized
+   * `safeResetK3s` in src/cli/cluster.js via `lxc exec`. No-op when K3s or the
+   * engine mirror is missing. Bounded by `timeout 300`.
+   * @param {string} vmName
+   * @param {'drain'|'full'} resetMode - `drain` preserves K3s for next boot
+   *   (`--shutdown`); `full` uninstalls (`--vm-delete` / `--reset` / `--purge`).
+   * @private
+   */
+  static _resetK3sInVm(vmName, resetMode) {
+    if (UnderpostLxd._vmState(vmName) !== 'Running') return;
+    const m = resetMode === 'drain' ? 'drain' : 'full';
+    const probe = `if test -x /usr/local/bin/k3s && test -d ${ENGINE_ROOT_IN_VM}/bin; then echo yes; else echo no; fi`;
+    const probeOut = shellExec(`${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c '${probe}'`, {
+      stdout: true,
+    }).trim();
+    if (probeOut !== 'yes') {
+      logger.info(`  [${vmName}] No K3s+engine detected (probe=${probeOut}); skipping K3s reset.`);
+      return;
+    }
+    logger.info(`  [${vmName}] Resetting K3s (resetMode=${m}) via 'node bin cluster --reset --k3s --reset-mode=${m}'`);
+    UnderpostLxd._execVmNodeCommand(
+      vmName,
+      `cd ${ENGINE_ROOT_IN_VM} && node bin cluster --dev --reset --k3s --reset-mode=${m}`,
+      { timeoutSeconds: 300 },
+    );
+  }
+
+  /**
+   * Safely deletes a single VM. Pre-conditions gate every step; absence is a
+   * no-op, but unexpected failures propagate.
+   *
+   *   1. If VM is absent → log and return.
+   *   2. Remove every proxy device (clears iptables NAT before the VM goes away).
+   *   3. If state is Running/Frozen → graceful stop with 30 s timeout.
+   *   4. Delete the VM.
+   *
+   * @param {string} vmName
+   * @private
+   */
+  static _safeDeleteVm(vmName) {
+    const state = UnderpostLxd._vmState(vmName);
+    if (state === null) {
+      logger.info(`VM '${vmName}' does not exist. Nothing to do.`);
+      return;
+    }
+    logger.info(`Deleting VM '${vmName}' (current state: ${state})...`);
+    UnderpostLxd._removeProxyDevices(vmName);
+    if (state === 'Running' || state === 'Frozen') {
+      UnderpostLxd._resetK3sInVm(vmName, 'full');
+      logger.info(`  Stopping VM: ${vmName}`);
+      shellExec(`${UnderpostLxd._lxcCmd()} stop ${vmName} --timeout 60`);
+    }
+    logger.info(`  Deleting VM: ${vmName}`);
+    shellExec(`${UnderpostLxd._lxcCmd()} delete ${vmName}`);
+    logger.info(`VM ${vmName} deleted.`);
+  }
+
+  /**
+   * Host-safe reset. Wipes per-VM state and the network/profile this CLI owns.
+   * Leaves the LXD snap and storage pools intact so the host stays bootable
+   * even if the daemon has internal issues. Use `--purge` for snap removal.
+   *
+   *   Phase 1: Remove proxy devices from every VM (clears iptables NAT rules).
+   *   Phase 2: Stop running VMs gracefully (30 s timeout each).
+   *   Phase 3: Delete every VM.
+   *   Phase 4: Drop `admin-profile` and the `lxdbr0` network if they exist.
+   *
+   * @private
+   */
+  static _safeReset() {
+    logger.info('=== LXD RESET (host-safe) ===');
+    const vmList = UnderpostLxd._listVms();
+
+    logger.info(`Phase 1/4: Removing proxy devices from ${vmList.length} VM(s)...`);
+    for (const vmName of vmList) {
+      UnderpostLxd._removeProxyDevices(vmName);
+    }
+
+    logger.info('Phase 2/4: Full K3s teardown + stopping running VMs gracefully...');
+    for (const vmName of vmList) {
+      const state = UnderpostLxd._vmState(vmName);
+      if (state === 'Running' || state === 'Frozen') {
+        UnderpostLxd._resetK3sInVm(vmName, 'full');
+        logger.info(`  Stopping VM: ${vmName}`);
+        shellExec(`${UnderpostLxd._lxcCmd()} stop ${vmName} --timeout 60`);
+      } else if (state !== null) {
+        logger.info(`  VM ${vmName} already in state: ${state}`);
+      }
+    }
+
+    logger.info('Phase 3/4: Deleting all VMs...');
+    for (const vmName of vmList) {
+      if (UnderpostLxd._vmExists(vmName)) {
+        logger.info(`  Deleting VM: ${vmName}`);
+        shellExec(`${UnderpostLxd._lxcCmd()} delete ${vmName}`);
+      }
+    }
+
+    logger.info(`Phase 4/4: Removing ${ADMIN_PROFILE} and ${BRIDGE_NETWORK} if present...`);
+    if (UnderpostLxd._profileExists(ADMIN_PROFILE)) {
+      shellExec(`${UnderpostLxd._lxcCmd()} profile delete ${ADMIN_PROFILE}`);
+    }
+    if (UnderpostLxd._networkExists(BRIDGE_NETWORK)) {
+      shellExec(`${UnderpostLxd._lxcCmd()} network delete ${BRIDGE_NETWORK}`);
+    }
+
+    logger.info('=== LXD RESET COMPLETE ===');
+    logger.info('Snap and storage pools were NOT touched. Use --purge to remove the LXD snap.');
+  }
+
+  /**
+   * Removes the LXD snap. ALWAYS preceded by `lxd shutdown --timeout 60` so
+   * the daemon flushes the ZFS pool cleanly. Without that flush, removing the
+   * snap while VMs are running and the pool is dirty has historically left the
+   * host unbootable. This is the safe variant.
+   *
+   * @private
+   */
+  static _safePurge() {
+    logger.info('=== LXD PURGE (DESTRUCTIVE) ===');
+    if (!UnderpostLxd._snapInstalled('lxd')) {
+      logger.info('LXD snap is not installed. Nothing to purge.');
+      return;
+    }
+    // Drain K3s inside every VM before lxd shutdown so containerd unmounts
+    // cleanly and the ZFS pool isn't dirty when the daemon flushes.
+    const vmList = UnderpostLxd._listVms();
+    if (vmList.length > 0) {
+      logger.info(`Phase 1/3: Full K3s teardown inside ${vmList.length} VM(s)...`);
+      for (const vmName of vmList) UnderpostLxd._resetK3sInVm(vmName, 'full');
+    } else {
+      logger.info('Phase 1/3: No VMs to process.');
+    }
+    logger.info('Phase 2/3: Asking LXD daemon to shut down cleanly (60s timeout)...');
+    // `lxd` lives at /snap/bin/lxd which is not in sudo's secure_path on most
+    // distros. Forward PATH explicitly so sudo can resolve the binary.
+    shellExec(`sudo env PATH="$PATH:/snap/bin" lxd shutdown --timeout 60`);
+    logger.info('Phase 3/3: Removing LXD snap and ALL its data (instances, storage pools)...');
+    shellExec(`sudo snap remove lxd --purge`);
+    logger.info('=== LXD PURGE COMPLETE ===');
+  }
+
+  /**
+   * Pre-host-reboot procedure. Gracefully stops every running VM, then asks
+   * the LXD daemon to shut down. Run this before `reboot` / `poweroff` so the
+   * storage pool is clean on next boot.
+   *
+   * @private
+   */
+  static _gracefulShutdownAll() {
+    logger.info('=== LXD GRACEFUL SHUTDOWN (pre-host-reboot) ===');
+    const vmList = UnderpostLxd._listVms();
+    for (const vmName of vmList) {
+      const state = UnderpostLxd._vmState(vmName);
+      if (state === 'Running' || state === 'Frozen') {
+        UnderpostLxd._resetK3sInVm(vmName, 'drain');
+        logger.info(`  Stopping VM: ${vmName} (timeout 60s)`);
+        shellExec(`${UnderpostLxd._lxcCmd()} stop ${vmName} --timeout 60`);
+      } else {
+        logger.info(`  VM ${vmName} already in state: ${state}`);
+      }
+    }
+    if (UnderpostLxd._snapInstalled('lxd')) {
+      logger.info('Asking LXD daemon to shut down cleanly (timeout 60s)...');
+      // sudo's secure_path excludes /snap/bin on most distros — forward PATH.
+      shellExec(`sudo env PATH="$PATH:/snap/bin" lxd shutdown --timeout 60`);
+    }
+    logger.info('=== HOST IS SAFE TO REBOOT/POWEROFF ===');
+  }
+
+  /**
+   * Symmetric counterpart to `_gracefulShutdownAll`. Brings the lab back up:
+   *
+   *   1. Start the LXD daemon via `snap start lxd` (idempotent).
+   *   2. Wait up to 30 s for `lxc info` to respond, so we don't race the
+   *      daemon's socket-bring-up.
+   *   3. Start every VM that exists. Skips VMs that are already Running.
+   *
+   * VMs created with `admin-profile` have `boot.autostart=false` by design
+   * (host-safety), so this command is how you explicitly bring them online.
+   *
+   * @private
+   */
+  static _restoreAll() {
+    logger.info('=== LXD RESTORE (bring lab back up) ===');
+    if (!UnderpostLxd._snapInstalled('lxd')) {
+      throw new Error('LXD snap is not installed; nothing to restore.');
+    }
+    logger.info('Starting LXD daemon...');
+    shellExec(`sudo snap start lxd`);
+
+    // Wait for the daemon's REST socket to be responsive before issuing
+    // instance commands. `lxc info` (no args) is the cheapest readiness probe.
+    logger.info('Waiting for LXD daemon to become responsive...');
+    let ready = false;
+    for (let i = 0; i < 15; i++) {
+      try {
+        shellExec(`lxc info`, { stdout: true });
+        ready = true;
+        break;
+      } catch (err) {
+        if (i === 0) logger.info(`  (daemon not ready yet: ${err.message.split('\n')[0]})`);
+      }
+      shellExec(`sleep 2`);
+    }
+    if (!ready) {
+      throw new Error('LXD daemon did not become responsive within 30s.');
+    }
+    logger.info('LXD daemon is responsive.');
+
+    const vmList = UnderpostLxd._listVms();
+    logger.info(`Starting ${vmList.length} VM(s)...`);
+    for (const vmName of vmList) {
+      const state = UnderpostLxd._vmState(vmName);
+      if (state === 'Running') {
+        logger.info(`  ${vmName} already running.`);
+      } else {
+        logger.info(`  Starting VM: ${vmName} (was: ${state})`);
+        shellExec(`${UnderpostLxd._lxcCmd()} start ${vmName}`);
+      }
+    }
+    logger.info('=== LXD RESTORE COMPLETE ===');
+  }
+
+  /**
+   * Replicates `/home/dd/engine` on the host into the VM, respecting the
+   * project `.gitignore`. If `engine-private/` exists on the host it is
+   * pushed in a second pass (it is gitignored at the root by design).
+   *
+   * Idempotent: replaces only the contents of `ENGINE_ROOT_IN_VM`, not the
+   * directory inode (avoids races with running watchers / shells inside the VM).
+   *
+   * @param {string} vmName
+   * @private
+   */
+  static async _bootstrapEngineSource(vmName) {
+    if (!UnderpostLxd._vmExists(vmName)) {
+      throw new Error(`Cannot bootstrap engine into '${vmName}': VM does not exist.`);
+    }
+    if (!fs.existsSync(ENGINE_ROOT_ON_HOST)) {
+      throw new Error(`Host engine source missing at ${ENGINE_ROOT_ON_HOST}.`);
+    }
+
+    const includesFile = `/tmp/lxd-push-${vmName}-${Date.now()}.txt`;
+    const files = await new Promise((resolve, reject) =>
+      walk(
+        { path: ENGINE_ROOT_ON_HOST, ignoreFiles: ['.gitignore'], includeEmpty: false, follow: false },
+        (err, result) => (err ? reject(err) : resolve(result)),
+      ),
+    );
+    fs.writeFileSync(includesFile, files.join('\n'));
+
+    shellExec(
+      `${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'mkdir -p ${ENGINE_ROOT_IN_VM} && find ${ENGINE_ROOT_IN_VM} -mindepth 1 -delete'`,
+    );
+    shellExec(
+      `tar -C ${ENGINE_ROOT_ON_HOST} -cf - --files-from=${includesFile} | ${UnderpostLxd._lxcCmd()} exec ${vmName} -- tar -C ${ENGINE_ROOT_IN_VM} -xf -`,
+    );
+    fs.removeSync(includesFile);
+
+    const privateSrcPath = `${ENGINE_ROOT_ON_HOST}/engine-private`;
+    if (fs.existsSync(privateSrcPath)) {
+      const privateFiles = await new Promise((resolve, reject) =>
+        walk(
+          { path: privateSrcPath, ignoreFiles: ['.gitignore'], includeEmpty: false, follow: false },
+          (err, result) => (err ? reject(err) : resolve(result)),
+        ),
+      );
+      const privateIncludes = `/tmp/lxd-push-${vmName}-private-${Date.now()}.txt`;
+      fs.writeFileSync(privateIncludes, privateFiles.join('\n'));
+      shellExec(
+        `${UnderpostLxd._lxcCmd()} exec ${vmName} -- bash -c 'mkdir -p ${ENGINE_ROOT_IN_VM}/engine-private && find ${ENGINE_ROOT_IN_VM}/engine-private -mindepth 1 -delete'`,
+      );
+      shellExec(
+        `tar -C ${privateSrcPath} -cf - --files-from=${privateIncludes} | ${UnderpostLxd._lxcCmd()} exec ${vmName} -- tar -C ${ENGINE_ROOT_IN_VM}/engine-private -xf -`,
+      );
+      fs.removeSync(privateIncludes);
+    }
+
+    logger.info(`  Engine source mirrored into ${vmName}:${ENGINE_ROOT_IN_VM}`);
+  }
 }
 
 export default UnderpostLxd;