underpost 3.2.80 → 3.3.0

package/src/db/mongo/MongoBootstrap.js

@@ -11,16 +11,29 @@
 import fs from 'fs-extra';
 import { loggerFactory } from '../../server/logger.js';
 import { shellExec } from '../../server/process.js';
+import { crictlCommandFactory } from '../../server/cri.js';
+import { resolveReplicaCount } from '../../server/conf.js';
+// Cyclic by construction (index -> cluster -> MongoBootstrap -> index), same as cluster.js.
+// Safe because the binding is only dereferenced inside method bodies, never at module scope.
+import Underpost from '../../index.js';
 import {
+  MONGODB_DATA_ROOT,
   MONGODB_DEFAULT_REPLICA_COUNT,
   MONGODB_DEFAULT_REPLICA_SET,
   MONGODB_SERVICE_NAME,
   MONGODB_STATEFULSET_NAME,
+  MONGODB_STORAGE_CLASS_NAME,
+  MONGODB_STORAGE_CLASS_PROVISIONER,
   resolveMongoReplicaHosts,
 } from './MongooseDB.js';
 
 const logger = loggerFactory(import.meta);
 
+// Teardown sweeps ordinals rather than a live replica count: the deployed count is not knowable
+// during a reset, and a member left behind from a larger previous deployment keeps its volume and
+// stale replica-set config, which a later deploy then trips over.
+const MONGODB_ORDINAL_SWEEP = 10;
+
 /**
  * @typedef {Object} MongoBootstrapOptions
  * @property {string} [namespace='default'] - Kubernetes namespace.
@@ -135,6 +148,30 @@ class MongoBootstrap {
       `  return true;`,
       `};`,
 
+      // Recovery for a node holding a config it is not a member of. A forced
+      // reconfig is the only way out: it is accepted on a non-primary node,
+      // which an ordinary reconfig is not. Auth is best-effort — when the set
+      // already has users the unauthenticated pass fails here and the caller's
+      // authenticated pass performs the recovery.
+      //
+      // The recovery config has ONE member. Forcing the full member list back
+      // in place leaves the node needing a majority of the others to elect it,
+      // and they are typically holding their own stale configs from the same
+      // dead cluster, so no election ever completes. A single-member set needs
+      // only its own vote and becomes writable at once; reconfigure() then
+      // widens it — the same two-step the pristine path takes after rs.initiate.
+      `const currentConfigVersion = () => { try { return rs.conf().version || 0; } catch(e) { return 0; } };`,
+      `const forceReconfig = () => {`,
+      `  try { ensureAdminAuth(); } catch(e) {}`,
+      `  const soloConfig = {`,
+      `    _id: desiredConfig._id,`,
+      `    version: currentConfigVersion() + 1,`,
+      `    members: [{ _id: 0, host: "localhost:" + mePort }],`,
+      `  };`,
+      `  rs.reconfig(soloConfig, { force: true });`,
+      `  print("SUCCESS_FORCE_RECONFIGURED");`,
+      `};`,
+
       `const reconfigure = () => {`,
       `  if (!ensureAdminAuth()) return false;`,
       `  const cur = rs.conf();`,
@@ -149,28 +186,39 @@ class MongoBootstrap {
       `  return true;`,
       `};`,
 
-      // Determine if replica set is already initialized
-      `let initialized = false;`,
+      // Classify before acting. Three distinct states reach this script:
+      //   pristine — no config yet (fresh volume)
+      //   orphaned — a config exists but this node is not a member of it, which
+      //              is what a volume retained from an earlier cluster leaves
+      //              behind; mongod parks in REMOVED and never elects itself
+      //   live     — a usable config, or one hidden behind auth
+      // The distinction matters for ordering: an orphaned node must be force
+      // reconfigured BEFORE waiting for a primary, because it can never become
+      // writable on its own. Waiting first is an unconditional timeout.
+      `const matchesAny = (msg, list) => list.some(s => msg.includes(s));`,
+      `let state = "live";`,
       `try {`,
       `  const s = rs.status();`,
-      `  if (s && s.ok === 1) initialized = true;`,
+      `  if (!s || s.ok !== 1) state = "pristine";`,
       `} catch(e) {`,
       `  const msg = String(e);`,
-      `  if (msg.includes("requires authentication") || msg.includes("Unauthorized") || msg.includes("not authorized")) {`,
-      `    initialized = true;`,
-      `  } else if (!msg.includes("NotYetInitialized") && !msg.includes("no replset config") && !msg.includes("maps to this node")) {`,
-      `    throw e;`,
-      `  }`,
+      `  const ORPHANED = ["not a member of it", "InvalidReplicaSetConfig", "maps to this node"];`,
+      `  if (matchesAny(msg, ORPHANED)) state = "orphaned";`,
+      `  else if (matchesAny(msg, ["NotYetInitialized", "no replset config"])) state = "pristine";`,
+      `  else if (matchesAny(msg, ["requires authentication", "Unauthorized", "not authorized"])) state = "live";`,
+      `  else throw e;`,
       `}`,
+      `print("REPLSET_STATE_" + state.toUpperCase());`,
 
-      // Not initialized: bootstrap via localhost
-      `if (!initialized) {`,
+      `if (state === "pristine") {`,
       `  try {`,
       `    rs.initiate({ _id: desiredConfig._id, members: [{ _id: 0, host: "localhost:" + mePort }] });`,
       `  } catch(e) {`,
       `    const msg = String(e);`,
       `    if (!msg.includes("already initialized") && !msg.includes("AlreadyInitialized")) throw e;`,
       `  }`,
+      `} else if (state === "orphaned") {`,
+      `  forceReconfig();`,
       `}`,
 
       // Wait for primary, create user, then reconfig to full host list
@@ -189,7 +237,7 @@ class MongoBootstrap {
   static findNodesMissingMongoMount(kindNodes) {
     return kindNodes.filter((node) => {
       const inspect = shellExec(
-        `sudo docker inspect ${node} --format '{{range .Mounts}}{{if eq .Destination "/data/mongodb"}}yes{{end}}{{end}}'`,
+        `sudo docker inspect ${node} --format '{{range .Mounts}}{{if eq .Destination "${MONGODB_DATA_ROOT}"}}yes{{end}}{{end}}'`,
         { stdout: true, silent: true, silentOnError: true },
       );
       return !inspect.trim().includes('yes');
@@ -210,7 +258,7 @@ class MongoBootstrap {
       logger.info('No Kind nodes detected for hostPath cleanup.');
       return;
     }
-    const basePath = '/data/mongodb';
+    const basePath = MONGODB_DATA_ROOT;
     for (const node of nodes) {
       const prepareCmd = Array.from(
         { length: replicaCount },
@@ -232,12 +280,14 @@ class MongoBootstrap {
    * an empty /data/db regardless of bind-mount staleness.
    *
    * @param {string[]} kindNodes - List of Kind node container names.
-   * @param {string} [basePath='/data/mongodb'] - The base path containing v0/v1/v2 subdirs.
+   * @param {number} [replicaCount=3] - Number of replica ordinal directories to clean. Must track
+   *   the deployed replica count, or members above the third start on stale data.
+   * @param {string} [basePath='/data/mongodb'] - The base path containing the v<ordinal> subdirs.
    */
-  static remountKindMongoVolume(kindNodes, basePath = '/data/mongodb') {
+  static remountKindMongoVolume(kindNodes, replicaCount = MONGODB_DEFAULT_REPLICA_COUNT, basePath = MONGODB_DATA_ROOT) {
     for (const node of kindNodes) {
       logger.info(`Cleaning MongoDB data dirs inside Kind node '${node}'...`);
-      for (let i = 0; i < 3; i++) {
+      for (let i = 0; i < replicaCount; i++) {
         const dir = `${basePath}/v${i}`;
         // Ensure directory exists, wipe all contents (including hidden files), set open permissions
         // so the pod's initContainer chown can run without issues.
@@ -264,23 +314,174 @@ class MongoBootstrap {
 
   /**
    * Creates or updates Kubernetes secrets required by the MongoDB statefulset.
+   *
+   * Prefers the SOPS/Age encrypted store, exactly like the MariaDB/MySQL/PostgreSQL branches of
+   * cluster init: when `engine-private/secrets/<ns>/<name>.enc.yaml` exists it is decrypted
+   * straight into `kubectl apply`, and only otherwise is the secret seeded from its plaintext
+   * origin seed file.
+   *
+   * The seed path uses `--from-literal`, which places the credential in the command string and so
+   * in the process table and the command log; `disableLog` keeps it out of the log at least. The
+   * encrypted path has no such exposure — the value only ever crosses an anonymous pipe.
    * @param {string} namespace - Target namespace.
    * @param {string} enginePrivateRoot - Path to engine-private directory.
    */
   static ensureMongoSecrets(namespace, enginePrivateRoot) {
-    const keyfile = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-keyfile`);
-    const { username, password } = MongoBootstrap.readMongoCredentials(enginePrivateRoot);
+    if (!Underpost.secret.sops.applyIfPresent('mongodb-keyfile', namespace)) {
+      const keyfile = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-keyfile`);
+      shellExec(
+        `sudo kubectl create secret generic mongodb-keyfile` +
+          ` --from-literal=mongodb-keyfile="${keyfile.replace(/'/g, "'\\''")}"` +
+          ` --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
+        { disableLog: true },
+      );
+      logger.info(`Seeded mongodb-keyfile from ${enginePrivateRoot}/mongodb-keyfile`);
+    }
 
-    shellExec(
-      `sudo kubectl create secret generic mongodb-keyfile` +
-        ` --from-literal=mongodb-keyfile="${keyfile.replace(/'/g, "'\\''")}"` +
-        ` --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
-    );
-    shellExec(
-      `sudo kubectl create secret generic mongodb-secret` +
-        ` --from-literal=username="${username}" --from-literal=password="${password}"` +
-        ` --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
-    );
+    if (!Underpost.secret.sops.applyIfPresent('mongodb-secret', namespace)) {
+      const { username, password } = MongoBootstrap.readMongoCredentials(enginePrivateRoot);
+      shellExec(
+        `sudo kubectl create secret generic mongodb-secret` +
+          ` --from-literal=username="${username}" --from-literal=password="${password}"` +
+          ` --dry-run=client -o yaml | kubectl apply -f - -n ${namespace}`,
+        { disableLog: true },
+      );
+      logger.info(`Seeded mongodb-secret from ${enginePrivateRoot}/mongodb-{username,password}`);
+    }
+  }
+
+  /**
+   * Renders one hostPath PersistentVolume per replica, each pre-bound to that member's claim.
+   *
+   * The volume set is a function of the replica count, so it is generated rather than read from a
+   * fixed manifest — a static file can only describe a fixed number, and any `--replicas` above
+   * it leaves the surplus members unbindable.
+   * @param {number} replicaCount - Number of members to provision volumes for.
+   * @param {string} namespace - Namespace the claims live in.
+   * @returns {string} Multi-document PersistentVolume YAML.
+   */
+  static buildReplicaVolumeManifest(replicaCount, namespace) {
+    return Array.from({ length: replicaCount }, (_, i) =>
+      [
+        'apiVersion: v1',
+        'kind: PersistentVolume',
+        'metadata:',
+        `  name: ${MONGODB_STATEFULSET_NAME}-pv-${i}`,
+        '  labels:',
+        `    app: ${MONGODB_STATEFULSET_NAME}`,
+        'spec:',
+        '  capacity:',
+        '    storage: 5Gi',
+        '  accessModes:',
+        '    - ReadWriteOnce',
+        '  persistentVolumeReclaimPolicy: Retain',
+        `  storageClassName: ${MONGODB_STORAGE_CLASS_NAME}`,
+        // claimRef pins each volume to exactly one member, so ordinals can never cross-bind and
+        // land two mongod processes on one data directory.
+        '  claimRef:',
+        `    namespace: ${namespace}`,
+        `    name: ${MONGODB_STATEFULSET_NAME}-storage-${MONGODB_STATEFULSET_NAME}-${i}`,
+        '  hostPath:',
+        `    path: ${MONGODB_DATA_ROOT}/v${i}`,
+        '    type: DirectoryOrCreate',
+      ].join('\n'),
+    ).join('\n---\n');
+  }
+
+  /**
+   * Applies the generated replica volumes, and removes any volume left over from a larger previous
+   * replica count so a scale-down does not strand a PV bound to a claim that no longer exists.
+   * @param {string} namespace - Target namespace.
+   * @param {number} replicaCount - Number of members to provision volumes for.
+   */
+  static applyReplicaVolumes(namespace, replicaCount) {
+    const manifest = MongoBootstrap.buildReplicaVolumeManifest(replicaCount, namespace);
+    shellExec(`kubectl apply -f - <<'UNDERPOST_MONGO_PV_EOF'\n${manifest}\nUNDERPOST_MONGO_PV_EOF`);
+    logger.info(`Applied ${replicaCount} MongoDB replica volume(s)`);
+
+    const stale = shellExec(`kubectl get pv -l app=${MONGODB_STATEFULSET_NAME} -o name 2>/dev/null || true`, {
+      stdout: true,
+      silent: true,
+      silentOnError: true,
+    })
+      .split('\n')
+      .map((name) => name.replace('persistentvolume/', '').trim())
+      .filter((name) => {
+        const ordinal = name.startsWith(`${MONGODB_STATEFULSET_NAME}-pv-`)
+          ? Number(name.slice(`${MONGODB_STATEFULSET_NAME}-pv-`.length))
+          : NaN;
+        return Number.isInteger(ordinal) && ordinal >= replicaCount;
+      });
+    for (const name of stale) {
+      shellExec(`kubectl delete pv ${name} --ignore-not-found`);
+      logger.info(`Removed stale replica volume ${name} (beyond replica count ${replicaCount})`);
+    }
+  }
+
+  /**
+   * Reads the claim-to-volume-to-path mapping for every replica and logs it. Non-throwing, so it
+   * can enrich a failure path without masking the original error.
+   * @param {string} namespace - Target namespace.
+   * @param {number} replicaCount - Expected number of members.
+   * @returns {Array<{claim: string, volume: string, path: string}>} Bindings in ordinal order.
+   */
+  static reportReplicaVolumeBindings(namespace, replicaCount) {
+    const bindings = [];
+    for (let i = 0; i < replicaCount; i++) {
+      const claim = `${MONGODB_STATEFULSET_NAME}-storage-${MONGODB_STATEFULSET_NAME}-${i}`;
+      const volume = shellExec(
+        `kubectl get pvc ${claim} -n ${namespace} -o jsonpath='{.spec.volumeName}' 2>/dev/null || true`,
+        { stdout: true, silent: true, silentOnError: true },
+      ).trim();
+      const path = volume
+        ? shellExec(`kubectl get pv ${volume} -o jsonpath='{.spec.hostPath.path}' 2>/dev/null || true`, {
+            stdout: true,
+            silent: true,
+            silentOnError: true,
+          }).trim()
+        : '';
+      bindings.push({ claim, volume, path });
+    }
+    logger.info('MongoDB replica volume bindings', bindings);
+    return bindings;
+  }
+
+  /**
+   * Verifies every replica PVC bound to its own distinct hostPath.
+   *
+   * Two members sharing one directory is unrecoverable at the mongod level — the second dies on
+   * `WiredTiger.lock: fcntl: Resource temporarily unavailable` after the readiness wait has
+   * already burned its timeout, and the real cause (a volume binding, not MongoDB) is invisible
+   * in the pod logs. Checking the binding directly turns that into an immediate, named failure.
+   * @param {string} namespace - Target namespace.
+   * @param {number} replicaCount - Expected number of members.
+   * @throws {Error} When a claim is unbound, or two claims share a backing path.
+   */
+  static assertReplicaVolumeBindings(namespace, replicaCount) {
+    const bindings = MongoBootstrap.reportReplicaVolumeBindings(namespace, replicaCount);
+
+    const unbound = bindings.filter((binding) => !binding.volume);
+    if (unbound.length > 0)
+      throw new Error(
+        `MongoDB claims are not bound to a PersistentVolume: ${unbound.map((b) => b.claim).join(', ')}. ` +
+          `Expected one volume per replica (${replicaCount} total); check that the generated PVs applied ` +
+          `and that each claimRef matches.`,
+      );
+
+    // Only hostPath-backed volumes expose a path to compare; a dynamically provisioned volume
+    // reports none, which is itself worth surfacing since these PVs are meant to be static.
+    const byPath = bindings.reduce((acc, binding) => {
+      if (binding.path) (acc[binding.path] = acc[binding.path] || []).push(binding.claim);
+      return acc;
+    }, {});
+    const shared = Object.entries(byPath).filter(([, claims]) => claims.length > 1);
+    if (shared.length > 0)
+      throw new Error(
+        `MongoDB members would share a data directory, which mongod cannot survive: ` +
+          shared.map(([path, claims]) => `${path} <- ${claims.join(' + ')}`).join('; ') +
+          `. Delete the claims and PVs, then redeploy: ` +
+          `kubectl delete pvc -n ${namespace} -l app=mongodb; kubectl delete pv -l app=mongodb`,
+      );
   }
 
   /**
@@ -323,7 +524,15 @@ class MongoBootstrap {
     } = options;
 
     const enginePrivateRoot = `${process.cwd()}/engine-private`;
-    const effectiveReplicaCount = Math.max(Number(replicaCount) || MONGODB_DEFAULT_REPLICA_COUNT, 3);
+    // No upward clamp: an explicit `--replicas 2` must deploy two members, not be silently
+    // raised to the default.
+    const effectiveReplicaCount = resolveReplicaCount(replicaCount, MONGODB_DEFAULT_REPLICA_COUNT);
+    if (effectiveReplicaCount % 2 === 0)
+      logger.warn(
+        `Deploying ${effectiveReplicaCount} MongoDB members. An even-sized replica set has no ` +
+          `majority when one member is down, so the set becomes read-only on a single failure. ` +
+          `Odd counts (3, 5) are recommended.`,
+      );
     const mongoRootUsername = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-username`);
     const mongoRootPassword = MongoBootstrap.readCredential(`${enginePrivateRoot}/mongodb-password`);
     const mongoReplicaHosts = resolveMongoReplicaHosts({
@@ -364,16 +573,7 @@ class MongoBootstrap {
         }
         shellExec(`rm -f ${tarPath}`);
       } else {
-        const criSock =
-          shellExec('test -S /var/run/crio/crio.sock && echo crio || echo containerd', {
-            stdout: true,
-            silent: true,
-          }).trim() === 'crio'
-            ? 'unix:///var/run/crio/crio.sock'
-            : 'unix:///run/containerd/containerd.sock';
-        shellExec(
-          `sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint ${criSock} pull mongo:latest`,
-        );
+        shellExec(crictlCommandFactory('pull mongo:latest', { k3s: clusterType === 'k3s' }));
       }
     }
 
@@ -386,24 +586,61 @@ class MongoBootstrap {
 
     // Clean data if reset or kind
     if (reset || isKind) {
+      // Delete the StatefulSet's PVCs by name. A label selector cannot reach them: the
+      // `volumeClaimTemplates` entry carries no labels, so `-l app=mongodb` matches nothing and
+      // silently leaves the previous run's PVCs Bound. With `persistentVolumeReclaimPolicy:
+      // Retain` those stale claims keep their old PVs, so the freshly created `mongodb-pv-N`
+      // (whose `claimRef` names the same PVCs) never binds and pods mount the previous volumes —
+      // which is how two members end up on one hostPath and mongod dies on the WiredTiger lock.
+      for (let i = 0; i < MONGODB_ORDINAL_SWEEP; i++)
+        shellExec(
+          `kubectl delete pvc ${MONGODB_STATEFULSET_NAME}-storage-${MONGODB_STATEFULSET_NAME}-${i} -n ${namespace} --ignore-not-found`,
+        );
       shellExec(`kubectl delete pvc -l app=mongodb -n ${namespace} --ignore-not-found`);
       shellExec(`kubectl delete pvc mongodb-pvc -n ${namespace} --ignore-not-found`);
       shellExec(`kubectl delete pv -l app=mongodb --ignore-not-found`);
       shellExec(`kubectl delete pv mongodb-pv --ignore-not-found`);
 
-      if (isKind) {
-        shellExec('sudo mkdir -p /data/mongodb');
-        for (let i = 0; i < effectiveReplicaCount; i++) {
-          shellExec(`sudo rm -rf /data/mongodb/v${i}`);
-          shellExec(`sudo mkdir -p /data/mongodb/v${i}`);
-        }
-        // Fix any stale bind mounts caused by prior deletion of /data/mongodb on the host
-        MongoBootstrap.remountKindMongoVolume(kindNodes);
+      // The data itself, for every cluster type. The PVs are hostPath with
+      // `persistentVolumeReclaimPolicy: Retain`, so deleting the objects frees
+      // nothing: the next run re-binds the same directories and mongod boots
+      // with the previous cluster's replica set config, ending up outside it.
+      // Only this node's copy is removed — on a multi-node cluster, wipe the
+      // other nodes' /data/mongodb before rebuilding there.
+      logger.info('Removing retained MongoDB hostPath data', {
+        path: MONGODB_DATA_ROOT,
+        replicas: effectiveReplicaCount,
+      });
+      shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}`);
+      for (let i = 0; i < effectiveReplicaCount; i++) {
+        shellExec(`sudo rm -rf ${MONGODB_DATA_ROOT}/v${i}`);
+        shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}/v${i}`);
       }
+      // Fix any stale bind mounts caused by prior deletion of /data/mongodb on the host
+      if (isKind) MongoBootstrap.remountKindMongoVolume(kindNodes, effectiveReplicaCount);
     }
 
     // Apply manifests
+    // A StorageClass `provisioner` is immutable, so a plain apply fails against a class created
+    // with a different one. Clusters provisioned before the switch to `kubernetes.io/no-provisioner`
+    // carry the dynamic `rancher.io/local-path`; recreate in that case. Deleting the class is safe
+    // — bound PVs and PVCs reference it by name only and are untouched.
+    const storageClassProvisioner = shellExec(
+      `kubectl get storageclass ${MONGODB_STORAGE_CLASS_NAME} -o jsonpath='{.provisioner}' 2>/dev/null || true`,
+      { stdout: true, silent: true, silentOnError: true },
+    ).trim();
+    if (storageClassProvisioner && storageClassProvisioner !== MONGODB_STORAGE_CLASS_PROVISIONER) {
+      logger.warn(
+        `StorageClass ${MONGODB_STORAGE_CLASS_NAME} uses provisioner '${storageClassProvisioner}'; recreating as ` +
+          `'${MONGODB_STORAGE_CLASS_PROVISIONER}' so the static replica PVs bind deterministically.`,
+      );
+      shellExec(`kubectl delete storageclass ${MONGODB_STORAGE_CLASS_NAME} --ignore-not-found`);
+    }
     shellExec(`kubectl apply -f ${underpostRoot}/manifests/mongodb/storage-class.yaml -n ${namespace}`);
+    // One PV per member, generated from the effective replica count. A static manifest can only
+    // ever describe a fixed number, so any `--replicas` above it leaves the extra members with no
+    // volume to bind and the StatefulSet stalls forever on Pending.
+    MongoBootstrap.applyReplicaVolumes(namespace, effectiveReplicaCount);
     shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb -n ${namespace}`);
     shellExec(
       `kubectl scale statefulset/${MONGODB_STATEFULSET_NAME} --replicas=${effectiveReplicaCount} -n ${namespace}`,
@@ -412,12 +649,17 @@ class MongoBootstrap {
     // Wait for all pods
     const failedCount = await MongoBootstrap.waitForPods(namespace, effectiveReplicaCount);
     if (failedCount > 0) {
+      // Surface the volume topology before failing: a stalled rollout is far more often a binding
+      // problem than a MongoDB one, and the mapping names it immediately.
+      MongoBootstrap.reportReplicaVolumeBindings(namespace, effectiveReplicaCount);
       throw new Error(
         `MongoDB replica pods did not reach Running state in time. ` +
           `Ensure podManagementPolicy is set to OrderedReady in statefulset.yaml.`,
       );
     }
 
+    MongoBootstrap.assertReplicaVolumeBindings(namespace, effectiveReplicaCount);
+
     // Build the bootstrap script
     const defaultHosts = Array.from(
       { length: effectiveReplicaCount },
@@ -528,13 +770,12 @@ class MongoBootstrap {
       // Phase 4: Delete MongoDB PVCs and PVs (both current and legacy mongodb-4.4)
       logger.info('Phase 4/6: Deleting MongoDB PersistentVolumeClaims and PersistentVolumes...');
       // Delete PVCs from volumeClaimTemplates
-      for (let i = 0; i < 10; i++) {
+      for (let i = 0; i < MONGODB_ORDINAL_SWEEP; i++) {
         shellExec(`kubectl delete pvc mongodb-storage-mongodb-${i} -n ${namespace} --ignore-not-found`);
       }
       shellExec(`kubectl delete pvc mongodb-pvc -n ${namespace} --ignore-not-found`);
-      shellExec(`kubectl delete pv mongodb-pv-0 --ignore-not-found`);
-      shellExec(`kubectl delete pv mongodb-pv-1 --ignore-not-found`);
-      shellExec(`kubectl delete pv mongodb-pv-2 --ignore-not-found`);
+      for (let i = 0; i < MONGODB_ORDINAL_SWEEP; i++)
+        shellExec(`kubectl delete pv ${MONGODB_STATEFULSET_NAME}-pv-${i} --ignore-not-found`);
       shellExec(`kubectl delete pv mongodb-pv --ignore-not-found`);
       // Also catch any remaining PVs with the app=mongodb label
       shellExec(`kubectl delete pv -l app=mongodb --ignore-not-found`);
@@ -550,10 +791,10 @@ class MongoBootstrap {
       // IMPORTANT: Do NOT remove /data/mongodb itself — it is bind-mounted into Kind node
       // containers by inode. Removing it makes the bind mount stale; only clear subdirs.
       logger.info('Phase 5/6: Cleaning up MongoDB hostPath data...');
-      shellExec(`sudo mkdir -p /data/mongodb`);
-      for (let i = 0; i < 3; i++) {
-        shellExec(`sudo rm -rf /data/mongodb/v${i}`);
-        shellExec(`sudo mkdir -p /data/mongodb/v${i}`);
+      shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}`);
+      for (let i = 0; i < MONGODB_ORDINAL_SWEEP; i++) {
+        shellExec(`sudo rm -rf ${MONGODB_DATA_ROOT}/v${i}`);
+        shellExec(`sudo mkdir -p ${MONGODB_DATA_ROOT}/v${i}`);
       }
       // For Kind: repair any stale bind mounts via nsenter (overmounts with current host inode)
       if (isKind) {

package/src/db/mongo/MongooseDB.js

@@ -15,6 +15,30 @@ const MONGODB_STATEFULSET_NAME = 'mongodb';
 const MONGODB_DEFAULT_AUTH_SOURCE = 'admin';
 const MONGODB_DEFAULT_REPLICA_SET = 'rs0';
 const MONGODB_DEFAULT_REPLICA_COUNT = 3;
+// Node-local base path backing the hostPath PVs (`<root>/v<replica index>`),
+// generated one-per-replica by MongoBootstrap.applyReplicaVolumes().
+const MONGODB_DATA_ROOT = '/data/mongodb';
+// Replica volumes are hostPath PVs generated one per member, each pinned by `claimRef`. The class
+// must stay static: a dynamic provisioner competes with the static binder and makes
+// member-to-directory assignment non-deterministic.
+const MONGODB_STORAGE_CLASS_NAME = 'mongodb-storage-class';
+const MONGODB_STORAGE_CLASS_PROVISIONER = 'kubernetes.io/no-provisioner';
+/**
+ * Mongoose connection options for MongoDB, with sensible defaults for production and development environments.
+ * @type {import('mongoose').ConnectOptions}
+ */
+const MONGODB_CONNECTION_OPTIONS = {
+  autoIndex: process.env.NODE_ENV !== 'production',
+  heartbeatFrequencyMS: 10000,
+  maxConnecting: 2,
+  maxPoolSize: 10,
+  minPoolSize: 0,
+  retryReads: true,
+  retryWrites: true,
+  serverSelectionTimeoutMS: 30000,
+  socketTimeoutMS: 120000,
+  waitQueueTimeoutMS: 10000,
+};
 
 /**
  * Resolves MongoDB replica hosts from explicit input or StatefulSet defaults.
@@ -49,6 +73,26 @@ const resolveMongoReplicaHosts = ({ hostList = '', replicaCount = MONGODB_DEFAUL
  * 3. No built-in defaults — both `host` and `name` are required from the caller or environment.
  */
 class MongooseDBService {
+  /**
+   * Establishes a Mongoose connection to the specified MongoDB instance.
+   *
+   * @async
+   * @param {object|string} configOrHost - Either a db config object or a legacy host string.
+   * @param {string} [configOrHost.host] - Legacy single host or comma-separated host list.
+   * @param {string} [configOrHost.name] - The database name.
+   * @param {string} [configOrHost.replicaSet] - The MongoDB replica set name.
+   * @param {string} [configOrHost.authSource] - The authentication database.
+   * @param {string} [configOrHost.user] - The MongoDB username.
+   * @param {string} [configOrHost.password] - The MongoDB password.
+   * @param {string} [name] - Legacy database name when a host string is passed.
+   * @returns {Promise<mongoose.Connection>} A promise that resolves to the established Mongoose connection object.
+   * @throws {Error} If neither the argument nor the corresponding environment variable supplies a value.
+   */
+  async connect(configOrHost, name) {
+    const uri = this.buildUri(configOrHost, name);
+    // if (process.env.NODE_ENV === 'development') logger.info(`Connecting to MongoDB with URI`, uri);
+    return await mongoose.createConnection(uri, MONGODB_CONNECTION_OPTIONS).asPromise();
+  }
   /**
    * Normalizes Mongo host inputs into plain host:port entries.
    * @param {Array<string>|string} hosts - Host input as list or comma-separated string.
@@ -125,38 +169,6 @@ class MongooseDBService {
     return `mongodb://${credentials}${config.hosts.join(',')}/${config.dbName}${query.size ? `?${query.toString()}` : ''}`;
   }
 
-  /**
-   * Establishes a Mongoose connection to the specified MongoDB instance.
-   *
-   * @async
-   * @param {object|string} configOrHost - Either a db config object or a legacy host string.
-   * @param {string} [configOrHost.host] - Legacy single host or comma-separated host list.
-   * @param {string} [configOrHost.name] - The database name.
-   * @param {string} [configOrHost.replicaSet] - The MongoDB replica set name.
-   * @param {string} [configOrHost.authSource] - The authentication database.
-   * @param {string} [configOrHost.user] - The MongoDB username.
-   * @param {string} [configOrHost.password] - The MongoDB password.
-   * @param {string} [name] - Legacy database name when a host string is passed.
-   * @returns {Promise<mongoose.Connection>} A promise that resolves to the established Mongoose connection object.
-   * @throws {Error} If neither the argument nor the corresponding environment variable supplies a value.
-   */
-  async connect(configOrHost, name) {
-    const uri = this.buildUri(configOrHost, name);
-    if (process.env.NODE_ENV === 'development') logger.info(`Connecting to MongoDB with URI`, uri);
-    return await mongoose
-      .createConnection(uri, {
-        autoIndex: process.env.NODE_ENV !== 'production',
-        heartbeatFrequencyMS: 10000,
-        maxPoolSize: 20,
-        minPoolSize: 2,
-        retryReads: true,
-        retryWrites: true,
-        serverSelectionTimeoutMS: 5000,
-        socketTimeoutMS: 45000,
-      })
-      .asPromise();
-  }
-
   /**
    * Dynamically loads Mongoose models for a list of APIs and binds them to the given connection.
    *
@@ -174,6 +186,10 @@ class MongooseDBService {
       const { ProviderSchema } = await import(`../../api/${api}/${api}.model.js`);
       const keyModel = getCapVariableName(api); // Assuming this returns a capitalized model name
       models[keyModel] = conn.model(keyModel, ProviderSchema);
+      // Mongoose emits 'error' on the model when an autoIndex build fails; with no
+      // listener attached EventEmitter rethrows and takes the whole process down.
+      // A stale or conflicting index must degrade to a log, not kill startup.
+      models[keyModel].on('error', (error) => logger.error(`${keyModel} index build failed: ${error.message}`));
     }
 
     return models;
@@ -191,9 +207,12 @@ const MongooseDB = new MongooseDBService();
 export {
   MongooseDB,
   MongooseDBService as MongooseDBClass,
+  MONGODB_DATA_ROOT,
   MONGODB_DEFAULT_REPLICA_COUNT,
   MONGODB_DEFAULT_REPLICA_SET,
   MONGODB_SERVICE_NAME,
   MONGODB_STATEFULSET_NAME,
+  MONGODB_STORAGE_CLASS_NAME,
+  MONGODB_STORAGE_CLASS_PROVISIONER,
   resolveMongoReplicaHosts,
 };

package/src/index.js

@@ -26,6 +26,8 @@ import UnderpostStatic from './cli/static.js';
 import UnderpostTest from './cli/test.js';
 import UnderpostRelease from './cli/release.js';
 import UnderpostSystemProvisionig from './cli/system.js';
+import UnderpostVultr from './cli/vultr.js';
+import UnderpostWireguard from './cli/wireguard.js';
 
 import UnderpostDns from './server/dns.js';
 import UnderpostBackup from './server/backup.js';
@@ -45,7 +47,7 @@ class Underpost {
    * @type {String}
    * @memberof Underpost
    */
-  static version = 'v3.2.80';
+  static version = 'v3.3.0';
 
   /**
    * Required Node.js major version
@@ -252,6 +254,26 @@ class Underpost {
     return UnderpostSystemProvisionig.API;
   }
 
+  /**
+   * Edge hub WireGuard/HAProxy cli API
+   * @static
+   * @type {UnderpostWireguard.API}
+   * @memberof Underpost
+   */
+  static get wireguard() {
+    return UnderpostWireguard.API;
+  }
+
+  /**
+   * Vultr bandwidth guard cli API
+   * @static
+   * @type {UnderpostVultr.API}
+   * @memberof Underpost
+   */
+  static get vultr() {
+    return UnderpostVultr.API;
+  }
+
   /**
    * Dns server API
    * @static
@@ -352,6 +374,8 @@ export {
   UnderpostStartUp,
   UnderpostRelease,
   UnderpostTLS,
+  UnderpostVultr,
+  UnderpostWireguard,
 };
 
 export default Underpost;