underpost 3.2.8 → 3.2.10

package/src/cli/cluster.js

@@ -7,6 +7,8 @@
 import { getNpmRootPath } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
+import { MONGODB_DEFAULT_REPLICA_COUNT } from '../db/mongo/MongooseDB.js';
+import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
 import os from 'os';
 import fs from 'fs-extra';
 import Underpost from '../index.js';
@@ -22,6 +24,7 @@ const logger = loggerFactory(import.meta);
  */
 class UnderpostCluster {
   static API = {
+
     /**
      * @method init
      * @description Initializes and configures the Kubernetes cluster based on provided options.
@@ -41,6 +44,7 @@ class UnderpostCluster {
      * @param {boolean} [options.certManager=false] - Deploy Cert-Manager for certificate management.
      * @param {boolean} [options.listPods=false] - List Kubernetes pods.
      * @param {boolean} [options.reset=false] - Perform a comprehensive reset of Kubernetes and container environments.
+     * @param {boolean} [options.resetMongodb=false] - Perform a targeted reset of MongoDB components without restarting the entire cluster.
      * @param {boolean} [options.dev=false] - Run in development mode (adjusts paths).
      * @param {string} [options.nsUse=''] - Set the current kubectl namespace (creates namespace if it doesn't exist).
      * @param {string} [options.namespace='default'] - Kubernetes namespace for cluster operations.
@@ -78,6 +82,7 @@ class UnderpostCluster {
         certManager: false,
         listPods: false,
         reset: false,
+        resetMongodb: false,
         dev: false,
         nsUse: '',
         namespace: 'default',
@@ -120,6 +125,7 @@ class UnderpostCluster {
         const namespaceExists = shellExec(`kubectl get namespace ${options.nsUse} --ignore-not-found -o name`, {
           stdout: true,
           silent: true,
+          silentOnError: true,
         }).trim();
 
         if (!namespaceExists) {
@@ -145,6 +151,16 @@ class UnderpostCluster {
         });
       }
 
+      // Targeted MongoDB-only reset (does not restart the whole node)
+      if (options.resetMongodb) {
+        const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
+        return await MongoBootstrap.reset({
+          namespace: options.namespace,
+          clusterType,
+          underpostRoot,
+        });
+      }
+
       // Check if a cluster (Kind, Kubeadm, or K3s) is already initialized
       const alreadyKubeadmCluster = Underpost.kubectl.get('calico-kube-controllers')[0];
       const alreadyKindCluster = Underpost.kubectl.get('kube-apiserver-kind-control-plane')[0];
@@ -172,9 +188,19 @@ class UnderpostCluster {
           const podNetworkCidr = options.podNetworkCidr || '192.168.0.0/16';
           const controlPlaneEndpoint = options.controlPlaneEndpoint || `${os.hostname()}:6443`;
 
-          // Initialize kubeadm control plane
+          // Initialize kubeadm control plane.
+          // Use CRI-O socket when available, otherwise fall back to containerd.
+          const crioSocket = 'unix:///var/run/crio/crio.sock';
+          const containerdSocket = 'unix:///run/containerd/containerd.sock';
+          const criSocket =
+            shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
+              stdout: true,
+              silent: true,
+            }).trim() === 'crio'
+              ? crioSocket
+              : containerdSocket;
           shellExec(
-            `sudo kubeadm init --pod-network-cidr=${podNetworkCidr} --control-plane-endpoint="${controlPlaneEndpoint}"`,
+            `sudo kubeadm init --pod-network-cidr=${podNetworkCidr} --control-plane-endpoint="${controlPlaneEndpoint}" --cri-socket=${criSocket}`,
           );
           // Configure kubectl for the current user
           Underpost.cluster.chown('kubeadm'); // Pass 'kubeadm' to chown
@@ -198,15 +224,40 @@ class UnderpostCluster {
             `kubectl apply -f https://cdn.jsdelivr.net/gh/rancher/local-path-provisioner@master/deploy/local-path-storage.yaml`,
           );
         } else {
-          // Kind cluster initialization (if not using kubeadm or k3s)
+          // Kind cluster initialization (default for development)
           logger.info('Initializing Kind cluster...');
-          shellExec(
-            `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
-              options.dev ? '-dev' : ''
-            }.yaml`,
-          );
+          const devReplicaCount = Math.max(Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT, 3);
+          shellExec(`sudo mkdir -p /data/mongodb`);
+          for (let index = 0; index < devReplicaCount; index++) {
+            shellExec(`sudo mkdir -p /data/mongodb/v${index}`);
+          }
+          const kindCreateCmd = `cd ${underpostRoot}/manifests && kind create cluster --config kind-config-dev.yaml`;
+          try {
+            shellExec(kindCreateCmd);
+          } catch (error) {
+            const kindCreateErrText = `${error?.message || ''}\n${error?.stderr || ''}`;
+            if (kindCreateErrText.includes('all predefined address pools have been fully subnetted')) {
+              logger.warn('Docker address pool exhausted while creating Kind cluster. Running cleanup and retrying once...');
+              Underpost.cluster.recoverKindDockerNetworks();
+              try {
+                shellExec(kindCreateCmd);
+              } catch (retryError) {
+                const retryErrText = `${retryError?.message || ''}\n${retryError?.stderr || ''}`;
+                if (retryErrText.includes('all predefined address pools have been fully subnetted')) {
+                  logger.warn('Kind retry still failed from pool exhaustion. Applying Docker daemon address-pool config and retrying once more...');
+                  Underpost.cluster.ensureDockerDefaultAddressPools();
+                  shellExec(kindCreateCmd);
+                } else {
+                  throw retryError;
+                }
+              }
+            } else {
+              throw error;
+            }
+          }
           Underpost.cluster.chown('kind'); // Pass 'kind' to chown
         }
+        Underpost.cluster.natSetup({ underpostRoot });
       }
 
       // --- Optional Component Deployments (Databases, Ingress, Cert-Manager) ---
@@ -307,33 +358,16 @@ EOF
           );
         }
       } else if (options.mongodb) {
-        if (options.pullImage) Underpost.cluster.pullImage('mongo:latest', options);
-        shellExec(
-          `sudo kubectl create secret generic mongodb-keyfile --from-file=/home/dd/engine/engine-private/mongodb-keyfile --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-        );
-        shellExec(
-          `sudo kubectl create secret generic mongodb-secret --from-file=username=/home/dd/engine/engine-private/mongodb-username --from-file=password=/home/dd/engine/engine-private/mongodb-password --dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
-        );
-        shellExec(`kubectl delete statefulset mongodb -n ${options.namespace} --ignore-not-found`);
-        shellExec(`kubectl apply -f ${underpostRoot}/manifests/mongodb/storage-class.yaml -n ${options.namespace}`);
-        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb -n ${options.namespace}`);
-
-        const successInstance = await Underpost.test.statusMonitor('mongodb-0', 'Running', 'pods', 1000, 60 * 10);
-
-        if (successInstance) {
-          if (!options.mongoDbHost) options.mongoDbHost = 'mongodb-0.mongodb-service';
-          const mongoConfig = {
-            _id: 'rs0',
-            members: options.mongoDbHost.split(',').map((host, index) => ({ _id: index, host: `${host}:27017` })),
-          };
-
-          shellExec(
-            `sudo kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
-        --eval 'use admin' \
-        --eval 'rs.initiate(${JSON.stringify(mongoConfig)})' \
-        --eval 'rs.status()'`,
-          );
-        }
+        const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
+        await MongoBootstrap.initReplicaSet({
+          namespace: options.namespace,
+          replicaCount: Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT,
+          mongoDbHost: options.mongoDbHost || '',
+          pullImage: options.pullImage,
+          reset: options.reset,
+          clusterType,
+          underpostRoot,
+        });
       }
 
       if (options.contour) {
@@ -389,8 +423,17 @@ EOF
         );
         shellExec(`rm -f ${tarPath}`);
       } else if (options.kubeadm || options.k3s) {
-        // Kubeadm / K3s: use crictl to pull directly into containerd
-        shellExec(`sudo crictl pull ${image}`);
+        // Kubeadm / K3s: use crictl to pull directly into the active CRI runtime.
+        // crictl is not in sudo's secure_path; pass full PATH through env.
+        // Point crictl at CRI-O when the socket exists, otherwise fall back to containerd.
+        const criSock =
+          shellExec(`test -S /var/run/crio/crio.sock && echo crio || echo containerd`, {
+            stdout: true,
+            silent: true,
+          }).trim() === 'crio'
+            ? 'unix:///var/run/crio/crio.sock'
+            : 'unix:///run/containerd/containerd.sock';
+        shellExec(`sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint ${criSock} pull ${image}`);
       }
     },
 
@@ -400,7 +443,8 @@ EOF
      * This method ensures proper SELinux, Docker, Containerd, and Sysctl settings
      * are applied for a healthy Kubernetes environment. It explicitly avoids
      * iptables flushing commands to prevent conflicts with Kubernetes' own network management.
-     * @param {string} underpostRoot - The root directory of the underpost project.
+     * @param {object} [options] - Configuration options for host setup.
+     * @param {string} [options.underpostRoot] - The root path of the underpost project, used for locating scripts if needed.
      * @memberof UnderpostCluster
      */
     config(options = { underpostRoot: '.' }) {
@@ -432,6 +476,27 @@ EOF
       // Reload systemd daemon to pick up new unit files/changes
       shellExec(`sudo systemctl daemon-reload`);
 
+      // Increase inotify limits
+      shellExec(`sudo sysctl -w fs.inotify.max_user_watches=2099999999`);
+      shellExec(`sudo sysctl -w fs.inotify.max_user_instances=2099999999`);
+      shellExec(`sudo sysctl -w fs.inotify.max_queued_events=2099999999`);
+
+    },
+
+    /**
+     * @method natSetup
+     * @description Configures NAT and iptables settings for Kubernetes networking.
+     * This method enables necessary sysctl settings for bridge networking and applies iptables rules
+     * required for Kubernetes cluster communication. It is designed to work with kubeadm and k3s clusters, ensuring that
+     * traffic through Linux bridges is processed by iptables, which is crucial for CNI plugins to function correctly.
+     * The method also applies NAT iptables rules and configures firewalld for Kubernetes, which is required for multi-machine kubeadm inter-node communication.
+     * Note: This method should be called after the cluster is initialized and before deploying any workloads that require network communication.
+     * @param {object} [options] - Configuration options for NAT setup.
+     * @param {string} [options.underpostRoot] - The root path of the underpost project, used to locate the nat-iptables.sh script.
+     * @memberof UnderpostCluster
+     */
+    natSetup(options = { underpostRoot: '.' }) {
+      const { underpostRoot } = options;
       // Enable bridge-nf-call-iptables for Kubernetes networking
       // This ensures traffic through Linux bridges is processed by iptables (crucial for CNI)
       for (const iptableConfPath of [
@@ -446,19 +511,12 @@ net.bridge.bridge-nf-call-arptables = 1
 net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
           { silent: true },
         );
-
-      // Increase inotify limits
-      shellExec(`sudo sysctl -w fs.inotify.max_user_watches=2099999999`);
-      shellExec(`sudo sysctl -w fs.inotify.max_user_instances=2099999999`);
-      shellExec(`sudo sysctl -w fs.inotify.max_queued_events=2099999999`);
-
       // shellExec(`sudo sysctl --system`); // Apply sysctl changes immediately
-      // Apply NAT iptables rules.
+      // Apply NAT iptables rules and configure firewalld for Kubernetes.
+      // nat-iptables.sh enables firewalld and opens all required ports; do NOT stop it
+      // afterwards — keeping firewalld running with these rules is required for
+      // multi-machine kubeadm inter-node communication.
       shellExec(`${underpostRoot}/scripts/nat-iptables.sh`, { silent: true });
-
-      // Disable firewalld (common cause of network issues in Kubernetes)
-      shellExec(`sudo systemctl stop firewalld`); // Stop if running
-      shellExec(`sudo systemctl disable firewalld`); // Disable from starting on boot
     },
 
     /**
@@ -535,6 +593,10 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
         // Phase 1: Clean up Persistent Volumes with hostPath
         // This targets data created by Kubernetes Persistent Volumes that use hostPath.
         logger.info('Phase 1/7: Cleaning Kubernetes hostPath volumes...');
+        if ((options.clusterType || 'kind') === 'kind') {
+          logger.info('  -> Kind detected: cleaning node-local MongoDB hostPath directories...');
+          Underpost.cluster.cleanKindMongoHostPaths({ basePath: '/data/mongodb', replicaCount: 3 });
+        }
         if (options.removeVolumeHostPaths)
           try {
             const pvListJson = shellExec(`kubectl get pv -o json || echo '{"items":[]}'`, {
@@ -575,8 +637,11 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
         shellExec('sudo systemctl stop kubelet');
         shellExec('sudo systemctl stop docker');
         shellExec('sudo systemctl stop podman');
-        // Safely unmount pod filesystems to avoid errors.
-        shellExec('sudo umount -f /var/lib/kubelet/pods/*/*');
+        // Lazy-unmount all kubelet pod mounts to avoid 'Device or resource busy' on rm.
+        shellExec(
+          `sudo sh -c 'findmnt --raw --noheadings -o TARGET | grep /var/lib/kubelet | sort -r | xargs -r umount -l'`,
+          { silentOnError: true },
+        );
 
         // Phase 3: Execute official uninstallation commands (type-specific)
         const clusterType = options.clusterType || 'kind';
@@ -584,6 +649,14 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
           `Phase 3/7: Executing official reset/uninstallation commands for cluster type: '${clusterType}'...`,
         );
         if (clusterType === 'kubeadm') {
+          // Kill control plane processes that hold ports (6443, 10257, 10259, 2379, 2380)
+          // so the next `kubeadm init` does not fail with [ERROR Port-xxxx].
+          logger.info('  -> Stopping and killing control plane containers and processes...');
+          shellExec('sudo crictl rm -a -f', { silentOnError: true });
+          shellExec('sudo crictl rmp -a -f', { silentOnError: true });
+          shellExec('sudo systemctl stop etcd', { silentOnError: true });
+          for (const port of [6443, 10259, 10257, 2379, 2380])
+            shellExec(`sudo fuser -k ${port}/tcp`, { silentOnError: true });
           logger.info('  -> Executing kubeadm reset...');
           shellExec('sudo kubeadm reset --force');
         } else if (clusterType === 'k3s') {
@@ -592,7 +665,13 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
         } else {
           // Default: kind
           logger.info('  -> Deleting Kind clusters...');
-          shellExec('kind get clusters | xargs -r -t -n1 kind delete cluster');
+          shellExec(`clusters=$(kind get clusters)
+if [ -n "$clusters" ]; then
+  for c in $clusters; do
+    echo "Deleting cluster: $c"
+    kind delete cluster --name "$c"
+  done
+fi`);
         }
 
         // Phase 4: File system cleanup
@@ -600,7 +679,13 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
         // Remove any leftover configurations and data.
         shellExec('sudo rm -rf /etc/kubernetes/*');
         shellExec('sudo rm -rf /etc/cni/net.d/*');
+        // Second-pass lazy umount before rm to clear any remaining busy mounts.
+        shellExec(
+          `sudo sh -c 'findmnt --raw --noheadings -o TARGET | grep /var/lib/kubelet | sort -r | xargs -r umount -l'`,
+          { silentOnError: true },
+        );
         shellExec('sudo rm -rf /var/lib/kubelet/*');
+        shellExec('sudo rm -rf /var/lib/etcd');
         shellExec('sudo rm -rf /var/lib/cni/*');
         shellExec('sudo rm -rf /var/lib/docker/*');
         shellExec('sudo rm -rf /var/lib/containerd/*');
@@ -613,11 +698,14 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
         // Remove iptables rules and CNI network interfaces.
         shellExec('sudo iptables -F');
         shellExec('sudo iptables -t nat -F');
-        shellExec('sudo ip link del cni0');
-        shellExec('sudo ip link del flannel.1');
+        shellExec('sudo ip link del cni0', { silentOnError: true });
+        shellExec('sudo ip link del flannel.1', { silentOnError: true });
+        shellExec('sudo ip link del vxlan.calico', { silentOnError: true });
+        shellExec('sudo ip link del tunl0', { silentOnError: true });
 
         logger.info('Phase 6/7: Clean up images');
-        shellExec(`podman rmi $(podman images -qa) --force`);
+        shellExec('sudo podman rmi --all --force', { silentOnError: true });
+        shellExec('sudo crictl rmi --prune', { silentOnError: true });
 
         // Phase 6: Reload daemon and finalize
         logger.info('Phase 7/7: Reloading the system daemon and finalizing...');
@@ -687,6 +775,9 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
       // Install Podman
       shellExec(`sudo dnf -y install podman`);
 
+      // Install CRI-O (required for kubeadm with CRI-O socket)
+      shellExec(`node bin run install-crio`);
+
       // Install Kind (Kubernetes in Docker)
       shellExec(`[ $(uname -m) = ${archData.name} ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-${archData.alias}
 chmod +x ./kind
@@ -744,6 +835,14 @@ EOF`);
       console.log('Removing Podman...');
       shellExec(`sudo dnf -y remove podman`);
 
+      // Remove CRI-O
+      console.log('Removing CRI-O...');
+      shellExec('sudo systemctl stop crio', { silentOnError: true });
+      shellExec('sudo systemctl disable crio', { silentOnError: true });
+      shellExec(`sudo dnf -y remove cri-o`);
+      shellExec(`sudo rm -f /etc/yum.repos.d/cri-o.repo`);
+      shellExec(`sudo rm -f /etc/crictl.yaml`);
+
       // Remove Kubeadm, Kubelet, and Kubectl
       console.log('Removing Kubernetes tools...');
       shellExec(`sudo yum remove -y kubelet kubeadm kubectl`);
@@ -780,6 +879,95 @@ EOF`);
 
       console.log('Uninstall process completed.');
     },
+
+    /**
+    * @method cleanKindMongoHostPaths
+    * @description Best-effort cleanup of MongoDB hostPath directories inside Kind node containers.
+    * This prevents stale replica/auth state when hostPath data lives in node-local container filesystems.
+    * @param {object} [options]
+    * @param {string} [options.basePath='/data/mongodb'] - Node-internal base path for MongoDB data.
+    * @param {number} [options.replicaCount=3] - Number of replica ordinal directories (v0..vN-1).
+    * @memberof UnderpostCluster
+    */
+    cleanKindMongoHostPaths(options = { basePath: '/data/mongodb', replicaCount: 3 }) {
+      const basePath = options.basePath || '/data/mongodb';
+      const replicaCount = Math.max(Number(options.replicaCount) || 3, 1);
+      const nodesRaw = shellExec('kind get nodes', {
+        stdout: true,
+        silent: true,
+        silentOnError: true,
+      });
+      const nodes = nodesRaw
+        .split('\n')
+        .map((node) => node.trim())
+        .filter((node) => !!node);
+
+      if (nodes.length === 0) {
+        logger.info('No Kind nodes detected for node-local MongoDB hostPath cleanup.');
+        return;
+      }
+
+      for (const node of nodes) {
+        logger.info(
+          `Cleaning Kind node-local MongoDB paths '${basePath}/v0..v${replicaCount - 1}' on node '${node}'...`,
+        );
+        const prepareReplicaDirsCmd = Array.from(
+          { length: replicaCount },
+          (_, index) => `mkdir -p ${basePath}/v${index}; rm -rf ${basePath}/v${index}/*;`,
+        ).join(' ');
+        const verifyReplicaDirsCmd = Array.from(
+          { length: replicaCount },
+          (_, index) => `test -d ${basePath}/v${index};`,
+        ).join(' ');
+        shellExec(
+          `sudo docker exec ${node} sh -lc 'mkdir -p ${basePath}; ${prepareReplicaDirsCmd}'`,
+          { silentOnError: true },
+        );
+        shellExec(`sudo docker exec ${node} sh -lc '${verifyReplicaDirsCmd}'`);
+      }
+    },
+
+    /**
+     * @method recoverKindDockerNetworks
+     * @description Best-effort cleanup of stale Kind Docker resources when Docker bridge
+     * address pools are exhausted and new networks cannot be allocated.
+     * @memberof UnderpostCluster
+     */
+    recoverKindDockerNetworks() {
+      logger.warn('Attempting Docker network recovery for Kind (address pool exhaustion detected)...');
+      shellExec(`sudo docker ps -aq --filter label=io.x-k8s.kind.cluster | xargs -r sudo docker rm -f`, {
+        silentOnError: true,
+      });
+      shellExec(`sudo docker network ls -q --filter label=io.x-k8s.kind.cluster | xargs -r sudo docker network rm`, {
+        silentOnError: true,
+      });
+      shellExec(`sudo docker network rm kind`, { silentOnError: true });
+      shellExec(`sudo docker network prune -f`, { silentOnError: true });
+    },
+
+    /**
+     * @method ensureDockerDefaultAddressPools
+     * @description Writes a sane Docker default-address-pools config to reduce
+     * Kind network allocation failures on hosts with exhausted predefined pools.
+     * @memberof UnderpostCluster
+     */
+    ensureDockerDefaultAddressPools() {
+      logger.warn('Applying Docker default-address-pools workaround for Kind network creation...');
+      shellExec(`cat <<'EOF' | sudo tee /etc/docker/daemon.json
+{
+  "default-address-pools": [
+    {"base": "172.17.0.0/16", "size": 24},
+    {"base": "172.18.0.0/16", "size": 24},
+    {"base": "172.19.0.0/16", "size": 24},
+    {"base": "172.20.0.0/14", "size": 24},
+    {"base": "172.24.0.0/14", "size": 24}
+  ]
+}
+EOF`);
+      shellExec('sudo systemctl restart docker');
+      shellExec('sudo docker network prune -f', { silentOnError: true });
+    },
+
   };
 }
 export default UnderpostCluster;