underpost 2.8.6 → 2.8.7

package/src/api/user/user.service.js

@@ -225,8 +225,8 @@ const UserService = {
         } else throw new Error('invalid email or password');
 
       case 'guest': {
-        const user = await ValkeyAPI.valkeyObjectFactory('user', options);
-        await ValkeyAPI.setValkeyObject(user.email, user);
+        const user = await ValkeyAPI.valkeyObjectFactory(options, 'user');
+        await ValkeyAPI.setValkeyObject(options, user.email, user);
         return {
           token: hashJWT({ user: UserDto.auth.payload(user) }),
           user: selectDtoFactory(user, UserDto.select.get()),
@@ -325,15 +325,18 @@ const UserService = {
         return await User.find().select(UserDto.select.getAll());
 
       case 'auth': {
-        const user = (await ValkeyAPI.getValkeyObject(req.auth.user.email))
-          ? await ValkeyAPI.getValkeyObject(req.auth.user.email)
-          : await User.findOne({
-              _id: req.auth.user._id,
-            });
+        let user;
+        if (req.auth.user._id.match('guest')) {
+          user = await ValkeyAPI.getValkeyObject(options, req.auth.user.email);
+          if (!user) throw new Error('guest user expired');
+        } else
+          user = await User.findOne({
+            _id: req.auth.user._id,
+          });
 
         const file = await File.findOne({ _id: user.profileImageId });
 
-        if (!file && !(await ValkeyAPI.getValkeyObject(req.auth.user.email))) {
+        if (!file && !(await ValkeyAPI.getValkeyObject(options, req.auth.user.email))) {
           await User.findByIdAndUpdate(
             user._id,
             { profileImageId: await getDefaultProfileImageId(File) },
@@ -342,8 +345,8 @@ const UserService = {
             },
           );
         }
-        return (await ValkeyAPI.getValkeyObject(req.auth.user.email))
-          ? selectDtoFactory(await ValkeyAPI.getValkeyObject(req.auth.user.email), UserDto.select.get())
+        return (await ValkeyAPI.getValkeyObject(options, req.auth.user.email))
+          ? selectDtoFactory(await ValkeyAPI.getValkeyObject(options, req.auth.user.email), UserDto.select.get())
           : await User.findOne({
               _id: req.auth.user._id,
             }).select(UserDto.select.get());
@@ -378,7 +381,7 @@ const UserService = {
         switch (user.role) {
           case 'admin': {
             if (req.params.id) return await User.findByIdAndDelete(req.params.id);
-            else return await await User.deleteMany();
+            else return await User.deleteMany();
           }
           default:
             if (req.auth.user._id !== req.params.id) throw new Error(`Invalid token user id`);

package/src/cli/cluster.js

@@ -1,5 +1,4 @@
-import { timer } from '../client/components/core/CommonJs.js';
-import { cliSpinner, getNpmRootPath } from '../server/conf.js';
+import { getNpmRootPath } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import UnderpostDeploy from './deploy.js';
@@ -15,6 +14,7 @@ class UnderpostCluster {
         mongodb: false,
         mongodb4: false,
         mariadb: false,
+        postgresql: false,
         valkey: false,
         full: false,
         info: false,
@@ -23,10 +23,21 @@ class UnderpostCluster {
         reset: false,
         dev: false,
         nsUse: '',
+        infoCapacity: false,
+        infoCapacityPod: false,
+        istio: false,
+        pullImage: false,
       },
     ) {
+      // 1) Install kind, kubeadm, docker, podman
+      // 2) Check kubectl, kubelet, containerd.io
+      // 3) Install Nvidia drivers from Rocky Linux docs
+      // 4) Install LXD with MAAS from Rocky Linux docs
+      // 5) Install MAAS src from snap
       const npmRoot = getNpmRootPath();
       const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      if (options.infoCapacityPod === true) return logger.info('', UnderpostDeploy.API.resourcesFactory());
+      if (options.infoCapacity === true) return logger.info('', UnderpostCluster.API.getResourcesCapacity());
       if (options.reset === true) return await UnderpostCluster.API.reset();
       if (options.listPods === true) return console.table(UnderpostDeploy.API.get(podName ?? undefined));
 
@@ -63,26 +74,55 @@ class UnderpostCluster {
         shellExec(`kubectl get secrets --all-namespaces -o wide`);
         shellExec(`docker secret ls`);
         shellExec(`kubectl get crd --all-namespaces -o wide`);
+        shellExec(`sudo kubectl api-resources`);
         return;
       }
 
-      if (!UnderpostDeploy.API.get('kube-apiserver-kind-control-plane')[0]) {
+      if (
+        (!options.istio && !UnderpostDeploy.API.get('kube-apiserver-kind-control-plane')[0]) ||
+        (options.istio === true && !UnderpostDeploy.API.get('calico-kube-controllers')[0])
+      ) {
+        shellExec(`sudo setenforce 0`);
+        shellExec(`sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config`);
+        // sudo systemctl disable kubelet
+        // shellExec(`sudo systemctl enable --now kubelet`);
         shellExec(`containerd config default > /etc/containerd/config.toml`);
         shellExec(`sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
         // shellExec(`cp /etc/kubernetes/admin.conf ~/.kube/config`);
-        shellExec(`sudo systemctl restart kubelet`);
+        // shellExec(`sudo systemctl restart kubelet`);
         shellExec(`sudo service docker restart`);
         shellExec(`sudo systemctl enable --now containerd.service`);
-        shellExec(`sudo systemctl restart containerd`);
-        shellExec(
-          `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
-            options?.dev === true ? '-dev' : ''
-          }.yaml`,
-        );
-        shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+        shellExec(`sudo swapoff -a; sudo sed -i '/swap/d' /etc/fstab`);
+        if (options.istio === true) {
+          shellExec(`sysctl net.bridge.bridge-nf-call-iptables=1`);
+          shellExec(`sudo kubeadm init --pod-network-cidr=192.168.0.0/16`);
+          shellExec(`sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config`);
+          shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+          // https://docs.tigera.io/calico/latest/getting-started/kubernetes/quickstart
+          shellExec(
+            `sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.3/manifests/tigera-operator.yaml`,
+          );
+          // shellExec(
+          //   `wget https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/custom-resources.yaml`,
+          // );
+          shellExec(`sudo kubectl apply -f ./manifests/kubeadm-calico-config.yaml`);
+          shellExec(`sudo systemctl restart containerd`);
+        } else {
+          shellExec(`sudo systemctl restart containerd`);
+          shellExec(
+            `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
+              options?.dev === true ? '-dev' : ''
+            }.yaml`,
+          );
+          shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+        }
       } else logger.warn('Cluster already initialized');
 
       if (options.full === true || options.valkey === true) {
+        if (options.pullImage === true) {
+          shellExec(`docker pull valkey/valkey`);
+          shellExec(`sudo kind load docker-image valkey/valkey:latest`);
+        }
         shellExec(`kubectl delete statefulset service-valkey`);
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/valkey`);
       }
@@ -96,7 +136,21 @@ class UnderpostCluster {
         shellExec(`kubectl delete statefulset mariadb-statefulset`);
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb`);
       }
+      if (options.full === true || options.postgresql === true) {
+        if (options.pullImage === true) {
+          shellExec(`docker pull postgres:latest`);
+          shellExec(`sudo kind load docker-image postgres:latest`);
+        }
+        shellExec(
+          `sudo kubectl create secret generic postgres-secret --from-file=password=/home/dd/engine/engine-private/postgresql-password`,
+        );
+        shellExec(`kubectl apply -k ./manifests/postgresql`);
+      }
       if (options.mongodb4 === true) {
+        if (options.pullImage === true) {
+          shellExec(`docker pull mongo:4.4`);
+          shellExec(`sudo kind load docker-image mongo:4.4`);
+        }
         shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb-4.4`);
 
         const deploymentName = 'mongodb-deployment';
@@ -168,34 +222,167 @@ class UnderpostCluster {
         shellExec(`sudo kubectl apply -f ${underpostRoot}/manifests/${letsEncName}.yaml`);
       }
     },
+    // This function performs a comprehensive reset of Kubernetes and container environments
+    // on the host machine. Its primary goal is to clean up cluster components, temporary files,
+    // and container data, ensuring a clean state for re-initialization or fresh deployments,
+    // while also preventing the loss of the host machine's internet connectivity.
+
     reset() {
+      // Step 1: Delete all existing Kind (Kubernetes in Docker) clusters.
+      // 'kind get clusters' lists all Kind clusters.
+      // 'xargs -t -n1 kind delete cluster --name' then iterates through each cluster name
+      // and executes 'kind delete cluster --name <cluster_name>' to remove them.
       shellExec(`kind get clusters | xargs -t -n1 kind delete cluster --name`);
+
+      // Step 2: Reset the Kubernetes control-plane components installed by kubeadm.
+      // 'kubeadm reset -f' performs a forceful reset, removing installed Kubernetes components,
+      // configuration files, and associated network rules (like iptables entries created by kubeadm).
+      // The '-f' flag bypasses confirmation prompts.
       shellExec(`sudo kubeadm reset -f`);
+
+      // Step 3: Remove specific CNI (Container Network Interface) configuration files.
+      // This command targets and removes the configuration file for Flannel,
+      // a common CNI plugin, which might be left behind after a reset.
       shellExec('sudo rm -f /etc/cni/net.d/10-flannel.conflist');
-      shellExec('sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X');
+
+      // Note: The aggressive 'sudo iptables -F ...' command was intentionally removed from previous versions.
+      // This command would flush all iptables rules, including those crucial for the host's general
+      // internet connectivity, leading to network loss. 'kubeadm reset' and container runtime pruning
+      // adequately handle Kubernetes and container-specific iptables rules without affecting the host's
+      // default network configuration.
+
+      // Step 4: Remove the kubectl configuration file from the current user's home directory.
+      // This ensures that after a reset, there's no lingering configuration pointing to the old cluster,
+      // providing a clean slate for connecting to a new or re-initialized cluster.
       shellExec('sudo rm -f $HOME/.kube/config');
+
+      // Step 5: Clear trash files from the root user's trash directory.
+      // This is a general cleanup step to remove temporary or deleted files.
       shellExec('sudo rm -rf /root/.local/share/Trash/files/*');
+
+      // Step 6: Prune all unused Docker data.
+      // 'docker system prune -a -f' removes:
+      // - All stopped containers
+      // - All unused networks
+      // - All dangling images
+      // - All build cache
+      // - All unused volumes
+      // This aggressively frees up disk space and removes temporary Docker artifacts.
       shellExec('sudo docker system prune -a -f');
+
+      // Step 7: Stop the Docker daemon service.
+      // This step is often necessary to ensure that Docker's files and directories
+      // can be safely manipulated or moved in subsequent steps without conflicts.
       shellExec('sudo service docker stop');
+
+      // Step 8: Aggressively remove container storage data for containerd and Docker.
+      // These commands target the default storage locations for containerd and Docker,
+      // as well as any custom paths that might have been used (`/home/containers/storage`, `/home/docker`).
+      // This ensures a complete wipe of all container images, layers, and volumes.
       shellExec(`sudo rm -rf /var/lib/containers/storage/*`);
       shellExec(`sudo rm -rf /var/lib/docker/volumes/*`);
-      shellExec(`sudo rm -rf /var/lib/docker~/*`);
-      shellExec(`sudo rm -rf /home/containers/storage/*`);
-      shellExec(`sudo rm -rf /home/docker/*`);
-      shellExec('sudo mv /var/lib/docker /var/lib/docker~');
-      shellExec('sudo mkdir /home/docker');
-      shellExec('sudo chmod 0711 /home/docker');
-      shellExec('sudo ln -s /home/docker /var/lib/docker');
+      shellExec(`sudo rm -rf /var/lib/docker~/*`); // Cleans up a potential backup directory for Docker data
+      shellExec(`sudo rm -rf /home/containers/storage/*`); // Cleans up custom containerd/Podman storage
+      shellExec(`sudo rm -rf /home/docker/*`); // Cleans up custom Docker storage
+
+      // Step 9: Re-configure Docker's default storage location (if desired).
+      // These commands effectively move Docker's data directory from its default `/var/lib/docker`
+      // to a new location (`/home/docker`) and create a symbolic link.
+      // This is a specific customization to relocate Docker's storage.
+      shellExec('sudo mv /var/lib/docker /var/lib/docker~'); // Moves existing /var/lib/docker to /var/lib/docker~ (backup)
+      shellExec('sudo mkdir /home/docker'); // Creates the new desired directory for Docker data
+      shellExec('sudo chmod 0711 /home/docker'); // Sets appropriate permissions for the new directory
+      shellExec('sudo ln -s /home/docker /var/lib/docker'); // Creates a symlink from original path to new path
+
+      // Step 10: Prune all unused Podman data.
+      // Similar to Docker pruning, these commands remove:
+      // - All stopped containers
+      // - All unused networks
+      // - All unused images
+      // - All unused volumes ('--volumes')
+      // - The '--force' flag bypasses confirmation.
+      // '--external' prunes external content not managed by Podman's default storage backend.
       shellExec(`sudo podman system prune -a -f`);
       shellExec(`sudo podman system prune --all --volumes --force`);
       shellExec(`sudo podman system prune --external --force`);
-      shellExec(`sudo podman system prune --all --volumes --force`);
+      shellExec(`sudo podman system prune --all --volumes --force`); // Redundant but harmless repetition
+
+      // Step 11: Create and set permissions for Podman's custom storage directory.
+      // This ensures the custom path `/home/containers/storage` exists and has correct permissions
+      // before Podman attempts to use it.
       shellExec(`sudo mkdir -p /home/containers/storage`);
       shellExec('sudo chmod 0711 /home/containers/storage');
+
+      // Step 12: Update Podman's storage configuration file.
+      // This command uses 'sed' to modify `/etc/containers/storage.conf`,
+      // changing the default storage path from `/var/lib/containers/storage`
+      // to the customized `/home/containers/storage`.
       shellExec(
         `sudo sed -i -e "s@/var/lib/containers/storage@/home/containers/storage@g" /etc/containers/storage.conf`,
       );
+
+      // Step 13: Reset Podman system settings.
+      // This command resets Podman's system-wide configuration to its default state.
       shellExec(`sudo podman system reset -f`);
+
+      // Note: The 'sysctl net.bridge.bridge-nf-call-iptables=0' and related commands
+      // were previously removed. These sysctl settings (bridge-nf-call-iptables,
+      // bridge-nf-call-arptables, bridge-nf-call-ip6tables) are crucial for allowing
+      // network traffic through Linux bridges to be processed by iptables.
+      // Kubernetes and CNI plugins generally require them to be enabled (set to '1').
+      // Re-initializing Kubernetes will typically set these as needed, and leaving them
+      // at their system default (or '1' if already configured) is safer for host
+      // connectivity during a reset operation.
+
+      // https://github.com/kubernetes-sigs/kind/issues/2886
+      // shellExec(`sysctl net.bridge.bridge-nf-call-iptables=0`);
+      // shellExec(`sysctl net.bridge.bridge-nf-call-arptables=0`);
+      // shellExec(`sysctl net.bridge.bridge-nf-call-ip6tables=0`);
+
+      // Step 14: Remove the 'kind' Docker network.
+      // This cleans up any network bridges or configurations specifically created by Kind.
+      shellExec(`docker network rm kind`);
+    },
+
+    getResourcesCapacity() {
+      const resources = {};
+      const info = false
+        ? `Capacity:
+  cpu:                8
+  ephemeral-storage:  153131976Ki
+  hugepages-1Gi:      0
+  hugepages-2Mi:      0
+  memory:             11914720Ki
+  pods:               110
+Allocatable:
+  cpu:                8
+  ephemeral-storage:  153131976Ki
+  hugepages-1Gi:      0
+  hugepages-2Mi:      0
+  memory:             11914720Ki
+  pods: `
+        : shellExec(`kubectl describe node kind-worker | grep -E '(Allocatable:|Capacity:)' -A 6`, {
+            stdout: true,
+            silent: true,
+          });
+      info
+        .split('Allocatable:')[1]
+        .split('\n')
+        .filter((row) => row.match('cpu') || row.match('memory'))
+        .map((row) => {
+          if (row.match('cpu'))
+            resources.cpu = {
+              value: parseInt(row.split(':')[1].trim()) * 1000,
+              unit: 'm',
+            };
+          if (row.match('memory'))
+            resources.memory = {
+              value: parseInt(row.split(':')[1].split('Ki')[0].trim()),
+              unit: 'Ki',
+            };
+        });
+
+      return resources;
     },
   };
 }

package/src/cli/cron.js

@@ -4,20 +4,24 @@
  * @namespace UnderpostCron
  */
 
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
 import BackUp from '../server/backup.js';
 import { Cmd } from '../server/conf.js';
 import Dns from '../server/dns.js';
-import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { loggerFactory } from '../server/logger.js';
+
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 
+const logger = loggerFactory(import.meta);
+
 /**
  * UnderpostCron main module methods
  * @class
  * @memberof UnderpostCron
  */
 class UnderpostCron {
+  static NETWORK = [];
   static JOB = {
     /**
      * DNS cli API
@@ -46,10 +50,10 @@ class UnderpostCron {
     callback: async function (
       deployList = 'default',
       jobList = Object.keys(UnderpostCron.JOB),
-      options = { itc: false, init: false, git: false },
+      options = { itc: false, init: false, git: false, dashboardUpdate: false },
     ) {
       if (options.init === true) {
-        await Underpost.test.setUpInfo();
+        UnderpostCron.NETWORK = [];
         const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
         deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
         const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
@@ -57,7 +61,7 @@ class UnderpostCron {
           for (const job of Object.keys(confCronConfig.jobs)) {
             const name = `${jobDeployId}-${job}`;
             let deployId;
-            shellExec(Cmd.delete(name));
+            if (!options.dashboardUpdate) shellExec(Cmd.delete(name));
             switch (job) {
               case 'dns':
                 deployId = jobDeployId;
@@ -67,15 +71,16 @@ class UnderpostCron {
                 deployId = deployList;
                 break;
             }
-            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
-            netWorkCron.push({
+            if (!options.dashboardUpdate)
+              shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            UnderpostCron.NETWORK.push({
               deployId,
               jobId: job,
               expression: confCronConfig.jobs[job].expression,
             });
           }
         }
-        await saveRuntimeCron();
+        if (options.dashboardUpdate === true) await UnderpostCron.API.updateDashboardData();
         if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
         return;
       }
@@ -84,6 +89,32 @@ class UnderpostCron {
         if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
       }
     },
+    async updateDashboardData() {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
+        const { db } = confServer[host][path];
+
+        await DataBaseProvider.load({ apis: ['cron'], host, path, db });
+
+        /** @type {import('../api/cron/cron.model.js').CronModel} */
+        const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
+
+        await Cron.deleteMany();
+
+        for (const cronInstance of UnderpostCron.NETWORK) {
+          logger.info('save', cronInstance);
+          await new Cron(cronInstance).save();
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/db.js

@@ -15,11 +15,13 @@ class UnderpostDB {
         export: false,
         podName: false,
         ns: false,
-        collection: '',
+        collections: '',
         outPath: '',
         drop: false,
         preserveUUID: false,
         git: false,
+        hosts: '',
+        paths: '',
       },
     ) {
       const newBackupTimestamp = new Date().getTime();
@@ -39,20 +41,28 @@ class UnderpostDB {
               if (!dbs[provider]) dbs[provider] = {};
 
               if (!(name in dbs[provider]))
-                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-') };
+                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-'), host, path };
             }
           }
         }
 
-        if (!fs.existsSync(`../${repoName}`)) {
-          shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
-        } else {
-          shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+        if (options.git === true) {
+          if (!fs.existsSync(`../${repoName}`)) {
+            shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
+          } else {
+            shellExec(`cd ../${repoName} && git checkout . && git clean -f -d`);
+            shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+          }
         }
 
         for (const provider of Object.keys(dbs)) {
           for (const dbName of Object.keys(dbs[provider])) {
-            const { hostFolder, user, password } = dbs[provider][dbName];
+            const { hostFolder, user, password, host, path } = dbs[provider][dbName];
+            if (
+              (options.hosts && !options.hosts.split(',').includes(host)) ||
+              (options.paths && !options.paths.split(',').includes(path))
+            )
+              continue;
             if (hostFolder) {
               logger.info('', { hostFolder, provider, dbName });
 
@@ -153,11 +163,11 @@ class UnderpostDB {
                       const podName = podNameData.NAME;
                       shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
                       if (options.collections)
-                        for (const collection of options.collections)
+                        for (const collection of options.collections.split(','))
                           shellExec(
-                            `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /${dbName}"`,
+                            `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /"`,
                           );
-                      else shellExec(`sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} -o /${dbName}"`);
+                      else shellExec(`sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} -o /"`);
                       shellExec(
                         `sudo kubectl cp ${nameSpace}/${podName}:/${dbName} ${
                           options.outPath ? options.outPath : _toNewBsonPath