underpost 2.8.0 → 2.8.6

package/src/cli/cluster.js

@@ -0,0 +1,202 @@
+import { timer } from '../client/components/core/CommonJs.js';
+import { cliSpinner, getNpmRootPath } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import UnderpostDeploy from './deploy.js';
+import UnderpostTest from './test.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostCluster {
+  static API = {
+    async init(
+      podName,
+      options = {
+        mongodb: false,
+        mongodb4: false,
+        mariadb: false,
+        valkey: false,
+        full: false,
+        info: false,
+        certManager: false,
+        listPods: false,
+        reset: false,
+        dev: false,
+        nsUse: '',
+      },
+    ) {
+      const npmRoot = getNpmRootPath();
+      const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      if (options.reset === true) return await UnderpostCluster.API.reset();
+      if (options.listPods === true) return console.table(UnderpostDeploy.API.get(podName ?? undefined));
+
+      if (options.nsUse && typeof options.nsUse === 'string') {
+        shellExec(`kubectl config set-context --current --namespace=${options.nsUse}`);
+        return;
+      }
+      if (options.info === true) {
+        shellExec(`kubectl config get-contexts`); // config env persisente for manage multiple clusters
+        shellExec(`kubectl config get-clusters`);
+        shellExec(`kubectl get nodes -o wide`); // set of nodes of a cluster
+        shellExec(`kubectl config view | grep namespace`);
+        shellExec(`kubectl get ns -o wide`); // A namespace can have pods of different nodes
+        shellExec(`kubectl get pvc --all-namespaces -o wide`); // PersistentVolumeClaim -> request storage service
+        shellExec(`kubectl get pv --all-namespaces -o wide`); // PersistentVolume -> real storage
+        shellExec(`kubectl get cronjob --all-namespaces -o wide`);
+        shellExec(`kubectl get svc --all-namespaces -o wide`); // proxy dns gate way -> deployments, statefulsets, pods
+        shellExec(`kubectl get statefulsets --all-namespaces -o wide`); // set pods with data/volume persistence
+        shellExec(`kubectl get deployments --all-namespaces -o wide`); // set pods
+        shellExec(`kubectl get configmap --all-namespaces -o wide`);
+        shellExec(`kubectl get pods --all-namespaces -o wide`);
+        shellExec(
+          `kubectl get pod --all-namespaces -o="custom-columns=NAME:.metadata.name,INIT-CONTAINERS:.spec.initContainers[*].name,CONTAINERS:.spec.containers[*].name"`,
+        );
+        shellExec(
+          `kubectl get pods --all-namespaces -o=jsonpath='{range .items[*]}{"\\n"}{.metadata.name}{":\\t"}{range .spec.containers[*]}{.image}{", "}{end}{end}'`,
+        );
+        console.log();
+        logger.info('contour -------------------------------------------------');
+        for (const _k of ['Cluster', 'HTTPProxy', 'ClusterIssuer', 'Certificate']) {
+          shellExec(`kubectl get ${_k} --all-namespaces -o wide`);
+        }
+        logger.info('----------------------------------------------------------------');
+        shellExec(`kubectl get secrets --all-namespaces -o wide`);
+        shellExec(`docker secret ls`);
+        shellExec(`kubectl get crd --all-namespaces -o wide`);
+        return;
+      }
+
+      if (!UnderpostDeploy.API.get('kube-apiserver-kind-control-plane')[0]) {
+        shellExec(`containerd config default > /etc/containerd/config.toml`);
+        shellExec(`sed -i -e "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml`);
+        // shellExec(`cp /etc/kubernetes/admin.conf ~/.kube/config`);
+        shellExec(`sudo systemctl restart kubelet`);
+        shellExec(`sudo service docker restart`);
+        shellExec(`sudo systemctl enable --now containerd.service`);
+        shellExec(`sudo systemctl restart containerd`);
+        shellExec(
+          `cd ${underpostRoot}/manifests && kind create cluster --config kind-config${
+            options?.dev === true ? '-dev' : ''
+          }.yaml`,
+        );
+        shellExec(`sudo chown $(id -u):$(id -g) $HOME/.kube/config**`);
+      } else logger.warn('Cluster already initialized');
+
+      if (options.full === true || options.valkey === true) {
+        shellExec(`kubectl delete statefulset service-valkey`);
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/valkey`);
+      }
+      if (options.full === true || options.mariadb === true) {
+        shellExec(
+          `sudo kubectl create secret generic mariadb-secret --from-file=username=/home/dd/engine/engine-private/mariadb-username --from-file=password=/home/dd/engine/engine-private/mariadb-password`,
+        );
+        shellExec(
+          `sudo kubectl create secret generic github-secret --from-literal=GITHUB_TOKEN=${process.env.GITHUB_TOKEN}`,
+        );
+        shellExec(`kubectl delete statefulset mariadb-statefulset`);
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mariadb`);
+      }
+      if (options.mongodb4 === true) {
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb-4.4`);
+
+        const deploymentName = 'mongodb-deployment';
+
+        const successInstance = await UnderpostTest.API.statusMonitor(deploymentName);
+
+        if (successInstance) {
+          const mongoConfig = {
+            _id: 'rs0',
+            members: [{ _id: 0, host: '127.0.0.1:27017' }],
+          };
+
+          const [pod] = UnderpostDeploy.API.get(deploymentName);
+
+          shellExec(
+            `sudo kubectl exec -i ${pod.NAME} -- mongo --quiet \
+        --eval 'rs.initiate(${JSON.stringify(mongoConfig)})'`,
+          );
+        }
+
+        // await UnderpostTest.API.statusMonitor('mongodb-1');
+      } else if (options.full === true || options.mongodb === true) {
+        shellExec(
+          `sudo kubectl create secret generic mongodb-keyfile --from-file=/home/dd/engine/engine-private/mongodb-keyfile`,
+        );
+        shellExec(
+          `sudo kubectl create secret generic mongodb-secret --from-file=username=/home/dd/engine/engine-private/mongodb-username --from-file=password=/home/dd/engine/engine-private/mongodb-password`,
+        );
+        shellExec(`kubectl delete statefulset mongodb`);
+        shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb`);
+
+        const successInstance = await UnderpostTest.API.statusMonitor('mongodb-1');
+
+        if (successInstance) {
+          const mongoConfig = {
+            _id: 'rs0',
+            members: [
+              { _id: 0, host: 'mongodb-0.mongodb-service:27017', priority: 1 },
+              { _id: 1, host: 'mongodb-1.mongodb-service:27017', priority: 1 },
+            ],
+          };
+
+          shellExec(
+            `sudo kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
+        --eval 'use admin' \
+        --eval 'rs.initiate(${JSON.stringify(mongoConfig)})' \
+        --eval 'rs.status()'`,
+          );
+        }
+      }
+
+      if (options.full === true || options.contour === true)
+        shellExec(`kubectl apply -f https://projectcontour.io/quickstart/contour.yaml`);
+
+      if (options.full === true || options.certManager === true) {
+        if (!UnderpostDeploy.API.get('cert-manager').find((p) => p.STATUS === 'Running')) {
+          shellExec(`helm repo add jetstack https://charts.jetstack.io --force-update`);
+          shellExec(
+            `helm install cert-manager jetstack/cert-manager \
+--namespace cert-manager \
+--create-namespace \
+--version v1.17.0 \
+--set crds.enabled=true`,
+          );
+        }
+
+        const letsEncName = 'letsencrypt-prod';
+        shellExec(`sudo kubectl delete ClusterIssuer ${letsEncName}`);
+        shellExec(`sudo kubectl apply -f ${underpostRoot}/manifests/${letsEncName}.yaml`);
+      }
+    },
+    reset() {
+      shellExec(`kind get clusters | xargs -t -n1 kind delete cluster --name`);
+      shellExec(`sudo kubeadm reset -f`);
+      shellExec('sudo rm -f /etc/cni/net.d/10-flannel.conflist');
+      shellExec('sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X');
+      shellExec('sudo rm -f $HOME/.kube/config');
+      shellExec('sudo rm -rf /root/.local/share/Trash/files/*');
+      shellExec('sudo docker system prune -a -f');
+      shellExec('sudo service docker stop');
+      shellExec(`sudo rm -rf /var/lib/containers/storage/*`);
+      shellExec(`sudo rm -rf /var/lib/docker/volumes/*`);
+      shellExec(`sudo rm -rf /var/lib/docker~/*`);
+      shellExec(`sudo rm -rf /home/containers/storage/*`);
+      shellExec(`sudo rm -rf /home/docker/*`);
+      shellExec('sudo mv /var/lib/docker /var/lib/docker~');
+      shellExec('sudo mkdir /home/docker');
+      shellExec('sudo chmod 0711 /home/docker');
+      shellExec('sudo ln -s /home/docker /var/lib/docker');
+      shellExec(`sudo podman system prune -a -f`);
+      shellExec(`sudo podman system prune --all --volumes --force`);
+      shellExec(`sudo podman system prune --external --force`);
+      shellExec(`sudo podman system prune --all --volumes --force`);
+      shellExec(`sudo mkdir -p /home/containers/storage`);
+      shellExec('sudo chmod 0711 /home/containers/storage');
+      shellExec(
+        `sudo sed -i -e "s@/var/lib/containers/storage@/home/containers/storage@g" /etc/containers/storage.conf`,
+      );
+      shellExec(`sudo podman system reset -f`);
+    },
+  };
+}
+export default UnderpostCluster;

package/src/cli/cron.js

@@ -0,0 +1,90 @@
+/**
+ * UnderpostCron CLI index module
+ * @module src/cli/cron.js
+ * @namespace UnderpostCron
+ */
+
+import Underpost from '../index.js';
+import BackUp from '../server/backup.js';
+import { Cmd } from '../server/conf.js';
+import Dns from '../server/dns.js';
+import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+
+/**
+ * UnderpostCron main module methods
+ * @class
+ * @memberof UnderpostCron
+ */
+class UnderpostCron {
+  static JOB = {
+    /**
+     * DNS cli API
+     * @static
+     * @type {Dns}
+     * @memberof UnderpostCron
+     */
+    dns: Dns,
+    /**
+     * BackUp cli API
+     * @static
+     * @type {BackUp}
+     * @memberof UnderpostCron
+     */
+    backup: BackUp,
+  };
+  static API = {
+    /**
+     * Run the cron jobs
+     * @static
+     * @param {String} deployList - Comma separated deploy ids
+     * @param {String} jobList - Comma separated job ids
+     * @return {void}
+     * @memberof UnderpostCron
+     */
+    callback: async function (
+      deployList = 'default',
+      jobList = Object.keys(UnderpostCron.JOB),
+      options = { itc: false, init: false, git: false },
+    ) {
+      if (options.init === true) {
+        await Underpost.test.setUpInfo();
+        const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
+        deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
+        const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
+        if (confCronConfig.jobs && Object.keys(confCronConfig.jobs).length > 0) {
+          for (const job of Object.keys(confCronConfig.jobs)) {
+            const name = `${jobDeployId}-${job}`;
+            let deployId;
+            shellExec(Cmd.delete(name));
+            switch (job) {
+              case 'dns':
+                deployId = jobDeployId;
+                break;
+
+              default:
+                deployId = deployList;
+                break;
+            }
+            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            netWorkCron.push({
+              deployId,
+              jobId: job,
+              expression: confCronConfig.jobs[job].expression,
+            });
+          }
+        }
+        await saveRuntimeCron();
+        if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
+        return;
+      }
+      for (const _jobId of jobList.split(',')) {
+        const jobId = _jobId.trim();
+        if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
+      }
+    },
+  };
+}
+
+export default UnderpostCron;

package/src/cli/db.js

@@ -0,0 +1,212 @@
+import { mergeFile, splitFileFactory } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+import UnderpostDeploy from './deploy.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostDB {
+  static API = {
+    async callback(
+      deployList = 'default',
+      options = {
+        import: false,
+        export: false,
+        podName: false,
+        ns: false,
+        collection: '',
+        outPath: '',
+        drop: false,
+        preserveUUID: false,
+        git: false,
+      },
+    ) {
+      const newBackupTimestamp = new Date().getTime();
+      const nameSpace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+        const dbs = {};
+        const repoName = `engine-${deployId.split('dd-')[1]}-cron-backups`;
+
+        const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
+        for (const host of Object.keys(confServer)) {
+          for (const path of Object.keys(confServer[host])) {
+            const { db } = confServer[host][path];
+            if (db) {
+              const { provider, name, user, password } = db;
+              if (!dbs[provider]) dbs[provider] = {};
+
+              if (!(name in dbs[provider]))
+                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-') };
+            }
+          }
+        }
+
+        if (!fs.existsSync(`../${repoName}`)) {
+          shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
+        } else {
+          shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+        }
+
+        for (const provider of Object.keys(dbs)) {
+          for (const dbName of Object.keys(dbs[provider])) {
+            const { hostFolder, user, password } = dbs[provider][dbName];
+            if (hostFolder) {
+              logger.info('', { hostFolder, provider, dbName });
+
+              const backUpPath = `../${repoName}/${hostFolder}`;
+              if (!fs.existsSync(backUpPath)) fs.mkdirSync(backUpPath, { recursive: true });
+              shellExec(`cd ${backUpPath} && find . -type d -empty -delete`); // delete empty folders
+              const times = await fs.readdir(backUpPath);
+              const currentBackupTimestamp = Math.max(...times.map((t) => parseInt(t)).filter((t) => !isNaN(t)));
+              dbs[provider][dbName].currentBackupTimestamp = currentBackupTimestamp;
+              const removeBackupTimestamp = Math.min(...times.map((t) => parseInt(t)).filter((t) => !isNaN(t)));
+
+              const sqlContainerPath = `/home/${dbName}.sql`;
+              const _fromPartsParts = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}-parths.json`;
+              const _toSqlPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}.sql`;
+              const _toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
+              const _toBsonPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}`;
+              const _toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
+
+              if (options.import === true && fs.existsSync(_fromPartsParts) && !fs.existsSync(_toSqlPath)) {
+                const names = JSON.parse(fs.readFileSync(_fromPartsParts, 'utf8')).map((_path) => {
+                  return `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${_path.split('/').pop()}`;
+                });
+                logger.info('merge Back Up paths', {
+                  _fromPartsParts,
+                  _toSqlPath,
+                  names,
+                });
+                await mergeFile(names, _toSqlPath);
+              }
+
+              if (options.export === true && times.length >= 5) {
+                logger.info('remove', `../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                fs.removeSync(`../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                logger.info('create', `../${repoName}/${hostFolder}/${newBackupTimestamp}`);
+                fs.mkdirSync(`../${repoName}/${hostFolder}/${newBackupTimestamp}`, { recursive: true });
+              }
+
+              switch (provider) {
+                case 'mariadb': {
+                  const podNames =
+                    options.podName && typeof options.podName === 'string'
+                      ? options.podName.split(',')
+                      : UnderpostDeploy.API.get('mariadb'); // `mariadb-statefulset-0`;
+                  const serviceName = 'mariadb';
+                  for (const podNameData of [podNames[0]]) {
+                    const podName = podNameData.NAME;
+                    if (options.import === true) {
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}.sql"`);
+                      shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
+                      const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
+                      shellExec(
+                        `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
+                      );
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    }
+                    if (options.export === true) {
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf ${sqlContainerPath}"`);
+                      const cmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${sqlContainerPath}`;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:${sqlContainerPath} ${
+                          options.outPath ? options.outPath : _toNewSqlPath
+                        }`,
+                      );
+                      await splitFileFactory(dbName, options.outPath ? options.outPath : _toNewSqlPath);
+                    }
+                  }
+                  break;
+                }
+
+                case 'mongoose': {
+                  if (options.import === true) {
+                    const podNames =
+                      options.podName && typeof options.podName === 'string'
+                        ? options.podName.split(',')
+                        : UnderpostDeploy.API.get('mongo');
+                    // `mongodb-0`;
+                    for (const podNameData of [podNames[0]]) {
+                      const podName = podNameData.NAME;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
+                      shellExec(
+                        `sudo kubectl cp ${
+                          options.outPath ? options.outPath : _toBsonPath
+                        } ${nameSpace}/${podName}:/${dbName}`,
+                      );
+                      const cmd = `mongorestore -d ${dbName} /${dbName}${options.drop ? ' --drop' : ''}${
+                        options.preserveUUID ? ' --preserveUUID' : ''
+                      }`;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    }
+                  }
+                  if (options.export === true) {
+                    const podNames =
+                      options.podName && typeof options.podName === 'string'
+                        ? options.podName.split(',')
+                        : UnderpostDeploy.API.get('mongo'); // `backup-access`;
+                    for (const podNameData of [podNames[0]]) {
+                      const podName = podNameData.NAME;
+                      shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
+                      if (options.collections)
+                        for (const collection of options.collections)
+                          shellExec(
+                            `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /${dbName}"`,
+                          );
+                      else shellExec(`sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} -o /${dbName}"`);
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:/${dbName} ${
+                          options.outPath ? options.outPath : _toNewBsonPath
+                        }`,
+                      );
+                    }
+                    if (false) {
+                      const containerBaseBackupPath = '/backup';
+                      let timeFolder = shellExec(
+                        `sudo kubectl exec -i ${podName} -- sh -c "cd ${containerBaseBackupPath} && ls -a"`,
+                        {
+                          stdout: true,
+                          disableLog: false,
+                          silent: true,
+                        },
+                      ).split(`\n`);
+                      timeFolder = timeFolder[timeFolder.length - 2];
+                      if (timeFolder === '..') {
+                        logger.warn(`Cannot backup available`, { timeFolder });
+                      } else {
+                        shellExec(
+                          `sudo kubectl cp ${nameSpace}/${podName}:${containerBaseBackupPath}/${timeFolder}/${dbName} ${_toNewBsonPath}`,
+                        );
+                      }
+                    }
+                  }
+                  break;
+                }
+
+                default:
+                  break;
+              }
+            }
+          }
+        }
+        if (options.export === true && options.git === true) {
+          shellExec(`cd ../${repoName} && git add .`);
+          shellExec(
+            `underpost cmt ../${repoName} backup '' '${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
+              newBackupTimestamp,
+            ).toLocaleTimeString()}'`,
+          );
+          shellExec(`cd ../${repoName} && underpost push . ${process.env.GITHUB_USERNAME}/${repoName}`, {
+            disableLog: true,
+          });
+        }
+      }
+    },
+  };
+}
+
+export default UnderpostDB;