underpost 2.8.62 → 2.8.65

package/Dockerfile

@@ -1,11 +1,11 @@
 ARG BASE_DEBIAN=buster
 
+# USER root
+
 FROM debian:${BASE_DEBIAN}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-WORKDIR /home/dd
-
 # Set root password to root, format is 'user:password'.
 RUN echo 'root:root' | chpasswd
 
@@ -25,9 +25,6 @@ RUN mkdir -p /var/run/sshd
 # Allow root login via password
 RUN sed -ri 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
 
-# copy supervisor config file to start openssh-server
-COPY supervisord-openssh-server.conf /etc/supervisor/conf.d/supervisord-openssh-server.conf
-
 # install open ssl git and others tools
 RUN apt-get install -yq --no-install-recommends libssl-dev curl wget git gnupg
 
@@ -37,12 +34,14 @@ RUN apt-get install -y nodejs build-essential
 RUN node --version
 RUN npm --version
 
-RUN npm install -g underpost
-
-VOLUME [ "/home/dd/engine/logs" ]
+WORKDIR /home/dd
 
 EXPOSE 22
 
-EXPOSE 4000-4004
+EXPOSE 80
+
+EXPOSE 443
+
+EXPOSE 3000-3100
 
-CMD [ "underpost", "new", "service" ]
+EXPOSE 4000-4100

package/bin/build.js

@@ -47,7 +47,7 @@ if (process.argv.includes('conf')) {
     if (!fs.existsSync(`../${privateRepoName}`)) {
       shellExec(`cd .. && underpost clone ${privateGitUri}`, { silent: true });
     } else {
-      shellExec(`cd ../${privateRepoName} && underpost pull . ${privateGitUri}`);
+      shellExec(`cd ../${privateRepoName} && git checkout . && git clean -f -d && underpost pull . ${privateGitUri}`);
     }
     const toPath = `../${privateRepoName}/conf/${_confName}`;
     fs.removeSync(toPath);
@@ -176,7 +176,7 @@ const { DefaultConf } = await import(`../conf.${confName}.js`);
   if (!fs.existsSync(`${basePath}/images`)) fs.mkdirSync(`${basePath}/images`);
 
   const env = process.argv.includes('development') ? 'development' : 'production';
-  const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'secret.yaml'];
+  const deploymentsFiles = ['proxy.yaml', 'deployment.yaml', 'secret.yaml'];
   // remove engine-private of .dockerignore for local testing
   for (const file of deploymentsFiles) {
     if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {

package/bin/deploy.js

@@ -33,8 +33,9 @@ import { MongooseDB } from '../src/db/mongo/MongooseDB.js';
 import { Lampp } from '../src/runtime/lampp/Lampp.js';
 import { DefaultConf } from '../conf.js';
 import { JSONweb } from '../src/server/client-formatted.js';
-import ejs from 'easy-json-schema';
+
 import { Xampp } from '../src/runtime/xampp/Xampp.js';
+import { ejs } from '../src/server/json-schema.js';
 
 const logger = loggerFactory(import.meta);
 
@@ -673,8 +674,8 @@ try {
     }
 
     case 'version-build': {
-      const newVersion = process.argv[3];
       const originPackageJson = JSON.parse(fs.readFileSync(`package.json`, 'utf8'));
+      const newVersion = process.argv[3] ?? originPackageJson.version;
       const { version } = originPackageJson;
       originPackageJson.version = newVersion;
       fs.writeFileSync(`package.json`, JSON.stringify(originPackageJson, null, 4), 'utf8');
@@ -1048,6 +1049,43 @@ ${shellExec(`git log | grep Author: | sort -u`, { stdout: true }).split(`\n`).jo
       break;
     }
 
+    case 'update-instances': {
+      shellExec(`node bin deploy dd production --sync --build-manifest --info-router --dashboard-update`);
+      shellExec(`node bin cron --dashboard-update --init`);
+      const deployId = 'dd-core';
+      const host = 'www.nexodev.org';
+      const path = '/';
+
+      {
+        const outputPath = './engine-private/instances';
+        if (fs.existsSync(outputPath)) fs.mkdirSync(outputPath, { recursive: true });
+        const collection = 'instances';
+        if (process.argv.includes('export'))
+          shellExec(
+            `node bin db --export --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+          );
+        if (process.argv.includes('import'))
+          shellExec(
+            `node bin db --import --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+          );
+      }
+      {
+        const outputPath = './engine-private/crons';
+        if (fs.existsSync(outputPath)) fs.mkdirSync(outputPath, { recursive: true });
+        const collection = 'crons';
+        if (process.argv.includes('export'))
+          shellExec(
+            `node bin db --export --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+          );
+        if (process.argv.includes('import'))
+          shellExec(
+            `node bin db --import --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+          );
+      }
+
+      break;
+    }
+
     default:
       break;
   }

package/bin/index.js

@@ -22,6 +22,15 @@ program
   .description('Create a new project')
   .action(Underpost.repo.new);
 
+program
+  .command('start')
+  .argument('<deploy-id>', 'Deploy configuration id')
+  .argument('[env]', 'Optional environment, for default is development')
+  .option('--run', 'Run app servers and monitor health server')
+  .option('--build', 'Build app client')
+  .action(Underpost.start.callback)
+  .description('Start up server, build pipelines, or services');
+
 program
   .command('clone')
   .argument(`<uri>`, 'e.g. username/repository')
@@ -91,6 +100,8 @@ program
   .option('--ns-use <ns-name>', 'Switches current context to namespace')
   .option('--dev', 'init with dev cluster')
   .option('--list-pods', 'Display list pods information')
+  .option('--info-capacity', 'display current total machine capacity info')
+  .option('--info-capacity-pod', 'display current machine capacity pod info')
   .action(Underpost.cluster.init)
   .description('Manage cluster, for default initialization base kind cluster');
 
@@ -105,7 +116,10 @@ program
   .option('--info-util', 'Display kubectl util management commands')
   .option('--cert', 'Reset tls/ssl certificate secrets')
   .option('--build-manifest', 'Build kind yaml manifests: deployments, services, proxy and secrets')
-  .option('--version', 'Set custom version')
+  .option('--dashboard-update', 'Update dashboard instance data with current router config')
+  .option('--replicas <replicas>', 'Set custom number of replicas')
+  .option('--versions <deployment-versions>', 'Comma separated custom deployment versions')
+  .option('--traffic <traffic-versions>', 'Comma separated custom deployment traffic')
   .description('Manage deployment, for default deploy development pods')
   .action(Underpost.deploy.callback);
 
@@ -124,24 +138,17 @@ program
     if (args[1].init) return Underpost.secret[args[0]].init();
   });
 
-program
-  .command('dockerfile-node-script')
-  .argument('<deploy-id>', 'Deploy configuration id')
-  .argument('[env]', 'Optional environment, for default is development')
-  .option('--run', 'Run custom entry point script')
-  .option('--build', 'Build custom image container scripts')
-  .description('Dockerfile custom node build script')
-  .action(Underpost.image.dockerfile.script);
-
 program
   .command('dockerfile-image-build')
-  .argument('<deploy-id>', 'Deploy configuration id')
-  .argument('[env]', 'Optional environment, for default is development')
-  .argument('[path]', 'Absolute or relative directory, for default is current')
-  .option('--image-archive', 'Only load tar image from ./images')
-  .option('--podman-save', 'Save image from podman to ./images')
-  .option('--image-name <image-name>', 'Set custom image name')
-  .option('--image-version <image-version>', 'Set custom image version')
+  .option('--path [path]', 'Dockerfile path')
+  .option('--image-name [image-name]', 'Set image name')
+  .option('--image-path [image-path]', 'Set tar image path')
+  .option('--dockerfile-name [dockerfile-name]', 'set Dockerfile name')
+  .option('--podman-save', 'Export tar file from podman')
+  .option('--kind-load', 'Import tar image to Kind cluster')
+  .option('--secrets', 'Dockerfile env secrets')
+  .option('--secrets-path [secrets-path]', 'Dockerfile custom path env secrets')
+  .option('--no-cache', 'Build without using cache')
   .description('Build image from Dockerfile')
   .action(Underpost.image.dockerfile.build);
 
@@ -163,11 +170,13 @@ program
   .option('--import', 'Import container backups from repositories')
   .option('--export', 'Export container backups to repositories')
   .option('--pod-name <pod-name>', 'Optional pod context')
-  .option('--collection <collection>', 'Collection')
+  .option('--collections <collections>', 'Comma separated collections')
   .option('--out-path <out-path>', 'Custom out path backup')
   .option('--drop', 'Drop databases')
   .option('--preserveUUID', 'Preserve Ids')
   .option('--git', 'Upload to github')
+  .option('--hosts <hosts>', 'Comma separated hosts')
+  .option('--paths <paths>', 'Comma separated paths')
   .option('--ns <ns-name>', 'Optional name space context')
   .description('Manage databases')
   .action(Underpost.db.callback);
@@ -193,6 +202,7 @@ program
   .option('--itc', 'Inside container execution context')
   .option('--init', 'Init cron jobs for cron job default deploy id')
   .option('--git', 'Upload to github')
+  .option('--dashboard-update', 'Update dashboard cron data with current jobs config')
   .description('Cron jobs management')
   .action(Underpost.cron.callback);
 
@@ -220,4 +230,15 @@ program
   .option('--kind-type <kind-type>')
   .action(Underpost.test.callback);
 
+program
+  .command('monitor')
+  .argument('<deploy-id>', 'Deploy configuration id')
+  .argument('[env]', 'Optional environment, for default is development')
+  .option('--ms-interval <ms-interval>', 'Custom ms interval delta time')
+  .option('--now', 'Exec immediately monitor script')
+  .option('--single', 'Disable recurrence')
+  .option('--type <type>', 'Set custom monitor type')
+  .description('Monitor health server management')
+  .action(Underpost.monitor.callback);
+
 program.parse();

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.62'
+      engine.version: '2.8.65'
     networks:
       - load-balancer
 

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.62",
+    "version": "2.8.65",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -69,7 +69,6 @@
         "cors": "^2.8.5",
         "d3": "^7.9.0",
         "dotenv": "^16.3.1",
-        "easy-json-schema": "^0.0.2-beta",
         "easymde": "^2.18.0",
         "env-cmd": "^10.1.0",
         "express": "^4.18.2",
@@ -119,13 +118,7 @@
         "vanilla-jsoneditor": "^2.3.2",
         "winston": "^3.11.0"
     },
-    "devDependencies": {
-        "clean-jsdoc-theme": "^4.3.0",
-        "easy-json-schema": "^0.0.2-beta",
-        "mocha": "^10.8.2",
-        "plantuml": "^0.0.2",
-        "swagger-autogen": "^2.23.7"
-    },
+    "devDependencies": {},
     "publishConfig": {
         "provenance": true,
         "access": "public",

package/src/api/default/default.service.js

@@ -24,7 +24,7 @@ const DefaultService = {
     /** @type {import('./default.model.js').DefaultModel} */
     const Default = DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.Default;
     if (req.params.id) return await Default.findByIdAndDelete(req.params.id);
-    else return await await Default.deleteMany();
+    else return await Default.deleteMany();
   },
 };
 

package/src/api/user/user.service.js

@@ -225,8 +225,8 @@ const UserService = {
         } else throw new Error('invalid email or password');
 
       case 'guest': {
-        const user = await ValkeyAPI.valkeyObjectFactory('user', options);
-        await ValkeyAPI.setValkeyObject(user.email, user);
+        const user = await ValkeyAPI.valkeyObjectFactory(options, 'user');
+        await ValkeyAPI.setValkeyObject(options, user.email, user);
         return {
           token: hashJWT({ user: UserDto.auth.payload(user) }),
           user: selectDtoFactory(user, UserDto.select.get()),
@@ -325,15 +325,18 @@ const UserService = {
         return await User.find().select(UserDto.select.getAll());
 
       case 'auth': {
-        const user = (await ValkeyAPI.getValkeyObject(req.auth.user.email))
-          ? await ValkeyAPI.getValkeyObject(req.auth.user.email)
-          : await User.findOne({
-              _id: req.auth.user._id,
-            });
+        let user;
+        if (req.auth.user._id.match('guest')) {
+          user = await ValkeyAPI.getValkeyObject(options, req.auth.user.email);
+          if (!user) throw new Error('guest user expired');
+        } else
+          user = await User.findOne({
+            _id: req.auth.user._id,
+          });
 
         const file = await File.findOne({ _id: user.profileImageId });
 
-        if (!file && !(await ValkeyAPI.getValkeyObject(req.auth.user.email))) {
+        if (!file && !(await ValkeyAPI.getValkeyObject(options, req.auth.user.email))) {
           await User.findByIdAndUpdate(
             user._id,
             { profileImageId: await getDefaultProfileImageId(File) },
@@ -342,8 +345,8 @@ const UserService = {
             },
           );
         }
-        return (await ValkeyAPI.getValkeyObject(req.auth.user.email))
-          ? selectDtoFactory(await ValkeyAPI.getValkeyObject(req.auth.user.email), UserDto.select.get())
+        return (await ValkeyAPI.getValkeyObject(options, req.auth.user.email))
+          ? selectDtoFactory(await ValkeyAPI.getValkeyObject(options, req.auth.user.email), UserDto.select.get())
           : await User.findOne({
               _id: req.auth.user._id,
             }).select(UserDto.select.get());
@@ -378,7 +381,7 @@ const UserService = {
         switch (user.role) {
           case 'admin': {
             if (req.params.id) return await User.findByIdAndDelete(req.params.id);
-            else return await await User.deleteMany();
+            else return await User.deleteMany();
           }
           default:
             if (req.auth.user._id !== req.params.id) throw new Error(`Invalid token user id`);

package/src/cli/cluster.js

@@ -1,5 +1,4 @@
-import { timer } from '../client/components/core/CommonJs.js';
-import { cliSpinner, getNpmRootPath } from '../server/conf.js';
+import { getNpmRootPath } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import UnderpostDeploy from './deploy.js';
@@ -23,10 +22,14 @@ class UnderpostCluster {
         reset: false,
         dev: false,
         nsUse: '',
+        infoCapacity: false,
+        infoCapacityPod: false,
       },
     ) {
       const npmRoot = getNpmRootPath();
       const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
+      if (options.infoCapacityPod === true) return logger.info('', UnderpostDeploy.API.resourcesFactory());
+      if (options.infoCapacity === true) return logger.info('', UnderpostCluster.API.getResourcesCapacity());
       if (options.reset === true) return await UnderpostCluster.API.reset();
       if (options.listPods === true) return console.table(UnderpostDeploy.API.get(podName ?? undefined));
 
@@ -197,6 +200,46 @@ class UnderpostCluster {
       );
       shellExec(`sudo podman system reset -f`);
     },
+    getResourcesCapacity() {
+      const resources = {};
+      const info = false
+        ? `Capacity:
+  cpu:                8
+  ephemeral-storage:  153131976Ki
+  hugepages-1Gi:      0
+  hugepages-2Mi:      0
+  memory:             11914720Ki
+  pods:               110
+Allocatable:
+  cpu:                8
+  ephemeral-storage:  153131976Ki
+  hugepages-1Gi:      0
+  hugepages-2Mi:      0
+  memory:             11914720Ki
+  pods: `
+        : shellExec(`kubectl describe node kind-worker | grep -E '(Allocatable:|Capacity:)' -A 6`, {
+            stdout: true,
+            silent: true,
+          });
+      info
+        .split('Allocatable:')[1]
+        .split('\n')
+        .filter((row) => row.match('cpu') || row.match('memory'))
+        .map((row) => {
+          if (row.match('cpu'))
+            resources.cpu = {
+              value: parseInt(row.split(':')[1].trim()) * 1000,
+              unit: 'm',
+            };
+          if (row.match('memory'))
+            resources.memory = {
+              value: parseInt(row.split(':')[1].split('Ki')[0].trim()),
+              unit: 'Ki',
+            };
+        });
+
+      return resources;
+    },
   };
 }
 export default UnderpostCluster;

package/src/cli/cron.js

@@ -4,20 +4,24 @@
  * @namespace UnderpostCron
  */
 
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
 import BackUp from '../server/backup.js';
 import { Cmd } from '../server/conf.js';
 import Dns from '../server/dns.js';
-import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { loggerFactory } from '../server/logger.js';
+
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 
+const logger = loggerFactory(import.meta);
+
 /**
  * UnderpostCron main module methods
  * @class
  * @memberof UnderpostCron
  */
 class UnderpostCron {
+  static NETWORK = [];
   static JOB = {
     /**
      * DNS cli API
@@ -46,10 +50,10 @@ class UnderpostCron {
     callback: async function (
       deployList = 'default',
       jobList = Object.keys(UnderpostCron.JOB),
-      options = { itc: false, init: false, git: false },
+      options = { itc: false, init: false, git: false, dashboardUpdate: false },
     ) {
       if (options.init === true) {
-        await Underpost.test.setUpInfo();
+        UnderpostCron.NETWORK = [];
         const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
         deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
         const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
@@ -57,7 +61,7 @@ class UnderpostCron {
           for (const job of Object.keys(confCronConfig.jobs)) {
             const name = `${jobDeployId}-${job}`;
             let deployId;
-            shellExec(Cmd.delete(name));
+            if (!options.dashboardUpdate) shellExec(Cmd.delete(name));
             switch (job) {
               case 'dns':
                 deployId = jobDeployId;
@@ -67,15 +71,16 @@ class UnderpostCron {
                 deployId = deployList;
                 break;
             }
-            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
-            netWorkCron.push({
+            if (!options.dashboardUpdate)
+              shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            UnderpostCron.NETWORK.push({
               deployId,
               jobId: job,
               expression: confCronConfig.jobs[job].expression,
             });
           }
         }
-        await saveRuntimeCron();
+        if (options.dashboardUpdate === true) await UnderpostCron.API.updateDashboardData();
         if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
         return;
       }
@@ -84,6 +89,32 @@ class UnderpostCron {
         if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
       }
     },
+    async updateDashboardData() {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
+        const { db } = confServer[host][path];
+
+        await DataBaseProvider.load({ apis: ['cron'], host, path, db });
+
+        /** @type {import('../api/cron/cron.model.js').CronModel} */
+        const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
+
+        await Cron.deleteMany();
+
+        for (const cronInstance of UnderpostCron.NETWORK) {
+          logger.info('save', cronInstance);
+          await new Cron(cronInstance).save();
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/db.js

@@ -15,11 +15,13 @@ class UnderpostDB {
         export: false,
         podName: false,
         ns: false,
-        collection: '',
+        collections: '',
         outPath: '',
         drop: false,
         preserveUUID: false,
         git: false,
+        hosts: '',
+        paths: '',
       },
     ) {
       const newBackupTimestamp = new Date().getTime();
@@ -39,20 +41,28 @@ class UnderpostDB {
               if (!dbs[provider]) dbs[provider] = {};
 
               if (!(name in dbs[provider]))
-                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-') };
+                dbs[provider][name] = { user, password, hostFolder: host + path.replaceAll('/', '-'), host, path };
             }
           }
         }
 
-        if (!fs.existsSync(`../${repoName}`)) {
-          shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
-        } else {
-          shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+        if (options.git === true) {
+          if (!fs.existsSync(`../${repoName}`)) {
+            shellExec(`cd .. && underpost clone ${process.env.GITHUB_USERNAME}/${repoName}`);
+          } else {
+            shellExec(`cd ../${repoName} && git checkout . && git clean -f -d`);
+            shellExec(`cd ../${repoName} && underpost pull . ${process.env.GITHUB_USERNAME}/${repoName}`);
+          }
         }
 
         for (const provider of Object.keys(dbs)) {
           for (const dbName of Object.keys(dbs[provider])) {
-            const { hostFolder, user, password } = dbs[provider][dbName];
+            const { hostFolder, user, password, host, path } = dbs[provider][dbName];
+            if (
+              (options.hosts && !options.hosts.split(',').includes(host)) ||
+              (options.paths && !options.paths.split(',').includes(path))
+            )
+              continue;
             if (hostFolder) {
               logger.info('', { hostFolder, provider, dbName });
 
@@ -153,7 +163,7 @@ class UnderpostDB {
                       const podName = podNameData.NAME;
                       shellExec(`sudo kubectl exec -i ${podName} -- sh -c "rm -rf /${dbName}"`);
                       if (options.collections)
-                        for (const collection of options.collections)
+                        for (const collection of options.collections.split(','))
                           shellExec(
                             `sudo kubectl exec -i ${podName} -- sh -c "mongodump -d ${dbName} --collection ${collection} -o /"`,
                           );