underpost 2.8.635 → 2.8.641

package/Dockerfile

@@ -1,11 +1,11 @@
 ARG BASE_DEBIAN=buster
 
+# USER root
+
 FROM debian:${BASE_DEBIAN}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-WORKDIR /home/dd
-
 # Set root password to root, format is 'user:password'.
 RUN echo 'root:root' | chpasswd
 
@@ -25,9 +25,6 @@ RUN mkdir -p /var/run/sshd
 # Allow root login via password
 RUN sed -ri 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
 
-# copy supervisor config file to start openssh-server
-COPY supervisord-openssh-server.conf /etc/supervisor/conf.d/supervisord-openssh-server.conf
-
 # install open ssl git and others tools
 RUN apt-get install -yq --no-install-recommends libssl-dev curl wget git gnupg
 
@@ -37,12 +34,14 @@ RUN apt-get install -y nodejs build-essential
 RUN node --version
 RUN npm --version
 
-RUN npm install -g underpost
-
-VOLUME [ "/home/dd/engine/logs" ]
+WORKDIR /home/dd
 
 EXPOSE 22
 
-EXPOSE 4000-4004
+EXPOSE 80
+
+EXPOSE 443
+
+EXPOSE 3000-3100
 
-CMD [ "underpost", "new", "service" ]
+EXPOSE 4000-4100

package/bin/build.js

@@ -47,7 +47,7 @@ if (process.argv.includes('conf')) {
     if (!fs.existsSync(`../${privateRepoName}`)) {
       shellExec(`cd .. && underpost clone ${privateGitUri}`, { silent: true });
     } else {
-      shellExec(`cd ../${privateRepoName} && underpost pull . ${privateGitUri}`);
+      shellExec(`cd ../${privateRepoName} && git checkout . && git clean -f -d && underpost pull . ${privateGitUri}`);
     }
     const toPath = `../${privateRepoName}/conf/${_confName}`;
     fs.removeSync(toPath);
@@ -176,7 +176,7 @@ const { DefaultConf } = await import(`../conf.${confName}.js`);
   if (!fs.existsSync(`${basePath}/images`)) fs.mkdirSync(`${basePath}/images`);
 
   const env = process.argv.includes('development') ? 'development' : 'production';
-  const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'secret.yaml'];
+  const deploymentsFiles = ['proxy.yaml', 'deployment.yaml', 'secret.yaml'];
   // remove engine-private of .dockerignore for local testing
   for (const file of deploymentsFiles) {
     if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {

package/bin/deploy.js

@@ -33,8 +33,9 @@ import { MongooseDB } from '../src/db/mongo/MongooseDB.js';
 import { Lampp } from '../src/runtime/lampp/Lampp.js';
 import { DefaultConf } from '../conf.js';
 import { JSONweb } from '../src/server/client-formatted.js';
-import ejs from 'easy-json-schema';
+
 import { Xampp } from '../src/runtime/xampp/Xampp.js';
+import { ejs } from '../src/server/json-schema.js';
 
 const logger = loggerFactory(import.meta);
 

package/bin/index.js

@@ -105,6 +105,7 @@ program
   .option('--info-util', 'Display kubectl util management commands')
   .option('--cert', 'Reset tls/ssl certificate secrets')
   .option('--build-manifest', 'Build kind yaml manifests: deployments, services, proxy and secrets')
+  .option('--dashboard-update', 'Update dashboard instance data with current router config')
   .option('--version', 'Set custom version')
   .description('Manage deployment, for default deploy development pods')
   .action(Underpost.deploy.callback);
@@ -135,13 +136,15 @@ program
 
 program
   .command('dockerfile-image-build')
-  .argument('<deploy-id>', 'Deploy configuration id')
-  .argument('[env]', 'Optional environment, for default is development')
-  .argument('[path]', 'Absolute or relative directory, for default is current')
-  .option('--image-archive', 'Only load tar image from ./images')
-  .option('--podman-save', 'Save image from podman to ./images')
-  .option('--image-name <image-name>', 'Set custom image name')
-  .option('--image-version <image-version>', 'Set custom image version')
+  .option('--path [path]', 'Dockerfile path')
+  .option('--image-name [image-name]', 'Set image name')
+  .option('--image-path [image-path]', 'Set tar image path')
+  .option('--dockerfile-name [dockerfile-name]', 'set Dockerfile name')
+  .option('--podman-save', 'Export tar file from podman')
+  .option('--kind-load', 'Import tar image to Kind cluster')
+  .option('--secrets', 'Dockerfile env secrets')
+  .option('--secrets-path [secrets-path]', 'Dockerfile custom path env secrets')
+  .option('--no-cache', 'Build without using cache')
   .description('Build image from Dockerfile')
   .action(Underpost.image.dockerfile.build);
 
@@ -193,6 +196,7 @@ program
   .option('--itc', 'Inside container execution context')
   .option('--init', 'Init cron jobs for cron job default deploy id')
   .option('--git', 'Upload to github')
+  .option('--dashboard-update', 'Update dashboard cron data with current jobs config')
   .description('Cron jobs management')
   .action(Underpost.cron.callback);
 
@@ -224,8 +228,10 @@ program
   .command('monitor')
   .argument('<deploy-id>', 'Deploy configuration id')
   .argument('[env]', 'Optional environment, for default is development')
+  .option('--ms-interval <ms-interval>', 'Custom ms interval delta time')
+  .option('--now', 'Exec immediately monitor script')
+  .option('--single', 'Disable recurrence')
   .description('Monitor health server management')
-  .option('--itc', 'Inside container execution context')
   .action(Underpost.monitor.callback);
 
 program.parse();

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.635'
+      engine.version: '2.8.641'
     networks:
       - load-balancer
 

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.635",
+    "version": "2.8.641",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -69,7 +69,6 @@
         "cors": "^2.8.5",
         "d3": "^7.9.0",
         "dotenv": "^16.3.1",
-        "easy-json-schema": "^0.0.2-beta",
         "easymde": "^2.18.0",
         "env-cmd": "^10.1.0",
         "express": "^4.18.2",
@@ -121,7 +120,6 @@
     },
     "devDependencies": {
         "clean-jsdoc-theme": "^4.3.0",
-        "easy-json-schema": "^0.0.2-beta",
         "mocha": "^10.8.2",
         "plantuml": "^0.0.2",
         "swagger-autogen": "^2.23.7"

package/src/api/default/default.service.js

@@ -24,7 +24,7 @@ const DefaultService = {
     /** @type {import('./default.model.js').DefaultModel} */
     const Default = DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.Default;
     if (req.params.id) return await Default.findByIdAndDelete(req.params.id);
-    else return await await Default.deleteMany();
+    else return await Default.deleteMany();
   },
 };
 

package/src/api/user/user.service.js

@@ -378,7 +378,7 @@ const UserService = {
         switch (user.role) {
           case 'admin': {
             if (req.params.id) return await User.findByIdAndDelete(req.params.id);
-            else return await await User.deleteMany();
+            else return await User.deleteMany();
           }
           default:
             if (req.auth.user._id !== req.params.id) throw new Error(`Invalid token user id`);

package/src/cli/cron.js

@@ -4,20 +4,24 @@
  * @namespace UnderpostCron
  */
 
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
 import BackUp from '../server/backup.js';
 import { Cmd } from '../server/conf.js';
 import Dns from '../server/dns.js';
-import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { loggerFactory } from '../server/logger.js';
+
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 
+const logger = loggerFactory(import.meta);
+
 /**
  * UnderpostCron main module methods
  * @class
  * @memberof UnderpostCron
  */
 class UnderpostCron {
+  static NETWORK = [];
   static JOB = {
     /**
      * DNS cli API
@@ -46,10 +50,10 @@ class UnderpostCron {
     callback: async function (
       deployList = 'default',
       jobList = Object.keys(UnderpostCron.JOB),
-      options = { itc: false, init: false, git: false },
+      options = { itc: false, init: false, git: false, dashboardUpdate: false },
     ) {
       if (options.init === true) {
-        await Underpost.test.setUpInfo();
+        UnderpostCron.NETWORK = [];
         const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
         deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
         const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
@@ -57,7 +61,7 @@ class UnderpostCron {
           for (const job of Object.keys(confCronConfig.jobs)) {
             const name = `${jobDeployId}-${job}`;
             let deployId;
-            shellExec(Cmd.delete(name));
+            if (!options.dashboardUpdate) shellExec(Cmd.delete(name));
             switch (job) {
               case 'dns':
                 deployId = jobDeployId;
@@ -67,15 +71,16 @@ class UnderpostCron {
                 deployId = deployList;
                 break;
             }
-            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
-            netWorkCron.push({
+            if (!options.dashboardUpdate)
+              shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            UnderpostCron.NETWORK.push({
               deployId,
               jobId: job,
               expression: confCronConfig.jobs[job].expression,
             });
           }
         }
-        await saveRuntimeCron();
+        if (options.dashboardUpdate === true) await UnderpostCron.API.updateDashboardData();
         if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
         return;
       }
@@ -84,6 +89,32 @@ class UnderpostCron {
         if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
       }
     },
+    async updateDashboardData() {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
+        const { db } = confServer[host][path];
+
+        await DataBaseProvider.load({ apis: ['cron'], host, path, db });
+
+        /** @type {import('../api/cron/cron.model.js').CronModel} */
+        const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
+
+        await Cron.deleteMany();
+
+        for (const cronInstance of UnderpostCron.NETWORK) {
+          logger.info('save', cronInstance);
+          await new Cron(cronInstance).save();
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/deploy.js

@@ -12,11 +12,12 @@ import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 import dotenv from 'dotenv';
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
 
 const logger = loggerFactory(import.meta);
 
 class UnderpostDeploy {
+  static NETWORK = {};
   static API = {
     sync(deployList) {
       const deployGroupId = 'dd.tmp';
@@ -50,7 +51,7 @@ class UnderpostDeploy {
         if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
 
         logger.info('port range', { deployId, fromPort, toPort });
-        // const customImg = `underpost-engine:${version && typeof version === 'string' ? version : Underpost.version}`;
+        // const customImg = `underpost-engine:${version && typeof version === 'string' ? version : '0.0.0'}`;
         // lifecycle:
         // postStart:
         //   exec:
@@ -85,11 +86,7 @@ spec:
             - -c
             - >
               npm install -g npm@11.2.0 &&
-              npm config delete proxy &&
-              npm config delete http-proxy &&
-              npm config delete https-proxy &&
-              npm config set registry http://registry.npmjs.org/ &&
-              npm install --unsafe-perm --ignore-scripts -g underpost &&
+              npm install -g underpost &&
               underpost secret underpost --create-from-file /etc/config/.env.${env} &&
               underpost dockerfile-node-script --build --run ${deployId} ${env}
           volumeMounts:
@@ -99,7 +96,7 @@ spec:
         - name: config-volume
           configMap:
             name: underpost-config
-# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : Underpost.version}
+# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : '0.0.0'}
 ---
 apiVersion: v1
 kind: Service
@@ -200,6 +197,7 @@ spec:
         expose: false,
         cert: false,
         version: '',
+        dashboardUpdate: false,
       },
     ) {
       if (options.infoUtil === true)
@@ -212,8 +210,9 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
         deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
       if (options.sync) UnderpostDeploy.API.sync(deployList);
       if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options.version);
-      if (options.infoRouter === true)
-        return logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      if (options.infoRouter === true) logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      if (options.dashboardUpdate === true) await UnderpostDeploy.API.updateDashboardData(deployList, env, options);
+      if (options.infoRouter === true) return;
       shellExec(`kubectl delete configmap underpost-config`);
       shellExec(
         `kubectl create configmap underpost-config --from-file=/home/dd/engine/engine-private/conf/dd-cron/.env.${env}`,
@@ -316,6 +315,60 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
 
       return result;
     },
+    async updateDashboardData(deployList, env, options) {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const { db } = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'))[host][
+          path
+        ];
+
+        await DataBaseProvider.load({ apis: ['instance'], host, path, db });
+
+        /** @type {import('../api/instance/instance.model.js').InstanceModel} */
+        const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
+
+        await Instance.deleteMany();
+
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
+          const confServer = loadReplicas(
+            JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+            'proxy',
+          );
+          const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+          const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
+
+          for (const host of Object.keys(confServer)) {
+            for (const { path, port } of pathPortAssignmentData[host]) {
+              if (!confServer[host][path]) continue;
+
+              const { client, runtime, apis } = confServer[host][path];
+
+              const body = {
+                deployId,
+                host,
+                path,
+                port,
+                client,
+                runtime,
+                apis,
+              };
+
+              logger.info('save', body);
+
+              await new Instance(body).save();
+            }
+          }
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/fs.js

@@ -38,7 +38,7 @@ class UnderpostFileStorage {
       options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false },
     ) {
       const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
-      const deleteFiles = UnderpostRepository.API.getDeleteFiles(path);
+      const deleteFiles = options.pull === true ? [] : UnderpostRepository.API.getDeleteFiles(path);
       for (const relativePath of deleteFiles) {
         const _path = path + '/' + relativePath;
         if (_path in storage) {
@@ -46,10 +46,6 @@ class UnderpostFileStorage {
           delete storage[_path];
         }
       }
-      const files =
-        options.git === true
-          ? UnderpostRepository.API.getChangedFiles(path)
-          : await fs.readdir(path, { recursive: true });
       if (options.pull === true) {
         for (const _path of Object.keys(storage)) {
           if (!fs.existsSync(_path) || options.force === true) {
@@ -57,7 +53,11 @@ class UnderpostFileStorage {
             await UnderpostFileStorage.API.pull(_path, options);
           } else logger.warn(`Pull path already exists`, _path);
         }
-      } else
+      } else {
+        const files =
+          options.git === true
+            ? UnderpostRepository.API.getChangedFiles(path)
+            : await fs.readdir(path, { recursive: true });
         for (const relativePath of files) {
           const _path = path + '/' + relativePath;
           if (fs.statSync(_path).isDirectory()) {
@@ -68,6 +68,7 @@ class UnderpostFileStorage {
             if (storage) storage[_path] = {};
           } else logger.warn('File already exists', _path);
         }
+      }
       UnderpostFileStorage.API.writeStorageConf(storage, storageConf);
       if (options.git === true) {
         shellExec(`cd ${path} && git add .`);

package/src/cli/image.js

@@ -1,9 +1,7 @@
 import fs from 'fs-extra';
-import Underpost from '../index.js';
 import { shellCd, shellExec } from '../server/process.js';
 import dotenv from 'dotenv';
 import { awaitDeployMonitor, getNpmRootPath } from '../server/conf.js';
-import { timer } from '../client/components/core/CommonJs.js';
 import { loggerFactory } from '../server/logger.js';
 import UnderpostMonitor from './monitor.js';
 
@@ -18,41 +16,49 @@ class UnderpostImage {
         shellExec(`sudo podman pull docker.io/library/debian:buster`);
       },
       build(
-        deployId = 'default',
-        env = 'development',
-        path = '.',
-        options = { imageArchive: false, podmanSave: false, imageName: '', imageVersion: '' },
+        options = {
+          path: '',
+          imageName: '',
+          imagePath: '',
+          dockerfileName: '',
+          podmanSave: false,
+          kindLoad: false,
+          secrets: false,
+          secretsPath: '',
+          noCache: false,
+        },
       ) {
-        const imgName = `${
-          options.imageName && typeof options.imageName === 'string' ? options.imageName : `${deployId}-${env}`
-        }:${
-          options.imageVersion && typeof options.imageVersion === 'string' ? options.imageVersion : Underpost.version
-        }`;
-        const podManImg = `localhost/${imgName}`;
-        const imagesStoragePath = `/images`;
-        if (!fs.existsSync(`${path}${imagesStoragePath}`))
-          fs.mkdirSync(`${path}${imagesStoragePath}`, { recursive: true });
-        const tarFile = `.${imagesStoragePath}/${imgName.replace(':', '_')}.tar`;
-
-        let secrets = ' ';
+        const { path, imageName, imagePath, dockerfileName, podmanSave, secrets, secretsPath, kindLoad, noCache } =
+          options;
+        const podManImg = `localhost/${imageName}`;
+        if (imagePath && typeof imagePath === 'string' && !fs.existsSync(imagePath))
+          fs.mkdirSync(imagePath, { recursive: true });
+        const tarFile = `${imagePath}/${imageName.replace(':', '_')}.tar`;
+        let secretsInput = ' ';
         let secretDockerInput = '';
-
-        const envObj = dotenv.parse(fs.readFileSync(`${getNpmRootPath()}/underpost/.env`, 'utf8'));
-
-        for (const key of Object.keys(envObj)) {
-          continue;
-          secrets += ` && export ${key}="${envObj[key]}" `; // $(cat gitlab-token.txt)
-          secretDockerInput += ` --secret id=${key},env=${key} \ `;
+        let cache = '';
+        if (secrets === true) {
+          const envObj = dotenv.parse(
+            fs.readFileSync(
+              secretsPath && typeof secretsPath === 'string' ? secretsPath : `${getNpmRootPath()}/underpost/.env`,
+              'utf8',
+            ),
+          );
+          for (const key of Object.keys(envObj)) {
+            secretsInput += ` && export ${key}="${envObj[key]}" `; // $(cat gitlab-token.txt)
+            secretDockerInput += ` --secret id=${key},env=${key} \ `;
+          }
         }
-        // --rm --no-cache
-        if (options.imageArchive !== true) {
+        if (noCache === true) cache += ' --rm --no-cache';
+        if (path && typeof path === 'string')
           shellExec(
-            `cd ${path}${secrets}&& sudo podman build -f ./Dockerfile -t ${imgName} --pull=never --cap-add=CAP_AUDIT_WRITE${secretDockerInput}`,
+            `cd ${path}${secretsInput}&& sudo podman build -f ./${
+              dockerfileName && typeof dockerfileName === 'string' ? dockerfileName : 'Dockerfile'
+            } -t ${imageName} --pull=never --cap-add=CAP_AUDIT_WRITE${cache}${secretDockerInput}`,
           );
-        }
-        if (options.imageArchive !== true || options.podmanSave === true)
-          shellExec(`cd ${path} && podman save -o ${tarFile} ${podManImg}`);
-        shellExec(`cd ${path} && sudo kind load image-archive ${tarFile}`);
+
+        if (podmanSave === true) shellExec(`podman save -o ${tarFile} ${podManImg}`);
+        if (kindLoad === true) shellExec(`sudo kind load image-archive ${tarFile}`);
       },
       async script(deployId = 'default', env = 'development', options = { run: false, build: false }) {
         if (options.build === true) {
@@ -63,7 +69,8 @@ class UnderpostImage {
           shellExec(`cd ${buildBasePath}/engine && underpost clone underpostnet/${repoName}-private`);
           shellExec(`cd ${buildBasePath}/engine && sudo mv ./${repoName}-private ./engine-private`);
           shellCd(`${buildBasePath}/engine`);
-          shellExec(`underpost install`);
+          shellExec(`npm install`);
+          shellExec(`node bin/deploy conf ${deployId} ${env}`);
           if (fs.existsSync('./engine-private/itc-scripts')) {
             const itcScripts = await fs.readdir('./engine-private/itc-scripts');
             for (const itcScript of itcScripts)
@@ -102,7 +109,6 @@ class UnderpostImage {
               }
               break;
           }
-          shellExec(`node bin/deploy conf ${deployId} ${env}`);
           shellExec(`node bin/deploy build-full-client ${deployId}`);
         }
         if (options.run === true) {