underpost 2.8.62 → 2.8.64

package/Dockerfile

@@ -1,11 +1,11 @@
 ARG BASE_DEBIAN=buster
 
+# USER root
+
 FROM debian:${BASE_DEBIAN}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-WORKDIR /home/dd
-
 # Set root password to root, format is 'user:password'.
 RUN echo 'root:root' | chpasswd
 
@@ -25,9 +25,6 @@ RUN mkdir -p /var/run/sshd
 # Allow root login via password
 RUN sed -ri 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
 
-# copy supervisor config file to start openssh-server
-COPY supervisord-openssh-server.conf /etc/supervisor/conf.d/supervisord-openssh-server.conf
-
 # install open ssl git and others tools
 RUN apt-get install -yq --no-install-recommends libssl-dev curl wget git gnupg
 
@@ -37,12 +34,14 @@ RUN apt-get install -y nodejs build-essential
 RUN node --version
 RUN npm --version
 
-RUN npm install -g underpost
-
-VOLUME [ "/home/dd/engine/logs" ]
+WORKDIR /home/dd
 
 EXPOSE 22
 
-EXPOSE 4000-4004
+EXPOSE 80
+
+EXPOSE 443
+
+EXPOSE 3000-3100
 
-CMD [ "underpost", "new", "service" ]
+EXPOSE 4000-4100

package/bin/build.js

@@ -47,7 +47,7 @@ if (process.argv.includes('conf')) {
     if (!fs.existsSync(`../${privateRepoName}`)) {
       shellExec(`cd .. && underpost clone ${privateGitUri}`, { silent: true });
     } else {
-      shellExec(`cd ../${privateRepoName} && underpost pull . ${privateGitUri}`);
+      shellExec(`cd ../${privateRepoName} && git checkout . && git clean -f -d && underpost pull . ${privateGitUri}`);
     }
     const toPath = `../${privateRepoName}/conf/${_confName}`;
     fs.removeSync(toPath);
@@ -176,7 +176,7 @@ const { DefaultConf } = await import(`../conf.${confName}.js`);
   if (!fs.existsSync(`${basePath}/images`)) fs.mkdirSync(`${basePath}/images`);
 
   const env = process.argv.includes('development') ? 'development' : 'production';
-  const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'secret.yaml'];
+  const deploymentsFiles = ['proxy.yaml', 'deployment.yaml', 'secret.yaml'];
   // remove engine-private of .dockerignore for local testing
   for (const file of deploymentsFiles) {
     if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {

package/bin/deploy.js

@@ -673,8 +673,8 @@ try {
     }
 
     case 'version-build': {
-      const newVersion = process.argv[3];
       const originPackageJson = JSON.parse(fs.readFileSync(`package.json`, 'utf8'));
+      const newVersion = process.argv[3] ?? originPackageJson.version;
       const { version } = originPackageJson;
       originPackageJson.version = newVersion;
       fs.writeFileSync(`package.json`, JSON.stringify(originPackageJson, null, 4), 'utf8');

package/bin/index.js

@@ -105,6 +105,7 @@ program
   .option('--info-util', 'Display kubectl util management commands')
   .option('--cert', 'Reset tls/ssl certificate secrets')
   .option('--build-manifest', 'Build kind yaml manifests: deployments, services, proxy and secrets')
+  .option('--dashboard-update', 'Update dashboard instance data with current router config')
   .option('--version', 'Set custom version')
   .description('Manage deployment, for default deploy development pods')
   .action(Underpost.deploy.callback);
@@ -135,13 +136,15 @@ program
 
 program
   .command('dockerfile-image-build')
-  .argument('<deploy-id>', 'Deploy configuration id')
-  .argument('[env]', 'Optional environment, for default is development')
-  .argument('[path]', 'Absolute or relative directory, for default is current')
-  .option('--image-archive', 'Only load tar image from ./images')
-  .option('--podman-save', 'Save image from podman to ./images')
-  .option('--image-name <image-name>', 'Set custom image name')
-  .option('--image-version <image-version>', 'Set custom image version')
+  .option('--path [path]', 'Dockerfile path')
+  .option('--image-name [image-name]', 'Set image name')
+  .option('--image-path [image-path]', 'Set tar image path')
+  .option('--dockerfile-name [dockerfile-name]', 'set Dockerfile name')
+  .option('--podman-save', 'Export tar file from podman')
+  .option('--kind-load', 'Import tar image to Kind cluster')
+  .option('--secrets', 'Dockerfile env secrets')
+  .option('--secrets-path [secrets-path]', 'Dockerfile custom path env secrets')
+  .option('--no-cache', 'Build without using cache')
   .description('Build image from Dockerfile')
   .action(Underpost.image.dockerfile.build);
 
@@ -193,6 +196,7 @@ program
   .option('--itc', 'Inside container execution context')
   .option('--init', 'Init cron jobs for cron job default deploy id')
   .option('--git', 'Upload to github')
+  .option('--dashboard-update', 'Update dashboard cron data with current jobs config')
   .description('Cron jobs management')
   .action(Underpost.cron.callback);
 
@@ -220,4 +224,14 @@ program
   .option('--kind-type <kind-type>')
   .action(Underpost.test.callback);
 
+program
+  .command('monitor')
+  .argument('<deploy-id>', 'Deploy configuration id')
+  .argument('[env]', 'Optional environment, for default is development')
+  .option('--ms-interval <ms-interval>', 'Custom ms interval delta time')
+  .option('--now', 'Exec immediately monitor script')
+  .option('--single', 'Disable recurrence')
+  .description('Monitor health server management')
+  .action(Underpost.monitor.callback);
+
 program.parse();

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.62'
+      engine.version: '2.8.64'
     networks:
       - load-balancer
 

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.62",
+    "version": "2.8.64",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",

package/src/api/default/default.service.js

@@ -24,7 +24,7 @@ const DefaultService = {
     /** @type {import('./default.model.js').DefaultModel} */
     const Default = DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.Default;
     if (req.params.id) return await Default.findByIdAndDelete(req.params.id);
-    else return await await Default.deleteMany();
+    else return await Default.deleteMany();
   },
 };
 

package/src/api/user/user.service.js

@@ -378,7 +378,7 @@ const UserService = {
         switch (user.role) {
           case 'admin': {
             if (req.params.id) return await User.findByIdAndDelete(req.params.id);
-            else return await await User.deleteMany();
+            else return await User.deleteMany();
           }
           default:
             if (req.auth.user._id !== req.params.id) throw new Error(`Invalid token user id`);

package/src/cli/cron.js

@@ -4,20 +4,24 @@
  * @namespace UnderpostCron
  */
 
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
 import BackUp from '../server/backup.js';
 import { Cmd } from '../server/conf.js';
 import Dns from '../server/dns.js';
-import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { loggerFactory } from '../server/logger.js';
+
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 
+const logger = loggerFactory(import.meta);
+
 /**
  * UnderpostCron main module methods
  * @class
  * @memberof UnderpostCron
  */
 class UnderpostCron {
+  static NETWORK = [];
   static JOB = {
     /**
      * DNS cli API
@@ -46,10 +50,10 @@ class UnderpostCron {
     callback: async function (
       deployList = 'default',
       jobList = Object.keys(UnderpostCron.JOB),
-      options = { itc: false, init: false, git: false },
+      options = { itc: false, init: false, git: false, dashboardUpdate: false },
     ) {
       if (options.init === true) {
-        await Underpost.test.setUpInfo();
+        UnderpostCron.NETWORK = [];
         const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
         deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
         const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
@@ -57,7 +61,7 @@ class UnderpostCron {
           for (const job of Object.keys(confCronConfig.jobs)) {
             const name = `${jobDeployId}-${job}`;
             let deployId;
-            shellExec(Cmd.delete(name));
+            if (!options.dashboardUpdate) shellExec(Cmd.delete(name));
             switch (job) {
               case 'dns':
                 deployId = jobDeployId;
@@ -67,15 +71,16 @@ class UnderpostCron {
                 deployId = deployList;
                 break;
             }
-            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
-            netWorkCron.push({
+            if (!options.dashboardUpdate)
+              shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            UnderpostCron.NETWORK.push({
               deployId,
               jobId: job,
               expression: confCronConfig.jobs[job].expression,
             });
           }
         }
-        await saveRuntimeCron();
+        if (options.dashboardUpdate === true) await UnderpostCron.API.updateDashboardData();
         if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
         return;
       }
@@ -84,6 +89,32 @@ class UnderpostCron {
         if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
       }
     },
+    async updateDashboardData() {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
+        const { db } = confServer[host][path];
+
+        await DataBaseProvider.load({ apis: ['cron'], host, path, db });
+
+        /** @type {import('../api/cron/cron.model.js').CronModel} */
+        const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
+
+        await Cron.deleteMany();
+
+        for (const cronInstance of UnderpostCron.NETWORK) {
+          logger.info('save', cronInstance);
+          await new Cron(cronInstance).save();
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/deploy.js

@@ -3,18 +3,21 @@ import {
   buildPortProxyRouter,
   buildProxyRouter,
   Config,
+  deployRangePortFactory,
   getDataDeploy,
   loadReplicas,
+  pathPortAssignmentFactory,
 } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 import dotenv from 'dotenv';
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
 
 const logger = loggerFactory(import.meta);
 
 class UnderpostDeploy {
+  static NETWORK = {};
   static API = {
     sync(deployList) {
       const deployGroupId = 'dd.tmp';
@@ -36,21 +39,29 @@ class UnderpostDeploy {
       for (const _deployId of deployList.split(',')) {
         const deployId = _deployId.trim();
         if (!deployId) continue;
-
-        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
-        const ports = Object.values(router).map((p) => parseInt(p.split(':')[2]));
-        const fromPort = Math.min(...ports);
-        const toPort = Math.max(...ports);
         const confServer = loadReplicas(
           JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
           'proxy',
         );
+        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+        const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
+        const { fromPort, toPort } = deployRangePortFactory(router);
 
         fs.mkdirSync(`./engine-private/conf/${deployId}/build/${env}`, { recursive: true });
         if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
 
         logger.info('port range', { deployId, fromPort, toPort });
-        // const customImg = `underpost-engine:${version && typeof version === 'string' ? version : Underpost.version}`;
+        // const customImg = `underpost-engine:${version && typeof version === 'string' ? version : '0.0.0'}`;
+        // lifecycle:
+        // postStart:
+        //   exec:
+        //     command:
+        //       - /bin/sh
+        //       - -c
+        //       - >
+        //         sleep 20 &&
+        //         npm install -g underpost &&
+        //         underpost secret underpost --create-from-file /etc/config/.env.${env}
         const deploymentYamlParts = `apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -74,17 +85,10 @@ spec:
             - /bin/sh
             - -c
             - >
+              npm install -g npm@11.2.0 &&
+              npm install -g underpost &&
+              underpost secret underpost --create-from-file /etc/config/.env.${env} &&
               underpost dockerfile-node-script --build --run ${deployId} ${env}
-          lifecycle:
-            postStart:
-              exec:
-                command:
-                  - /bin/sh
-                  - -c
-                  - >
-                    sleep 20 &&
-                    npm install -g underpost
-                    underpost secret underpost --create-from-file /etc/config/.env.${env}
           volumeMounts:
             - name: config-volume
               mountPath: /etc/config
@@ -92,7 +96,7 @@ spec:
         - name: config-volume
           configMap:
             name: underpost-config
-# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : Underpost.version}
+# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : '0.0.0'}
 ---
 apiVersion: v1
 kind: Service
@@ -134,27 +138,8 @@ spec:
     kind: ClusterIssuer
   secretName: ${host}`;
 
-          const pathPortConditions = [];
-          for (const path of Object.keys(confServer[host])) {
-            const { peer } = confServer[host][path];
-            if (!router[`${host}${path === '/' ? '' : path}`]) continue;
-            const port = parseInt(router[`${host}${path === '/' ? '' : path}`].split(':')[2]);
-            // logger.info('', { host, port, path });
-            pathPortConditions.push({
-              port,
-              path,
-            });
-
-            if (peer) {
-              //  logger.info('', { host, port: port + 1, path: '/peer' });
-              pathPortConditions.push({
-                port: port + 1,
-                path: '/peer',
-              });
-            }
-          }
-
-          // logger.info('', { host, pathPortConditions });
+          const pathPortAssignment = pathPortAssignmentData[host];
+          // logger.info('', { host, pathPortAssignment });
           proxyYaml += `
 ---
 apiVersion: projectcontour.io/v1
@@ -171,7 +156,7 @@ spec:
       secretName: ${host}`
           }
   routes:`;
-          for (const conditionObj of pathPortConditions) {
+          for (const conditionObj of pathPortAssignment) {
             const { path, port } = conditionObj;
             proxyYaml += `
     - conditions:
@@ -212,6 +197,7 @@ spec:
         expose: false,
         cert: false,
         version: '',
+        dashboardUpdate: false,
       },
     ) {
       if (options.infoUtil === true)
@@ -224,8 +210,9 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
         deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
       if (options.sync) UnderpostDeploy.API.sync(deployList);
       if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options.version);
-      if (options.infoRouter === true)
-        return logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      if (options.infoRouter === true) logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      if (options.dashboardUpdate === true) await UnderpostDeploy.API.updateDashboardData(deployList, env, options);
+      if (options.infoRouter === true) return;
       shellExec(`kubectl delete configmap underpost-config`);
       shellExec(
         `kubectl create configmap underpost-config --from-file=/home/dd/engine/engine-private/conf/dd-cron/.env.${env}`,
@@ -328,6 +315,60 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
 
       return result;
     },
+    async updateDashboardData(deployList, env, options) {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const { db } = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'))[host][
+          path
+        ];
+
+        await DataBaseProvider.load({ apis: ['instance'], host, path, db });
+
+        /** @type {import('../api/instance/instance.model.js').InstanceModel} */
+        const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
+
+        await Instance.deleteMany();
+
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
+          const confServer = loadReplicas(
+            JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+            'proxy',
+          );
+          const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+          const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
+
+          for (const host of Object.keys(confServer)) {
+            for (const { path, port } of pathPortAssignmentData[host]) {
+              if (!confServer[host][path]) continue;
+
+              const { client, runtime, apis } = confServer[host][path];
+
+              const body = {
+                deployId,
+                host,
+                path,
+                port,
+                client,
+                runtime,
+                apis,
+              };
+
+              logger.info('save', body);
+
+              await new Instance(body).save();
+            }
+          }
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/fs.js

@@ -38,7 +38,7 @@ class UnderpostFileStorage {
       options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false },
     ) {
       const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
-      const deleteFiles = UnderpostRepository.API.getDeleteFiles(path);
+      const deleteFiles = options.pull === true ? [] : UnderpostRepository.API.getDeleteFiles(path);
       for (const relativePath of deleteFiles) {
         const _path = path + '/' + relativePath;
         if (_path in storage) {
@@ -46,10 +46,6 @@ class UnderpostFileStorage {
           delete storage[_path];
         }
       }
-      const files =
-        options.git === true
-          ? UnderpostRepository.API.getChangedFiles(path)
-          : await fs.readdir(path, { recursive: true });
       if (options.pull === true) {
         for (const _path of Object.keys(storage)) {
           if (!fs.existsSync(_path) || options.force === true) {
@@ -57,7 +53,11 @@ class UnderpostFileStorage {
             await UnderpostFileStorage.API.pull(_path, options);
           } else logger.warn(`Pull path already exists`, _path);
         }
-      } else
+      } else {
+        const files =
+          options.git === true
+            ? UnderpostRepository.API.getChangedFiles(path)
+            : await fs.readdir(path, { recursive: true });
         for (const relativePath of files) {
           const _path = path + '/' + relativePath;
           if (fs.statSync(_path).isDirectory()) {
@@ -68,6 +68,7 @@ class UnderpostFileStorage {
             if (storage) storage[_path] = {};
           } else logger.warn('File already exists', _path);
         }
+      }
       UnderpostFileStorage.API.writeStorageConf(storage, storageConf);
       if (options.git === true) {
         shellExec(`cd ${path} && git add .`);