underpost 2.8.47 → 2.8.51

package/Dockerfile

@@ -45,4 +45,4 @@ EXPOSE 22
 
 EXPOSE 4000-4004
 
-CMD [ "underpost", "new", "engine" ]
+CMD [ "underpost", "new", "service" ]

package/bin/build.js

@@ -3,7 +3,7 @@ import { loggerFactory } from '../src/server/logger.js';
 import { shellExec } from '../src/server/process.js';
 import dotenv from 'dotenv';
 import { getCapVariableName } from '../src/client/components/core/CommonJs.js';
-import { buildProxyRouter, buildPortProxyRouter, Config, getPathsSSR, buildKindPorts } from '../src/server/conf.js';
+import { getPathsSSR } from '../src/server/conf.js';
 
 const baseConfPath = './engine-private/conf/dd-cron/.env.production';
 if (fs.existsSync(baseConfPath)) dotenv.config({ path: baseConfPath, override: true });
@@ -20,21 +20,13 @@ const logger = loggerFactory(import.meta);
 const confName = process.argv[2];
 const basePath = '../pwa-microservices-template';
 const repoName = `engine-${confName.split('dd-')[1]}`;
-const privateRepoName = `${repoName}-private`;
-const privateRepoNameBackUp = `${repoName}-cron-backups`;
-const gitPrivateUrl = `https://${process.env.GITHUB_TOKEN}@github.com/underpostnet/${privateRepoName}.git`;
-const gitPrivateBackUpUrl = `https://${process.env.GITHUB_TOKEN}@github.com/underpostnet/${privateRepoNameBackUp}.git`;
 
 logger.info('', {
   confName,
   repoName,
-  privateRepoName,
-  privateRepoNameBackUp,
   basePath,
 });
 
-if (process.argv.includes('info')) process.exit(0);
-
 if (process.argv.includes('clean')) {
   if (fs.existsSync(`${basePath}/images`)) fs.copySync(`${basePath}/images`, `./images`);
   shellExec(`cd ${basePath} && git checkout .`);
@@ -42,173 +34,40 @@ if (process.argv.includes('clean')) {
   process.exit(0);
 }
 
-if (process.argv.includes('proxy')) {
-  const env = process.argv.includes('development') ? 'development' : 'production';
-  process.env.NODE_ENV = env;
-  process.env.PORT = process.env.NODE_ENV === 'development' ? 4000 : 3000;
-  process.argv[2] = 'proxy';
-  process.argv[3] = fs.readFileSync('./engine-private/deploy/dd-router', 'utf8').trim();
-
-  await Config.build();
-  process.env.NODE_ENV = 'production';
-  const router = buildPortProxyRouter(443, buildProxyRouter());
-  const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${confName}/conf.server.json`, 'utf8'));
-  const confHosts = Object.keys(confServer);
-
-  for (const host of Object.keys(router)) {
-    if (!confHosts.find((_host) => host.match(_host))) {
-      delete router[host];
-    }
-  }
-
-  const ports = Object.values(router).map((p) => p.split(':')[2]);
-
-  const fromPort = ports[0];
-  const toPort = ports[ports.length - 1];
-
-  logger.info('port range', { fromPort, toPort, router });
-
-  const deploymentYamlFilePath = `./engine-private/conf/${confName}/build/${env}/deployment.yaml`;
-
-  const deploymentYamlParts = fs.readFileSync(deploymentYamlFilePath, 'utf8').split('ports:');
-  deploymentYamlParts[1] =
-    buildKindPorts(fromPort, toPort) +
-    `  type: LoadBalancer
-`;
-
-  fs.writeFileSync(
-    deploymentYamlFilePath,
-    deploymentYamlParts.join(`ports:
-`),
-  );
-
-  let proxyYaml = '';
-  let secretYaml = '';
-
-  for (const host of Object.keys(confServer)) {
-    if (env === 'production')
-      secretYaml += `
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: ${host}
-spec:
-  commonName: ${host}
-  dnsNames:
-  - ${host}
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
-  secretName: ${host}`;
-
-    const pathPortConditions = [];
-    for (const path of Object.keys(confServer[host])) {
-      const { peer } = confServer[host][path];
-      const port = parseInt(router[`${host}${path === '/' ? '' : path}`].split(':')[2]);
-      // logger.info('', { host, port, path });
-      pathPortConditions.push({
-        port,
-        path,
-      });
-
-      if (peer) {
-        //  logger.info('', { host, port: port + 1, path: '/peer' });
-        pathPortConditions.push({
-          port: port + 1,
-          path: '/peer',
-        });
-      }
-    }
-    // logger.info('', { host, pathPortConditions });
-    proxyYaml += `
----
-apiVersion: projectcontour.io/v1
-kind: HTTPProxy
-metadata:
-  name: ${host}
-spec:
-  virtualhost:
-    fqdn: ${host}${
-      env === 'development'
-        ? ''
-        : `
-    tls:
-      secretName: ${host}`
-    }
-  routes:`;
-    for (const conditionObj of pathPortConditions) {
-      const { path, port } = conditionObj;
-      proxyYaml += `
-    - conditions:
-        - prefix: ${path}
-      enableWebsockets: true
-      services:
-        - name: ${confName}-${env}-service
-          port: ${port}`;
-    }
-  }
-  const yamlPath = `./engine-private/conf/${confName}/build/${env}/proxy.yaml`;
-  fs.writeFileSync(yamlPath, proxyYaml, 'utf8');
-  if (env === 'production') {
-    const yamlPath = `./engine-private/conf/${confName}/build/${env}/secret.yaml`;
-    fs.writeFileSync(yamlPath, secretYaml, 'utf8');
-  }
-
-  process.exit(0);
-}
 if (process.argv.includes('conf')) {
-  if (!fs.existsSync(`../${privateRepoName}`)) {
-    shellExec(`cd .. && git clone ${gitPrivateUrl}`, { silent: true });
-  } else {
-    shellExec(`cd ../${privateRepoName} && git pull`);
-  }
-  const toPath = `../${privateRepoName}/conf/${confName}`;
-  fs.removeSync(toPath);
-  fs.mkdirSync(toPath, { recursive: true });
-  fs.copySync(`./engine-private/conf/${confName}`, toPath);
-  shellExec(
-    `cd ../${privateRepoName}` +
-      ` && git add .` +
-      ` && git commit -m "ci(engine-core-conf): ⚙️ Update ${confName} conf"` +
-      ` && git push`,
-  );
-  process.exit(0);
-}
-
-if (process.argv.includes('cron-backups')) {
-  if (!fs.existsSync(`../${privateRepoNameBackUp}`)) {
-    shellExec(`cd .. && git clone ${gitPrivateBackUpUrl}`, { silent: true });
-  } else {
-    shellExec(`cd ../${privateRepoNameBackUp} && git pull`);
-  }
-  const serverConf = JSON.parse(fs.readFileSync(`./engine-private/conf/${confName}/conf.server.json`, 'utf8'));
-  for (const host of Object.keys(serverConf)) {
-    for (let path of Object.keys(serverConf[host])) {
-      path = path.replaceAll('/', '-');
-      const toPath = `../${privateRepoNameBackUp}/${host}${path}`;
-      const fromPath = `./engine-private/cron-backups/${host}${path}`;
-      if (fs.existsSync(fromPath)) {
-        if (fs.existsSync(toPath)) fs.removeSync(toPath);
-        logger.info('Build', { fromPath, toPath });
-        fs.copySync(fromPath, toPath);
-      }
+  for (const _confName of (confName === 'dd'
+    ? fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8')
+    : confName
+  ).split(',')) {
+    const _repoName = `engine-${_confName.split('dd-')[1]}`;
+    const privateRepoName = `${_repoName}-private`;
+    const privateGitUri = `${process.env.GITHUB_USERNAME}/${privateRepoName}`;
+
+    if (!fs.existsSync(`../${privateRepoName}`)) {
+      shellExec(`cd .. && underpost clone ${privateGitUri}`, { silent: true });
+    } else {
+      shellExec(`cd ../${privateRepoName} && underpost pull . ${privateGitUri}`);
+    }
+    const toPath = `../${privateRepoName}/conf/${_confName}`;
+    fs.removeSync(toPath);
+    fs.mkdirSync(toPath, { recursive: true });
+    fs.copySync(`./engine-private/conf/${_confName}`, toPath);
+    if (fs.existsSync(`./engine-private/replica`)) {
+      const replicas = await fs.readdir(`./engine-private/replica`);
+      for (const replica of replicas)
+        if (replica.match(_confName))
+          fs.copySync(`./engine-private/replica/${replica}`, `../${privateRepoName}/replica/${replica}`);
     }
+    shellExec(
+      `cd ../${privateRepoName}` +
+        ` && git add .` +
+        ` && underpost cmt . ci engine-core-conf 'Update ${_confName} conf'` +
+        ` && underpost push . ${privateGitUri}`,
+    );
   }
-  shellExec(
-    `cd ../${privateRepoNameBackUp}` +
-      ` && git add .` +
-      ` && git commit -m "ci(engine-core-cron-backups): ⚙️ Update ${confName} cron backups"` +
-      ` && git push`,
-  );
   process.exit(0);
 }
 
-if (process.argv.includes('test')) {
-  fs.mkdirSync(`${basePath}/engine-private/conf`, { recursive: true });
-  fs.copySync(`./engine-private/conf/${confName}`, `${basePath}/engine-private/conf/${confName}`);
-}
-
 const { DefaultConf } = await import(`../conf.${confName}.js`);
 
 {
@@ -294,27 +153,9 @@ const { DefaultConf } = await import(`../conf.${confName}.js`);
   const env = process.argv.includes('development') ? 'development' : 'production';
   const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'secret.yaml'];
   // remove engine-private of .dockerignore for local testing
-
-  if (process.argv.includes('engine')) {
-    fs.removeSync(`${basePath}/manifests/deployment`);
-
-    if (!fs.existsSync(`./manifests/deployment/${confName}-${env}`))
-      fs.mkdirSync(`./manifests/deployment/${confName}-${env}`);
-
-    for (const file of deploymentsFiles) {
-      if (fs.existsSync(`./engine-private/conf/${confName}/build/${env}/${file}`)) {
-        fs.copyFileSync(`./engine-private/conf/${confName}/build/${env}/${file}`, `${basePath}/${file}`);
-        fs.copyFileSync(
-          `./engine-private/conf/${confName}/build/${env}/${file}`,
-          `./manifests/deployment/${confName}-${env}/${file}`,
-        );
-      }
-    }
-  } else {
-    for (const file of deploymentsFiles) {
-      if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {
-        fs.copyFileSync(`./manifests/deployment/${confName}-${env}/${file}`, `${basePath}/${file}`);
-      }
+  for (const file of deploymentsFiles) {
+    if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {
+      fs.copyFileSync(`./manifests/deployment/${confName}-${env}/${file}`, `${basePath}/${file}`);
     }
   }
 }

package/bin/deploy.js

@@ -141,28 +141,7 @@ try {
       loadConf(process.argv[3], process.argv[4]);
       break;
     }
-    case 'run':
-      {
-        if (process.argv.includes('replicas')) {
-          const deployGroupId = getDeployGroupId();
-          const dataDeploy = getDataDeploy({
-            deployId: process.argv[3],
-            buildSingleReplica: true,
-            deployGroupId,
-          });
-          if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
-          await deployRun(dataDeploy);
-        } else {
-          loadConf(process.argv[3]);
-          shellExec(`npm start ${process.argv.includes('maintenance') ? 'maintenance' : ''}`);
-        }
-      }
-      break;
 
-    case 'remove-await-deploy': {
-      if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
-      break;
-    }
     case 'new-nodejs-app':
       {
         const deployId = process.argv[3];
@@ -229,6 +208,7 @@ try {
       break;
     case 'build-full-client':
       {
+        dotenv.config({ override: true });
         if (!process.argv[3]) process.argv[3] = 'default';
         const { deployId, folder } = loadConf(process.argv[3]);
 
@@ -254,7 +234,7 @@ try {
                   serverConf[host][path].replicas.map((replica) => buildReplicaId({ deployId, replica })),
                 );
 
-                shellExec(Cmd.replica(deployId, host, path));
+                // shellExec(Cmd.replica(deployId, host, path));
               }
             }
           }
@@ -263,7 +243,7 @@ try {
         await buildClient();
 
         for (const replicaDeployId of deployIdSingleReplicas) {
-          shellExec(Cmd.conf(replicaDeployId));
+          shellExec(Cmd.conf(replicaDeployId, process.env.NODE_ENV));
           shellExec(Cmd.build(replicaDeployId));
         }
       }
@@ -456,12 +436,13 @@ try {
     case 'run-macro':
       {
         if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
-        const dataDeploy = getDataDeploy({ deployGroupId: process.argv[3], buildSingleReplica: true });
+        const dataDeploy = getDataDeploy({
+          deployGroupId: process.argv[3],
+          buildSingleReplica: true,
+          deployIdConcat: ['dd-proxy', 'dd-cron'],
+        });
         if (!process.argv[4]) await setUpProxyMaintenanceServer({ deployGroupId: process.argv[3] });
-        await deployRun(
-          process.argv[4] ? dataDeploy.filter((d) => d.deployId.match(process.argv[4])) : dataDeploy,
-          true,
-        );
+        await deployRun(process.argv[4] ? dataDeploy.filter((d) => d.deployId.match(process.argv[4])) : dataDeploy);
       }
       break;
 

package/bin/index.js

@@ -8,8 +8,8 @@ import { getNpmRootPath, loadConf } from '../src/server/conf.js';
 import fs from 'fs-extra';
 import { commitData } from '../src/client/components/core/CommonJs.js';
 import UnderpostScript from '../src/cli/script.js';
-import { shellExec } from '../src/server/process.js';
 import UnderpostDB from '../src/cli/db.js';
+import UnderpostCron from '../src/cli/cron.js';
 
 const npmRoot = getNpmRootPath();
 const underpostRoot = `${npmRoot}/underpost/.env`;
@@ -63,7 +63,7 @@ program
 
 program
   .command('env')
-  .argument('<deploy-id>', 'deploy configuration id')
+  .argument('<deploy-id>', `deploy configuration id, if 'clean' restore default`)
   .argument('[env]', 'Optional environment, for default is production')
   .description('Set environment variables files and conf related to <deploy-id>')
   .action(loadConf);
@@ -99,9 +99,12 @@ program
 
 program
   .command('deploy')
-  .argument('<deploy-list>', 'Deploy id list, e.g. default-a, default-b')
+  .argument('<deploy-list>', 'Deploy id list, e.g. default-a,default-b')
   .argument('[env]', 'Optional environment, for default is development')
   .option('--remove', 'Delete deployments and services')
+  .option('--sync', 'Sync deployments env, ports, and replicas')
+  .option('--info-router', 'Display router structure')
+  .option('--build-manifest', 'Build kind yaml manifests: deployments, services, proxy and secrets')
   .description('Manage deployment, for default deploy development pods')
   .action(Underpost.deploy.callback);
 
@@ -124,6 +127,7 @@ program
   .command('dockerfile-node-script')
   .argument('<deploy-id>', 'Deploy configuration id')
   .argument('[env]', 'Optional environment, for default is development')
+  .option('--run', 'Run custom entry point script')
   .description('Dockerfile custom node build script')
   .action(Underpost.image.dockerfile.script);
 
@@ -150,7 +154,7 @@ program
 
 program
   .command('db')
-  .argument('<deploy-list>', 'Deploy id list, e.g. default-a, default-b')
+  .argument('<deploy-list>', 'Deploy id list, e.g. default-a,default-b')
   .option('--import', 'Import container backups from repositories')
   .option('--export', 'Export container backups to repositories')
   .description('Manage databases')
@@ -166,12 +170,22 @@ program
   )
   .action((...args) => Underpost.script[args[0]](args[1], args[2]));
 
+program
+  .command('cron')
+  .argument('[deploy-list]', 'Deploy id list, e.g. default-a,default-b')
+  .argument('[job-list]', `Deploy id list, e.g. ${Object.keys(UnderpostCron.JOB)}, for default all available jobs`)
+  .option('--disable-kind-cluster', 'Disable kind cluster configuration')
+  .option('--init', 'Init cron jobs for cron job default deploy id')
+  .description('Cron jobs management')
+  .action(Underpost.cron.callback);
+
 program
   .command('test')
-  .argument('[deploy-list]', 'Deploy id list, e.g. default-a, default-b')
+  .argument('[deploy-list]', 'Deploy id list, e.g. default-a,default-b')
   .description('Manage Test, for default run current underpost default test')
   .option('--inside-container', 'Inside container execution context')
   .option('--sh', 'Copy to clipboard, container entrypoint shell command')
+  .option('--logs', 'Display container logs')
   .action(Underpost.test.callback);
 
 program.parse();

package/bin/util.js

@@ -112,14 +112,6 @@ try {
       fs.writeFileSync('b64-image', `data:image/jpg;base64,${fs.readFileSync(process.argv[3]).toString('base64')}`);
       break;
 
-    case 'clean-env': {
-      shellExec(`git checkout package.json`);
-      shellExec(`git checkout .env.production`);
-      shellExec(`git checkout .env.development`);
-      shellExec(`git checkout .env.test`);
-      shellExec(`git checkout jsdoc.json`);
-      break;
-    }
     case 'get-keys': {
       const sentence = fs.existsSync('./_')
         ? fs.readFileSync('./_', 'utf8')

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.47'
+      engine.version: '2.8.51'
     networks:
       - load-balancer
 

package/manifests/mongodb/backup-cronjob.yaml

@@ -16,25 +16,27 @@ spec:
                 - -c
                 - |
                   # Perform backup
-                  mongodump -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
+                  mongodump --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
                   # Remove backups older than 7 days
                   find /backup -type d -mtime +7 -exec rm -rf {} +
               volumeMounts:
                 - name: backup-storage
                   mountPath: /backup
-              env:
-                - name: MONGO_INITDB_ROOT_USERNAME
-                  valueFrom:
-                    secretKeyRef:
-                      name: mongodb-secret
-                      key: username
-                - name: MONGO_INITDB_ROOT_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: mongodb-secret
-                      key: password
           restartPolicy: Never
           volumes:
             - name: backup-storage
               persistentVolumeClaim:
                 claimName: backup-pvc
+#     mongodump -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
+
+# env:
+#   - name: MONGO_INITDB_ROOT_USERNAME
+#     valueFrom:
+#       secretKeyRef:
+#         name: mongodb-secret
+#         key: username
+#   - name: MONGO_INITDB_ROOT_PASSWORD
+#     valueFrom:
+#       secretKeyRef:
+#         name: mongodb-secret
+#         key: password

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.47",
+    "version": "2.8.51",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",

package/src/api/core/core.service.js

@@ -9,7 +9,7 @@ const CoreService = {
     /** @type {import('./core.model.js').CoreModel} */
     const Core = DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.Core;
     if (req.path.startsWith('/sh')) {
-      if (req.body.print) return shellExec(req.body.sh, { stdout: true });
+      if (req.body.stdout) return shellExec(req.body.sh, { stdout: true });
       shellExec(req.body.sh, { async: true });
       return 'Command "' + req.body.sh + '" running';
     }

package/src/cli/cron.js

@@ -0,0 +1,90 @@
+/**
+ * UnderpostCron CLI index module
+ * @module src/cli/cron.js
+ * @namespace UnderpostCron
+ */
+
+import Underpost from '../index.js';
+import BackUp from '../server/backup.js';
+import { Cmd } from '../server/conf.js';
+import Dns from '../server/dns.js';
+import { netWorkCron, saveRuntimeCron } from '../server/network.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+
+/**
+ * UnderpostCron main module methods
+ * @class
+ * @memberof UnderpostCron
+ */
+class UnderpostCron {
+  static JOB = {
+    /**
+     * DNS cli API
+     * @static
+     * @type {Dns}
+     * @memberof UnderpostCron
+     */
+    dns: Dns,
+    /**
+     * BackUp cli API
+     * @static
+     * @type {BackUp}
+     * @memberof UnderpostCron
+     */
+    backup: BackUp,
+  };
+  static API = {
+    /**
+     * Run the cron jobs
+     * @static
+     * @param {String} deployList - Comma separated deploy ids
+     * @param {String} jobList - Comma separated job ids
+     * @return {void}
+     * @memberof UnderpostCron
+     */
+    callback: async function (
+      deployList = 'default',
+      jobList = Object.keys(UnderpostCron.JOB),
+      options = { disableKindCluster: false, init: false },
+    ) {
+      if (options.init === true) {
+        await Underpost.test.setUpInfo();
+        const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
+        deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
+        const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
+        if (confCronConfig.jobs && Object.keys(confCronConfig.jobs).length > 0) {
+          for (const job of Object.keys(confCronConfig.jobs)) {
+            const name = `${jobDeployId}-${job}`;
+            let deployId;
+            shellExec(Cmd.delete(name));
+            switch (job) {
+              case 'dns':
+                deployId = jobDeployId;
+                break;
+
+              default:
+                deployId = deployList;
+                break;
+            }
+            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
+            netWorkCron.push({
+              deployId,
+              jobId: job,
+              expression: confCronConfig.jobs[job].expression,
+            });
+          }
+        }
+        await saveRuntimeCron();
+        if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
+        return;
+      }
+      for (const _jobId of jobList.split(',')) {
+        const jobId = _jobId.trim();
+        if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
+      }
+    },
+  };
+}
+
+export default UnderpostCron;