underpost 2.8.453 → 2.8.482

package/Dockerfile

@@ -4,7 +4,7 @@ FROM debian:${BASE_DEBIAN}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-WORKDIR /code
+WORKDIR /home/dd
 
 # Set root password to root, format is 'user:password'.
 RUN echo 'root:root' | chpasswd
@@ -39,10 +39,10 @@ RUN npm --version
 
 RUN npm install -g underpost
 
-VOLUME [ "/code/app/logs" ]
+VOLUME [ "/home/dd/engine/logs" ]
 
 EXPOSE 22
 
 EXPOSE 4000-4004
 
-CMD [ "underpost", "new", "app" ]
+CMD [ "underpost", "new", "service" ]

package/bin/build.js

@@ -3,7 +3,7 @@ import { loggerFactory } from '../src/server/logger.js';
 import { shellExec } from '../src/server/process.js';
 import dotenv from 'dotenv';
 import { getCapVariableName } from '../src/client/components/core/CommonJs.js';
-import { buildProxyRouter, buildPortProxyRouter, Config, getPathsSSR, buildKindPorts } from '../src/server/conf.js';
+import { getPathsSSR } from '../src/server/conf.js';
 
 const baseConfPath = './engine-private/conf/dd-cron/.env.production';
 if (fs.existsSync(baseConfPath)) dotenv.config({ path: baseConfPath, override: true });
@@ -20,21 +20,13 @@ const logger = loggerFactory(import.meta);
 const confName = process.argv[2];
 const basePath = '../pwa-microservices-template';
 const repoName = `engine-${confName.split('dd-')[1]}`;
-const privateRepoName = `${repoName}-private`;
-const privateRepoNameBackUp = `${repoName}-cron-backups`;
-const gitPrivateUrl = `https://${process.env.GITHUB_TOKEN}@github.com/underpostnet/${privateRepoName}.git`;
-const gitPrivateBackUpUrl = `https://${process.env.GITHUB_TOKEN}@github.com/underpostnet/${privateRepoNameBackUp}.git`;
 
 logger.info('', {
   confName,
   repoName,
-  privateRepoName,
-  privateRepoNameBackUp,
   basePath,
 });
 
-if (process.argv.includes('info')) process.exit(0);
-
 if (process.argv.includes('clean')) {
   if (fs.existsSync(`${basePath}/images`)) fs.copySync(`${basePath}/images`, `./images`);
   shellExec(`cd ${basePath} && git checkout .`);
@@ -42,173 +34,40 @@ if (process.argv.includes('clean')) {
   process.exit(0);
 }
 
-if (process.argv.includes('proxy')) {
-  const env = process.argv.includes('development') ? 'development' : 'production';
-  process.env.NODE_ENV = env;
-  process.env.PORT = process.env.NODE_ENV === 'development' ? 4000 : 3000;
-  process.argv[2] = 'proxy';
-  process.argv[3] = fs.readFileSync('./engine-private/deploy/dd-router', 'utf8').trim();
-
-  await Config.build();
-  process.env.NODE_ENV = 'production';
-  const router = buildPortProxyRouter(443, buildProxyRouter());
-  const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${confName}/conf.server.json`, 'utf8'));
-  const confHosts = Object.keys(confServer);
-
-  for (const host of Object.keys(router)) {
-    if (!confHosts.find((_host) => host.match(_host))) {
-      delete router[host];
-    }
-  }
-
-  const ports = Object.values(router).map((p) => p.split(':')[2]);
-
-  const fromPort = ports[0];
-  const toPort = ports[ports.length - 1];
-
-  logger.info('port range', { fromPort, toPort, router });
-
-  const deploymentYamlFilePath = `./engine-private/conf/${confName}/build/${env}/deployment.yaml`;
-
-  const deploymentYamlParts = fs.readFileSync(deploymentYamlFilePath, 'utf8').split('ports:');
-  deploymentYamlParts[1] =
-    buildKindPorts(fromPort, toPort) +
-    `  type: LoadBalancer
-`;
-
-  fs.writeFileSync(
-    deploymentYamlFilePath,
-    deploymentYamlParts.join(`ports:
-`),
-  );
-
-  let proxyYaml = '';
-  let secretYaml = '';
-
-  for (const host of Object.keys(confServer)) {
-    if (env === 'production')
-      secretYaml += `
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: ${host}
-spec:
-  commonName: ${host}
-  dnsNames:
-  - ${host}
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
-  secretName: ${host}`;
-
-    const pathPortConditions = [];
-    for (const path of Object.keys(confServer[host])) {
-      const { peer } = confServer[host][path];
-      const port = parseInt(router[`${host}${path === '/' ? '' : path}`].split(':')[2]);
-      // logger.info('', { host, port, path });
-      pathPortConditions.push({
-        port,
-        path,
-      });
-
-      if (peer) {
-        //  logger.info('', { host, port: port + 1, path: '/peer' });
-        pathPortConditions.push({
-          port: port + 1,
-          path: '/peer',
-        });
-      }
-    }
-    // logger.info('', { host, pathPortConditions });
-    proxyYaml += `
----
-apiVersion: projectcontour.io/v1
-kind: HTTPProxy
-metadata:
-  name: ${host}
-spec:
-  virtualhost:
-    fqdn: ${host}${
-      env === 'development'
-        ? ''
-        : `
-    tls:
-      secretName: ${host}`
-    }
-  routes:`;
-    for (const conditionObj of pathPortConditions) {
-      const { path, port } = conditionObj;
-      proxyYaml += `
-    - conditions:
-        - prefix: ${path}
-      enableWebsockets: true
-      services:
-        - name: ${confName}-${env}-service
-          port: ${port}`;
-    }
-  }
-  const yamlPath = `./engine-private/conf/${confName}/build/${env}/proxy.yaml`;
-  fs.writeFileSync(yamlPath, proxyYaml, 'utf8');
-  if (env === 'production') {
-    const yamlPath = `./engine-private/conf/${confName}/build/${env}/secret.yaml`;
-    fs.writeFileSync(yamlPath, secretYaml, 'utf8');
-  }
-
-  process.exit(0);
-}
 if (process.argv.includes('conf')) {
-  if (!fs.existsSync(`../${privateRepoName}`)) {
-    shellExec(`cd .. && git clone ${gitPrivateUrl}`, { silent: true });
-  } else {
-    shellExec(`cd ../${privateRepoName} && git pull`);
-  }
-  const toPath = `../${privateRepoName}/conf/${confName}`;
-  fs.removeSync(toPath);
-  fs.mkdirSync(toPath, { recursive: true });
-  fs.copySync(`./engine-private/conf/${confName}`, toPath);
-  shellExec(
-    `cd ../${privateRepoName}` +
-      ` && git add .` +
-      ` && git commit -m "ci(engine-core-conf): ⚙️ Update ${confName} conf"` +
-      ` && git push`,
-  );
-  process.exit(0);
-}
-
-if (process.argv.includes('cron-backups')) {
-  if (!fs.existsSync(`../${privateRepoNameBackUp}`)) {
-    shellExec(`cd .. && git clone ${gitPrivateBackUpUrl}`, { silent: true });
-  } else {
-    shellExec(`cd ../${privateRepoNameBackUp} && git pull`);
-  }
-  const serverConf = JSON.parse(fs.readFileSync(`./engine-private/conf/${confName}/conf.server.json`, 'utf8'));
-  for (const host of Object.keys(serverConf)) {
-    for (let path of Object.keys(serverConf[host])) {
-      path = path.replaceAll('/', '-');
-      const toPath = `../${privateRepoNameBackUp}/${host}${path}`;
-      const fromPath = `./engine-private/cron-backups/${host}${path}`;
-      if (fs.existsSync(fromPath)) {
-        if (fs.existsSync(toPath)) fs.removeSync(toPath);
-        logger.info('Build', { fromPath, toPath });
-        fs.copySync(fromPath, toPath);
-      }
+  for (const _confName of (confName === 'dd'
+    ? fs.readFileSync(`./engine-private/deploy/dd-router`, 'utf8')
+    : confName
+  ).split(',')) {
+    const _repoName = `engine-${_confName.split('dd-')[1]}`;
+    const privateRepoName = `${_repoName}-private`;
+    const privateGitUri = `${process.env.GITHUB_USERNAME}/${privateRepoName}`;
+
+    if (!fs.existsSync(`../${privateRepoName}`)) {
+      shellExec(`cd .. && underpost clone ${privateGitUri}`, { silent: true });
+    } else {
+      shellExec(`cd ../${privateRepoName} && underpost pull . ${privateGitUri}`);
+    }
+    const toPath = `../${privateRepoName}/conf/${_confName}`;
+    fs.removeSync(toPath);
+    fs.mkdirSync(toPath, { recursive: true });
+    fs.copySync(`./engine-private/conf/${_confName}`, toPath);
+    if (fs.existsSync(`./engine-private/replica`)) {
+      const replicas = await fs.readdir(`./engine-private/replica`);
+      for (const replica of replicas)
+        if (replica.match(_confName))
+          fs.copySync(`./engine-private/replica/${replica}`, `../${privateRepoName}/replica/${replica}`);
     }
+    shellExec(
+      `cd ../${privateRepoName}` +
+        ` && git add .` +
+        ` && underpost cmt . ci engine-core-conf 'Update ${_confName} conf'` +
+        ` && underpost push . ${privateGitUri}`,
+    );
   }
-  shellExec(
-    `cd ../${privateRepoNameBackUp}` +
-      ` && git add .` +
-      ` && git commit -m "ci(engine-core-cron-backups): ⚙️ Update ${confName} cron backups"` +
-      ` && git push`,
-  );
   process.exit(0);
 }
 
-if (process.argv.includes('test')) {
-  fs.mkdirSync(`${basePath}/engine-private/conf`, { recursive: true });
-  fs.copySync(`./engine-private/conf/${confName}`, `${basePath}/engine-private/conf/${confName}`);
-}
-
 const { DefaultConf } = await import(`../conf.${confName}.js`);
 
 {
@@ -294,27 +153,9 @@ const { DefaultConf } = await import(`../conf.${confName}.js`);
   const env = process.argv.includes('development') ? 'development' : 'production';
   const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'secret.yaml'];
   // remove engine-private of .dockerignore for local testing
-
-  if (process.argv.includes('engine')) {
-    fs.removeSync(`${basePath}/manifests/deployment`);
-
-    if (!fs.existsSync(`./manifests/deployment/${confName}-${env}`))
-      fs.mkdirSync(`./manifests/deployment/${confName}-${env}`);
-
-    for (const file of deploymentsFiles) {
-      if (fs.existsSync(`./engine-private/conf/${confName}/build/${env}/${file}`)) {
-        fs.copyFileSync(`./engine-private/conf/${confName}/build/${env}/${file}`, `${basePath}/${file}`);
-        fs.copyFileSync(
-          `./engine-private/conf/${confName}/build/${env}/${file}`,
-          `./manifests/deployment/${confName}-${env}/${file}`,
-        );
-      }
-    }
-  } else {
-    for (const file of deploymentsFiles) {
-      if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {
-        fs.copyFileSync(`./manifests/deployment/${confName}-${env}/${file}`, `${basePath}/${file}`);
-      }
+  for (const file of deploymentsFiles) {
+    if (fs.existsSync(`./manifests/deployment/${confName}-${env}/${file}`)) {
+      fs.copyFileSync(`./manifests/deployment/${confName}-${env}/${file}`, `${basePath}/${file}`);
     }
   }
 }

package/bin/deploy.js

@@ -159,10 +159,6 @@ try {
       }
       break;
 
-    case 'remove-await-deploy': {
-      if (fs.existsSync(`./tmp/await-deploy`)) fs.remove(`./tmp/await-deploy`);
-      break;
-    }
     case 'new-nodejs-app':
       {
         const deployId = process.argv[3];
@@ -229,6 +225,7 @@ try {
       break;
     case 'build-full-client':
       {
+        dotenv.config({ override: true });
         if (!process.argv[3]) process.argv[3] = 'default';
         const { deployId, folder } = loadConf(process.argv[3]);
 
@@ -254,20 +251,7 @@ try {
                   serverConf[host][path].replicas.map((replica) => buildReplicaId({ deployId, replica })),
                 );
 
-                shellExec(Cmd.replica(deployId, host, path));
-              }
-              if (serverConf[host][path].db) {
-                switch (serverConf[host][path].db.provider) {
-                  case 'mariadb':
-                    {
-                      shellExec(`node bin/db ${host}${path} create ${deployId}`);
-                    }
-
-                    break;
-
-                  default:
-                    break;
-                }
+                // shellExec(Cmd.replica(deployId, host, path));
               }
             }
           }
@@ -276,7 +260,7 @@ try {
         await buildClient();
 
         for (const replicaDeployId of deployIdSingleReplicas) {
-          shellExec(Cmd.conf(replicaDeployId));
+          shellExec(Cmd.conf(replicaDeployId, process.env.NODE_ENV));
           shellExec(Cmd.build(replicaDeployId));
         }
       }

package/bin/file.js

@@ -1,7 +1,13 @@
 import fs from 'fs-extra';
 
 import { loggerFactory } from '../src/server/logger.js';
-import { cap, getCapVariableName, getDirname, newInstance } from '../src/client/components/core/CommonJs.js';
+import {
+  cap,
+  getCapVariableName,
+  getDirname,
+  newInstance,
+  uniqueArray,
+} from '../src/client/components/core/CommonJs.js';
 import { shellCd, shellExec } from '../src/server/process.js';
 import walk from 'ignore-walk';
 import { validateTemplatePath } from '../src/server/conf.js';
@@ -81,10 +87,12 @@ try {
           '.github/workflows/engine.core.ci.yml',
           '.github/workflows/engine.cyberia.ci.yml',
           './manifests/deployment/dd-lampp-development',
+          './manifests/deployment/dd-cyberia-development',
+          './manifests/deployment/dd-core-development',
           'bin/web3.js',
           'bin/cyberia.js',
         ]) {
-          fs.removeSync('../pwa-microservices-template/' + deletePath);
+          if (fs.existsSync(deletePath)) fs.removeSync('../pwa-microservices-template/' + deletePath);
         }
         const originPackageJson = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
         const templatePackageJson = JSON.parse(fs.readFileSync('../pwa-microservices-template/package.json', 'utf8'));
@@ -102,6 +110,9 @@ try {
         templatePackageJson.description = description;
         templatePackageJson.scripts.dev = dev;
         templatePackageJson.scripts.build = build;
+        templatePackageJson.keywords = uniqueArray(
+          ['pwa', 'microservices', 'template', 'builder'].concat(templatePackageJson.keywords),
+        );
         delete templatePackageJson.scripts['update-template'];
         fs.writeFileSync(
           '../pwa-microservices-template/package.json',

package/bin/index.js

@@ -9,6 +9,7 @@ import fs from 'fs-extra';
 import { commitData } from '../src/client/components/core/CommonJs.js';
 import UnderpostScript from '../src/cli/script.js';
 import { shellExec } from '../src/server/process.js';
+import UnderpostDB from '../src/cli/db.js';
 
 const npmRoot = getNpmRootPath();
 const underpostRoot = `${npmRoot}/underpost/.env`;
@@ -96,6 +97,17 @@ program
   })
   .description('Manage cluster, for default initialization base kind cluster');
 
+program
+  .command('deploy')
+  .argument('<deploy-list>', 'Deploy id list, e.g. default-a,default-b')
+  .argument('[env]', 'Optional environment, for default is development')
+  .option('--remove', 'Delete deployments and services')
+  .option('--sync', 'Sync deployments env, ports, and replicas')
+  .option('--info-router', 'Display router structure')
+  .option('--build-manifest', 'Build kind yaml manifests: deployments, services, proxy and secrets')
+  .description('Manage deployment, for default deploy development pods')
+  .action(Underpost.deploy.callback);
+
 program
   .command('secret')
   .argument('<platform>', `Options: ${Object.keys(Underpost.secret)}`)
@@ -115,6 +127,7 @@ program
   .command('dockerfile-node-script')
   .argument('<deploy-id>', 'Deploy configuration id')
   .argument('[env]', 'Optional environment, for default is development')
+  .option('--run', 'Run custom entry point script')
   .description('Dockerfile custom node build script')
   .action(Underpost.image.dockerfile.script);
 
@@ -139,6 +152,14 @@ program
     fs.copySync(`${npmRoot}/underpost/node_modules`, './node_modules');
   });
 
+program
+  .command('db')
+  .argument('<deploy-list>', 'Deploy id list, e.g. default-a,default-b')
+  .option('--import', 'Import container backups from repositories')
+  .option('--export', 'Export container backups to repositories')
+  .description('Manage databases')
+  .action(UnderpostDB.API.callback);
+
 program
   .command('script')
   .argument('operator', `Options: ${Object.keys(UnderpostScript.API)}`)
@@ -149,6 +170,13 @@ program
   )
   .action((...args) => Underpost.script[args[0]](args[1], args[2]));
 
-program.command('test').description('Run tests').action(Underpost.test.run);
+program
+  .command('test')
+  .argument('[deploy-list]', 'Deploy id list, e.g. default-a,default-b')
+  .description('Manage Test, for default run current underpost default test')
+  .option('--inside-container', 'Inside container execution context')
+  .option('--sh', 'Copy to clipboard, container entrypoint shell command')
+  .option('--logs', 'Display container logs')
+  .action(Underpost.test.callback);
 
 program.parse();

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.453'
+      engine.version: '2.8.482'
     networks:
       - load-balancer
 

package/manifests/mongodb/backup-cronjob.yaml

@@ -16,25 +16,27 @@ spec:
                 - -c
                 - |
                   # Perform backup
-                  mongodump -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
+                  mongodump --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
                   # Remove backups older than 7 days
                   find /backup -type d -mtime +7 -exec rm -rf {} +
               volumeMounts:
                 - name: backup-storage
                   mountPath: /backup
-              env:
-                - name: MONGO_INITDB_ROOT_USERNAME
-                  valueFrom:
-                    secretKeyRef:
-                      name: mongodb-secret
-                      key: username
-                - name: MONGO_INITDB_ROOT_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: mongodb-secret
-                      key: password
           restartPolicy: Never
           volumes:
             - name: backup-storage
               persistentVolumeClaim:
                 claimName: backup-pvc
+#     mongodump -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --host=mongodb-service --port=27017 --out=/backup/$(date +\%Y-\%m-\%dT\%H-\%M-\%S)
+
+# env:
+#   - name: MONGO_INITDB_ROOT_USERNAME
+#     valueFrom:
+#       secretKeyRef:
+#         name: mongodb-secret
+#         key: username
+#   - name: MONGO_INITDB_ROOT_PASSWORD
+#     valueFrom:
+#       secretKeyRef:
+#         name: mongodb-secret
+#         key: password

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.453",
+    "version": "2.8.482",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -32,6 +32,10 @@
         "url": "git+https://github.com/underpostnet/pwa-microservices-template.git"
     },
     "keywords": [
+        "pwa",
+        "microservices",
+        "template",
+        "builder",
         "engine",
         "server",
         "proxy",

package/src/cli/cluster.js

@@ -43,7 +43,7 @@ class UnderpostCluster {
         return;
       }
       const testClusterInit = shellExec(`kubectl get pods --all-namespaces -o wide`, {
-        disableLogging: true,
+        disableLog: true,
         silent: true,
         stdout: true,
       });
@@ -93,7 +93,7 @@ class UnderpostCluster {
                 silent: true,
                 stdout: true,
                 disableLog: true,
-              }).match(`mongodb-0                                    1/1     Running`)
+              }).match(`mongodb-1                                    1/1     Running`)
             )
               return resolve();
             return monitor();
@@ -110,7 +110,7 @@ class UnderpostCluster {
         };
 
         shellExec(
-          `kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
+          `sudo kubectl exec -i mongodb-0 -- mongosh --quiet --json=relaxed \
           --eval 'use admin' \
           --eval 'rs.initiate(${JSON.stringify(mongoConfig)})' \
           --eval 'rs.status()'`,