underpost 2.8.46 → 2.8.47

package/Dockerfile

@@ -4,7 +4,7 @@ FROM debian:${BASE_DEBIAN}
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-WORKDIR /code
+WORKDIR /home/dd
 
 # Set root password to root, format is 'user:password'.
 RUN echo 'root:root' | chpasswd
@@ -39,10 +39,10 @@ RUN npm --version
 
 RUN npm install -g underpost
 
-VOLUME [ "/code/app/logs" ]
+VOLUME [ "/home/dd/engine/logs" ]
 
 EXPOSE 22
 
 EXPOSE 4000-4004
 
-CMD [ "underpost", "new", "app" ]
+CMD [ "underpost", "new", "engine" ]

package/bin/file.js

@@ -1,7 +1,13 @@
 import fs from 'fs-extra';
 
 import { loggerFactory } from '../src/server/logger.js';
-import { cap, getCapVariableName, getDirname, newInstance } from '../src/client/components/core/CommonJs.js';
+import {
+  cap,
+  getCapVariableName,
+  getDirname,
+  newInstance,
+  uniqueArray,
+} from '../src/client/components/core/CommonJs.js';
 import { shellCd, shellExec } from '../src/server/process.js';
 import walk from 'ignore-walk';
 import { validateTemplatePath } from '../src/server/conf.js';
@@ -81,10 +87,12 @@ try {
           '.github/workflows/engine.core.ci.yml',
           '.github/workflows/engine.cyberia.ci.yml',
           './manifests/deployment/dd-lampp-development',
+          './manifests/deployment/dd-cyberia-development',
+          './manifests/deployment/dd-core-development',
           'bin/web3.js',
           'bin/cyberia.js',
         ]) {
-          fs.removeSync('../pwa-microservices-template/' + deletePath);
+          if (fs.existsSync(deletePath)) fs.removeSync('../pwa-microservices-template/' + deletePath);
         }
         const originPackageJson = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
         const templatePackageJson = JSON.parse(fs.readFileSync('../pwa-microservices-template/package.json', 'utf8'));
@@ -102,8 +110,8 @@ try {
         templatePackageJson.description = description;
         templatePackageJson.scripts.dev = dev;
         templatePackageJson.scripts.build = build;
-        templatePackageJson.keywords = ['pwa', 'microservices', 'template', 'builder'].concat(
-          templatePackageJson.keywords,
+        templatePackageJson.keywords = uniqueArray(
+          ['pwa', 'microservices', 'template', 'builder'].concat(templatePackageJson.keywords),
         );
         delete templatePackageJson.scripts['update-template'];
         fs.writeFileSync(

package/bin/index.js

@@ -97,6 +97,14 @@ program
   })
   .description('Manage cluster, for default initialization base kind cluster');
 
+program
+  .command('deploy')
+  .argument('<deploy-list>', 'Deploy id list, e.g. default-a, default-b')
+  .argument('[env]', 'Optional environment, for default is development')
+  .option('--remove', 'Delete deployments and services')
+  .description('Manage deployment, for default deploy development pods')
+  .action(Underpost.deploy.callback);
+
 program
   .command('secret')
   .argument('<platform>', `Options: ${Object.keys(Underpost.secret)}`)
@@ -142,11 +150,11 @@ program
 
 program
   .command('db')
-  .option('--import <deploy-id-list>', 'Import databases to containers from deploy id list, e.g. default-a, default-b')
+  .argument('<deploy-list>', 'Deploy id list, e.g. default-a, default-b')
+  .option('--import', 'Import container backups from repositories')
+  .option('--export', 'Export container backups to repositories')
   .description('Manage databases')
-  .action((...args) => {
-    if (args && args[0].import) return UnderpostDB.API.import(...args);
-  });
+  .action(UnderpostDB.API.callback);
 
 program
   .command('script')
@@ -158,6 +166,12 @@ program
   )
   .action((...args) => Underpost.script[args[0]](args[1], args[2]));
 
-program.command('test').description('Run tests').action(Underpost.test.run);
+program
+  .command('test')
+  .argument('[deploy-list]', 'Deploy id list, e.g. default-a, default-b')
+  .description('Manage Test, for default run current underpost default test')
+  .option('--inside-container', 'Inside container execution context')
+  .option('--sh', 'Copy to clipboard, container entrypoint shell command')
+  .action(Underpost.test.callback);
 
 program.parse();

package/docker-compose.yml

@@ -58,7 +58,7 @@ services:
           cpus: '0.25'
           memory: 20M
     labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.46'
+      engine.version: '2.8.47'
     networks:
       - load-balancer
 

package/package.json

@@ -2,7 +2,7 @@
     "type": "module",
     "main": "src/index.js",
     "name": "underpost",
-    "version": "2.8.46",
+    "version": "2.8.47",
     "description": "pwa api rest template",
     "scripts": {
         "start": "env-cmd -f .env.production node --max-old-space-size=8192 src/server",
@@ -32,10 +32,6 @@
         "url": "git+https://github.com/underpostnet/pwa-microservices-template.git"
     },
     "keywords": [
-        "pwa",
-        "microservices",
-        "template",
-        "builder",
         "pwa",
         "microservices",
         "template",

package/src/cli/cluster.js

@@ -43,7 +43,7 @@ class UnderpostCluster {
         return;
       }
       const testClusterInit = shellExec(`kubectl get pods --all-namespaces -o wide`, {
-        disableLogging: true,
+        disableLog: true,
         silent: true,
         stdout: true,
       });

package/src/cli/db.js

@@ -1,4 +1,4 @@
-import { mergeFile } from '../server/conf.js';
+import { mergeFile, splitFileFactory } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
@@ -7,8 +7,10 @@ const logger = loggerFactory(import.meta);
 
 class UnderpostDB {
   static API = {
-    async import(options = { import: 'default' }) {
-      for (const _deployId of options.import.split(',')) {
+    async callback(deployList = 'default', options = { import: false, export: false }) {
+      const newBackupTimestamp = new Date().getTime();
+      const nameSpace = 'default';
+      for (const _deployId of deployList.split(',')) {
         const deployId = _deployId.trim();
         if (!deployId) continue;
         const dbs = {};
@@ -38,18 +40,22 @@ class UnderpostDB {
           for (const dbName of Object.keys(dbs[provider])) {
             const { hostFolder, user, password } = dbs[provider][dbName];
             if (hostFolder) {
-              logger.info('import', { hostFolder, provider, dbName });
+              logger.info('', { hostFolder, provider, dbName });
 
               const backUpPath = `../${repoName}/${hostFolder}`;
               const times = await fs.readdir(backUpPath);
               const currentBackupTimestamp = Math.max(...times.map((t) => parseInt(t)));
               dbs[provider][dbName].currentBackupTimestamp = currentBackupTimestamp;
+              const removeBackupTimestamp = Math.min(...times.map((t) => parseInt(t)));
 
+              const sqlContainerPath = `/home/${dbName}.sql`;
               const _fromPartsParts = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}-parths.json`;
               const _toSqlPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}.sql`;
+              const _toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
               const _toBsonPath = `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${dbName}`;
+              const _toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
 
-              if (fs.existsSync(_fromPartsParts) && !fs.existsSync(_toSqlPath)) {
+              if (options.import === true && fs.existsSync(_fromPartsParts) && !fs.existsSync(_toSqlPath)) {
                 const names = JSON.parse(fs.readFileSync(_fromPartsParts, 'utf8')).map((_path) => {
                   return `../${repoName}/${hostFolder}/${currentBackupTimestamp}/${_path.split('/').pop()}`;
                 });
@@ -61,26 +67,59 @@ class UnderpostDB {
                 await mergeFile(names, _toSqlPath);
               }
 
+              if (options.export === true && times.length >= 5) {
+                fs.removeSync(`../${repoName}/${hostFolder}/${removeBackupTimestamp}`);
+                fs.mkdirSync(`../${repoName}/${hostFolder}/${newBackupTimestamp}`, { recursive: true });
+              }
+
               switch (provider) {
                 case 'mariadb': {
                   const podName = `mariadb-statefulset-0`;
-                  const nameSpace = 'default';
                   const serviceName = 'mariadb';
-                  shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
-                  const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
-                  shellExec(
-                    `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
-                  );
-                  shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                  if (options.import === true) {
+                    shellExec(`sudo kubectl cp ${_toSqlPath} ${nameSpace}/${podName}:/${dbName}.sql`);
+                    const cmd = `mariadb -u ${user} -p${password} ${dbName} < /${dbName}.sql`;
+                    shellExec(
+                      `kubectl exec -i ${podName} -- ${serviceName} -p${password} -e 'CREATE DATABASE ${dbName};'`,
+                    );
+                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                  }
+                  if (options.export === true) {
+                    const cmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${sqlContainerPath}`;
+                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                    shellExec(`sudo kubectl cp ${nameSpace}/${podName}:${sqlContainerPath} ${_toNewSqlPath}`);
+                    await splitFileFactory(dbName, _toNewSqlPath);
+                  }
                   break;
                 }
 
                 case 'mongoose': {
-                  const podName = `mongodb-0`;
-                  const nameSpace = 'default';
-                  shellExec(`sudo kubectl cp ${_toBsonPath} ${nameSpace}/${podName}:/${dbName}`);
-                  const cmd = `mongorestore -d ${dbName} /${dbName}`;
-                  shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                  if (options.import === true) {
+                    const podName = `mongodb-0`;
+                    shellExec(`sudo kubectl cp ${_toBsonPath} ${nameSpace}/${podName}:/${dbName}`);
+                    const cmd = `mongorestore -d ${dbName} /${dbName}`;
+                    shellExec(`sudo kubectl exec -i ${podName} -- sh -c "${cmd}"`);
+                  }
+                  if (options.export === true) {
+                    const podName = `backup-access`;
+                    const containerBaseBackupPath = '/backup';
+                    let timeFolder = shellExec(
+                      `sudo kubectl exec -i ${podName} -- sh -c "cd ${containerBaseBackupPath} && ls -a"`,
+                      {
+                        stdout: true,
+                        disableLog: false,
+                        silent: true,
+                      },
+                    ).split(`\n`);
+                    timeFolder = timeFolder[timeFolder.length - 2];
+                    if (timeFolder === '..') {
+                      logger.warn(`Cannot backup available`, { timeFolder });
+                    } else {
+                      shellExec(
+                        `sudo kubectl cp ${nameSpace}/${podName}:${containerBaseBackupPath}/${timeFolder}/${dbName} ${_toNewBsonPath}`,
+                      );
+                    }
+                  }
                   break;
                 }
 
@@ -90,6 +129,17 @@ class UnderpostDB {
             }
           }
         }
+        if (options.export === true) {
+          shellExec(`cd ../${repoName} && git add .`);
+          shellExec(
+            `underpost cmt ../${repoName} backup '' '${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
+              newBackupTimestamp,
+            ).toLocaleTimeString()}'`,
+          );
+          shellExec(`cd ../${repoName} && underpost push . ${process.env.GITHUB_USERNAME}/${repoName}`, {
+            disableLog: true,
+          });
+        }
       }
     },
   };

package/src/cli/deploy.js

@@ -0,0 +1,95 @@
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostDeploy {
+  static API = {
+    callback: (deployList = 'default', env = 'development', options = { remove: false }) => {
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+
+        shellExec(`sudo kubectl delete svc ${deployId}-${env}-service`);
+        shellExec(`sudo kubectl delete deployment ${deployId}-${env}`);
+
+        const etcHost = (
+          concat,
+        ) => `127.0.0.1  ${concat} localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6`;
+        let concatHots = '';
+
+        const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
+        for (const host of Object.keys(confServer)) {
+          shellExec(`sudo kubectl delete HTTPProxy ${host}`);
+          if (!options.remove === true && env === 'development') concatHots += ` ${host}`;
+        }
+
+        if (!options.remove === true) {
+          shellExec(`sudo kubectl apply -f ./manifests/deployment/${deployId}-${env}/deployment.yaml`);
+          shellExec(`sudo kubectl apply -f ./manifests/deployment/${deployId}-${env}/proxy.yaml`);
+        }
+
+        let renderHosts;
+
+        switch (process.platform) {
+          case 'linux':
+            {
+              switch (env) {
+                case 'development':
+                  renderHosts = etcHost(concatHots);
+                  fs.writeFileSync(`/etc/hosts`, renderHosts, 'utf8');
+
+                  break;
+
+                default:
+                  break;
+              }
+            }
+            break;
+
+          default:
+            break;
+        }
+        logger.info(
+          `
+` + renderHosts,
+        );
+      }
+    },
+    getPods(deployId) {
+      const raw = shellExec(`sudo kubectl get pods --all-namespaces -o wide`, {
+        stdout: true,
+        disableLog: false,
+        silent: true,
+      });
+
+      const heads = raw
+        .split(`\n`)[0]
+        .split(' ')
+        .filter((_r) => _r.trim());
+
+      const pods = raw
+        .split(`\n`)
+        .filter((r) => (deployId ? r.match(deployId) : r.trim() && !r.match('NAME')))
+        .map((r) => r.split(' ').filter((_r) => _r.trim()));
+
+      const result = [];
+
+      for (const row of pods) {
+        const pod = {};
+        let index = -1;
+        for (const head of heads) {
+          index++;
+          pod[head] = row[index];
+        }
+        result.push(pod);
+      }
+
+      return result;
+    },
+  };
+}
+
+export default UnderpostDeploy;

package/src/cli/image.js

@@ -1,7 +1,6 @@
 import fs from 'fs-extra';
 import Underpost from '../index.js';
 import { shellExec } from '../server/process.js';
-import { MariaDB } from '../db/mariadb/MariaDB.js';
 import dotenv from 'dotenv';
 import { getNpmRootPath } from '../server/conf.js';
 
@@ -45,17 +44,6 @@ class UnderpostImage {
           case 'dd-lampp':
             {
               const lamppPublicPath = '/xampp/htdocs/online';
-              if (process.argv.includes('test')) {
-                const { MARIADB_HOST, MARIADB_USER, MARIADB_PASSWORD, DD_LAMPP_TEST_DB_0 } = process.env;
-
-                await MariaDB.query({
-                  host: MARIADB_HOST,
-                  user: MARIADB_USER,
-                  password: MARIADB_PASSWORD,
-                  query: `SHOW TABLES FROM ${DD_LAMPP_TEST_DB_0}`,
-                });
-                process.exit(0);
-              }
               shellExec(`sudo mkdir -p ${lamppPublicPath}`);
 
               {

package/src/cli/repository.js

@@ -54,6 +54,10 @@ class UnderpostRepository {
         empty: false,
       },
     ) {
+      if (commitType === 'reset') {
+        shellExec(`cd ${repoPath} && git reset --soft HEAD~${isNaN(parseInt(subModule)) ? 1 : parseInt(subModule)}`);
+        return;
+      }
       if (options.info) return logger.info('', commitData);
       const _message = `${commitType}${subModule ? `(${subModule})` : ''}${process.argv.includes('!') ? '!' : ''}: ${
         commitData[commitType].emoji

package/src/cli/test.js

@@ -1,6 +1,8 @@
+import { MariaDB } from '../db/mariadb/MariaDB.js';
 import { getNpmRootPath } from '../server/conf.js';
 import { actionInitLog, loggerFactory } from '../server/logger.js';
-import { shellExec } from '../server/process.js';
+import { pbcopy, shellExec } from '../server/process.js';
+import UnderpostDeploy from './deploy.js';
 
 const logger = loggerFactory(import.meta);
 
@@ -26,6 +28,52 @@ class UnderpostTest {
       actionInitLog();
       shellExec(`cd ${getNpmRootPath()}/underpost && npm run test`);
     },
+    async callback(deployList = '', options = { insideContainer: false, sh: false }) {
+      if (options.sh === true) {
+        const [pod] = UnderpostDeploy.API.getPods(deployList);
+        if (pod) return pbcopy(`sudo kubectl exec -it ${pod.NAME} -- sh`);
+        return logger.warn(`Couldn't find pods in deployment`, deployList);
+      }
+      if (deployList) {
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
+          if (options.insideContainer === true)
+            switch (deployId) {
+              case 'dd-lampp':
+                {
+                  const { MARIADB_HOST, MARIADB_USER, MARIADB_PASSWORD, DD_LAMPP_TEST_DB_0 } = process.env;
+
+                  await MariaDB.query({
+                    host: MARIADB_HOST,
+                    user: MARIADB_USER,
+                    password: MARIADB_PASSWORD,
+                    query: `SHOW TABLES FROM ${DD_LAMPP_TEST_DB_0}`,
+                  });
+                }
+                break;
+
+              default:
+                {
+                  shellExec('npm run test');
+                }
+
+                break;
+            }
+          else {
+            const pods = UnderpostDeploy.API.getPods(deployId);
+            if (pods.length > 0)
+              for (const deployData of pods) {
+                const { NAME } = deployData;
+                shellExec(
+                  `sudo kubectl exec -i ${NAME} -- sh -c "cd /home/dd/engine && underpost test ${deployId} --inside-container"`,
+                );
+              }
+            else logger.warn(`Couldn't find pods in deployment`, { deployId });
+          }
+        }
+      } else return UnderpostTest.API.run();
+    },
   };
 }
 

package/src/index.js

@@ -6,6 +6,7 @@
 
 import UnderpostCluster from './cli/cluster.js';
 import UnderpostDB from './cli/db.js';
+import UnderpostDeploy from './cli/deploy.js';
 import UnderpostRootEnv from './cli/env.js';
 import UnderpostImage from './cli/image.js';
 import UnderpostRepository from './cli/repository.js';
@@ -25,7 +26,7 @@ class Underpost {
    * @type {String}
    * @memberof Underpost
    */
-  static version = 'v2.8.46';
+  static version = 'v2.8.47';
   /**
    * Repository cli API
    * @static
@@ -82,6 +83,13 @@ class Underpost {
    * @memberof Underpost
    */
   static db = UnderpostDB.API;
+  /**
+   * Deployment cli API
+   * @static
+   * @type {UnderpostDeploy.API}
+   * @memberof Underpost
+   */
+  static deploy = UnderpostDeploy.API;
 }
 
 const up = Underpost;

package/src/server/network.js

@@ -120,7 +120,14 @@ const saveRuntimeCron = async () => {
 
 const listenServerFactory = (logic = async () => {}) => {
   return {
-    listen: async (...args) => (logic ? await logic(...args) : undefined, args[1]()),
+    listen: async (...args) => (
+      setTimeout(() => {
+        const message = 'Listen server factory timeout';
+        logger.error(message);
+        throw new Error(message);
+      }, 80000000), // ~ 55 days
+      (logic ? await logic(...args) : undefined, args[1]())
+    ),
   };
 };