underpost 2.8.62 → 2.8.65

package/src/cli/deploy.js

@@ -3,22 +3,41 @@ import {
   buildPortProxyRouter,
   buildProxyRouter,
   Config,
+  deployRangePortFactory,
   getDataDeploy,
   loadReplicas,
+  pathPortAssignmentFactory,
 } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 import dotenv from 'dotenv';
-import Underpost from '../index.js';
+import { DataBaseProvider } from '../db/DataBaseProvider.js';
+import UnderpostRootEnv from './env.js';
+import UnderpostCluster from './cluster.js';
 
 const logger = loggerFactory(import.meta);
 
 class UnderpostDeploy {
+  static NETWORK = {};
   static API = {
-    sync(deployList) {
+    sync(deployList, { versions, replicas }) {
       const deployGroupId = 'dd.tmp';
       fs.writeFileSync(`./engine-private/deploy/${deployGroupId}`, deployList, 'utf8');
+      const totalPods = deployList.split(',').length * versions.split(',').length * parseInt(replicas);
+      const limitFactor = 0.95;
+      const reserveFactor = 0.35;
+      const resources = UnderpostCluster.API.getResourcesCapacity();
+      const memory = parseInt(resources.memory.value / totalPods);
+      const cpu = parseInt(resources.cpu.value / totalPods);
+      UnderpostRootEnv.API.set(
+        'resources.requests.memory',
+        `${parseInt(memory * reserveFactor)}${resources.memory.unit}`,
+      );
+      UnderpostRootEnv.API.set('resources.requests.cpu', `${parseInt(cpu * reserveFactor)}${resources.cpu.unit}`);
+      UnderpostRootEnv.API.set('resources.limits.memory', `${parseInt(memory * limitFactor)}${resources.memory.unit}`);
+      UnderpostRootEnv.API.set('resources.limits.cpu', `${parseInt(cpu * limitFactor)}${resources.cpu.unit}`);
+      UnderpostRootEnv.API.set('total-pods', totalPods);
       return getDataDeploy({
         buildSingleReplica: true,
         deployGroupId,
@@ -32,59 +51,51 @@ class UnderpostDeploy {
       await Config.build(undefined, 'proxy', deployList);
       return buildPortProxyRouter(env === 'development' ? 80 : 443, buildProxyRouter());
     },
-    async buildManifest(deployList, env, version) {
-      for (const _deployId of deployList.split(',')) {
-        const deployId = _deployId.trim();
-        if (!deployId) continue;
-
-        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
-        const ports = Object.values(router).map((p) => parseInt(p.split(':')[2]));
-        const fromPort = Math.min(...ports);
-        const toPort = Math.max(...ports);
-        const confServer = loadReplicas(
-          JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
-          'proxy',
-        );
-
-        fs.mkdirSync(`./engine-private/conf/${deployId}/build/${env}`, { recursive: true });
-        if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
-
-        logger.info('port range', { deployId, fromPort, toPort });
-        // const customImg = `underpost-engine:${version && typeof version === 'string' ? version : Underpost.version}`;
-        const deploymentYamlParts = `apiVersion: apps/v1
+    deploymentYamlServiceFactory({ deployId, env, port, deploymentVersions }) {
+      return deploymentVersions
+        .map(
+          (version, i) => `    - name: ${deployId}-${env}-${version}-service
+          port: ${port}
+          weight: ${i === 0 ? 100 : 0}
+    `,
+        )
+        .join('');
+    },
+    deploymentYamlPartsFactory({ deployId, env, suffix, resources, replicas }) {
+      return `apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: ${deployId}-${env}
+  name: ${deployId}-${env}-${suffix}
   labels:
-    app: ${deployId}-${env}
+    app: ${deployId}-${env}-${suffix}
 spec:
-  replicas: 2
+  replicas: ${replicas}
   selector:
     matchLabels:
-      app: ${deployId}-${env}
+      app: ${deployId}-${env}-${suffix}
   template:
     metadata:
       labels:
-        app: ${deployId}-${env}
+        app: ${deployId}-${env}-${suffix}
     spec:
       containers:
-        - name: ${deployId}-${env}
+        - name: ${deployId}-${env}-${suffix}
           image: localhost/debian:underpost
+          resources:
+            requests:
+              memory: "${resources.requests.memory}"
+              cpu: "${resources.requests.cpu}"
+            limits:
+              memory: "${resources.limits.memory}"
+              cpu: "${resources.limits.cpu}"
           command:
             - /bin/sh
             - -c
             - >
-              underpost dockerfile-node-script --build --run ${deployId} ${env}
-          lifecycle:
-            postStart:
-              exec:
-                command:
-                  - /bin/sh
-                  - -c
-                  - >
-                    sleep 20 &&
-                    npm install -g underpost
-                    underpost secret underpost --create-from-file /etc/config/.env.${env}
+              npm install -g npm@11.2.0 &&
+              npm install -g underpost &&
+              underpost secret underpost --create-from-file /etc/config/.env.${env} &&
+              underpost start --build --run ${deployId} ${env}
           volumeMounts:
             - name: config-volume
               mountPath: /etc/config
@@ -92,27 +103,50 @@ spec:
         - name: config-volume
           configMap:
             name: underpost-config
-# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : Underpost.version}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: ${deployId}-${env}-service
+  name: ${deployId}-${env}-${suffix}-service
 spec:
   selector:
-    app: ${deployId}-${env}
+    app: ${deployId}-${env}-${suffix}
   ports:
-  type: LoadBalancer`.split('ports:');
-        deploymentYamlParts[1] =
-          buildKindPorts(fromPort, toPort) +
-          `  type: LoadBalancer
-`;
+{{ports}}  type: LoadBalancer`;
+    },
+    async buildManifest(deployList, env, options) {
+      const resources = UnderpostDeploy.API.resourcesFactory();
+      const replicas = options.replicas;
 
-        fs.writeFileSync(
-          `./engine-private/conf/${deployId}/build/${env}/deployment.yaml`,
-          deploymentYamlParts.join(`ports:
-`),
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+        const confServer = loadReplicas(
+          JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+          'proxy',
         );
+        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+        const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
+        const { fromPort, toPort } = deployRangePortFactory(router);
+        const deploymentVersions = options.versions.split(',');
+        fs.mkdirSync(`./engine-private/conf/${deployId}/build/${env}`, { recursive: true });
+        if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
+
+        logger.info('port range', { deployId, fromPort, toPort });
+
+        let deploymentYamlParts = '';
+        for (const deploymentVersion of deploymentVersions) {
+          deploymentYamlParts += `---
+${UnderpostDeploy.API.deploymentYamlPartsFactory({
+  deployId,
+  env,
+  suffix: deploymentVersion,
+  resources,
+  replicas,
+}).replace('{{ports}}', buildKindPorts(fromPort, toPort))}
+`;
+        }
+        fs.writeFileSync(`./engine-private/conf/${deployId}/build/${env}/deployment.yaml`, deploymentYamlParts, 'utf8');
 
         let proxyYaml = '';
         let secretYaml = '';
@@ -134,27 +168,8 @@ spec:
     kind: ClusterIssuer
   secretName: ${host}`;
 
-          const pathPortConditions = [];
-          for (const path of Object.keys(confServer[host])) {
-            const { peer } = confServer[host][path];
-            if (!router[`${host}${path === '/' ? '' : path}`]) continue;
-            const port = parseInt(router[`${host}${path === '/' ? '' : path}`].split(':')[2]);
-            // logger.info('', { host, port, path });
-            pathPortConditions.push({
-              port,
-              path,
-            });
-
-            if (peer) {
-              //  logger.info('', { host, port: port + 1, path: '/peer' });
-              pathPortConditions.push({
-                port: port + 1,
-                path: '/peer',
-              });
-            }
-          }
-
-          // logger.info('', { host, pathPortConditions });
+          const pathPortAssignment = pathPortAssignmentData[host];
+          // logger.info('', { host, pathPortAssignment });
           proxyYaml += `
 ---
 apiVersion: projectcontour.io/v1
@@ -171,15 +186,20 @@ spec:
       secretName: ${host}`
           }
   routes:`;
-          for (const conditionObj of pathPortConditions) {
+          for (const conditionObj of pathPortAssignment) {
             const { path, port } = conditionObj;
             proxyYaml += `
     - conditions:
         - prefix: ${path}
       enableWebsockets: true
       services:
-        - name: ${deployId}-${env}-service
-          port: ${port}`;
+    ${UnderpostDeploy.API.deploymentYamlServiceFactory({
+      deployId,
+      env,
+      port,
+      deploymentVersions:
+        options.traffic && typeof options.traffic === 'string' ? options.traffic.split(',') : ['blue'],
+    })}`;
           }
         }
         const yamlPath = `./engine-private/conf/${deployId}/build/${env}/proxy.yaml`;
@@ -211,7 +231,10 @@ spec:
         infoUtil: false,
         expose: false,
         cert: false,
-        version: '',
+        versions: '',
+        traffic: '',
+        dashboardUpdate: false,
+        replicas: '',
       },
     ) {
       if (options.infoUtil === true)
@@ -222,10 +245,13 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
         `);
       if (deployList === 'dd' && fs.existsSync(`./engine-private/deploy/dd.router`))
         deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
-      if (options.sync) UnderpostDeploy.API.sync(deployList);
-      if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options.version);
-      if (options.infoRouter === true)
-        return logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      if (!(options.versions && typeof options.versions === 'string')) options.versions = 'blue,green';
+      if (!options.replicas) options.replicas = 2;
+      if (options.sync) UnderpostDeploy.API.sync(deployList, options);
+      if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options);
+      if (options.infoRouter === true) logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
+      if (options.dashboardUpdate === true) await UnderpostDeploy.API.updateDashboardData(deployList, env, options);
+      if (options.infoRouter === true) return;
       shellExec(`kubectl delete configmap underpost-config`);
       shellExec(
         `kubectl create configmap underpost-config --from-file=/home/dd/engine/engine-private/conf/dd-cron/.env.${env}`,
@@ -328,6 +354,73 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
 
       return result;
     },
+    resourcesFactory() {
+      return {
+        requests: {
+          memory: UnderpostRootEnv.API.get('resources.requests.memory'),
+          cpu: UnderpostRootEnv.API.get('resources.requests.cpu'),
+        },
+        limits: {
+          memory: UnderpostRootEnv.API.get('resources.limits.memory'),
+          cpu: UnderpostRootEnv.API.get('resources.limits.cpu'),
+        },
+        totalPods: UnderpostRootEnv.API.get('total-pods'),
+      };
+    },
+    async updateDashboardData(deployList, env, options) {
+      try {
+        const deployId = process.env.DEFAULT_DEPLOY_ID;
+        const host = process.env.DEFAULT_DEPLOY_HOST;
+        const path = process.env.DEFAULT_DEPLOY_PATH;
+        const { db } = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'))[host][
+          path
+        ];
+
+        await DataBaseProvider.load({ apis: ['instance'], host, path, db });
+
+        /** @type {import('../api/instance/instance.model.js').InstanceModel} */
+        const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
+
+        await Instance.deleteMany();
+
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
+          const confServer = loadReplicas(
+            JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+            'proxy',
+          );
+          const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+          const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
+
+          for (const host of Object.keys(confServer)) {
+            for (const { path, port } of pathPortAssignmentData[host]) {
+              if (!confServer[host][path]) continue;
+
+              const { client, runtime, apis } = confServer[host][path];
+
+              const body = {
+                deployId,
+                host,
+                path,
+                port,
+                client,
+                runtime,
+                apis,
+              };
+
+              logger.info('save', body);
+
+              await new Instance(body).save();
+            }
+          }
+        }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+      } catch (error) {
+        logger.error(error, error.stack);
+      }
+    },
   };
 }
 

package/src/cli/fs.js

@@ -38,7 +38,7 @@ class UnderpostFileStorage {
       options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false },
     ) {
       const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
-      const deleteFiles = UnderpostRepository.API.getDeleteFiles(path);
+      const deleteFiles = options.pull === true ? [] : UnderpostRepository.API.getDeleteFiles(path);
       for (const relativePath of deleteFiles) {
         const _path = path + '/' + relativePath;
         if (_path in storage) {
@@ -46,10 +46,6 @@ class UnderpostFileStorage {
           delete storage[_path];
         }
       }
-      const files =
-        options.git === true
-          ? UnderpostRepository.API.getChangedFiles(path)
-          : await fs.readdir(path, { recursive: true });
       if (options.pull === true) {
         for (const _path of Object.keys(storage)) {
           if (!fs.existsSync(_path) || options.force === true) {
@@ -57,7 +53,11 @@ class UnderpostFileStorage {
             await UnderpostFileStorage.API.pull(_path, options);
           } else logger.warn(`Pull path already exists`, _path);
         }
-      } else
+      } else {
+        const files =
+          options.git === true
+            ? UnderpostRepository.API.getChangedFiles(path)
+            : await fs.readdir(path, { recursive: true });
         for (const relativePath of files) {
           const _path = path + '/' + relativePath;
           if (fs.statSync(_path).isDirectory()) {
@@ -68,6 +68,7 @@ class UnderpostFileStorage {
             if (storage) storage[_path] = {};
           } else logger.warn('File already exists', _path);
         }
+      }
       UnderpostFileStorage.API.writeStorageConf(storage, storageConf);
       if (options.git === true) {
         shellExec(`cd ${path} && git add .`);

package/src/cli/image.js

@@ -1,11 +1,9 @@
 import fs from 'fs-extra';
-import Underpost from '../index.js';
 import { shellCd, shellExec } from '../server/process.js';
 import dotenv from 'dotenv';
-import { getNpmRootPath } from '../server/conf.js';
-import { timer } from '../client/components/core/CommonJs.js';
-import UnderpostRootEnv from './env.js';
+import { awaitDeployMonitor, getNpmRootPath } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
+import UnderpostMonitor from './monitor.js';
 
 dotenv.config();
 
@@ -18,109 +16,49 @@ class UnderpostImage {
         shellExec(`sudo podman pull docker.io/library/debian:buster`);
       },
       build(
-        deployId = 'default',
-        env = 'development',
-        path = '.',
-        options = { imageArchive: false, podmanSave: false, imageName: '', imageVersion: '' },
+        options = {
+          path: '',
+          imageName: '',
+          imagePath: '',
+          dockerfileName: '',
+          podmanSave: false,
+          kindLoad: false,
+          secrets: false,
+          secretsPath: '',
+          noCache: false,
+        },
       ) {
-        const imgName = `${
-          options.imageName && typeof options.imageName === 'string' ? options.imageName : `${deployId}-${env}`
-        }:${
-          options.imageVersion && typeof options.imageVersion === 'string' ? options.imageVersion : Underpost.version
-        }`;
-        const podManImg = `localhost/${imgName}`;
-        const imagesStoragePath = `/images`;
-        if (!fs.existsSync(`${path}${imagesStoragePath}`))
-          fs.mkdirSync(`${path}${imagesStoragePath}`, { recursive: true });
-        const tarFile = `.${imagesStoragePath}/${imgName.replace(':', '_')}.tar`;
-
-        let secrets = ' ';
+        const { path, imageName, imagePath, dockerfileName, podmanSave, secrets, secretsPath, kindLoad, noCache } =
+          options;
+        const podManImg = `localhost/${imageName}`;
+        if (imagePath && typeof imagePath === 'string' && !fs.existsSync(imagePath))
+          fs.mkdirSync(imagePath, { recursive: true });
+        const tarFile = `${imagePath}/${imageName.replace(':', '_')}.tar`;
+        let secretsInput = ' ';
         let secretDockerInput = '';
-
-        const envObj = dotenv.parse(fs.readFileSync(`${getNpmRootPath()}/underpost/.env`, 'utf8'));
-
-        for (const key of Object.keys(envObj)) {
-          continue;
-          secrets += ` && export ${key}="${envObj[key]}" `; // $(cat gitlab-token.txt)
-          secretDockerInput += ` --secret id=${key},env=${key} \ `;
+        let cache = '';
+        if (secrets === true) {
+          const envObj = dotenv.parse(
+            fs.readFileSync(
+              secretsPath && typeof secretsPath === 'string' ? secretsPath : `${getNpmRootPath()}/underpost/.env`,
+              'utf8',
+            ),
+          );
+          for (const key of Object.keys(envObj)) {
+            secretsInput += ` && export ${key}="${envObj[key]}" `; // $(cat gitlab-token.txt)
+            secretDockerInput += ` --secret id=${key},env=${key} \ `;
+          }
         }
-        // --rm --no-cache
-        if (options.imageArchive !== true) {
+        if (noCache === true) cache += ' --rm --no-cache';
+        if (path && typeof path === 'string')
           shellExec(
-            `cd ${path}${secrets}&& sudo podman build -f ./Dockerfile -t ${imgName} --pull=never --cap-add=CAP_AUDIT_WRITE${secretDockerInput}`,
+            `cd ${path}${secretsInput}&& sudo podman build -f ./${
+              dockerfileName && typeof dockerfileName === 'string' ? dockerfileName : 'Dockerfile'
+            } -t ${imageName} --pull=never --cap-add=CAP_AUDIT_WRITE${cache}${secretDockerInput}`,
           );
-        }
-        if (options.imageArchive !== true || options.podmanSave === true)
-          shellExec(`cd ${path} && podman save -o ${tarFile} ${podManImg}`);
-        shellExec(`cd ${path} && sudo kind load image-archive ${tarFile}`);
-      },
-      async script(deployId = 'default', env = 'development', options = { run: false, build: false }) {
-        if (options.build === true) {
-          const buildBasePath = `/home/dd`;
-          const repoName = `engine-${deployId.split('-')[1]}`;
-          shellExec(`cd ${buildBasePath} && underpost clone underpostnet/${repoName}`);
-          shellExec(`cd ${buildBasePath} && sudo mv ./${repoName} ./engine`);
-          shellExec(`cd ${buildBasePath}/engine && underpost clone underpostnet/${repoName}-private`);
-          shellExec(`cd ${buildBasePath}/engine && sudo mv ./${repoName}-private ./engine-private`);
-          shellCd(`${buildBasePath}/engine`);
-          shellExec(`underpost install`);
-          const itcScripts = fs.readdir('./engine-private/itc-scripts');
-          for (const itcScript of itcScripts)
-            if (itcScript.match(deployId)) shellExec(`node ./engine-private/itc-scripts/${itcScript}`);
-        }
-        switch (deployId) {
-          default:
-            {
-              {
-                const originPath = `./src/db/mongo/MongooseDB.js`;
-                fs.writeFileSync(
-                  originPath,
-                  fs.readFileSync(originPath, 'utf8').replaceAll(
-                    `connect: async (host, name) => {`,
-                    `connect: async (host, name) => {
-    host = 'mongodb://mongodb-0.mongodb-service:27017';
-        `,
-                  ),
-                  'utf8',
-                );
-              }
 
-              {
-                const originPath = `./src/server/valkey.js`;
-                fs.writeFileSync(
-                  originPath,
-                  fs.readFileSync(originPath, 'utf8').replaceAll(
-                    `    // port: 6379,
-    // host: 'service-valkey.default.svc.cluster.local',`,
-                    `     port: 6379,
-    host: 'service-valkey.default.svc.cluster.local',`,
-                  ),
-                  'utf8',
-                );
-              }
-            }
-            break;
-        }
-        shellExec(`node bin/deploy conf ${deployId} ${env}`);
-        shellExec(`node bin/deploy build-full-client ${deployId}`);
-        if (options.run === true) {
-          const runCmd = env === 'production' ? 'run prod-img' : 'run dev-img';
-          if (fs.existsSync(`./engine-private/replica`)) {
-            const replicas = await fs.readdir(`./engine-private/replica`);
-            for (const replica of replicas) {
-              shellExec(`node bin/deploy conf ${replica} ${env}`);
-              shellExec(`npm ${runCmd} deploy deploy-id:${replica}`, { async: true });
-              fs.writeFileSync(`./tmp/await-deploy`, '', 'utf8');
-              const monitor = async () => {
-                await timer(1000);
-                if (fs.existsSync(`./tmp/await-deploy`)) return await monitor();
-              };
-              await monitor();
-            }
-            shellExec(`node bin/deploy conf ${deployId} ${env}`);
-          }
-          shellExec(`npm ${runCmd} deploy deploy-id:${deployId}`);
-        }
+        if (podmanSave === true) shellExec(`podman save -o ${tarFile} ${podManImg}`);
+        if (kindLoad === true) shellExec(`sudo kind load image-archive ${tarFile}`);
       },
     },
   };