underpost 2.8.64 → 2.8.67

package/src/cli/deploy.js

@@ -13,15 +13,31 @@ import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 import dotenv from 'dotenv';
 import { DataBaseProvider } from '../db/DataBaseProvider.js';
+import UnderpostRootEnv from './env.js';
+import UnderpostCluster from './cluster.js';
 
 const logger = loggerFactory(import.meta);
 
 class UnderpostDeploy {
   static NETWORK = {};
   static API = {
-    sync(deployList) {
-      const deployGroupId = 'dd.tmp';
+    sync(deployList, { versions, replicas }) {
+      const deployGroupId = 'dd.router';
       fs.writeFileSync(`./engine-private/deploy/${deployGroupId}`, deployList, 'utf8');
+      const totalPods = deployList.split(',').length * versions.split(',').length * parseInt(replicas);
+      const limitFactor = 0.8;
+      const reserveFactor = 0.05;
+      const resources = UnderpostCluster.API.getResourcesCapacity();
+      const memory = parseInt(resources.memory.value / totalPods);
+      const cpu = parseInt(resources.cpu.value / totalPods);
+      UnderpostRootEnv.API.set(
+        'resources.requests.memory',
+        `${parseInt(memory * reserveFactor)}${resources.memory.unit}`,
+      );
+      UnderpostRootEnv.API.set('resources.requests.cpu', `${parseInt(cpu * reserveFactor)}${resources.cpu.unit}`);
+      UnderpostRootEnv.API.set('resources.limits.memory', `${parseInt(memory * limitFactor)}${resources.memory.unit}`);
+      UnderpostRootEnv.API.set('resources.limits.cpu', `${parseInt(cpu * limitFactor)}${resources.cpu.unit}`);
+      UnderpostRootEnv.API.set('total-pods', totalPods);
       return getDataDeploy({
         buildSingleReplica: true,
         deployGroupId,
@@ -35,52 +51,43 @@ class UnderpostDeploy {
       await Config.build(undefined, 'proxy', deployList);
       return buildPortProxyRouter(env === 'development' ? 80 : 443, buildProxyRouter());
     },
-    async buildManifest(deployList, env, version) {
-      for (const _deployId of deployList.split(',')) {
-        const deployId = _deployId.trim();
-        if (!deployId) continue;
-        const confServer = loadReplicas(
-          JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
-          'proxy',
-        );
-        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
-        const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
-        const { fromPort, toPort } = deployRangePortFactory(router);
-
-        fs.mkdirSync(`./engine-private/conf/${deployId}/build/${env}`, { recursive: true });
-        if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
-
-        logger.info('port range', { deployId, fromPort, toPort });
-        // const customImg = `underpost-engine:${version && typeof version === 'string' ? version : '0.0.0'}`;
-        // lifecycle:
-        // postStart:
-        //   exec:
-        //     command:
-        //       - /bin/sh
-        //       - -c
-        //       - >
-        //         sleep 20 &&
-        //         npm install -g underpost &&
-        //         underpost secret underpost --create-from-file /etc/config/.env.${env}
-        const deploymentYamlParts = `apiVersion: apps/v1
+    deploymentYamlServiceFactory({ deployId, env, port, deploymentVersions }) {
+      return deploymentVersions
+        .map(
+          (version, i) => `    - name: ${deployId}-${env}-${version}-service
+          port: ${port}
+          weight: ${i === 0 ? 100 : 0}
+    `,
+        )
+        .join('');
+    },
+    deploymentYamlPartsFactory({ deployId, env, suffix, resources, replicas }) {
+      return `apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: ${deployId}-${env}
+  name: ${deployId}-${env}-${suffix}
   labels:
-    app: ${deployId}-${env}
+    app: ${deployId}-${env}-${suffix}
 spec:
-  replicas: 2
+  replicas: ${replicas}
   selector:
     matchLabels:
-      app: ${deployId}-${env}
+      app: ${deployId}-${env}-${suffix}
   template:
     metadata:
       labels:
-        app: ${deployId}-${env}
+        app: ${deployId}-${env}-${suffix}
     spec:
       containers:
-        - name: ${deployId}-${env}
+        - name: ${deployId}-${env}-${suffix}
           image: localhost/debian:underpost
+          resources:
+            requests:
+              memory: "${resources.requests.memory}"
+              cpu: "${resources.requests.cpu}"
+            limits:
+              memory: "${resources.limits.memory}"
+              cpu: "${resources.limits.cpu}"
           command:
             - /bin/sh
             - -c
@@ -88,7 +95,7 @@ spec:
               npm install -g npm@11.2.0 &&
               npm install -g underpost &&
               underpost secret underpost --create-from-file /etc/config/.env.${env} &&
-              underpost dockerfile-node-script --build --run ${deployId} ${env}
+              underpost start --build --run ${deployId} ${env}
           volumeMounts:
             - name: config-volume
               mountPath: /etc/config
@@ -96,27 +103,50 @@ spec:
         - name: config-volume
           configMap:
             name: underpost-config
-# image: localhost/${deployId}-${env}:${version && typeof version === 'string' ? version : '0.0.0'}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: ${deployId}-${env}-service
+  name: ${deployId}-${env}-${suffix}-service
 spec:
   selector:
-    app: ${deployId}-${env}
+    app: ${deployId}-${env}-${suffix}
   ports:
-  type: LoadBalancer`.split('ports:');
-        deploymentYamlParts[1] =
-          buildKindPorts(fromPort, toPort) +
-          `  type: LoadBalancer
-`;
+{{ports}}  type: LoadBalancer`;
+    },
+    async buildManifest(deployList, env, options) {
+      const resources = UnderpostDeploy.API.resourcesFactory();
+      const replicas = options.replicas;
 
-        fs.writeFileSync(
-          `./engine-private/conf/${deployId}/build/${env}/deployment.yaml`,
-          deploymentYamlParts.join(`ports:
-`),
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        if (!deployId) continue;
+        const confServer = loadReplicas(
+          JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+          'proxy',
         );
+        const router = await UnderpostDeploy.API.routerFactory(deployId, env);
+        const pathPortAssignmentData = pathPortAssignmentFactory(router, confServer);
+        const { fromPort, toPort } = deployRangePortFactory(router);
+        const deploymentVersions = options.versions.split(',');
+        fs.mkdirSync(`./engine-private/conf/${deployId}/build/${env}`, { recursive: true });
+        if (env === 'development') fs.mkdirSync(`./manifests/deployment/${deployId}-${env}`, { recursive: true });
+
+        logger.info('port range', { deployId, fromPort, toPort });
+
+        let deploymentYamlParts = '';
+        for (const deploymentVersion of deploymentVersions) {
+          deploymentYamlParts += `---
+${UnderpostDeploy.API.deploymentYamlPartsFactory({
+  deployId,
+  env,
+  suffix: deploymentVersion,
+  resources,
+  replicas,
+}).replace('{{ports}}', buildKindPorts(fromPort, toPort))}
+`;
+        }
+        fs.writeFileSync(`./engine-private/conf/${deployId}/build/${env}/deployment.yaml`, deploymentYamlParts, 'utf8');
 
         let proxyYaml = '';
         let secretYaml = '';
@@ -163,8 +193,13 @@ spec:
         - prefix: ${path}
       enableWebsockets: true
       services:
-        - name: ${deployId}-${env}-service
-          port: ${port}`;
+    ${UnderpostDeploy.API.deploymentYamlServiceFactory({
+      deployId,
+      env,
+      port,
+      deploymentVersions:
+        options.traffic && typeof options.traffic === 'string' ? options.traffic.split(',') : ['blue'],
+    })}`;
           }
         }
         const yamlPath = `./engine-private/conf/${deployId}/build/${env}/proxy.yaml`;
@@ -185,6 +220,14 @@ spec:
         }
       }
     },
+    getCurrentTraffic(deployId) {
+      // kubectl get deploy,sts,svc,configmap,secret -n default -o yaml --export > default.yaml
+      const hostTest = Object.keys(
+        JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
+      )[0];
+      const info = shellExec(`sudo kubectl get HTTPProxy/${hostTest} -o yaml`, { silent: true, stdout: true });
+      return info.match('blue') ? 'blue' : info.match('green') ? 'green' : null;
+    },
     async callback(
       deployList = 'default',
       env = 'development',
@@ -196,8 +239,13 @@ spec:
         infoUtil: false,
         expose: false,
         cert: false,
-        version: '',
+        versions: '',
+        traffic: '',
         dashboardUpdate: false,
+        replicas: '',
+        disableUpdateDeployment: false,
+        infoTraffic: false,
+        rebuildClientsBundle: false,
       },
     ) {
       if (options.infoUtil === true)
@@ -205,11 +253,28 @@ spec:
 kubectl rollout restart deployment/deployment-name
 kubectl rollout undo deployment/deployment-name
 kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
-        `);
+kubectl get pods -w
+kubectl patch statefulset service-valkey --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value":"valkey/valkey:latest"}]'
+kubectl patch statefulset service-valkey -p '{"spec":{"template":{"spec":{"containers":[{"name":"service-valkey","imagePullPolicy":"Never"}]}}}}'
+`);
       if (deployList === 'dd' && fs.existsSync(`./engine-private/deploy/dd.router`))
         deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
-      if (options.sync) UnderpostDeploy.API.sync(deployList);
-      if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options.version);
+      if (options.infoTraffic === true) {
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          logger.info('', {
+            deployId,
+            env,
+            traffic: UnderpostDeploy.API.getCurrentTraffic(deployId),
+          });
+        }
+        return;
+      }
+      if (options.rebuildClientsBundle === true) await UnderpostDeploy.API.rebuildClientsBundle(deployList);
+      if (!(options.versions && typeof options.versions === 'string')) options.versions = 'blue,green';
+      if (!options.replicas) options.replicas = 1;
+      if (options.sync) UnderpostDeploy.API.sync(deployList, options);
+      if (options.buildManifest === true) await UnderpostDeploy.API.buildManifest(deployList, env, options);
       if (options.infoRouter === true) logger.info('router', await UnderpostDeploy.API.routerFactory(deployList, env));
       if (options.dashboardUpdate === true) await UnderpostDeploy.API.updateDashboardData(deployList, env, options);
       if (options.infoRouter === true) return;
@@ -236,8 +301,12 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
           shellExec(`sudo kubectl port-forward -n default svc/${svc.NAME} ${port}:${port}`, { async: true });
           continue;
         }
-        shellExec(`sudo kubectl delete svc ${deployId}-${env}-service`);
-        shellExec(`sudo kubectl delete deployment ${deployId}-${env}`);
+
+        if (!options.disableUpdateDeployment)
+          for (const version of options.versions.split(',')) {
+            shellExec(`sudo kubectl delete svc ${deployId}-${env}-${version}-service`);
+            shellExec(`sudo kubectl delete deployment ${deployId}-${env}-${version}`);
+          }
 
         const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
         for (const host of Object.keys(confServer)) {
@@ -252,7 +321,7 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
             : `manifests/deployment/${deployId}-${env}`;
 
         if (!options.remove === true) {
-          shellExec(`sudo kubectl apply -f ./${manifestsPath}/deployment.yaml`);
+          if (!options.disableUpdateDeployment) shellExec(`sudo kubectl apply -f ./${manifestsPath}/deployment.yaml`);
           shellExec(`sudo kubectl apply -f ./${manifestsPath}/proxy.yaml`);
           if (env === 'production' && options.cert === true)
             shellExec(`sudo kubectl apply -f ./${manifestsPath}/secret.yaml`);
@@ -315,6 +384,36 @@ kubectl scale statefulsets <stateful-set-name> --replicas=<new-replicas>
 
       return result;
     },
+    rebuildClientsBundle(deployList) {
+      for (const _deployId of deployList.split(',')) {
+        const deployId = _deployId.trim();
+        const repoName = `engine-${deployId.split('-')[1]}`;
+
+        shellExec(`underpost script set ${deployId}-client-build '
+cd /home/dd/engine &&
+git checkout . &&
+underpost pull . underpostnet/${repoName} &&
+underpost pull ./engine-private underpostnet/${repoName}-private &&
+underpost env ${deployId} production &&
+node bin/deploy build-full-client ${deployId}
+'`);
+
+        shellExec(`node bin script run ${deployId}-client-build --itc --pod-name ${deployId}`);
+      }
+    },
+    resourcesFactory() {
+      return {
+        requests: {
+          memory: UnderpostRootEnv.API.get('resources.requests.memory'),
+          cpu: UnderpostRootEnv.API.get('resources.requests.cpu'),
+        },
+        limits: {
+          memory: UnderpostRootEnv.API.get('resources.limits.memory'),
+          cpu: UnderpostRootEnv.API.get('resources.limits.cpu'),
+        },
+        totalPods: UnderpostRootEnv.API.get('total-pods'),
+      };
+    },
     async updateDashboardData(deployList, env, options) {
       try {
         const deployId = process.env.DEFAULT_DEPLOY_ID;

package/src/cli/fs.js

@@ -24,7 +24,7 @@ class UnderpostFileStorage {
     getStorageConf(options) {
       let storage, storageConf;
       if (options.deployId && typeof options.deployId === 'string') {
-        storageConf = `./engine-private/conf/${options.deployId}/storage.json`;
+        storageConf = options.storageFilePath ?? `./engine-private/conf/${options.deployId}/storage.json`;
         if (!fs.existsSync(storageConf)) fs.writeFileSync(storageConf, JSON.stringify({}), 'utf8');
         storage = JSON.parse(fs.readFileSync(storageConf, 'utf8'));
       }
@@ -35,7 +35,15 @@ class UnderpostFileStorage {
     },
     async recursiveCallback(
       path,
-      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false },
+      options = {
+        rm: false,
+        recursive: false,
+        deployId: '',
+        force: false,
+        pull: false,
+        git: false,
+        storageFilePath: '',
+      },
     ) {
       const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
       const deleteFiles = options.pull === true ? [] : UnderpostRepository.API.getDeleteFiles(path);
@@ -85,7 +93,10 @@ class UnderpostFileStorage {
       if (options.rm === true) return await UnderpostFileStorage.API.delete(path, options);
       return await UnderpostFileStorage.API.upload(path, options);
     },
-    async upload(path, options = { rm: false, recursive: false, deployId: '', force: false, pull: false }) {
+    async upload(
+      path,
+      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, storageFilePath: '' },
+    ) {
       UnderpostFileStorage.API.cloudinaryConfig();
       const { storage, storageConf } = UnderpostFileStorage.API.getStorageConf(options);
       // path = UnderpostFileStorage.API.file2Zip(path);

package/src/cli/image.js

@@ -60,74 +60,6 @@ class UnderpostImage {
         if (podmanSave === true) shellExec(`podman save -o ${tarFile} ${podManImg}`);
         if (kindLoad === true) shellExec(`sudo kind load image-archive ${tarFile}`);
       },
-      async script(deployId = 'default', env = 'development', options = { run: false, build: false }) {
-        if (options.build === true) {
-          const buildBasePath = `/home/dd`;
-          const repoName = `engine-${deployId.split('-')[1]}`;
-          shellExec(`cd ${buildBasePath} && underpost clone underpostnet/${repoName}`);
-          shellExec(`cd ${buildBasePath} && sudo mv ./${repoName} ./engine`);
-          shellExec(`cd ${buildBasePath}/engine && underpost clone underpostnet/${repoName}-private`);
-          shellExec(`cd ${buildBasePath}/engine && sudo mv ./${repoName}-private ./engine-private`);
-          shellCd(`${buildBasePath}/engine`);
-          shellExec(`npm install`);
-          shellExec(`node bin/deploy conf ${deployId} ${env}`);
-          if (fs.existsSync('./engine-private/itc-scripts')) {
-            const itcScripts = await fs.readdir('./engine-private/itc-scripts');
-            for (const itcScript of itcScripts)
-              if (itcScript.match(deployId)) shellExec(`node ./engine-private/itc-scripts/${itcScript}`);
-          }
-          switch (deployId) {
-            default:
-              {
-                {
-                  const originPath = `./src/db/mongo/MongooseDB.js`;
-                  fs.writeFileSync(
-                    originPath,
-                    fs.readFileSync(originPath, 'utf8').replaceAll(
-                      `connect: async (host, name) => {`,
-                      `connect: async (host, name) => {
-    host = 'mongodb://mongodb-0.mongodb-service:27017';
-          `,
-                    ),
-                    'utf8',
-                  );
-                }
-
-                {
-                  const originPath = `./src/server/valkey.js`;
-                  fs.writeFileSync(
-                    originPath,
-                    fs.readFileSync(originPath, 'utf8').replaceAll(
-                      `    // port: 6379,
-    // host: 'service-valkey.default.svc.cluster.local',`,
-                      `     port: 6379,
-    host: 'service-valkey.default.svc.cluster.local',`,
-                    ),
-                    'utf8',
-                  );
-                }
-              }
-              break;
-          }
-          shellExec(`node bin/deploy build-full-client ${deployId}`);
-        }
-        if (options.run === true) {
-          const runCmd = env === 'production' ? 'run prod-img' : 'run dev-img';
-          if (fs.existsSync(`./engine-private/replica`)) {
-            const replicas = await fs.readdir(`./engine-private/replica`);
-            for (const replica of replicas) {
-              if (!replica.match(deployId)) continue;
-              shellExec(`node bin/deploy conf ${replica} ${env}`);
-              shellExec(`npm ${runCmd} deploy deploy-id:${replica}`, { async: true });
-              await awaitDeployMonitor(true);
-            }
-          }
-          shellExec(`node bin/deploy conf ${deployId} ${env}`);
-          shellExec(`npm ${runCmd} deploy deploy-id:${deployId}`, { async: true });
-          await awaitDeployMonitor(true);
-          await UnderpostMonitor.API.callback(deployId, env, { itc: true });
-        }
-      },
     },
   };
 }