underpost 2.8.6 → 2.8.8

package/src/server/network.js

@@ -1,123 +1,18 @@
-import fs from 'fs-extra';
-
-import { publicIp, publicIpv4, publicIpv6 } from 'public-ip';
+import Underpost from '../index.js';
 import { actionInitLog, loggerFactory } from './logger.js';
-import { DataBaseProvider } from '../db/DataBaseProvider.js';
-import { getDeployId } from './conf.js';
-
-// Network Address Translation Management
-
-// import dotenv from 'dotenv';
-// dotenv.config();
 
 const logger = loggerFactory(import.meta);
 
-const ip = {
-  public: {
-    get: async () => await publicIp(), // => 'fe80::200:f8ff:fe21:67cf'
-    ipv4: async () => await publicIpv4(), // => '46.5.21.123'
-    ipv6: async () => await publicIpv6(), // => 'fe80::200:f8ff:fe21:67cf'
-  },
-};
-
-let ipInstance = '';
-const networkRouter = {};
-
 const logRuntimeRouter = () => {
   const displayLog = {};
 
-  for (const host of Object.keys(networkRouter))
-    for (const path of Object.keys(networkRouter[host]))
-      displayLog[networkRouter[host][path].publicHost] = networkRouter[host][path].local;
+  for (const host of Object.keys(Underpost.deployNetwork))
+    for (const path of Object.keys(Underpost.deployNetwork[host]))
+      displayLog[Underpost.deployNetwork[host][path].publicHost] = Underpost.deployNetwork[host][path].local;
 
   logger.info('Runtime network', displayLog);
 };
 
-const saveRuntimeRouter = async () => {
-  try {
-    const deployId = process.env.DEFAULT_DEPLOY_ID;
-    const host = process.env.DEFAULT_DEPLOY_HOST;
-    const path = process.env.DEFAULT_DEPLOY_PATH;
-    const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-    if (!deployId || !host || !path || !fs.existsSync(confServerPath)) {
-      // logger.warn('default deploy instance not found');
-      return;
-    }
-    const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
-    const { db } = confServer[host][path];
-
-    let closeConn;
-    if (!DataBaseProvider.instance[`${host}${path}`]) {
-      await DataBaseProvider.load({ apis: ['instance'], host, path, db });
-      closeConn = true;
-    }
-
-    /** @type {import('../api/instance/instance.model.js').InstanceModel} */
-    const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
-
-    for (const _host of Object.keys(networkRouter)) {
-      for (const _path of Object.keys(networkRouter[_host])) {
-        const body = {
-          host: _host,
-          path: _path,
-          deployId: getDeployId(),
-          client: networkRouter[_host][_path].client,
-          runtime: networkRouter[_host][_path].runtime,
-          port: networkRouter[_host][_path].port,
-          apis: networkRouter[_host][_path].apis,
-        };
-        const instance = await Instance.findOne({ deployId: body.deployId, host: _host, path: _path });
-        if (instance) {
-          await Instance.findByIdAndUpdate(instance._id, body);
-        } else {
-          await new Instance(body).save();
-        }
-      }
-    }
-
-    if (closeConn) await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
-  } catch (error) {
-    logger.error(error, error.stack);
-  }
-};
-
-const netWorkCron = [];
-
-const saveRuntimeCron = async () => {
-  try {
-    const deployId = process.env.DEFAULT_DEPLOY_ID;
-    const host = process.env.DEFAULT_DEPLOY_HOST;
-    const path = process.env.DEFAULT_DEPLOY_PATH;
-    const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-    const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
-    const { db } = confServer[host][path];
-
-    let closeConn;
-    if (!DataBaseProvider.instance[`${host}${path}`]) {
-      await DataBaseProvider.load({ apis: ['cron'], host, path, db });
-      closeConn = true;
-    }
-
-    /** @type {import('../api/cron/cron.model.js').CronModel} */
-    const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
-
-    // await Cron.insertMany(netWorkCron);
-
-    for (const cronInstance of netWorkCron) {
-      const cron = await Cron.findOne({ deployId: cronInstance.deployId, jobId: cronInstance.jobId });
-      if (cron) {
-        await Cron.findByIdAndUpdate(cron._id, cronInstance);
-      } else {
-        await new Cron(cronInstance).save();
-      }
-    }
-
-    if (closeConn) await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
-  } catch (error) {
-    logger.error(error, error.stack);
-  }
-};
-
 const listenServerFactory = (logic = async () => {}) => {
   return {
     listen: async (...args) => (
@@ -150,13 +45,12 @@ const listenPortController = async (server, port, metadata) =>
       if (error.length > 0) throw new Error('Listen port controller requires values: ' + error.join(', '));
 
       server.listen(port, () => {
-        if (!networkRouter[host]) networkRouter[host] = {};
-        networkRouter[host][path] = {
+        if (!Underpost.deployNetwork[host]) Underpost.deployNetwork[host] = {};
+        Underpost.deployNetwork[host][path] = {
           meta,
           client,
           runtime,
           port,
-          public: `http://${ipInstance}:${port}${path}`,
           publicHost:
             port === 80
               ? `http://${host}${path}`
@@ -175,13 +69,4 @@ const listenPortController = async (server, port, metadata) =>
     }
   });
 
-export {
-  ip,
-  listenPortController,
-  networkRouter,
-  netWorkCron,
-  saveRuntimeRouter,
-  logRuntimeRouter,
-  listenServerFactory,
-  saveRuntimeCron,
-};
+export { listenPortController, logRuntimeRouter, listenServerFactory };

package/src/server/peer.js

@@ -2,7 +2,7 @@ import { PeerServer } from 'peer';
 import dotenv from 'dotenv';
 import { loggerFactory } from './logger.js';
 import fs from 'fs-extra';
-import { listenServerFactory } from './network.js';
+import UnderpostStartUp from './start.js';
 
 dotenv.config();
 
@@ -25,7 +25,7 @@ const createPeerServer = async ({ port, devPort, origins, host, path }) => {
     // cert: fs.readFileSync(''),
     // ca: fs.readFileSync(''),
   };
-  const peerServer = listenServerFactory(() => PeerServer(options));
+  const peerServer = UnderpostStartUp.API.listenServerFactory(() => PeerServer(options));
 
   return { options, peerServer, meta: import.meta };
 };

package/src/server/proxy.js

@@ -5,9 +5,9 @@ import dotenv from 'dotenv';
 
 import { createProxyMiddleware } from 'http-proxy-middleware';
 import { loggerFactory, loggerMiddleware } from './logger.js';
-import { listenPortController } from './network.js';
 import { createSslServer, sslRedirectMiddleware } from './ssl.js';
 import { buildPortProxyRouter, buildProxyRouter, maintenanceMiddleware } from './conf.js';
+import UnderpostStartUp from './start.js';
 
 dotenv.config();
 
@@ -71,11 +71,11 @@ const buildProxy = async () => {
         switch (port) {
           case 443:
             const { ServerSSL } = await createSslServer(app, hosts);
-            await listenPortController(ServerSSL, port, runningData);
+            await UnderpostStartUp.API.listenPortController(ServerSSL, port, runningData);
             break;
 
           default:
-            await listenPortController(app, port, runningData);
+            await UnderpostStartUp.API.listenPortController(app, port, runningData);
 
             break;
         }
@@ -83,7 +83,7 @@ const buildProxy = async () => {
         break;
 
       default:
-        await listenPortController(app, port, runningData);
+        await UnderpostStartUp.API.listenPortController(app, port, runningData);
 
         break;
     }

package/src/server/runtime.js

@@ -9,7 +9,7 @@ import compression from 'compression';
 
 import { createServer } from 'http';
 import { getRootDirectory } from './process.js';
-import { listenPortController, saveRuntimeRouter, logRuntimeRouter, listenServerFactory } from './network.js';
+import UnderpostStartUp from './start.js';
 import { loggerFactory, loggerMiddleware } from './logger.js';
 import { getCapVariableName, newInstance } from '../client/components/core/CommonJs.js';
 import { Xampp } from '../runtime/xampp/Xampp.js';
@@ -21,6 +21,7 @@ import { Lampp } from '../runtime/lampp/Lampp.js';
 import { getDeployId } from './conf.js';
 import { JSONweb, ssrFactory } from './client-formatted.js';
 import Underpost from '../index.js';
+import { createValkeyConnection } from './valkey.js';
 
 dotenv.config();
 
@@ -41,7 +42,6 @@ const buildRuntime = async () => {
   // logger.info('promCounterOption', promCounterOption);
 
   const requestCounter = new promClient.Counter(promCounterOption);
-  const ipInstance = ''; // await ip.public.ipv4();
   const initPort = parseInt(process.env.PORT) + 1;
   let currentPort = initPort;
   const confServer = JSON.parse(fs.readFileSync(`./conf/conf.server.json`, 'utf8'));
@@ -68,6 +68,7 @@ const buildRuntime = async () => {
         peer,
         singleReplica,
         replicas,
+        valkey,
       } = confServer[host][path];
 
       if (singleReplica && replicas && replicas.length > 0 && !singleReplicaHosts.includes(host)) {
@@ -141,6 +142,8 @@ const buildRuntime = async () => {
           // if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
           //   $_SERVER['HTTPS'] = 'on';
           // }
+          // For plugins:
+          // define( 'FS_METHOD', 'direct' );
 
           // ErrorDocument 404 /custom_404.html
           // ErrorDocument 500 /custom_50x.html
@@ -183,7 +186,11 @@ const buildRuntime = async () => {
           // RewriteCond %{HTTP_HOST} ^(?:www\.)?(.+)$ [NC]
           // RewriteRule ^ https://%1%{REQUEST_URI} [L,NE,R=301]
 
-          await listenPortController(listenServerFactory(), port, runningData);
+          await UnderpostStartUp.API.listenPortController(
+            UnderpostStartUp.API.listenServerFactory(),
+            port,
+            runningData,
+          );
           break;
         case 'xampp':
           if (!Xampp.enabled()) continue;
@@ -230,7 +237,11 @@ const buildRuntime = async () => {
           // if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
           //   $_SERVER['HTTPS'] = 'on';
           // }
-          await listenPortController(listenServerFactory(), port, runningData);
+          await UnderpostStartUp.API.listenPortController(
+            UnderpostStartUp.API.listenServerFactory(),
+            port,
+            runningData,
+          );
           break;
         case 'nodejs':
           const app = express();
@@ -283,7 +294,7 @@ const buildRuntime = async () => {
             currentPort += 2;
             const staticPort = newInstance(currentPort);
 
-            await listenPortController(app, staticPort, runningData);
+            await UnderpostStartUp.API.listenPortController(app, staticPort, runningData);
             currentPort++;
             continue;
           }
@@ -334,7 +345,7 @@ const buildRuntime = async () => {
             //   }),
             // );
 
-            await listenPortController(app, port, runningData);
+            await UnderpostStartUp.API.listenPortController(app, port, runningData);
             break;
           }
 
@@ -354,6 +365,9 @@ const buildRuntime = async () => {
 
           if (db && apis) await DataBaseProvider.load({ apis, host, path, db });
 
+          // valkey server
+          if (valkey) await createValkeyConnection({ host, path }, valkey);
+
           if (mailer) {
             const mailerSsrConf = confSSR[getCapVariableName(client)];
             await MailerProvider.load({
@@ -442,7 +456,7 @@ const buildRuntime = async () => {
                 port,
                 origins,
               });
-              await listenPortController(listenServerFactory(), port, {
+              await UnderpostStartUp.API.listenPortController(UnderpostStartUp.API.listenServerFactory(), port, {
                 runtime: 'nodejs',
                 client: null,
                 host,
@@ -462,7 +476,7 @@ const buildRuntime = async () => {
               path,
             });
 
-            await listenPortController(peerServer, peerPort, {
+            await UnderpostStartUp.API.listenPortController(peerServer, peerPort, {
               runtime: 'nodejs',
               client: null,
               host,
@@ -471,7 +485,7 @@ const buildRuntime = async () => {
             });
           }
 
-          await listenPortController(server, port, runningData);
+          await UnderpostStartUp.API.listenPortController(server, port, runningData);
 
           break;
         default:
@@ -484,8 +498,7 @@ const buildRuntime = async () => {
   if (Xampp.enabled() && Xampp.router) Xampp.initService();
   if (Lampp.enabled() && Lampp.router) Lampp.initService();
 
-  saveRuntimeRouter();
-  logRuntimeRouter();
+  UnderpostStartUp.API.logRuntimeRouter();
 };
 
 export { buildRuntime };

package/src/server/start.js

@@ -0,0 +1,122 @@
+import UnderpostDeploy from '../cli/deploy.js';
+import fs from 'fs-extra';
+import { awaitDeployMonitor } from './conf.js';
+import { actionInitLog, loggerFactory } from './logger.js';
+import { shellCd, shellExec } from './process.js';
+import UnderpostRootEnv from '../cli/env.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostStartUp {
+  static API = {
+    logRuntimeRouter: () => {
+      const displayLog = {};
+
+      for (const host of Object.keys(UnderpostDeploy.NETWORK))
+        for (const path of Object.keys(UnderpostDeploy.NETWORK[host]))
+          displayLog[UnderpostDeploy.NETWORK[host][path].publicHost] = UnderpostDeploy.NETWORK[host][path].local;
+
+      logger.info('Runtime network', displayLog);
+    },
+    listenServerFactory: (logic = async () => {}) => {
+      return {
+        listen: async (...args) => {
+          const msDelta = 1000;
+          const msMax = 30 * 24 * 60 * 60 * 1000; // ~ 1 month
+          let msCount = 0;
+          setInterval(() => {
+            msCount += msDelta;
+            if (msCount >= msMax) {
+              const message = 'Listen server factory timeout';
+              logger.error(message);
+              throw new Error(message);
+            }
+          }, msDelta);
+          return logic ? await logic(...args) : undefined, args[1]();
+        },
+      };
+    },
+    listenPortController: async (server, port, metadata) =>
+      new Promise((resolve) => {
+        try {
+          if (port === ':') {
+            server.listen(port, actionInitLog);
+            return resolve(true);
+          }
+
+          const { host, path, client, runtime, meta } = metadata;
+          const error = [];
+          if (port === undefined) error.push(`port`);
+          if (host === undefined) error.push(`host`);
+          if (path === undefined) error.push(`path`);
+          if (client === undefined) error.push(`client`);
+          if (runtime === undefined) error.push(`runtime`);
+          if (meta === undefined) error.push(`meta`);
+          if (error.length > 0) throw new Error('Listen port controller requires values: ' + error.join(', '));
+
+          server.listen(port, () => {
+            if (!UnderpostDeploy.NETWORK[host]) UnderpostDeploy.NETWORK[host] = {};
+            UnderpostDeploy.NETWORK[host][path] = {
+              meta,
+              client,
+              runtime,
+              port,
+              publicHost:
+                port === 80
+                  ? `http://${host}${path}`
+                  : port === 443
+                  ? `https://${host}${path}`
+                  : `http://${host}:${port}${path}`,
+              local: `http://localhost:${port}${path}`,
+              apis: metadata.apis,
+            };
+
+            return resolve(true);
+          });
+        } catch (error) {
+          logger.error(error, { metadata, port, stack: error.stack });
+          resolve(false);
+        }
+      }),
+
+    async callback(deployId = 'default', env = 'development', options = { build: false, run: false }) {
+      if (options.build === true) await UnderpostStartUp.API.build(deployId, env);
+      if (options.run === true) await UnderpostStartUp.API.run(deployId, env);
+    },
+    async build(deployId = 'default', env = 'development') {
+      const buildBasePath = `/home/dd`;
+      const repoName = `engine-${deployId.split('-')[1]}`;
+      shellExec(`cd ${buildBasePath} && underpost clone underpostnet/${repoName}`);
+      shellExec(`cd ${buildBasePath} && sudo mv ./${repoName} ./engine`);
+      shellExec(`cd ${buildBasePath}/engine && underpost clone underpostnet/${repoName}-private`);
+      shellExec(`cd ${buildBasePath}/engine && sudo mv ./${repoName}-private ./engine-private`);
+      shellCd(`${buildBasePath}/engine`);
+      shellExec(`npm install`);
+      shellExec(`node bin/deploy conf ${deployId} ${env}`);
+      if (fs.existsSync('./engine-private/itc-scripts')) {
+        const itcScripts = await fs.readdir('./engine-private/itc-scripts');
+        for (const itcScript of itcScripts)
+          if (itcScript.match(deployId)) shellExec(`node ./engine-private/itc-scripts/${itcScript}`);
+      }
+      shellExec(`node bin/deploy build-full-client ${deployId}`);
+    },
+    async run(deployId = 'default', env = 'development') {
+      const runCmd = env === 'production' ? 'run prod-img' : 'run dev-img';
+      if (fs.existsSync(`./engine-private/replica`)) {
+        const replicas = await fs.readdir(`./engine-private/replica`);
+        for (const replica of replicas) {
+          if (!replica.match(deployId)) continue;
+          shellExec(`node bin/deploy conf ${replica} ${env}`);
+          shellExec(`npm ${runCmd} deploy deploy-id:${replica}`, { async: true });
+          await awaitDeployMonitor(true);
+        }
+      }
+      shellExec(`node bin/deploy conf ${deployId} ${env}`);
+      shellExec(`npm ${runCmd} deploy deploy-id:${deployId}`, { async: true });
+      await awaitDeployMonitor(true);
+      UnderpostRootEnv.API.set('container-status', `${deployId}-${env}-running-deployment`);
+    },
+  };
+}
+
+export default UnderpostStartUp;

package/src/server/valkey.js

@@ -5,12 +5,24 @@ import { loggerFactory } from './logger.js';
 
 const logger = loggerFactory(import.meta);
 
+const ValkeyInstances = {};
+
 let valkeyEnabled = true;
 
 const disableValkeyErrorMessage = 'valkey is not enabled';
 
 const isValkeyEnable = () => valkeyEnabled;
 
+const createValkeyConnection = async (
+  instance = { host: '', port: 0 },
+  valkeyServerConnectionOptions = { host: '', port: 0 },
+) => {
+  ValkeyInstances[`${instance.host}${instance.path}`] = await ValkeyAPI.valkeyClientFactory(
+    valkeyServerConnectionOptions,
+  );
+  return ValkeyInstances[`${instance.host}${instance.path}`];
+};
+
 const selectDtoFactory = (payload, select) => {
   const result = {};
   for (const key of Object.keys(select)) {
@@ -19,15 +31,17 @@ const selectDtoFactory = (payload, select) => {
   return result;
 };
 
-const valkeyClientFactory = async () => {
+const valkeyClientFactory = async (options) => {
   const valkey = new Valkey({
     // port: 6379,
-    // host: 'service-valkey.default.svc.cluster.local',
+    // host: 'valkey-service.default.svc.cluster.local',
+    port: options?.port ? options.port : undefined,
+    host: options?.host ? options.host : undefined,
     retryStrategy: (attempt) => {
       if (attempt === 1) {
         valkey.disconnect();
         valkeyEnabled = false;
-        logger.warn('Valkey service not enabled', { valkeyEnabled });
+        logger.warn('Valkey service not enabled', { ...options, valkeyEnabled });
         return;
       }
       return 1000; // 1 second interval attempt
@@ -46,12 +60,12 @@ const valkeyClientFactory = async () => {
   return valkey;
 };
 
-const getValkeyObject = async (key = '') => {
+const getValkeyObject = async (options = { host: '', port: 0 }, key = '') => {
   if (!valkeyEnabled) {
     logger.warn(disableValkeyErrorMessage + ' get', key);
     return null;
   }
-  const object = await valkey.get(key);
+  const object = await ValkeyInstances[`${options.host}${options.path}`].get(key);
   try {
     return JSON.parse(object);
   } catch (error) {
@@ -60,19 +74,19 @@ const getValkeyObject = async (key = '') => {
   }
 };
 
-const setValkeyObject = async (key = '', payload = {}) => {
+const setValkeyObject = async (options = { host: '', port: 0 }, key = '', payload = {}) => {
   if (!valkeyEnabled) throw new Error(disableValkeyErrorMessage);
-  return await valkey.set(key, JSON.stringify(payload));
+  return await ValkeyInstances[`${options.host}${options.path}`].set(key, JSON.stringify(payload));
 };
 
-const updateValkeyObject = async (key = '', payload = {}) => {
+const updateValkeyObject = async (options = { host: '', port: 0 }, key = '', payload = {}) => {
   if (!valkeyEnabled) throw new Error(disableValkeyErrorMessage);
-  const object = await getValkeyObject(key, valkey);
+  const object = await getValkeyObject(key);
   object.updatedAt = new Date().toISOString();
-  return await valkey.set(key, JSON.stringify({ ...object, ...payload }));
+  return await ValkeyInstances[`${options.host}${options.path}`].set(key, JSON.stringify({ ...object, ...payload }));
 };
 
-const valkeyObjectFactory = async (module = '', options = { host: 'localhost', object: {} }) => {
+const valkeyObjectFactory = async (options = { host: 'localhost', object: {} }, module = '') => {
   if (!valkeyEnabled) throw new Error(disableValkeyErrorMessage);
   const idoDate = new Date().toISOString();
   options.object = options.object || {};
@@ -112,10 +126,9 @@ const ValkeyAPI = {
   setValkeyObject,
   valkeyObjectFactory,
   updateValkeyObject,
+  createValkeyConnection,
 };
 
-const valkey = await ValkeyAPI.valkeyClientFactory();
-
 export {
   valkeyClientFactory,
   selectDtoFactory,
@@ -124,5 +137,6 @@ export {
   valkeyObjectFactory,
   updateValkeyObject,
   isValkeyEnable,
+  createValkeyConnection,
   ValkeyAPI,
 };