underpost 2.8.1 → 2.8.7

package/src/server/runtime.js

@@ -9,17 +9,19 @@ import compression from 'compression';
 
 import { createServer } from 'http';
 import { getRootDirectory } from './process.js';
-import { network, listenPortController, saveRuntimeRouter, logRuntimeRouter, listenServerFactory } from './network.js';
+import UnderpostStartUp from './start.js';
 import { loggerFactory, loggerMiddleware } from './logger.js';
 import { getCapVariableName, newInstance } from '../client/components/core/CommonJs.js';
 import { Xampp } from '../runtime/xampp/Xampp.js';
 import { MailerProvider } from '../mailer/MailerProvider.js';
 import { DataBaseProvider } from '../db/DataBaseProvider.js';
-import { createProxyMiddleware } from 'http-proxy-middleware';
+// import { createProxyMiddleware } from 'http-proxy-middleware';
 import { createPeerServer } from './peer.js';
 import { Lampp } from '../runtime/lampp/Lampp.js';
 import { getDeployId } from './conf.js';
-import { ssrFactory } from './client-formatted.js';
+import { JSONweb, ssrFactory } from './client-formatted.js';
+import Underpost from '../index.js';
+import { createValkeyConnection } from './valkey.js';
 
 dotenv.config();
 
@@ -31,20 +33,6 @@ const buildRuntime = async () => {
   const collectDefaultMetrics = promClient.collectDefaultMetrics;
   collectDefaultMetrics();
 
-  if (fs.existsSync(`/root/.bashrc`) && !fs.readFileSync(`/root/.bashrc`, 'utf8').match(`underpost-engine`)) {
-    fs.writeFileSync(
-      `/root/.bashrc`,
-      `${fs.readFileSync(`/root/.bashrc`, 'utf8')}
-` +
-        `export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm underpost-engine bash_completion
-
-export PATH=$PATH:/opt/lampp/bin`,
-      'utf8',
-    );
-  }
-
   const promCounterOption = {
     name: `${deployId.replaceAll('-', '_')}_http_requests_total`,
     help: 'Total number of HTTP requests',
@@ -54,8 +42,6 @@ export PATH=$PATH:/opt/lampp/bin`,
   // logger.info('promCounterOption', promCounterOption);
 
   const requestCounter = new promClient.Counter(promCounterOption);
-
-  const ipInstance = ''; // await ip.public.ipv4();
   const initPort = parseInt(process.env.PORT) + 1;
   let currentPort = initPort;
   const confServer = JSON.parse(fs.readFileSync(`./conf/conf.server.json`, 'utf8'));
@@ -82,6 +68,7 @@ export PATH=$PATH:/opt/lampp/bin`,
         peer,
         singleReplica,
         replicas,
+        valkey,
       } = confServer[host][path];
 
       if (singleReplica && replicas && replicas.length > 0 && !singleReplicaHosts.includes(host)) {
@@ -101,12 +88,13 @@ export PATH=$PATH:/opt/lampp/bin`,
         apis,
       };
 
-      let redirectUrl;
-      let redirectTarget;
-      if (redirect) {
-        redirectUrl = new URL(redirect);
-        redirectTarget = redirect[redirect.length - 1] === '/' ? redirect.slice(0, -1) : redirect;
-      }
+      const redirectTarget = redirect
+        ? redirect[redirect.length - 1] === '/'
+          ? redirect.slice(0, -1)
+          : redirect
+        : undefined;
+
+      // if (redirect) logger.info('redirect', new URL(redirect));
 
       switch (runtime) {
         case 'lampp':
@@ -154,6 +142,8 @@ export PATH=$PATH:/opt/lampp/bin`,
           // if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
           //   $_SERVER['HTTPS'] = 'on';
           // }
+          // For plugins:
+          // define( 'FS_METHOD', 'direct' );
 
           // ErrorDocument 404 /custom_404.html
           // ErrorDocument 500 /custom_50x.html
@@ -196,7 +186,11 @@ export PATH=$PATH:/opt/lampp/bin`,
           // RewriteCond %{HTTP_HOST} ^(?:www\.)?(.+)$ [NC]
           // RewriteRule ^ https://%1%{REQUEST_URI} [L,NE,R=301]
 
-          await listenPortController(listenServerFactory(), port, runningData);
+          await UnderpostStartUp.API.listenPortController(
+            UnderpostStartUp.API.listenServerFactory(),
+            port,
+            runningData,
+          );
           break;
         case 'xampp':
           if (!Xampp.enabled()) continue;
@@ -243,7 +237,11 @@ export PATH=$PATH:/opt/lampp/bin`,
           // if (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] === 'https') {
           //   $_SERVER['HTTPS'] = 'on';
           // }
-          await listenPortController(listenServerFactory(), port, runningData);
+          await UnderpostStartUp.API.listenPortController(
+            UnderpostStartUp.API.listenServerFactory(),
+            port,
+            runningData,
+          );
           break;
         case 'nodejs':
           const app = express();
@@ -295,8 +293,8 @@ export PATH=$PATH:/opt/lampp/bin`,
             logger.info('Build static server runtime', `${host}${path}`);
             currentPort += 2;
             const staticPort = newInstance(currentPort);
-            await network.port.portClean(staticPort);
-            await listenPortController(app, staticPort, runningData);
+
+            await UnderpostStartUp.API.listenPortController(app, staticPort, runningData);
             currentPort++;
             continue;
           }
@@ -346,8 +344,8 @@ export PATH=$PATH:/opt/lampp/bin`,
             //     changeOrigin: true,
             //   }),
             // );
-            await network.port.portClean(port);
-            await listenPortController(app, port, runningData);
+
+            await UnderpostStartUp.API.listenPortController(app, port, runningData);
             break;
           }
 
@@ -367,6 +365,9 @@ export PATH=$PATH:/opt/lampp/bin`,
 
           if (db && apis) await DataBaseProvider.load({ apis, host, path, db });
 
+          // valkey server
+          await createValkeyConnection({ host, path }, valkey);
+
           if (mailer) {
             const mailerSsrConf = confSSR[getCapVariableName(client)];
             await MailerProvider.load({
@@ -398,6 +399,13 @@ export PATH=$PATH:/opt/lampp/bin`,
             ssrPath,
             ssrHeadComponents: '',
             ssrBodyComponents: (await ssrFactory(`./src/client/ssr/body/404.js`))(),
+            renderPayload: {
+              apiBasePath: process.env.BASE_API,
+              version: Underpost.version,
+            },
+            renderApi: {
+              JSONweb,
+            },
           });
           const path404 = `${directory ? directory : `${getRootDirectory()}${rootHostPath}`}/404/index.html`;
           const page404 = fs.existsSync(path404) ? `${path === '/' ? '' : path}/404` : undefined;
@@ -414,6 +422,13 @@ export PATH=$PATH:/opt/lampp/bin`,
             ssrPath,
             ssrHeadComponents: '',
             ssrBodyComponents: (await ssrFactory(`./src/client/ssr/body/500.js`))(),
+            renderPayload: {
+              apiBasePath: process.env.BASE_API,
+              version: Underpost.version,
+            },
+            renderApi: {
+              JSONweb,
+            },
           });
           const path500 = `${directory ? directory : `${getRootDirectory()}${rootHostPath}`}/500/index.html`;
           const page500 = fs.existsSync(path500) ? `${path === '/' ? '' : path}/500` : undefined;
@@ -441,7 +456,7 @@ export PATH=$PATH:/opt/lampp/bin`,
                 port,
                 origins,
               });
-              await listenPortController(listenServerFactory(), port, {
+              await UnderpostStartUp.API.listenPortController(UnderpostStartUp.API.listenServerFactory(), port, {
                 runtime: 'nodejs',
                 client: null,
                 host,
@@ -460,8 +475,8 @@ export PATH=$PATH:/opt/lampp/bin`,
               host,
               path,
             });
-            await network.port.portClean(peerPort);
-            await listenPortController(peerServer, peerPort, {
+
+            await UnderpostStartUp.API.listenPortController(peerServer, peerPort, {
               runtime: 'nodejs',
               client: null,
               host,
@@ -470,8 +485,7 @@ export PATH=$PATH:/opt/lampp/bin`,
             });
           }
 
-          await network.port.portClean(port);
-          await listenPortController(server, port, runningData);
+          await UnderpostStartUp.API.listenPortController(server, port, runningData);
 
           break;
         default:
@@ -481,11 +495,10 @@ export PATH=$PATH:/opt/lampp/bin`,
     }
   }
 
-  if (Xampp.enabled() && Xampp.router) await Xampp.initService({ daemon: true });
-  if (Lampp.enabled() && Lampp.router) await Lampp.initService({ daemon: true });
+  if (Xampp.enabled() && Xampp.router) Xampp.initService();
+  if (Lampp.enabled() && Lampp.router) Lampp.initService();
 
-  saveRuntimeRouter();
-  logRuntimeRouter();
+  UnderpostStartUp.API.logRuntimeRouter();
 };
 
 export { buildRuntime };

package/src/server/ssl.js

@@ -70,7 +70,7 @@ const validateSecureContext = (host) => {
 const buildSecureContext = (host) => {
   return {
     key: fs.readFileSync(`./engine-private/ssl/${host}/key.key`, 'utf8'),
-    cert: fs.readFileSync(`./engine-private/ssl/${host}/crt.crt`, 'utf8'),
+    cert: fs.readFileSync(`./engine-private/ssl/${host}/ca_bundle.crt`, 'utf8'),
     ca: fs.readFileSync(`./engine-private/ssl/${host}/ca_bundle.crt`, 'utf8'),
   };
 };

package/src/server/start.js

@@ -0,0 +1,122 @@
+import UnderpostDeploy from '../cli/deploy.js';
+import fs from 'fs-extra';
+import { awaitDeployMonitor } from './conf.js';
+import { actionInitLog, loggerFactory } from './logger.js';
+import { shellCd, shellExec } from './process.js';
+import UnderpostRootEnv from '../cli/env.js';
+
+const logger = loggerFactory(import.meta);
+
+class UnderpostStartUp {
+  static API = {
+    logRuntimeRouter: () => {
+      const displayLog = {};
+
+      for (const host of Object.keys(UnderpostDeploy.NETWORK))
+        for (const path of Object.keys(UnderpostDeploy.NETWORK[host]))
+          displayLog[UnderpostDeploy.NETWORK[host][path].publicHost] = UnderpostDeploy.NETWORK[host][path].local;
+
+      logger.info('Runtime network', displayLog);
+    },
+    listenServerFactory: (logic = async () => {}) => {
+      return {
+        listen: async (...args) => {
+          const msDelta = 1000;
+          const msMax = 30 * 24 * 60 * 60 * 1000; // ~ 1 month
+          let msCount = 0;
+          setInterval(() => {
+            msCount += msDelta;
+            if (msCount >= msMax) {
+              const message = 'Listen server factory timeout';
+              logger.error(message);
+              throw new Error(message);
+            }
+          }, msDelta);
+          return logic ? await logic(...args) : undefined, args[1]();
+        },
+      };
+    },
+    listenPortController: async (server, port, metadata) =>
+      new Promise((resolve) => {
+        try {
+          if (port === ':') {
+            server.listen(port, actionInitLog);
+            return resolve(true);
+          }
+
+          const { host, path, client, runtime, meta } = metadata;
+          const error = [];
+          if (port === undefined) error.push(`port`);
+          if (host === undefined) error.push(`host`);
+          if (path === undefined) error.push(`path`);
+          if (client === undefined) error.push(`client`);
+          if (runtime === undefined) error.push(`runtime`);
+          if (meta === undefined) error.push(`meta`);
+          if (error.length > 0) throw new Error('Listen port controller requires values: ' + error.join(', '));
+
+          server.listen(port, () => {
+            if (!UnderpostDeploy.NETWORK[host]) UnderpostDeploy.NETWORK[host] = {};
+            UnderpostDeploy.NETWORK[host][path] = {
+              meta,
+              client,
+              runtime,
+              port,
+              publicHost:
+                port === 80
+                  ? `http://${host}${path}`
+                  : port === 443
+                  ? `https://${host}${path}`
+                  : `http://${host}:${port}${path}`,
+              local: `http://localhost:${port}${path}`,
+              apis: metadata.apis,
+            };
+
+            return resolve(true);
+          });
+        } catch (error) {
+          logger.error(error, { metadata, port, stack: error.stack });
+          resolve(false);
+        }
+      }),
+
+    async callback(deployId = 'default', env = 'development', options = { build: false, run: false }) {
+      if (options.build === true) await UnderpostStartUp.API.build(deployId, env);
+      if (options.run === true) await UnderpostStartUp.API.run(deployId, env);
+    },
+    async build(deployId = 'default', env = 'development') {
+      const buildBasePath = `/home/dd`;
+      const repoName = `engine-${deployId.split('-')[1]}`;
+      shellExec(`cd ${buildBasePath} && underpost clone underpostnet/${repoName}`);
+      shellExec(`cd ${buildBasePath} && sudo mv ./${repoName} ./engine`);
+      shellExec(`cd ${buildBasePath}/engine && underpost clone underpostnet/${repoName}-private`);
+      shellExec(`cd ${buildBasePath}/engine && sudo mv ./${repoName}-private ./engine-private`);
+      shellCd(`${buildBasePath}/engine`);
+      shellExec(`npm install`);
+      shellExec(`node bin/deploy conf ${deployId} ${env}`);
+      if (fs.existsSync('./engine-private/itc-scripts')) {
+        const itcScripts = await fs.readdir('./engine-private/itc-scripts');
+        for (const itcScript of itcScripts)
+          if (itcScript.match(deployId)) shellExec(`node ./engine-private/itc-scripts/${itcScript}`);
+      }
+      shellExec(`node bin/deploy build-full-client ${deployId}`);
+    },
+    async run(deployId = 'default', env = 'development') {
+      const runCmd = env === 'production' ? 'run prod-img' : 'run dev-img';
+      if (fs.existsSync(`./engine-private/replica`)) {
+        const replicas = await fs.readdir(`./engine-private/replica`);
+        for (const replica of replicas) {
+          if (!replica.match(deployId)) continue;
+          shellExec(`node bin/deploy conf ${replica} ${env}`);
+          shellExec(`npm ${runCmd} deploy deploy-id:${replica}`, { async: true });
+          await awaitDeployMonitor(true);
+        }
+      }
+      shellExec(`node bin/deploy conf ${deployId} ${env}`);
+      shellExec(`npm ${runCmd} deploy deploy-id:${deployId}`, { async: true });
+      await awaitDeployMonitor(true);
+      UnderpostRootEnv.API.set('container-status', `${deployId}-${env}-running-deployment`);
+    },
+  };
+}
+
+export default UnderpostStartUp;

package/src/server/valkey.js

@@ -5,12 +5,24 @@ import { loggerFactory } from './logger.js';
 
 const logger = loggerFactory(import.meta);
 
+const ValkeyInstances = {};
+
 let valkeyEnabled = true;
 
 const disableValkeyErrorMessage = 'valkey is not enabled';
 
 const isValkeyEnable = () => valkeyEnabled;
 
+const createValkeyConnection = async (
+  instance = { host: '', port: 0 },
+  valkeyServerConnectionOptions = { host: '', port: 0 },
+) => {
+  ValkeyInstances[`${instance.host}${instance.path}`] = await ValkeyAPI.valkeyClientFactory(
+    valkeyServerConnectionOptions,
+  );
+  return ValkeyInstances[`${instance.host}${instance.path}`];
+};
+
 const selectDtoFactory = (payload, select) => {
   const result = {};
   for (const key of Object.keys(select)) {
@@ -19,8 +31,12 @@ const selectDtoFactory = (payload, select) => {
   return result;
 };
 
-const valkeyClientFactory = async () => {
+const valkeyClientFactory = async (options) => {
   const valkey = new Valkey({
+    // port: 6379,
+    // host: 'service-valkey.default.svc.cluster.local',
+    port: options?.port ? options.port : undefined,
+    host: options?.port ? options.host : undefined,
     retryStrategy: (attempt) => {
       if (attempt === 1) {
         valkey.disconnect();
@@ -44,12 +60,12 @@ const valkeyClientFactory = async () => {
   return valkey;
 };
 
-const getValkeyObject = async (key = '') => {
+const getValkeyObject = async (options = { host: '', port: 0 }, key = '') => {
   if (!valkeyEnabled) {
     logger.warn(disableValkeyErrorMessage + ' get', key);
     return null;
   }
-  const object = await valkey.get(key);
+  const object = await ValkeyInstances[`${options.host}${options.path}`].get(key);
   try {
     return JSON.parse(object);
   } catch (error) {
@@ -58,19 +74,19 @@ const getValkeyObject = async (key = '') => {
   }
 };
 
-const setValkeyObject = async (key = '', payload = {}) => {
+const setValkeyObject = async (options = { host: '', port: 0 }, key = '', payload = {}) => {
   if (!valkeyEnabled) throw new Error(disableValkeyErrorMessage);
-  return await valkey.set(key, JSON.stringify(payload));
+  return await ValkeyInstances[`${options.host}${options.path}`].set(key, JSON.stringify(payload));
 };
 
-const updateValkeyObject = async (key = '', payload = {}) => {
+const updateValkeyObject = async (options = { host: '', port: 0 }, key = '', payload = {}) => {
   if (!valkeyEnabled) throw new Error(disableValkeyErrorMessage);
-  const object = await getValkeyObject(key, valkey);
+  const object = await getValkeyObject(key);
   object.updatedAt = new Date().toISOString();
-  return await valkey.set(key, JSON.stringify({ ...object, ...payload }));
+  return await ValkeyInstances[`${options.host}${options.path}`].set(key, JSON.stringify({ ...object, ...payload }));
 };
 
-const valkeyObjectFactory = async (module = '', options = { host: 'localhost', object: {} }) => {
+const valkeyObjectFactory = async (options = { host: 'localhost', object: {} }, module = '') => {
   if (!valkeyEnabled) throw new Error(disableValkeyErrorMessage);
   const idoDate = new Date().toISOString();
   options.object = options.object || {};
@@ -110,10 +126,9 @@ const ValkeyAPI = {
   setValkeyObject,
   valkeyObjectFactory,
   updateValkeyObject,
+  createValkeyConnection,
 };
 
-const valkey = await ValkeyAPI.valkeyClientFactory();
-
 export {
   valkeyClientFactory,
   selectDtoFactory,
@@ -122,5 +137,6 @@ export {
   valkeyObjectFactory,
   updateValkeyObject,
   isValkeyEnable,
+  createValkeyConnection,
   ValkeyAPI,
 };

package/test/api.test.js

@@ -2,7 +2,6 @@
 
 import axios from 'axios';
 import dotenv from 'dotenv';
-import https from 'https';
 
 import { expect } from 'chai';
 import { loggerFactory } from '../src/server/logger.js';
@@ -20,13 +19,6 @@ const BASE_URL =
     ? `http://localhost:${PORT}/${process.env.BASE_API}`
     : `https://www.nexodev.org/api`;
 
-axios.defaults.baseURL = BASE_URL;
-
-const httpsAgent = new https.Agent({
-  rejectUnauthorized: false,
-});
-axios.defaults.httpsAgent = httpsAgent;
-
 describe(`GET 'Test' API Request `, async () => {
   {
     const url = `${BASE_URL}/test/youtube-id/?url=https://www.youtube.com/watch?v=o4f42SbyDMk`;

package/src/dns.js

@@ -1,22 +0,0 @@
-'use strict';
-
-// https://nodejs.org/api
-// https://expressjs.com/en/4x/api.html
-
-import dotenv from 'dotenv';
-import { loggerFactory } from './server/logger.js';
-import { Dns } from './server/dns.js';
-import { ProcessController } from './server/process.js';
-import { Config } from './server/conf.js';
-
-dotenv.config();
-
-await Config.build();
-
-const logger = loggerFactory(import.meta);
-
-await logger.setUpInfo();
-
-await Dns.InitIpDaemon();
-
-ProcessController.init(logger);

package/src/server/prompt-optimizer.js

@@ -1,28 +0,0 @@
-// https://github.com/xenova/transformers.js/blob/f43d3dd348fd7b293008802590bb3a1afa218dc7/src/models.js#L10
-
-import { AutoModelForSeq2SeqLM, AutoTokenizer } from '@xenova/transformers';
-import { loggerFactory } from './logger.js';
-import dotenv from 'dotenv';
-
-dotenv.config();
-
-const logger = loggerFactory(import.meta);
-
-const tokenizer = await AutoTokenizer.from_pretrained('Xenova/t5-small');
-
-const model = await AutoModelForSeq2SeqLM.from_pretrained('Xenova/t5-small');
-
-const prompt = 'translate English to German: I love transformers!';
-
-logger.info('input', { prompt });
-
-const tokenizerData = await tokenizer(prompt);
-
-const { input_ids } = tokenizerData;
-
-const outputs = await model.generate(input_ids);
-
-for (const output of outputs) {
-  const decoded = tokenizer.decode(output, { skip_special_tokens: true });
-  logger.info('decoded', { decoded });
-}

package/startup.js

@@ -1,11 +0,0 @@
-import { shellExec } from './src/server/process.js';
-
-// /usr/bin/supervisord -n
-// /usr/sbin/sshd -D
-shellExec(`/usr/bin/supervisord -n`, { async: true });
-
-shellExec(`sudo /opt/lampp/lampp start`, { async: true });
-
-shellExec(`/usr/bin/mongod -f /etc/mongod.conf`, { async: true });
-
-shellExec(`underpost new app server`);