underpost 2.8.852 → 2.8.854

package/src/server/conf.js

@@ -770,152 +770,22 @@ const validateTemplatePath = (absolutePath = '') => {
   return true;
 };
 
-const deployTest = async (dataDeploy = [{ deployId: 'default' }]) => {
-  const failed = [];
-  for (const deploy of dataDeploy) {
-    const deployServerConfPath = fs.existsSync(`./engine-private/replica/${deploy.deployId}/conf.server.json`)
-      ? `./engine-private/replica/${deploy.deployId}/conf.server.json`
-      : `./engine-private/conf/${deploy.deployId}/conf.server.json`;
-    const serverConf = loadReplicas(JSON.parse(fs.readFileSync(deployServerConfPath, 'utf8')));
-    let fail = false;
-    for (const host of Object.keys(serverConf))
-      for (const path of Object.keys(serverConf[host])) {
-        const { singleReplica } = serverConf[host][path];
-        if (singleReplica) continue;
-        const urlTest = `https://${host}${path}`;
-        try {
-          const result = await axios.get(urlTest, { timeout: 10000 });
-          const test = result.data.split('<title>');
-          if (test[1])
-            logger.info('Success deploy', {
-              ...deploy,
-              result: test[1].split('</title>')[0],
-              urlTest,
-            });
-          else {
-            logger.error('Error deploy', {
-              ...deploy,
-              result: result.data,
-              urlTest,
-            });
-            fail = true;
-          }
-        } catch (error) {
-          logger.error('Error deploy', {
-            ...deploy,
-            message: error.message,
-            urlTest,
-          });
-          fail = true;
-        }
-      }
-    if (fail) failed.push(deploy);
-  }
-  return { failed };
-};
-
 const awaitDeployMonitor = async (init = false, deltaMs = 1000) => {
   if (init) fs.writeFileSync(`./tmp/await-deploy`, '', 'utf8');
   await timer(deltaMs);
   if (fs.existsSync(`./tmp/await-deploy`)) return await awaitDeployMonitor();
 };
 
-const getDeployGroupId = () => {
-  const deployGroupIndexArg = process.argv.findIndex((a) => a.match(`deploy-group:`));
-  if (deployGroupIndexArg > -1) return process.argv[deployGroupIndexArg].split(':')[1].trim();
-  return 'dd';
-};
-
 const getDeployId = () => {
   const deployIndexArg = process.argv.findIndex((a) => a.match(`deploy-id:`));
   if (deployIndexArg > -1) return process.argv[deployIndexArg].split(':')[1].trim();
-  for (const deployId of process.argv) {
-    if (fs.existsSync(`./engine-private/conf/${deployId}`)) return deployId;
-    else if (fs.existsSync(`./engine-private/replica/${deployId}`)) return deployId;
-  }
-  return 'default';
+  return 'dd-default';
 };
 
 const getCronBackUpFolder = (host = '', path = '') => {
   return `${host}${path.replace(/\\/g, '/').replace(`/`, '-')}`;
 };
 
-const execDeploy = async (options = { deployId: 'default' }, currentAttempt = 1) => {
-  const { deployId } = options;
-  shellExec(Cmd.delete(deployId));
-  shellExec(Cmd.conf(deployId));
-  shellExec(Cmd.run(deployId));
-  const maxTime = 1000 * 60;
-  const minTime = 20 * 1000;
-  const intervalTime = 1000;
-  return await new Promise(async (resolve) => {
-    let currentTime = 0;
-    const attempt = () => {
-      if (currentTime >= minTime && !fs.existsSync(`./tmp/await-deploy`)) {
-        clearInterval(processMonitor);
-        return resolve(true);
-      }
-      cliSpinner(
-        intervalTime,
-        `[deploy.js] `,
-        ` Load instance | attempt:${currentAttempt} | elapsed time ${currentTime / 1000}s / ${maxTime / 1000}s`,
-        'yellow',
-        'material',
-      );
-      currentTime += intervalTime;
-      if (currentTime >= maxTime) {
-        clearInterval(processMonitor);
-        return resolve(false);
-      }
-    };
-    const processMonitor = setInterval(attempt, intervalTime);
-  });
-};
-
-const deployRun = async (dataDeploy, currentAttempt = 1) => {
-  if (!fs.existsSync(`./tmp`)) fs.mkdirSync(`./tmp`, { recursive: true });
-  await fixDependencies();
-  const maxAttempts = 3;
-  for (const deploy of dataDeploy) {
-    let currentAttempt = 1;
-    const attempt = async () => {
-      const success = await execDeploy(deploy, currentAttempt);
-      currentAttempt++;
-      if (!success && currentAttempt <= maxAttempts) await attempt();
-    };
-    await attempt();
-  }
-  const { failed } = await deployTest(dataDeploy);
-  if (failed.length > 0) {
-    for (const deploy of failed) logger.error(deploy.deployId, Cmd.run(deploy.deployId));
-    if (currentAttempt === maxAttempts) return logger.error(`max deploy attempts exceeded`);
-    await read({ prompt: 'Press enter to retry failed processes\n' });
-    currentAttempt++;
-    await deployRun(failed, currentAttempt);
-  } else logger.info(`Deploy process successfully`);
-};
-
-const restoreMacroDb = async (deployGroupId = '', deployId = null) => {
-  const dataDeploy = await getDataDeploy({ deployGroupId, buildSingleReplica: false });
-  for (const deployGroup of dataDeploy) {
-    if (deployId && deployGroup.deployId !== deployId) continue;
-    if (!deployGroup.replicaHost) {
-      const deployServerConfPath = `./engine-private/conf/${deployGroup.deployId}/conf.server.json`;
-      const serverConf = JSON.parse(fs.readFileSync(deployServerConfPath, 'utf8'));
-
-      for (const host of Object.keys(serverConf)) {
-        for (const path of Object.keys(serverConf[host])) {
-          const { db, singleReplica } = serverConf[host][path];
-          if (db && !singleReplica) {
-            const cmd = `node bin/db ${host}${path} import ${deployGroup.deployId} cron`;
-            shellExec(cmd);
-          }
-        }
-      }
-    }
-  }
-};
-
 const mergeFile = async (parts = [], outputFilePath) => {
   await new Promise((resolve) => {
     splitFile
@@ -977,99 +847,6 @@ const rebuildConfFactory = ({ deployId, valkey, mongo }) => {
   return { hosts };
 };
 
-const getRestoreCronCmd = async (options = { host: '', path: '', conf: {}, deployId: '' }) => {
-  const { host, path, conf, deployId } = options;
-  const { runtime, db, git, directory } = conf[host][path];
-  const { provider, name, user, password = '', backupPath = '' } = db;
-
-  if (['xampp', 'lampp'].includes(runtime)) {
-    logger.info('Create database', `node bin/db ${host}${path} create ${deployId}`);
-    shellExec(`node bin/db ${host}${path} create ${deployId}`);
-  }
-
-  if (git) {
-    if (directory && !fs.existsSync(directory)) fs.mkdirSync(directory, { recursive: true });
-
-    shellExec(`git clone ${git}`);
-
-    // fs.mkdirSync(`./public/${host}${path}`, { recursive: true });
-
-    if (fs.existsSync(`./${git.split('/').pop()}`))
-      fs.moveSync(`./${git.split('/').pop()}`, directory ? directory : `./public/${host}${path}`, {
-        overwrite: true,
-      });
-  }
-
-  let cmd, currentBackupTimestamp, baseBackUpPath;
-
-  if (process.argv.includes('cron')) {
-    baseBackUpPath = `${process.cwd()}/engine-private/cron-backups/${getCronBackUpFolder(host, path)}`;
-
-    const files = await fs.readdir(baseBackUpPath, { withFileTypes: true });
-
-    currentBackupTimestamp = files
-      .map((fileObj) => parseInt(fileObj.name))
-      .sort((a, b) => a - b)
-      .reverse()[0];
-  }
-
-  switch (provider) {
-    case 'mariadb':
-      {
-        if (process.argv.includes('cron')) {
-          cmd = `mysql -u ${user} -p${password} ${name} < ${baseBackUpPath}/${currentBackupTimestamp}/${name}.sql`;
-          if (fs.existsSync(`${baseBackUpPath}/${currentBackupTimestamp}/${name}-parths.json`)) {
-            const names = JSON.parse(
-              fs.readFileSync(`${baseBackUpPath}/${currentBackupTimestamp}/${name}-parths.json`, 'utf8'),
-            ).map((p) => p.replaceAll(`\\`, '/').replaceAll('C:/', '/').replaceAll('c:/', '/'));
-
-            await mergeFile(names, `${baseBackUpPath}/${currentBackupTimestamp}/${name}.sql`);
-          }
-        } else {
-          cmd = `mysql -u ${user} -p${password} ${name} < ${
-            backupPath ? backupPath : `./engine-private/sql-backups/${name}.sql`
-          }`;
-          if (
-            fs.existsSync(
-              `${
-                backupPath ? backupPath.split('/').slice(0, -1).join('/') : `./engine-private/sql-backups`
-              }/${name}-parths.json`,
-            )
-          ) {
-            const names = JSON.parse(
-              fs.readFileSync(
-                `${
-                  backupPath ? backupPath.split('/').slice(0, -1).join('/') : `./engine-private/sql-backups`
-                }/${name}-parths.json`,
-                'utf8',
-              ),
-            ).map((p) => p.replaceAll(`\\`, '/').replaceAll('C:/', '/').replaceAll('c:/', '/'));
-
-            await mergeFile(
-              names,
-              `${
-                backupPath ? backupPath.split('/').slice(0, -1).join('/') : `./engine-private/sql-backups`
-              }/${name}.sql`,
-            );
-          }
-        }
-      }
-      break;
-
-    case 'mongoose':
-      {
-        if (process.argv.includes('cron')) {
-          cmd = `mongorestore -d ${name} ${baseBackUpPath}/${currentBackupTimestamp}/${name}`;
-        } else cmd = `mongorestore -d ${name} ${backupPath ? backupPath : `./engine-private/mongodb-backup/${name}`}`;
-      }
-      break;
-  }
-
-  // logger.info('Restore', cmd);
-
-  return cmd;
-};
-
 const getPathsSSR = (conf) => {
   const paths = ['src/client/ssr/Render.js'];
   for (const o of conf.head) paths.push(`src/client/ssr/head/${o}.js`);
@@ -1093,37 +870,6 @@ const Cmd = {
     }${options?.git ? `--git ` : ''}${deployList} ${jobList}`,
 };
 
-const fixDependencies = async () => {
-  return;
-  // sed -i "$line_number s,.*,$new_text," "$file"
-  // sed -i "$line_number c \\$new_text" "$file"
-  const dep = fs.readFileSync(`./node_modules/peer/dist/module.mjs`, 'utf8');
-  const errorLine = `import {WebSocketServer as $hSjDC$WebSocketServer} from "ws";`;
-
-  fs.writeFileSync(
-    `./node_modules/peer/dist/module.mjs`,
-    dep.replaceAll(
-      errorLine,
-      `import WebSocketServer from "ws";
-    let $hSjDC$WebSocketServer = WebSocketServer.Server;`,
-    ),
-    'utf8',
-  );
-};
-
-const maintenanceMiddleware = (req, res, port, proxyRouter) => {
-  if (process.argv.includes('maintenance') && globalThis.defaultHtmlSrcMaintenance) {
-    if (req.method.toUpperCase() === 'GET') {
-      res.set('Content-Type', 'text/html');
-      return res.status(503).send(globalThis.defaultHtmlSrcMaintenance);
-    }
-    return res.status(503).json({
-      status: 'error',
-      message: 'Server is under maintenance',
-    });
-  }
-};
-
 const splitFileFactory = async (name, _path) => {
   const stats = fs.statSync(_path);
   const maxSizeInBytes = 1024 * 1024 * 50; // 50 mb
@@ -1152,14 +898,6 @@ const splitFileFactory = async (name, _path) => {
   return false;
 };
 
-const setUpProxyMaintenanceServer = ({ deployGroupId }) => {
-  shellExec(`pm2 kill`);
-  shellExec(`node bin/deploy valkey-service`);
-  const proxyDeployId = fs.readFileSync(`./engine-private/deploy/${deployGroupId}.proxy`, 'utf8').trim();
-  shellExec(`node bin/deploy conf ${proxyDeployId} production`);
-  shellExec(`npm start ${proxyDeployId} maintenance`);
-};
-
 const getNpmRootPath = () =>
   shellExec(`npm root -g`, {
     stdout: true,
@@ -1258,17 +996,9 @@ export {
   getDataDeploy,
   validateTemplatePath,
   buildReplicaId,
-  restoreMacroDb,
-  getDeployGroupId,
-  execDeploy,
-  deployRun,
   getCronBackUpFolder,
-  getRestoreCronCmd,
   mergeFile,
-  fixDependencies,
   getDeployId,
-  maintenanceMiddleware,
-  setUpProxyMaintenanceServer,
   getPathsSSR,
   buildKindPorts,
   buildPortProxyRouter,
@@ -1276,7 +1006,6 @@ export {
   getNpmRootPath,
   getUnderpostRootPath,
   writeEnv,
-  deployTest,
   pathPortAssignmentFactory,
   deployRangePortFactory,
   awaitDeployMonitor,

package/src/server/proxy.js

@@ -6,7 +6,7 @@ import dotenv from 'dotenv';
 import { createProxyMiddleware } from 'http-proxy-middleware';
 import { loggerFactory, loggerMiddleware } from './logger.js';
 import { createSslServer, sslRedirectMiddleware } from './ssl.js';
-import { buildPortProxyRouter, buildProxyRouter, maintenanceMiddleware } from './conf.js';
+import { buildPortProxyRouter, buildProxyRouter } from './conf.js';
 import UnderpostStartUp from './start.js';
 
 dotenv.config();
@@ -48,7 +48,6 @@ const buildProxy = async () => {
       onProxyReq: (proxyReq, req, res, options) => {
         // https://wtools.io/check-http-status-code
         // http://nexodev.org
-        maintenanceMiddleware(req, res, port, proxyRouter);
         sslRedirectMiddleware(req, res, port, proxyRouter);
       },
       pathRewrite: {

package/docker-compose.yml

@@ -1,67 +0,0 @@
-# https://docs.docker.com/compose/compose-file/compose-file-v3
-# https://docs.docker.com/engine/reference/commandline/compose/
-version: '3'
-services:
-  prometheus:
-    image: prom/prometheus
-    ports:
-      - 9090:9090
-    volumes:
-      - ./prometheus_data:/prometheus
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-    networks:
-      - load-balancer
-
-  grafana:
-    image: grafana/grafana
-    ports:
-      - 3000:3000
-    volumes:
-      - ./grafana_data:/var/lib/grafana
-    #  - ./grafana.ini:/etc/grafana/grafana.ini
-    networks:
-      - load-balancer
-    depends_on:
-      - prometheus
-
-  underpost-engine:
-    build:
-      dockerfile: ./Dockerfile
-      context: . # workdir path
-    # image: underpost-engine
-    # container_name: <name> ignore for replicas
-    ports:
-      - '22'
-      - '80' # host port allocated dynamically, host ports are unique independent of replicas
-      - '443'
-      - '3306'
-      - '27017'
-      - '4001-4002:3001'
-      - '3002-3020'
-    volumes:
-      - ./logs:/code/logs
-    deploy:
-      mode: replicated
-      replicas: 2
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-        max_attempts: 3
-        window: 120s
-      resources:
-        limits:
-          cpus: '2'
-          memory: 400M
-        reservations:
-          cpus: '0.25'
-          memory: 20M
-    labels: # labels in Compose file instead of Dockerfile
-      engine.version: '2.8.852'
-    networks:
-      - load-balancer
-
-networks:
-  load-balancer:
-    driver: bridge

package/prometheus.yml

@@ -1,36 +0,0 @@
-# my global config
-global:
-  scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 5s # Evaluate rules every 15 seconds. The default is every 1 minute.
-  # scrape_timeout is set to the global default (10s).
-
-# Alertmanager configuration
-alerting:
-  alertmanagers:
-    - static_configs:
-        - targets:
-          # - alertmanager:9093
-
-# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
-rule_files:
-  # - "first_rules.yml"
-  # - "second_rules.yml"
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
-scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: 'prometheus_service'
-
-    # metrics_path defaults to '/metrics'
-    # scheme defaults to 'http'.
-
-    static_configs:
-      - targets: ['host.docker.internal:9090']
-
-  - job_name: 'nodejs_service'
-    static_configs:
-      - targets: ['host.docker.internal:4001', 'host.docker.internal:4002']
-      # - targets: ['localhost:4001', 'localhost:4002']
-      # - targets: ["host.docker.internal:3002"] # Windows
-      # - targets: ["docker.for.mac.localhost:9090"] # macOs