underpost 2.95.3 → 2.96.0

package/src/cli/cron.js

@@ -4,12 +4,10 @@
  * @namespace UnderpostCron
  */
 
-import { DataBaseProvider } from '../db/DataBaseProvider.js';
 import BackUp from '../server/backup.js';
 import { Cmd } from '../server/conf.js';
 import Dns from '../server/dns.js';
 import { loggerFactory } from '../server/logger.js';
-
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
 
@@ -37,57 +35,191 @@ class UnderpostCron {
      */
     backup: BackUp,
   };
+
   static API = {
     /**
      * Run the cron jobs
      * @static
      * @param {String} deployList - Comma separated deploy ids
      * @param {String} jobList - Comma separated job ids
+     * @param {Object} options - Options for cron execution
      * @return {void}
      * @memberof UnderpostCron
      */
     callback: async function (
       deployList = 'default',
-      jobList = Object.keys(UnderpostCron.JOB),
-      options = { itc: false, init: false, git: false },
+      jobList = Object.keys(UnderpostCron.JOB).join(','),
+      options = { initPm2Cronjobs: false, git: false, updatePackageScripts: false },
     ) {
-      if (options.init === true) {
-        const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
-        deployList = fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
-        const confCronConfig = JSON.parse(fs.readFileSync(`./engine-private/conf/${jobDeployId}/conf.cron.json`));
-        if (confCronConfig.jobs && Object.keys(confCronConfig.jobs).length > 0) {
-          for (const job of Object.keys(confCronConfig.jobs)) {
-            const name = `${jobDeployId}-${job}`;
-            let deployId;
-            shellExec(Cmd.delete(name));
-            deployId = UnderpostCron.API.getRelatedDeployId(job);
-            shellExec(Cmd.cron(deployId, job, name, confCronConfig.jobs[job].expression, options));
-          }
-        }
+      if (options.updatePackageScripts === true) {
+        await UnderpostCron.API.updatePackageScripts(deployList);
         return;
       }
+
+      if (options.initPm2Cronjobs === true) {
+        await UnderpostCron.API.initCronJobs(options);
+        return;
+      }
+
+      // Execute the requested jobs
       for (const _jobId of jobList.split(',')) {
         const jobId = _jobId.trim();
-        if (UnderpostCron.JOB[jobId]) await UnderpostCron.JOB[jobId].callback(deployList, options);
+        if (UnderpostCron.JOB[jobId]) {
+          logger.info(`Executing cron job: ${jobId}`);
+          await UnderpostCron.JOB[jobId].callback(deployList, options);
+        } else {
+          logger.warn(`Unknown cron job: ${jobId}`);
+        }
+      }
+    },
+
+    /**
+     * Initialize PM2 cron jobs from configuration
+     * @static
+     * @param {Object} options - Initialization options
+     * @memberof UnderpostCron
+     */
+    initCronJobs: async function (options = { git: false }) {
+      logger.info('Initializing PM2 cron jobs');
+
+      // Read cron job deployment ID from dd.cron file (e.g., "dd-cron")
+      const jobDeployId = fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
+      const confCronPath = `./engine-private/conf/${jobDeployId}/conf.cron.json`;
+
+      if (!fs.existsSync(confCronPath)) {
+        logger.warn(`Cron configuration not found: ${confCronPath}`);
+        return;
+      }
+
+      const confCronConfig = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
+
+      if (!confCronConfig.jobs || Object.keys(confCronConfig.jobs).length === 0) {
+        logger.info('No cron jobs configured');
+        return;
+      }
+
+      // Delete all existing cron jobs
+      for (const job of Object.keys(confCronConfig.jobs)) {
+        const name = `${jobDeployId}-${job}`;
+        logger.info(`Removing existing PM2 process: ${name}`);
+        shellExec(Cmd.delete(name));
+      }
+
+      // Create PM2 cron jobs for each configured job
+      for (const job of Object.keys(confCronConfig.jobs)) {
+        const jobConfig = confCronConfig.jobs[job];
+
+        if (jobConfig.enabled === false) {
+          logger.info(`Skipping disabled job: ${job}`);
+          continue;
+        }
+
+        const name = `${jobDeployId}-${job}`;
+        const deployIdList = UnderpostCron.API.getRelatedDeployIdList(job);
+        const expression = jobConfig.expression || '0 0 * * *'; // Default: daily at midnight
+        const instances = jobConfig.instances || 1; // Default: 1 instance
+
+        logger.info(`Creating PM2 cron job: ${name} with expression: ${expression}, instances: ${instances}`);
+        shellExec(Cmd.cron(deployIdList, job, name, expression, options, instances));
       }
+
+      logger.info('PM2 cron jobs initialization completed');
     },
 
     /**
-     * Get the related deploy id for the given job id
+     * Update package.json start scripts for specified deploy-ids
      * @static
-     * @param {String} jobId - The job id
-     * @return {String} The related deploy id
+     * @param {String} deployList - Comma separated deploy ids
      * @memberof UnderpostCron
      */
-    getRelatedDeployId(jobId) {
-      switch (jobId) {
-        case 'dns':
-          return fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
-        case 'backup':
-          return fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').trim();
-        default:
-          return fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim();
+    updatePackageScripts: async function (deployList = 'default') {
+      logger.info('Updating package.json start scripts for deploy-id configurations');
+
+      // Resolve deploy list
+      if ((!deployList || deployList === 'dd') && fs.existsSync(`./engine-private/deploy/dd.router`)) {
+        deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').trim();
+      }
+
+      const confDir = './engine-private/conf';
+      if (!fs.existsSync(confDir)) {
+        logger.warn(`Configuration directory not found: ${confDir}`);
+        return;
       }
+
+      // Parse deploy list into array
+      const deployIds = deployList
+        .split(',')
+        .map((id) => id.trim())
+        .filter((id) => id);
+
+      for (const deployId of deployIds) {
+        const packageJsonPath = `${confDir}/${deployId}/package.json`;
+        const confCronPath = `${confDir}/${deployId}/conf.cron.json`;
+
+        // Only update if both package.json and conf.cron.json exist
+        if (!fs.existsSync(packageJsonPath)) {
+          logger.info(`Skipping ${deployId}: package.json not found`);
+          continue;
+        }
+
+        if (!fs.existsSync(confCronPath)) {
+          logger.info(`Skipping ${deployId}: conf.cron.json not found`);
+          continue;
+        }
+
+        const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+        const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
+
+        // Build start script based on cron jobs configuration
+        if (confCron.jobs && Object.keys(confCron.jobs).length > 0) {
+          const hasEnabledJobs = Object.values(confCron.jobs).some((job) => job.enabled !== false);
+
+          if (hasEnabledJobs) {
+            // Update start script with PM2 cron jobs initialization
+            const startScript = 'pm2 flush && pm2 reloadLogs && node bin cron --init-pm2-cronjobs --git';
+
+            if (!packageJson.scripts) {
+              packageJson.scripts = {};
+            }
+
+            packageJson.scripts.start = startScript;
+
+            // Write updated package.json
+            fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 4) + '\n', 'utf8');
+            logger.info(`Updated package.json for ${deployId} with cron start script`);
+          } else {
+            logger.info(`Skipping ${deployId}: no enabled cron jobs`);
+          }
+        } else {
+          logger.info(`Skipping ${deployId}: no cron jobs configured`);
+        }
+      }
+
+      logger.info('Package.json start scripts update completed');
+    },
+
+    /**
+     * Get the related deploy id list for the given job id
+     * @static
+     * @param {String} jobId - The job id (e.g., 'dns', 'backup')
+     * @return {String} Comma-separated list of deploy ids to process
+     * @memberof UnderpostCron
+     */
+    getRelatedDeployIdList(jobId) {
+      // Backup job uses dd.router file (contains multiple deploy-ids)
+      // Other jobs use dd.cron file (contains single deploy-id)
+      const deployFilePath =
+        jobId === 'backup' ? './engine-private/deploy/dd.router' : './engine-private/deploy/dd.cron';
+
+      if (!fs.existsSync(deployFilePath)) {
+        logger.warn(`Deploy file not found: ${deployFilePath}, using default`);
+        return fs.existsSync('./engine-private/deploy/dd.cron')
+          ? fs.readFileSync('./engine-private/deploy/dd.cron', 'utf8').trim()
+          : 'dd-cron';
+      }
+
+      // Return the deploy-id list from the file (may be single or comma-separated)
+      return fs.readFileSync(deployFilePath, 'utf8').trim();
     },
   };
 }

package/src/cli/db.js

@@ -31,7 +31,6 @@ const MAX_BACKUP_RETENTION = 5;
  * @property {boolean} [import=false] - Flag to import data from a backup
  * @property {boolean} [export=false] - Flag to export data to a backup
  * @property {string} [podName=''] - Comma-separated list of pod names or patterns
- * @property {string} [nodeName=''] - Comma-separated list of node names for pod filtering
  * @property {string} [ns='default'] - Kubernetes namespace
  * @property {string} [collections=''] - Comma-separated list of collections to include
  * @property {string} [outPath=''] - Output path for backup files
@@ -40,7 +39,6 @@ const MAX_BACKUP_RETENTION = 5;
  * @property {boolean} [git=false] - Flag to enable Git integration
  * @property {string} [hosts=''] - Comma-separated list of hosts to include
  * @property {string} [paths=''] - Comma-separated list of paths to include
- * @property {string} [labelSelector=''] - Kubernetes label selector for pods
  * @property {boolean} [allPods=false] - Flag to target all matching pods
  * @property {boolean} [primaryPod=false] - Flag to automatically detect and use MongoDB primary pod
  * @property {boolean} [stats=false] - Flag to display collection/table statistics
@@ -78,43 +76,17 @@ const MAX_BACKUP_RETENTION = 5;
  */
 class UnderpostDB {
   static API = {
-    /**
-     * Helper: Validates namespace name
-     * @private
-     * @param {string} namespace - Namespace to validate
-     * @returns {boolean} True if valid
-     */
-    _validateNamespace(namespace) {
-      if (!namespace || typeof namespace !== 'string') return false;
-      // Kubernetes namespace naming rules: lowercase alphanumeric, -, max 63 chars
-      return /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(namespace) && namespace.length <= 63;
-    },
-
-    /**
-     * Helper: Validates pod name
-     * @private
-     * @param {string} podName - Pod name to validate
-     * @returns {boolean} True if valid
-     */
-    _validatePodName(podName) {
-      if (!podName || typeof podName !== 'string') return false;
-      // Kubernetes pod naming rules: lowercase alphanumeric, -, max 253 chars
-      return /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(podName) && podName.length <= 253;
-    },
-
     /**
      * Helper: Gets filtered pods based on criteria
      * @private
      * @param {Object} criteria - Filter criteria
      * @param {string} [criteria.podNames] - Comma-separated pod name patterns
-     * @param {string} [criteria.nodeNames] - Comma-separated node names
      * @param {string} [criteria.namespace='default'] - Kubernetes namespace
-     * @param {string} [criteria.labelSelector] - Label selector
      * @param {string} [criteria.deployId] - Deployment ID pattern
      * @returns {Array<PodInfo>} Filtered pod list
      */
     _getFilteredPods(criteria = {}) {
-      const { podNames, nodeNames, namespace = 'default', labelSelector, deployId } = criteria;
+      const { podNames, namespace = 'default', deployId } = criteria;
 
       try {
         // Get all pods using UnderpostDeploy.API.get
@@ -132,25 +104,6 @@ class UnderpostDB {
           });
         }
 
-        // Filter by node names if specified (only if NODE is not '<none>')
-        if (nodeNames) {
-          const nodes = nodeNames.split(',').map((n) => n.trim());
-          pods = pods.filter((pod) => {
-            // Skip filtering if NODE is '<none>' or undefined
-            if (!pod.NODE || pod.NODE === '<none>') {
-              return true;
-            }
-            return nodes.includes(pod.NODE);
-          });
-        }
-
-        // Filter by label selector if specified
-        if (labelSelector) {
-          // Note: UnderpostDeploy.API.get doesn't support label selectors directly
-          // This would require a separate kubectl command
-          logger.warn('Label selector filtering requires additional implementation');
-        }
-
         logger.info(`Found ${pods.length} pod(s) matching criteria`, { criteria, podNames: pods.map((p) => p.NAME) });
         return pods;
       } catch (error) {
@@ -717,9 +670,6 @@ class UnderpostDB {
      * @param {string} [options.podName='mongodb-0'] - Initial pod name to query replica set status
      * @returns {string|null} Primary pod name or null if not found
      * @memberof UnderpostDB
-     * @example
-     * const primaryPod = UnderpostDB.API.getMongoPrimaryPodName({ namespace: 'production' });
-     * console.log(primaryPod); // 'mongodb-1'
      */
     getMongoPrimaryPodName(options = { namespace: 'default', podName: 'mongodb-0' }) {
       const { namespace = 'default', podName = 'mongodb-0' } = options;
@@ -765,7 +715,6 @@ class UnderpostDB {
      * @param {boolean} [options.import=false] - Whether to perform import operation
      * @param {boolean} [options.export=false] - Whether to perform export operation
      * @param {string} [options.podName=''] - Comma-separated pod name patterns to target
-     * @param {string} [options.nodeName=''] - Comma-separated node names to target
      * @param {string} [options.ns='default'] - Kubernetes namespace
      * @param {string} [options.collections=''] - Comma-separated MongoDB collections for export
      * @param {string} [options.outPath=''] - Output path for backups
@@ -774,7 +723,6 @@ class UnderpostDB {
      * @param {boolean} [options.git=false] - Whether to use Git for backup versioning
      * @param {string} [options.hosts=''] - Comma-separated list of hosts to filter databases
      * @param {string} [options.paths=''] - Comma-separated list of paths to filter databases
-     * @param {string} [options.labelSelector=''] - Label selector for pod filtering
      * @param {boolean} [options.allPods=false] - Whether to target all pods in deployment
      * @param {boolean} [options.primaryPod=false] - Whether to target MongoDB primary pod only
      * @param {boolean} [options.stats=false] - Whether to display database statistics
@@ -788,7 +736,6 @@ class UnderpostDB {
         import: false,
         export: false,
         podName: '',
-        nodeName: '',
         ns: 'default',
         collections: '',
         outPath: '',
@@ -797,7 +744,6 @@ class UnderpostDB {
         git: false,
         hosts: '',
         paths: '',
-        labelSelector: '',
         allPods: false,
         primaryPod: false,
         stats: false,
@@ -808,12 +754,6 @@ class UnderpostDB {
       const newBackupTimestamp = new Date().getTime();
       const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
 
-      // Validate namespace
-      if (!UnderpostDB.API._validateNamespace(namespace)) {
-        logger.error('Invalid namespace format', { namespace });
-        throw new Error(`Invalid namespace: ${namespace}`);
-      }
-
       if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
 
       logger.info('Starting database operation', {
@@ -823,6 +763,11 @@ class UnderpostDB {
         export: options.export,
       });
 
+      // Track processed repositories to avoid duplicate Git operations
+      const processedRepos = new Set();
+      // Track processed host+path combinations to avoid duplicates
+      const processedHostPaths = new Set();
+
       for (const _deployId of deployList.split(',')) {
         const deployId = _deployId.trim();
         if (!deployId) continue;
@@ -831,7 +776,7 @@ class UnderpostDB {
 
         /** @type {Object.<string, Object.<string, DatabaseConfig>>} */
         const dbs = {};
-        const repoName = `engine-${deployId.split('dd-')[1]}-cron-backups`;
+        const repoName = `engine-${deployId.includes('dd-') ? deployId.split('dd-')[1] : deployId}-cron-backups`;
 
         // Load server configuration
         const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
@@ -863,31 +808,42 @@ class UnderpostDB {
           }
         }
 
-        // Handle Git operations
-        if (options.git === true) {
-          UnderpostDB.API._manageGitRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
-          UnderpostDB.API._manageGitRepo({ repoName, operation: 'pull' });
-        }
+        // Handle Git operations - execute only once per repository
+        if (!processedRepos.has(repoName)) {
+          logger.info('Processing Git operations for repository', { repoName, deployId });
+          if (options.git === true) {
+            UnderpostDB.API._manageGitRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
+            UnderpostDB.API._manageGitRepo({ repoName, operation: 'pull' });
+          }
+
+          if (options.macroRollbackExport) {
+            // Only clone if not already done by git option above
+            if (options.git !== true) {
+              UnderpostDB.API._manageGitRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
+              UnderpostDB.API._manageGitRepo({ repoName, operation: 'pull' });
+            }
 
-        if (options.macroRollbackExport) {
-          UnderpostDB.API._manageGitRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
-          UnderpostDB.API._manageGitRepo({ repoName, operation: 'pull' });
-
-          const nCommits = parseInt(options.macroRollbackExport);
-          const repoPath = `../${repoName}`;
-          const username = process.env.GITHUB_USERNAME;
-
-          if (fs.existsSync(repoPath) && username) {
-            logger.info('Executing macro rollback export', { repoName, nCommits });
-            shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
-            shellExec(`cd ${repoPath} && git reset`);
-            shellExec(`cd ${repoPath} && git checkout .`);
-            shellExec(`cd ${repoPath} && git clean -f -d`);
-            shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
-          } else {
-            if (!username) logger.error('GITHUB_USERNAME environment variable not set');
-            logger.warn('Repository not found for macro rollback', { repoPath });
+            const nCommits = parseInt(options.macroRollbackExport);
+            const repoPath = `../${repoName}`;
+            const username = process.env.GITHUB_USERNAME;
+
+            if (fs.existsSync(repoPath) && username) {
+              logger.info('Executing macro rollback export', { repoName, nCommits });
+              shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
+              shellExec(`cd ${repoPath} && git reset`);
+              shellExec(`cd ${repoPath} && git checkout .`);
+              shellExec(`cd ${repoPath} && git clean -f -d`);
+              shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
+            } else {
+              if (!username) logger.error('GITHUB_USERNAME environment variable not set');
+              logger.warn('Repository not found for macro rollback', { repoPath });
+            }
           }
+
+          processedRepos.add(repoName);
+          logger.info('Repository marked as processed', { repoName });
+        } else {
+          logger.info('Skipping Git operations for already processed repository', { repoName, deployId });
         }
 
         // Process each database provider
@@ -895,6 +851,15 @@ class UnderpostDB {
           for (const dbName of Object.keys(dbs[provider])) {
             const { hostFolder, user, password, host, path } = dbs[provider][dbName];
 
+            // Create unique identifier for host+path combination
+            const hostPathKey = `${deployId}:${host}:${path}`;
+
+            // Skip if this host+path combination was already processed
+            if (processedHostPaths.has(hostPathKey)) {
+              logger.info('Skipping already processed host/path', { dbName, host, path, deployId });
+              continue;
+            }
+
             // Filter by hosts and paths if specified
             if (
               (options.hosts &&
@@ -917,7 +882,7 @@ class UnderpostDB {
               continue;
             }
 
-            logger.info('Processing database', { hostFolder, provider, dbName });
+            logger.info('Processing database', { hostFolder, provider, dbName, deployId });
 
             const backUpPath = `../${repoName}/${hostFolder}`;
             const backupInfo = UnderpostDB.API._manageBackupTimestamps(
@@ -949,16 +914,14 @@ class UnderpostDB {
             let targetPods = [];
             const podCriteria = {
               podNames: options.podName,
-              nodeNames: options.nodeName,
               namespace,
-              labelSelector: options.labelSelector,
               deployId: provider === 'mariadb' ? 'mariadb' : 'mongo',
             };
 
             targetPods = UnderpostDB.API._getFilteredPods(podCriteria);
 
             // Fallback to default if no custom pods specified
-            if (targetPods.length === 0 && !options.podName && !options.nodeName) {
+            if (targetPods.length === 0 && !options.podName) {
               const defaultPods = UnderpostDeploy.API.get(
                 provider === 'mariadb' ? 'mariadb' : 'mongo',
                 'pods',
@@ -1094,16 +1057,20 @@ class UnderpostDB {
                   break;
               }
             }
+
+            // Mark this host+path combination as processed
+            processedHostPaths.add(hostPathKey);
           }
         }
 
-        // Commit and push to Git if enabled
-        if (options.export === true && options.git === true) {
+        // Commit and push to Git if enabled - execute only once per repository
+        if (options.export === true && options.git === true && !processedRepos.has(`${repoName}-committed`)) {
           const commitMessage = `${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
             newBackupTimestamp,
           ).toLocaleTimeString()}`;
           UnderpostDB.API._manageGitRepo({ repoName, operation: 'commit', message: commitMessage });
           UnderpostDB.API._manageGitRepo({ repoName, operation: 'push' });
+          processedRepos.add(`${repoName}-committed`);
         }
       }
 
@@ -1267,7 +1234,7 @@ class UnderpostDB {
         for (const jobId of Object.keys(confCron.jobs)) {
           const body = {
             jobId,
-            deployId: UnderpostCron.API.getRelatedDeployId(jobId),
+            deployId: UnderpostCron.API.getRelatedDeployIdList(jobId),
             expression: confCron.jobs[jobId].expression,
             enabled: confCron.jobs[jobId].enabled,
           };

package/src/cli/env.js

@@ -74,17 +74,38 @@ class UnderpostRootEnv {
     /**
      * @method list
      * @description Lists all environment variables in the underpost root environment.
+     * @param {string} key - Not used for list operation.
+     * @param {string} value - Not used for list operation.
+     * @param {object} options - Options for listing environment variables.
+     * @param {string} [options.filter] - Filter keyword to match against keys or values.
      * @memberof UnderpostEnv
      */
-    list() {
+    list(key, value, options = {}) {
       const exeRootPath = `${getNpmRootPath()}/underpost`;
       const envPath = `${exeRootPath}/.env`;
       if (!fs.existsSync(envPath)) {
         logger.warn(`Empty environment variables`);
         return {};
       }
-      const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
-      logger.info('underpost root', env);
+      let env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
+
+      // Apply filter if provided
+      if (options.filter) {
+        const filterKeyword = options.filter.toLowerCase();
+        const filtered = {};
+        for (const [envKey, envValue] of Object.entries(env)) {
+          const keyMatch = envKey.toLowerCase().includes(filterKeyword);
+          const valueMatch = String(envValue).toLowerCase().includes(filterKeyword);
+          if (keyMatch || valueMatch) {
+            filtered[envKey] = envValue;
+          }
+        }
+        env = filtered;
+        logger.info(`underpost root (filtered by: ${options.filter})`, env);
+      } else {
+        logger.info('underpost root', env);
+      }
+
       return env;
     },
     /**