underpost 3.2.8 → 3.2.10

package/src/cli/db.js

@@ -10,8 +10,9 @@ import { mergeFile, splitFileFactory, loadConfServerJson, resolveConfSecrets } f
 import { loggerFactory } from '../server/logger.js';
 import { shellExec } from '../server/process.js';
 import fs from 'fs-extra';
-import { DataBaseProvider } from '../db/DataBaseProvider.js';
+import { DataBaseProviderService } from '../db/DataBaseProvider.js';
 import { loadReplicas, pathPortAssignmentFactory, loadCronDeployEnv } from '../server/conf.js';
+import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
 import Underpost from '../index.js';
 import { timer } from '../client/components/core/CommonJs.js';
 const logger = loggerFactory(import.meta);
@@ -229,9 +230,12 @@ class UnderpostDB {
      * @param {string} params.bsonPath - BSON directory path.
      * @param {boolean} params.drop - Whether to drop existing database.
      * @param {boolean} params.preserveUUID - Whether to preserve UUIDs.
+      * @param {string} [params.user=''] - MongoDB username for authenticated restore.
+      * @param {string} [params.password=''] - MongoDB password for authenticated restore.
+      * @param {string} [params.authDatabase='admin'] - Auth database for restore command.
      * @return {boolean} Success status.
      */
-    _importMongoDB({ pod, namespace, dbName, bsonPath, drop, preserveUUID }) {
+    _importMongoDB({ pod, namespace, dbName, bsonPath, drop, preserveUUID, user = '', password = '', authDatabase = 'admin' }) {
       try {
         const podName = pod.NAME;
         const containerBsonPath = `/${dbName}`;
@@ -268,9 +272,10 @@ class UnderpostDB {
         }
 
         // Restore database
-        const restoreCmd = `mongorestore -d ${dbName} ${containerBsonPath}${drop ? ' --drop' : ''}${
-          preserveUUID ? ' --preserveUUID' : ''
-        }`;
+        const authFlags = user && password
+          ? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
+          : '';
+        const restoreCmd = `mongorestore -d ${dbName} ${containerBsonPath}${drop ? ' --drop' : ''}${preserveUUID ? ' --preserveUUID' : ''}${authFlags}`;
         Underpost.kubectl.exec({ podName, namespace, command: restoreCmd });
 
         logger.info('Successfully imported MongoDB database', { podName, dbName });
@@ -291,9 +296,12 @@ class UnderpostDB {
      * @param {string} params.dbName - Database name.
      * @param {string} params.outputPath - Output directory path.
      * @param {string} [params.collections=''] - Comma-separated collection list.
+      * @param {string} [params.user=''] - MongoDB username for authenticated dump.
+      * @param {string} [params.password=''] - MongoDB password for authenticated dump.
+      * @param {string} [params.authDatabase='admin'] - Auth database for dump command.
      * @return {boolean} Success status.
      */
-    _exportMongoDB({ pod, namespace, dbName, outputPath, collections = '' }) {
+    _exportMongoDB({ pod, namespace, dbName, outputPath, collections = '', user = '', password = '', authDatabase = 'admin' }) {
       try {
         const podName = pod.NAME;
         const containerBsonPath = `/${dbName}`;
@@ -308,14 +316,18 @@ class UnderpostDB {
         });
 
         // Dump database or specific collections
+        const authFlags = user && password
+          ? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
+          : '';
+
         if (collections) {
           const collectionList = collections.split(',').map((c) => c.trim());
           for (const collection of collectionList) {
-            const dumpCmd = `mongodump -d ${dbName} --collection ${collection} -o /`;
+            const dumpCmd = `mongodump -d ${dbName} --collection ${collection} -o /${authFlags}`;
             Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
           }
         } else {
-          const dumpCmd = `mongodump -d ${dbName} -o /`;
+          const dumpCmd = `mongodump -d ${dbName} -o /${authFlags}`;
           Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
         }
 
@@ -347,9 +359,12 @@ class UnderpostDB {
      * @param {string} params.podName - Pod name.
      * @param {string} params.namespace - Namespace.
      * @param {string} params.dbName - Database name.
+      * @param {string} [params.user=''] - MongoDB username for authenticated stats query.
+      * @param {string} [params.password=''] - MongoDB password for authenticated stats query.
+      * @param {string} [params.authDatabase='admin'] - Auth database for stats query.
      * @return {Object|null} Collection statistics or null on error.
      */
-    _getMongoStats({ podName, namespace, dbName }) {
+    _getMongoStats({ podName, namespace, dbName, user = '', password = '', authDatabase = 'admin' }) {
       try {
         logger.info('Getting MongoDB collection statistics', { podName, dbName });
 
@@ -357,8 +372,11 @@ class UnderpostDB {
         const script = `db.getSiblingDB('${dbName}').getCollectionNames().map(function(c) { return { collection: c, count: db.getSiblingDB('${dbName}')[c].countDocuments() }; })`;
 
         // Execute the script
-        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet --eval "${script}"`;
-        const output = shellExec(command, { stdout: true, silent: true });
+        const authFlags = user && password
+          ? ` --authenticationDatabase ${JSON.stringify(authDatabase)} -u ${JSON.stringify(user)} -p ${JSON.stringify(password)}`
+          : '';
+        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet${authFlags} --eval "${script}"`;
+        const output = shellExec(command, { stdout: true, silent: true, silentOnError: true });
 
         if (!output || output.trim() === '') {
           logger.warn('No collections found or empty output');
@@ -415,7 +433,7 @@ class UnderpostDB {
         logger.info('Getting MariaDB table statistics', { podName, dbName });
 
         const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mariadb -u ${user} -p${password} ${dbName} -e "SELECT TABLE_NAME as 'table', TABLE_ROWS as 'count' FROM information_schema.TABLES WHERE TABLE_SCHEMA = '${dbName}' ORDER BY TABLE_NAME;" --skip-column-names --batch`;
-        const output = shellExec(command, { stdout: true, silent: true, disableLog: true });
+        const output = shellExec(command, { stdout: true, silent: true, disableLog: true, silentOnError: true });
 
         if (!output || output.trim() === '') {
           logger.warn('No tables found or empty output');
@@ -475,47 +493,7 @@ class UnderpostDB {
       console.log('='.repeat(70) + '\n');
     },
 
-    /**
-     * Gets MongoDB primary pod name from replica set status.
-     * @method getMongoPrimaryPodName
-     * @memberof UnderpostDB
-     * @param {Object} [options={}] - Options for getting primary pod.
-     * @param {string} [options.namespace='default'] - Kubernetes namespace.
-     * @param {string} [options.podName='mongodb-0'] - Initial pod name to query replica set status.
-     * @return {string|null} Primary pod name or null if not found.
-     */
-    getMongoPrimaryPodName(options = { namespace: 'default', podName: 'mongodb-0' }) {
-      const { namespace = 'default', podName = 'mongodb-0' } = options;
-
-      try {
-        logger.info('Checking for MongoDB primary pod', { namespace, checkingPod: podName });
-
-        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet --eval 'rs.status().members.filter(m => m.stateStr=="PRIMARY").map(m=>m.name)'`;
-        const output = shellExec(command, { stdout: true, silent: true });
-
-        if (!output || output.trim() === '') {
-          logger.warn('No primary pod found in replica set');
-          return null;
-        }
-
-        // Parse the output to get the primary pod name
-        // Output format: [ 'mongodb-0:27017' ] or [ 'mongodb-1.mongodb-service:27017' ] or similar
-        const match = output.match(/['"]([^'"]+)['"]/);
-        if (match && match[1]) {
-          let primaryName = match[1].split(':')[0]; // Extract pod name without port
-          // Remove service suffix if present (e.g., "mongodb-1.mongodb-service" -> "mongodb-1")
-          primaryName = primaryName.split('.')[0];
-          logger.info('Found MongoDB primary pod', { primaryPod: primaryName });
-          return primaryName;
-        }
 
-        logger.warn('Could not parse primary pod from replica set status', { output });
-        return null;
-      } catch (error) {
-        logger.error('Failed to get MongoDB primary pod', { error: error.message });
-        return null;
-      }
-    },
 
     /**
      * Main callback: Initiates database backup workflow.
@@ -626,7 +604,13 @@ class UnderpostDB {
         });
 
         if (options.primaryPodEnsure) {
-          const primaryPodName = Underpost.db.getMongoPrimaryPodName({ namespace, podName: options.primaryPodEnsure });
+          const primaryPodName = MongoBootstrap.getPrimaryPodName({
+            namespace,
+            podName: options.primaryPodEnsure,
+            username: process.env.MONGODB_USERNAME || process.env.DB_USER || '',
+            password: process.env.MONGODB_PASSWORD || process.env.DB_PASSWORD || '',
+            authDatabase: process.env.MONGODB_AUTH_DB || 'admin',
+          });
           if (!primaryPodName) {
             const baseCommand = options.dev ? 'node bin' : 'underpost';
             const baseClusterCommand = options.dev ? ' --dev' : '';
@@ -813,7 +797,13 @@ class UnderpostDB {
                   podsToProcess = [];
                 } else {
                   const firstPod = targetPods[0].NAME;
-                  const primaryPodName = Underpost.db.getMongoPrimaryPodName({ namespace, podName: firstPod });
+                  const primaryPodName = MongoBootstrap.getPrimaryPodName({
+                    namespace,
+                    podName: firstPod,
+                    username: user,
+                    password,
+                    authDatabase: 'admin',
+                  });
 
                   if (primaryPodName) {
                     const primaryPod = targetPods.find((p) => p.NAME === primaryPodName);
@@ -839,6 +829,8 @@ class UnderpostDB {
                 pods: podsToProcess.map((p) => p.NAME),
               });
 
+              let exportSucceeded = false;
+
               // Process each pod
               for (const pod of podsToProcess) {
                 logger.info('Processing pod', { podName: pod.NAME, node: pod.NODE, status: pod.STATUS });
@@ -871,7 +863,7 @@ class UnderpostDB {
 
                     if (options.export === true) {
                       const outputPath = options.outPath || toNewSqlPath;
-                      await Underpost.db._exportMariaDB({
+                      const success = await Underpost.db._exportMariaDB({
                         pod,
                         namespace,
                         dbName,
@@ -879,6 +871,7 @@ class UnderpostDB {
                         password,
                         outputPath,
                       });
+                      exportSucceeded = exportSucceeded || success;
                     }
                     break;
                   }
@@ -889,6 +882,9 @@ class UnderpostDB {
                         podName: pod.NAME,
                         namespace,
                         dbName,
+                        user,
+                        password,
+                        authDatabase: 'admin',
                       });
                       if (stats) {
                         Underpost.db._displayStats({ provider, dbName, stats });
@@ -904,18 +900,25 @@ class UnderpostDB {
                         bsonPath,
                         drop: options.drop,
                         preserveUUID: options.preserveUUID,
+                        user,
+                        password,
+                        authDatabase: 'admin',
                       });
                     }
 
                     if (options.export === true) {
                       const outputPath = options.outPath || toNewBsonPath;
-                      Underpost.db._exportMongoDB({
+                      const success = Underpost.db._exportMongoDB({
                         pod,
                         namespace,
                         dbName,
                         outputPath,
                         collections: options.collections,
+                        user,
+                        password,
+                        authDatabase: 'admin',
                       });
+                      exportSucceeded = exportSucceeded || success;
                     }
                     break;
                   }
@@ -926,6 +929,10 @@ class UnderpostDB {
                 }
               }
 
+              if (options.export === true && exportSucceeded === true) {
+                Underpost.db._enforceBackupRetention(`../${repoName}/${hostFolder}`);
+              }
+
               // Mark this host+path combination as processed
               processedHostPaths.add(hostPathKey);
             }
@@ -948,6 +955,43 @@ class UnderpostDB {
         throw error;
       }
     },
+    /**
+     * Helper: Removes old timestamp backup folders and keeps only the newest ones.
+     * @method _enforceBackupRetention
+     * @memberof UnderpostDB
+     * @param {string} backupDir - Path to host-folder backup directory.
+     * @param {number} [maxRetention=MAX_BACKUP_RETENTION] - Maximum folders to keep.
+     * @return {number} Number of removed backup folders.
+     */
+    _enforceBackupRetention(backupDir, maxRetention = MAX_BACKUP_RETENTION) {
+      try {
+        if (!fs.existsSync(backupDir)) return 0;
+
+        const timestamps = fs
+          .readdirSync(backupDir)
+          .filter((entry) => /^\d+$/.test(entry))
+          .sort((a, b) => parseInt(b, 10) - parseInt(a, 10));
+
+        if (timestamps.length <= maxRetention) return 0;
+
+        const staleTimestamps = timestamps.slice(maxRetention);
+        staleTimestamps.forEach((timestamp) => {
+          fs.removeSync(`${backupDir}/${timestamp}`);
+        });
+
+        logger.info('Pruned old backup timestamp folders', {
+          backupDir,
+          kept: maxRetention,
+          removed: staleTimestamps.length,
+          removedTimestamps: staleTimestamps,
+        });
+
+        return staleTimestamps.length;
+      } catch (error) {
+        logger.error('Failed to enforce backup retention', { backupDir, maxRetention, error: error.message });
+        return 0;
+      }
+    },
 
     /**
      * Creates cluster metadata for the specified deployment.
@@ -999,7 +1043,7 @@ class UnderpostDB {
         const retryDelay = 3000;
         for (let attempt = 1; attempt <= maxRetries; attempt++) {
           try {
-            await DataBaseProvider.load({ apis: ['instance', 'cron'], host, path, db });
+            await DataBaseProviderService.load({ apis: ['instance', 'cron'], host, path, db });
             break;
           } catch (err) {
             if (attempt === maxRetries) {
@@ -1013,7 +1057,7 @@ class UnderpostDB {
 
         try {
           /** @type {import('../api/instance/instance.model.js').InstanceModel} */
-          const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
+          const Instance = DataBaseProviderService.getModel('instance', { host, path });
 
           await Instance.deleteMany();
           logger.info('Cleared existing instance metadata');
@@ -1114,10 +1158,10 @@ class UnderpostDB {
 
           const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
 
-          await DataBaseProvider.load({ apis: ['cron'], host, path, db });
+          await DataBaseProviderService.load({ apis: ['cron'], host, path, db });
 
           /** @type {import('../api/cron/cron.model.js').CronModel} */
-          const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
+          const Cron = DataBaseProviderService.getModel('cron', { host, path });
 
           await Cron.deleteMany();
           logger.info('Cleared existing cron metadata');
@@ -1136,7 +1180,7 @@ class UnderpostDB {
           logger.error('Failed to create cron metadata', { error: error.message });
         }
 
-        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+        await DataBaseProviderService.getProvider({ host, path }, 'mongoose').close();
         logger.info('Cluster metadata creation completed');
       } catch (error) {
         logger.error('Cluster metadata creation failed', { error: error.message });
@@ -1229,7 +1273,7 @@ class UnderpostDB {
                 let dbProvider;
                 for (let attempt = 1; attempt <= 3; attempt++) {
                   try {
-                    dbProvider = await DataBaseProvider.load({ apis, host, path, db });
+                    dbProvider = await DataBaseProviderService.load({ apis, host, path, db });
                     break;
                   } catch (err) {
                     if (attempt === 3) throw err;