underpost 3.1.3 → 3.2.2

package/src/cli/db.js

@@ -13,23 +13,10 @@ import fs from 'fs-extra';
 import { DataBaseProvider } from '../db/DataBaseProvider.js';
 import { loadReplicas, pathPortAssignmentFactory, loadCronDeployEnv } from '../server/conf.js';
 import Underpost from '../index.js';
+import { timer } from '../client/components/core/CommonJs.js';
+import isInsideContainer from 'is-inside-container';
 const logger = loggerFactory(import.meta);
 
-/**
- * Redacts credentials from shell command strings before logging.
- * Masks passwords in `-p<password>`, `--password=<password>`, and `-P <password>` patterns.
- * @param {string} cmd - The raw command string.
- * @memberof UnderpostDB
- * @returns {string} The command with credentials replaced by `***`.
- */
-const sanitizeCommand = (cmd) => {
-  if (typeof cmd !== 'string') return cmd;
-  return cmd
-    .replace(/-p['"]?[^\s'"]+/g, '-p***')
-    .replace(/--password=['"]?[^\s'"]+/g, '--password=***')
-    .replace(/-P\s+['"]?[^\s'"]+/g, '-P ***');
-};
-
 /**
  * Constants for database operations
  * @constant {number} MAX_BACKUP_RETENTION - Maximum number of backups to retain
@@ -98,132 +85,6 @@ class UnderpostDB {
    * @memberof UnderpostDB
    */
   static API = {
-    /**
-     * Helper: Gets filtered pods based on criteria.
-     * @method _getFilteredPods
-     * @memberof UnderpostDB
-     * @param {Object} criteria - Filter criteria.
-     * @param {string} [criteria.podNames] - Comma-separated pod name patterns.
-     * @param {string} [criteria.namespace='default'] - Kubernetes namespace.
-     * @param {string} [criteria.deployId] - Deployment ID pattern.
-     * @return {Array<PodInfo>} Filtered pod list.
-     */
-    _getFilteredPods(criteria = {}) {
-      const { podNames, namespace = 'default', deployId } = criteria;
-
-      try {
-        // Get all pods using Underpost.deploy.get
-        let pods = Underpost.deploy.get(deployId || '', 'pods', namespace);
-
-        // Filter by pod names if specified
-        if (podNames) {
-          const patterns = podNames.split(',').map((p) => p.trim());
-          pods = pods.filter((pod) => {
-            return patterns.some((pattern) => {
-              // Support wildcards
-              const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
-              return regex.test(pod.NAME);
-            });
-          });
-        }
-
-        logger.info(`Found ${pods.length} pod(s) matching criteria`, { criteria, podNames: pods.map((p) => p.NAME) });
-        return pods;
-      } catch (error) {
-        logger.error('Error filtering pods', { error: error.message, criteria });
-        return [];
-      }
-    },
-
-    /**
-     * Helper: Executes kubectl command with error handling.
-     * @method _executeKubectl
-     * @memberof UnderpostDB
-     * @param {string} command - kubectl command to execute.
-     * @param {Object} [options={}] - Execution options.
-     * @param {string} [options.context=''] - Command context for logging.
-     * @return {string|null} Command output or null on error.
-     */
-    _executeKubectl(command, options = {}) {
-      const { context = '' } = options;
-
-      try {
-        logger.info(`Executing kubectl command`, { command: sanitizeCommand(command), context });
-        return shellExec(command, { stdout: true, disableLog: true });
-      } catch (error) {
-        logger.error(`kubectl command failed`, { command: sanitizeCommand(command), error: error.message, context });
-        throw error;
-      }
-    },
-
-    /**
-     * Helper: Copies file to pod.
-     * @method _copyToPod
-     * @memberof UnderpostDB
-     * @param {Object} params - Copy parameters.
-     * @param {string} params.sourcePath - Source file path.
-     * @param {string} params.podName - Target pod name.
-     * @param {string} params.namespace - Pod namespace.
-     * @param {string} params.destPath - Destination path in pod.
-     * @return {boolean} Success status.
-     */
-    _copyToPod({ sourcePath, podName, namespace, destPath }) {
-      try {
-        const command = `sudo kubectl cp ${sourcePath} ${namespace}/${podName}:${destPath}`;
-        Underpost.db._executeKubectl(command, { context: `copy to pod ${podName}` });
-        return true;
-      } catch (error) {
-        logger.error('Failed to copy file to pod', { sourcePath, podName, destPath, error: error.message });
-        return false;
-      }
-    },
-
-    /**
-     * Helper: Copies file from pod.
-     * @method _copyFromPod
-     * @memberof UnderpostDB
-     * @param {Object} params - Copy parameters.
-     * @param {string} params.podName - Source pod name.
-     * @param {string} params.namespace - Pod namespace.
-     * @param {string} params.sourcePath - Source path in pod.
-     * @param {string} params.destPath - Destination file path.
-     * @return {boolean} Success status.
-     */
-    _copyFromPod({ podName, namespace, sourcePath, destPath }) {
-      try {
-        const command = `sudo kubectl cp ${namespace}/${podName}:${sourcePath} ${destPath}`;
-        Underpost.db._executeKubectl(command, { context: `copy from pod ${podName}` });
-        return true;
-      } catch (error) {
-        logger.error('Failed to copy file from pod', { podName, sourcePath, destPath, error: error.message });
-        return false;
-      }
-    },
-
-    /**
-     * Helper: Executes command in pod.
-     * @method _execInPod
-     * @memberof UnderpostDB
-     * @param {Object} params - Execution parameters.
-     * @param {string} params.podName - Pod name.
-     * @param {string} params.namespace - Pod namespace.
-     * @param {string} params.command - Command to execute.
-     * @return {string|null} Command output or null.
-     */
-    _execInPod({ podName, namespace, command }) {
-      try {
-        const kubectlCmd = `sudo kubectl exec -n ${namespace} -i ${podName} -- sh -c "${command}"`;
-        return Underpost.db._executeKubectl(kubectlCmd, { context: `exec in pod ${podName}` });
-      } catch (error) {
-        logger.error('Failed to execute command in pod', {
-          podName,
-          command: sanitizeCommand(command),
-          error: error.message,
-        });
-        throw error;
-      }
-    },
-
     /**
      * Helper: Resolves the latest backup timestamp from an existing backup directory.
      * Scans the directory for numeric (epoch) sub-folders and returns the most recent one.
@@ -239,76 +100,6 @@ class UnderpostDB {
       return entries.sort((a, b) => parseInt(b) - parseInt(a))[0];
     },
 
-    /**
-     * Helper: Manages Git repository for backups.
-     * @method _manageGitRepo
-     * @memberof UnderpostDB
-     * @param {Object} params - Git parameters.
-     * @param {string} params.repoName - Repository name.
-     * @param {string} params.operation - Operation (clone, pull, commit, push).
-     * @param {string} [params.message=''] - Commit message.
-     * @param {boolean} [params.forceClone=false] - Force remove and re-clone repository.
-     * @return {boolean} Success status.
-     */
-    _manageGitRepo({ repoName, operation, message = '', forceClone = false }) {
-      try {
-        const username = process.env.GITHUB_USERNAME;
-        if (!username) {
-          logger.error('GITHUB_USERNAME environment variable not set');
-          return false;
-        }
-
-        const repoPath = `../${repoName}`;
-
-        switch (operation) {
-          case 'clone':
-            if (forceClone && fs.existsSync(repoPath)) {
-              logger.info(`Force clone enabled, removing existing repository: ${repoName}`);
-              fs.removeSync(repoPath);
-            }
-            if (!fs.existsSync(repoPath)) {
-              shellExec(`cd .. && underpost clone ${username}/${repoName}`);
-              logger.info(`Cloned repository: ${repoName}`);
-            }
-            break;
-
-          case 'pull':
-            if (fs.existsSync(repoPath)) {
-              shellExec(`cd ${repoPath} && git checkout . && git clean -f -d`);
-              shellExec(`cd ${repoPath} && underpost pull . ${username}/${repoName}`, {
-                silent: true,
-              });
-              logger.info(`Pulled repository: ${repoName}`);
-            }
-            break;
-
-          case 'commit':
-            if (fs.existsSync(repoPath)) {
-              shellExec(`cd ${repoPath} && git add .`);
-              shellExec(`underpost cmt ${repoPath} backup '' '${message}'`);
-              logger.info(`Committed to repository: ${repoName}`, { message });
-            }
-            break;
-
-          case 'push':
-            if (fs.existsSync(repoPath)) {
-              shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName}`, { silent: true });
-              logger.info(`Pushed repository: ${repoName}`);
-            }
-            break;
-
-          default:
-            logger.warn(`Unknown git operation: ${operation}`);
-            return false;
-        }
-
-        return true;
-      } catch (error) {
-        logger.error(`Git operation failed`, { repoName, operation, error: error.message });
-        return false;
-      }
-    },
-
     /**
      * Helper: Performs MariaDB import operation.
      * @method _importMariaDB
@@ -329,8 +120,20 @@ class UnderpostDB {
 
         logger.info('Importing MariaDB database', { podName, dbName });
 
+        // Always ensure the database exists first — required for WP even when no backup is available
+        Underpost.kubectl.run(
+          `kubectl exec -n ${namespace} -i ${podName} -- mariadb -p${password} -e 'CREATE DATABASE IF NOT EXISTS ${dbName};'`,
+          { context: `create database ${dbName}` },
+        );
+
+        // If no SQL file is available, the empty database is enough — return early
+        if (!sqlPath || !fs.existsSync(sqlPath)) {
+          logger.warn('No SQL backup file found — empty database ensured', { podName, dbName, sqlPath });
+          return true;
+        }
+
         // Remove existing SQL file in container
-        Underpost.db._execInPod({
+        Underpost.kubectl.exec({
           podName,
           namespace,
           command: `rm -rf ${containerSqlPath}`,
@@ -338,7 +141,7 @@ class UnderpostDB {
 
         // Copy SQL file to pod
         if (
-          !Underpost.db._copyToPod({
+          !Underpost.kubectl.cpTo({
             sourcePath: sqlPath,
             podName,
             namespace,
@@ -348,15 +151,9 @@ class UnderpostDB {
           return false;
         }
 
-        // Create database if it doesn't exist
-        Underpost.db._executeKubectl(
-          `kubectl exec -n ${namespace} -i ${podName} -- mariadb -p${password} -e 'CREATE DATABASE IF NOT EXISTS ${dbName};'`,
-          { context: `create database ${dbName}` },
-        );
-
         // Import SQL file
         const importCmd = `mariadb -u ${user} -p${password} ${dbName} < ${containerSqlPath}`;
-        Underpost.db._execInPod({ podName, namespace, command: importCmd });
+        Underpost.kubectl.exec({ podName, namespace, command: importCmd });
 
         logger.info('Successfully imported MariaDB database', { podName, dbName });
         return true;
@@ -387,7 +184,7 @@ class UnderpostDB {
         logger.info('Exporting MariaDB database', { podName, dbName });
 
         // Remove existing SQL file in container
-        Underpost.db._execInPod({
+        Underpost.kubectl.exec({
           podName,
           namespace,
           command: `rm -rf ${containerSqlPath}`,
@@ -395,11 +192,11 @@ class UnderpostDB {
 
         // Dump database
         const dumpCmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${containerSqlPath}`;
-        Underpost.db._execInPod({ podName, namespace, command: dumpCmd });
+        Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
 
         // Copy SQL file from pod
         if (
-          !Underpost.db._copyFromPod({
+          !Underpost.kubectl.cpFrom({
             podName,
             namespace,
             sourcePath: containerSqlPath,
@@ -442,8 +239,18 @@ class UnderpostDB {
 
         logger.info('Importing MongoDB database', { podName, dbName });
 
+        // If no BSON directory is available, skip — MongoDB creates the DB on first write
+        if (!bsonPath || !fs.existsSync(bsonPath)) {
+          logger.warn('No BSON backup directory found — database will be created on first write', {
+            podName,
+            dbName,
+            bsonPath,
+          });
+          return true;
+        }
+
         // Remove existing BSON directory in container
-        Underpost.db._execInPod({
+        Underpost.kubectl.exec({
           podName,
           namespace,
           command: `rm -rf ${containerBsonPath}`,
@@ -451,7 +258,7 @@ class UnderpostDB {
 
         // Copy BSON directory to pod
         if (
-          !Underpost.db._copyToPod({
+          !Underpost.kubectl.cpTo({
             sourcePath: bsonPath,
             podName,
             namespace,
@@ -465,7 +272,7 @@ class UnderpostDB {
         const restoreCmd = `mongorestore -d ${dbName} ${containerBsonPath}${drop ? ' --drop' : ''}${
           preserveUUID ? ' --preserveUUID' : ''
         }`;
-        Underpost.db._execInPod({ podName, namespace, command: restoreCmd });
+        Underpost.kubectl.exec({ podName, namespace, command: restoreCmd });
 
         logger.info('Successfully imported MongoDB database', { podName, dbName });
         return true;
@@ -495,7 +302,7 @@ class UnderpostDB {
         logger.info('Exporting MongoDB database', { podName, dbName, collections });
 
         // Remove existing BSON directory in container
-        Underpost.db._execInPod({
+        Underpost.kubectl.exec({
           podName,
           namespace,
           command: `rm -rf ${containerBsonPath}`,
@@ -506,16 +313,16 @@ class UnderpostDB {
           const collectionList = collections.split(',').map((c) => c.trim());
           for (const collection of collectionList) {
             const dumpCmd = `mongodump -d ${dbName} --collection ${collection} -o /`;
-            Underpost.db._execInPod({ podName, namespace, command: dumpCmd });
+            Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
           }
         } else {
           const dumpCmd = `mongodump -d ${dbName} -o /`;
-          Underpost.db._execInPod({ podName, namespace, command: dumpCmd });
+          Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
         }
 
         // Copy BSON directory from pod
         if (
-          !Underpost.db._copyFromPod({
+          !Underpost.kubectl.cpFrom({
             podName,
             namespace,
             sourcePath: containerBsonPath,
@@ -743,6 +550,7 @@ class UnderpostDB {
      * @param {boolean} [options.k3s=false] - k3s cluster flag.
      * @param {boolean} [options.kubeadm=false] - kubeadm cluster flag.
      * @param {boolean} [options.kind=false] - kind cluster flag.
+     * @param {boolean} [options.repoBackup=false] - Backs up repositories (git commit+push) inside deployment pods via kubectl exec.
      * @return {Promise<void>} Resolves when operation is complete.
      */
     async callback(
@@ -771,346 +579,381 @@ class UnderpostDB {
         k3s: false,
         kubeadm: false,
         kind: false,
+        repoBackup: false,
       },
     ) {
-      loadCronDeployEnv();
-      const newBackupTimestamp = new Date().getTime();
-      const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
-
-      if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
-
-      // Handle clean-fs-collection operation
-      if (options.cleanFsCollection || options.cleanFsDryRun) {
-        logger.info('Starting File collection cleanup operation', { deployList });
-        await Underpost.db.cleanFsCollection(deployList, {
-          hosts: options.hosts,
-          paths: options.paths,
-          dryRun: options.cleanFsDryRun,
-        });
-        return;
-      }
+      // Ensure engine-private is available (clone ephemerally if inside a deployment
+      // container where globalSecretClean has already removed it).
+      const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
+      const { ephemeral } = Underpost.repo.privateEngineRepoFactory(firstDeployId || undefined);
+      try {
+        loadCronDeployEnv();
+        const newBackupTimestamp = new Date().getTime();
+        const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
+
+        if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
+
+        // Handle repository backup (git commit+push inside deployment pod)
+        if (options.repoBackup) {
+          const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
+          for (const _deployId of deployList.split(',')) {
+            const deployId = _deployId.trim();
+            if (!deployId) continue;
+            logger.info('Starting pod repository backup', { deployId, namespace });
+            Underpost.repo.backupPodRepositories({
+              deployId,
+              namespace,
+              env: options.dev ? 'development' : 'production',
+            });
+          }
+          return;
+        }
 
-      logger.info('Starting database operation', {
-        deployList,
-        namespace,
-        import: options.import,
-        export: options.export,
-      });
+        // Handle clean-fs-collection operation
+        if (options.cleanFsCollection || options.cleanFsDryRun) {
+          logger.info('Starting File collection cleanup operation', { deployList });
+          await Underpost.db.cleanFsCollection(deployList, {
+            hosts: options.hosts,
+            paths: options.paths,
+            dryRun: options.cleanFsDryRun,
+          });
+          return;
+        }
 
-      if (options.primaryPodEnsure) {
-        const primaryPodName = Underpost.db.getMongoPrimaryPodName({ namespace, podName: options.primaryPodEnsure });
-        if (!primaryPodName) {
-          const baseCommand = options.dev ? 'node bin' : 'underpost';
-          const baseClusterCommand = options.dev ? ' --dev' : '';
-          let clusterFlag = options.k3s ? ' --k3s' : options.kubeadm ? ' --kubeadm' : '';
-          shellExec(`${baseCommand} cluster${baseClusterCommand}${clusterFlag} --mongodb`);
+        logger.info('Starting database operation', {
+          deployList,
+          namespace,
+          import: options.import,
+          export: options.export,
+        });
+
+        if (options.primaryPodEnsure) {
+          const primaryPodName = Underpost.db.getMongoPrimaryPodName({ namespace, podName: options.primaryPodEnsure });
+          if (!primaryPodName) {
+            const baseCommand = options.dev ? 'node bin' : 'underpost';
+            const baseClusterCommand = options.dev ? ' --dev' : '';
+            let clusterFlag = options.k3s ? ' --k3s' : options.kubeadm ? ' --kubeadm' : '';
+            shellExec(`${baseCommand} cluster${baseClusterCommand}${clusterFlag} --mongodb`);
+          }
+          return;
         }
-        return;
-      }
 
-      // Track processed repositories to avoid duplicate Git operations
-      const processedRepos = new Set();
-      // Track processed host+path combinations to avoid duplicates
-      const processedHostPaths = new Set();
+        // Track processed repositories to avoid duplicate Git operations
+        const processedRepos = new Set();
+        // Track processed host+path combinations to avoid duplicates
+        const processedHostPaths = new Set();
 
-      for (const _deployId of deployList.split(',')) {
-        const deployId = _deployId.trim();
-        if (!deployId) continue;
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
 
-        logger.info('Processing deployment', { deployId });
+          logger.info('Processing deployment', { deployId });
 
-        /** @type {Object.<string, Object.<string, DatabaseConfig>>} */
-        const dbs = {};
-        const repoName = `engine-${deployId.includes('dd-') ? deployId.split('dd-')[1] : deployId}-cron-backups`;
+          /** @type {Object.<string, Object.<string, DatabaseConfig>>} */
+          const dbs = {};
+          const repoName = `engine-${deployId.includes('dd-') ? deployId.split('dd-')[1] : deployId}-cron-backups`;
 
-        // Load server configuration
-        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-        if (!fs.existsSync(confServerPath)) {
-          logger.error('Configuration file not found', { path: confServerPath });
-          continue;
-        }
+          // Load server configuration
+          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+          if (!fs.existsSync(confServerPath)) {
+            logger.error('Configuration file not found', { path: confServerPath });
+            continue;
+          }
 
-        const confServer = loadConfServerJson(confServerPath, { resolve: true });
-
-        // Build database configuration map
-        for (const host of Object.keys(confServer)) {
-          for (const path of Object.keys(confServer[host])) {
-            const { db } = confServer[host][path];
-            if (db) {
-              const { provider, name, user, password } = db;
-              if (!dbs[provider]) dbs[provider] = {};
-
-              if (!(name in dbs[provider])) {
-                dbs[provider][name] = {
-                  user,
-                  password,
-                  hostFolder: host + path.replaceAll('/', '-'),
-                  host,
-                  path,
-                };
+          const confServer = loadConfServerJson(confServerPath, { resolve: true });
+
+          // Build database configuration map
+          for (const host of Object.keys(confServer)) {
+            for (const path of Object.keys(confServer[host])) {
+              const { db } = confServer[host][path];
+              if (db) {
+                const { provider, name, user, password } = db;
+                if (!dbs[provider]) dbs[provider] = {};
+
+                if (!(name in dbs[provider])) {
+                  dbs[provider][name] = {
+                    user,
+                    password,
+                    hostFolder: host + path.replaceAll('/', '-'),
+                    host,
+                    path,
+                  };
+                }
               }
             }
           }
-        }
-
-        // Handle Git operations - execute only once per repository
-        if (!processedRepos.has(repoName)) {
-          logger.info('Processing Git operations for repository', { repoName, deployId });
-          if (options.git === true) {
-            Underpost.db._manageGitRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
-            Underpost.db._manageGitRepo({ repoName, operation: 'pull' });
-          }
 
-          if (options.macroRollbackExport) {
-            // Only clone if not already done by git option above
-            if (options.git !== true) {
-              Underpost.db._manageGitRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
-              Underpost.db._manageGitRepo({ repoName, operation: 'pull' });
+          // Handle Git operations - execute only once per repository
+          if (!processedRepos.has(repoName)) {
+            logger.info('Processing Git operations for repository', { repoName, deployId });
+            if (options.git === true) {
+              Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
+              Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
             }
 
-            const nCommits = parseInt(options.macroRollbackExport);
-            const repoPath = `../${repoName}`;
-            const username = process.env.GITHUB_USERNAME;
-
-            if (fs.existsSync(repoPath) && username) {
-              logger.info('Executing macro rollback export', { repoName, nCommits });
-              shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
-              shellExec(`cd ${repoPath} && git reset`);
-              shellExec(`cd ${repoPath} && git checkout .`);
-              shellExec(`cd ${repoPath} && git clean -f -d`);
-              shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
-            } else {
-              if (!username) logger.error('GITHUB_USERNAME environment variable not set');
-              logger.warn('Repository not found for macro rollback', { repoPath });
-            }
-          }
+            if (options.macroRollbackExport) {
+              // Only clone if not already done by git option above
+              if (options.git !== true) {
+                Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
+                Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
+              }
 
-          processedRepos.add(repoName);
-          logger.info('Repository marked as processed', { repoName });
-        } else {
-          logger.info('Skipping Git operations for already processed repository', { repoName, deployId });
-        }
+              const nCommits = parseInt(options.macroRollbackExport);
+              const repoPath = `../${repoName}`;
+              const username = process.env.GITHUB_USERNAME;
+
+              if (fs.existsSync(repoPath) && username) {
+                logger.info('Executing macro rollback export', { repoName, nCommits });
+                shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
+                shellExec(`cd ${repoPath} && git reset`);
+                shellExec(`cd ${repoPath} && git checkout .`);
+                shellExec(`cd ${repoPath} && git clean -f -d`);
+                shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
+              } else {
+                if (!username) logger.error('GITHUB_USERNAME environment variable not set');
+                logger.warn('Repository not found for macro rollback', { repoPath });
+              }
+            }
 
-        // Process each database provider
-        for (const provider of Object.keys(dbs)) {
-          for (const dbName of Object.keys(dbs[provider])) {
-            const { hostFolder, user, password, host, path } = dbs[provider][dbName];
+            processedRepos.add(repoName);
+            logger.info('Repository marked as processed', { repoName });
+          } else {
+            logger.info('Skipping Git operations for already processed repository', { repoName, deployId });
+          }
 
-            // Create unique identifier for host+path combination
-            const hostPathKey = `${deployId}:${host}:${path}`;
+          // Process each database provider
+          for (const provider of Object.keys(dbs)) {
+            for (const dbName of Object.keys(dbs[provider])) {
+              const { hostFolder, user, password, host, path } = dbs[provider][dbName];
 
-            // Skip if this host+path combination was already processed
-            if (processedHostPaths.has(hostPathKey)) {
-              logger.info('Skipping already processed host/path', { dbName, host, path, deployId });
-              continue;
-            }
+              // Create unique identifier for host+path combination
+              const hostPathKey = `${deployId}:${host}:${path}`;
 
-            // Filter by hosts and paths if specified
-            if (
-              (options.hosts &&
-                !options.hosts
-                  .split(',')
-                  .map((h) => h.trim())
-                  .includes(host)) ||
-              (options.paths &&
-                !options.paths
-                  .split(',')
-                  .map((p) => p.trim())
-                  .includes(path))
-            ) {
-              logger.info('Skipping database due to host/path filter', { dbName, host, path });
-              continue;
-            }
+              // Skip if this host+path combination was already processed
+              if (processedHostPaths.has(hostPathKey)) {
+                logger.info('Skipping already processed host/path', { dbName, host, path, deployId });
+                continue;
+              }
 
-            if (!hostFolder) {
-              logger.warn('No hostFolder defined for database', { dbName, provider });
-              continue;
-            }
+              // Filter by hosts and paths if specified
+              if (
+                (options.hosts &&
+                  !options.hosts
+                    .split(',')
+                    .map((h) => h.trim())
+                    .includes(host)) ||
+                (options.paths &&
+                  !options.paths
+                    .split(',')
+                    .map((p) => p.trim())
+                    .includes(path))
+              ) {
+                logger.info('Skipping database due to host/path filter', { dbName, host, path });
+                continue;
+              }
 
-            logger.info('Processing database', { hostFolder, provider, dbName, deployId });
+              if (!hostFolder) {
+                logger.warn('No hostFolder defined for database', { dbName, provider });
+                continue;
+              }
 
-            const latestBackupTimestamp = Underpost.db._getLatestBackupTimestamp(`../${repoName}/${hostFolder}`);
+              logger.info('Processing database', { hostFolder, provider, dbName, deployId });
 
-            dbs[provider][dbName].currentBackupTimestamp = latestBackupTimestamp;
+              const latestBackupTimestamp = Underpost.db._getLatestBackupTimestamp(`../${repoName}/${hostFolder}`);
 
-            const currentTimestamp = latestBackupTimestamp || newBackupTimestamp;
-            const sqlContainerPath = `/home/${dbName}.sql`;
-            const fromPartsPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}-parths.json`;
-            const toSqlPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}.sql`;
-            const toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
-            const toBsonPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}`;
-            const toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
+              dbs[provider][dbName].currentBackupTimestamp = latestBackupTimestamp;
 
-            // Merge split SQL files if needed for import
-            if (options.import === true && fs.existsSync(fromPartsPath) && !fs.existsSync(toSqlPath)) {
-              const names = JSON.parse(fs.readFileSync(fromPartsPath, 'utf8')).map((_path) => {
-                return `../${repoName}/${hostFolder}/${currentTimestamp}/${_path.split('/').pop()}`;
-              });
-              logger.info('Merging backup parts', { fromPartsPath, toSqlPath, parts: names.length });
-              await mergeFile(names, toSqlPath);
-            }
+              const currentTimestamp = latestBackupTimestamp || newBackupTimestamp;
+              const sqlContainerPath = `/home/${dbName}.sql`;
+              const fromPartsPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}-parths.json`;
+              const toSqlPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}.sql`;
+              const toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
+              const toBsonPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}`;
+              const toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
 
-            // Get target pods based on provider and options
-            let targetPods = [];
-            const podCriteria = {
-              podNames: options.podName,
-              namespace,
-              deployId: provider === 'mariadb' ? 'mariadb' : 'mongo',
-            };
+              // Merge split SQL files if needed for import
+              if (options.import === true && fs.existsSync(fromPartsPath) && !fs.existsSync(toSqlPath)) {
+                const names = JSON.parse(fs.readFileSync(fromPartsPath, 'utf8')).map((_path) => {
+                  return `../${repoName}/${hostFolder}/${currentTimestamp}/${_path.split('/').pop()}`;
+                });
+                logger.info('Merging backup parts', { fromPartsPath, toSqlPath, parts: names.length });
+                await mergeFile(names, toSqlPath);
+              }
 
-            targetPods = Underpost.db._getFilteredPods(podCriteria);
+              // Get target pods based on provider and options
+              let targetPods = [];
+              const podCriteria = {
+                podNames: options.podName,
+                namespace,
+                deployId: provider === 'mariadb' ? 'mariadb' : 'mongo',
+              };
 
-            // Fallback to default if no custom pods specified
-            if (targetPods.length === 0 && !options.podName) {
-              const defaultPods = Underpost.deploy.get(provider === 'mariadb' ? 'mariadb' : 'mongo', 'pods', namespace);
-              console.log('defaultPods', defaultPods);
-              targetPods = defaultPods;
-            }
+              targetPods = Underpost.kubectl.getFilteredPods(podCriteria);
+
+              // Fallback to default if no custom pods specified
+              if (targetPods.length === 0 && !options.podName) {
+                const defaultPods = Underpost.kubectl.get(
+                  provider === 'mariadb' ? 'mariadb' : 'mongo',
+                  'pods',
+                  namespace,
+                );
+                console.log('defaultPods', defaultPods);
+                targetPods = defaultPods;
+              }
 
-            if (targetPods.length === 0) {
-              logger.warn('No pods found matching criteria', { provider, criteria: podCriteria });
-              continue;
-            }
+              if (targetPods.length === 0) {
+                logger.warn('No pods found matching criteria', { provider, criteria: podCriteria });
+                continue;
+              }
 
-            // Handle primary pod detection for MongoDB
-            let podsToProcess = [];
-            if (provider === 'mongoose' && !options.allPods) {
-              // For MongoDB, always use primary pod unless allPods is true
-              if (!targetPods || targetPods.length === 0) {
-                logger.warn('No MongoDB pods available to check for primary');
-                podsToProcess = [];
-              } else {
-                const firstPod = targetPods[0].NAME;
-                const primaryPodName = Underpost.db.getMongoPrimaryPodName({ namespace, podName: firstPod });
-
-                if (primaryPodName) {
-                  const primaryPod = targetPods.find((p) => p.NAME === primaryPodName);
-                  if (primaryPod) {
-                    podsToProcess = [primaryPod];
-                    logger.info('Using MongoDB primary pod', { primaryPod: primaryPodName });
+              // Handle primary pod detection for MongoDB
+              let podsToProcess = [];
+              if (provider === 'mongoose' && !options.allPods) {
+                // For MongoDB, always use primary pod unless allPods is true
+                if (!targetPods || targetPods.length === 0) {
+                  logger.warn('No MongoDB pods available to check for primary');
+                  podsToProcess = [];
+                } else {
+                  const firstPod = targetPods[0].NAME;
+                  const primaryPodName = Underpost.db.getMongoPrimaryPodName({ namespace, podName: firstPod });
+
+                  if (primaryPodName) {
+                    const primaryPod = targetPods.find((p) => p.NAME === primaryPodName);
+                    if (primaryPod) {
+                      podsToProcess = [primaryPod];
+                      logger.info('Using MongoDB primary pod', { primaryPod: primaryPodName });
+                    } else {
+                      logger.warn('Primary pod not in filtered list, using first pod', { primaryPodName });
+                      podsToProcess = [targetPods[0]];
+                    }
                   } else {
-                    logger.warn('Primary pod not in filtered list, using first pod', { primaryPodName });
+                    logger.warn('Could not detect primary pod, using first pod');
                     podsToProcess = [targetPods[0]];
                   }
-                } else {
-                  logger.warn('Could not detect primary pod, using first pod');
-                  podsToProcess = [targetPods[0]];
                 }
+              } else {
+                // For MariaDB or when allPods is true, limit to first pod unless allPods is true
+                podsToProcess = options.allPods === true ? targetPods : [targetPods[0]];
               }
-            } else {
-              // For MariaDB or when allPods is true, limit to first pod unless allPods is true
-              podsToProcess = options.allPods === true ? targetPods : [targetPods[0]];
-            }
 
-            logger.info(`Processing ${podsToProcess.length} pod(s) for ${provider}`, {
-              dbName,
-              pods: podsToProcess.map((p) => p.NAME),
-            });
+              logger.info(`Processing ${podsToProcess.length} pod(s) for ${provider}`, {
+                dbName,
+                pods: podsToProcess.map((p) => p.NAME),
+              });
 
-            // Process each pod
-            for (const pod of podsToProcess) {
-              logger.info('Processing pod', { podName: pod.NAME, node: pod.NODE, status: pod.STATUS });
-
-              switch (provider) {
-                case 'mariadb': {
-                  if (options.stats === true) {
-                    const stats = Underpost.db._getMariaDBStats({
-                      podName: pod.NAME,
-                      namespace,
-                      dbName,
-                      user,
-                      password,
-                    });
-                    if (stats) {
-                      Underpost.db._displayStats({ provider, dbName, stats });
+              // Process each pod
+              for (const pod of podsToProcess) {
+                logger.info('Processing pod', { podName: pod.NAME, node: pod.NODE, status: pod.STATUS });
+
+                switch (provider) {
+                  case 'mariadb': {
+                    if (options.stats === true) {
+                      const stats = Underpost.db._getMariaDBStats({
+                        podName: pod.NAME,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                      });
+                      if (stats) {
+                        Underpost.db._displayStats({ provider, dbName, stats });
+                      }
                     }
-                  }
 
-                  if (options.import === true) {
-                    Underpost.db._importMariaDB({
-                      pod,
-                      namespace,
-                      dbName,
-                      user,
-                      password,
-                      sqlPath: toSqlPath,
-                    });
-                  }
+                    if (options.import === true) {
+                      Underpost.db._importMariaDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                        sqlPath: toSqlPath,
+                      });
+                    }
 
-                  if (options.export === true) {
-                    const outputPath = options.outPath || toNewSqlPath;
-                    await Underpost.db._exportMariaDB({
-                      pod,
-                      namespace,
-                      dbName,
-                      user,
-                      password,
-                      outputPath,
-                    });
+                    if (options.export === true) {
+                      const outputPath = options.outPath || toNewSqlPath;
+                      await Underpost.db._exportMariaDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                        outputPath,
+                      });
+                    }
+                    break;
                   }
-                  break;
-                }
 
-                case 'mongoose': {
-                  if (options.stats === true) {
-                    const stats = Underpost.db._getMongoStats({
-                      podName: pod.NAME,
-                      namespace,
-                      dbName,
-                    });
-                    if (stats) {
-                      Underpost.db._displayStats({ provider, dbName, stats });
+                  case 'mongoose': {
+                    if (options.stats === true) {
+                      const stats = Underpost.db._getMongoStats({
+                        podName: pod.NAME,
+                        namespace,
+                        dbName,
+                      });
+                      if (stats) {
+                        Underpost.db._displayStats({ provider, dbName, stats });
+                      }
                     }
-                  }
 
-                  if (options.import === true) {
-                    const bsonPath = options.outPath || toBsonPath;
-                    Underpost.db._importMongoDB({
-                      pod,
-                      namespace,
-                      dbName,
-                      bsonPath,
-                      drop: options.drop,
-                      preserveUUID: options.preserveUUID,
-                    });
-                  }
+                    if (options.import === true) {
+                      const bsonPath = options.outPath || toBsonPath;
+                      Underpost.db._importMongoDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        bsonPath,
+                        drop: options.drop,
+                        preserveUUID: options.preserveUUID,
+                      });
+                    }
 
-                  if (options.export === true) {
-                    const outputPath = options.outPath || toNewBsonPath;
-                    Underpost.db._exportMongoDB({
-                      pod,
-                      namespace,
-                      dbName,
-                      outputPath,
-                      collections: options.collections,
-                    });
+                    if (options.export === true) {
+                      const outputPath = options.outPath || toNewBsonPath;
+                      Underpost.db._exportMongoDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        outputPath,
+                        collections: options.collections,
+                      });
+                    }
+                    break;
                   }
-                  break;
-                }
 
-                default:
-                  logger.warn('Unsupported database provider', { provider });
-                  break;
+                  default:
+                    logger.warn('Unsupported database provider', { provider });
+                    break;
+                }
               }
+
+              // Mark this host+path combination as processed
+              processedHostPaths.add(hostPathKey);
             }
+          }
 
-            // Mark this host+path combination as processed
-            processedHostPaths.add(hostPathKey);
+          // Commit and push to Git if enabled - execute only once per repository
+          if (options.export === true && options.git === true && !processedRepos.has(`${repoName}-committed`)) {
+            const commitMessage = `${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
+              newBackupTimestamp,
+            ).toLocaleTimeString()}`;
+            Underpost.repo.manageBackupRepo({ repoName, operation: 'commit', message: commitMessage });
+            Underpost.repo.manageBackupRepo({ repoName, operation: 'push' });
+            processedRepos.add(`${repoName}-committed`);
           }
         }
 
-        // Commit and push to Git if enabled - execute only once per repository
-        if (options.export === true && options.git === true && !processedRepos.has(`${repoName}-committed`)) {
-          const commitMessage = `${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
-            newBackupTimestamp,
-          ).toLocaleTimeString()}`;
-          Underpost.db._manageGitRepo({ repoName, operation: 'commit', message: commitMessage });
-          Underpost.db._manageGitRepo({ repoName, operation: 'push' });
-          processedRepos.add(`${repoName}-committed`);
+        logger.info('Database operation completed successfully');
+      } catch (error) {
+        logger.error('Database operation failed', { error: error.message });
+        throw error;
+      } finally {
+        if (ephemeral && isInsideContainer()) {
+          Underpost.repo.cleanupPrivateEngineRepo();
+          Underpost.env.clean();
         }
       }
-
-      logger.info('Database operation completed successfully');
     },
 
     /**
@@ -1130,160 +973,185 @@ class UnderpostDB {
       host = process.env.DEFAULT_DEPLOY_HOST,
       path = process.env.DEFAULT_DEPLOY_PATH,
     ) {
-      loadCronDeployEnv();
-      deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
-      host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
-      path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
+      const { ephemeral } = Underpost.repo.privateEngineRepoFactory(deployId || undefined);
+      try {
+        loadCronDeployEnv();
+        deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
+        host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
+        path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
 
-      logger.info('Creating cluster metadata', { deployId, host, path });
+        logger.info('Creating cluster metadata', { deployId, host, path });
 
-      const env = 'production';
-      const deployListPath = './engine-private/deploy/dd.router';
+        const env = 'production';
+        const deployListPath = './engine-private/deploy/dd.router';
 
-      if (!fs.existsSync(deployListPath)) {
-        logger.error('Deploy router file not found', { path: deployListPath });
-        throw new Error(`Deploy router file not found: ${deployListPath}`);
-      }
+        if (!fs.existsSync(deployListPath)) {
+          logger.error('Deploy router file not found', { path: deployListPath });
+          throw new Error(`Deploy router file not found: ${deployListPath}`);
+        }
 
-      const deployList = fs.readFileSync(deployListPath, 'utf8').split(',');
+        const deployList = fs.readFileSync(deployListPath, 'utf8').split(',');
 
-      const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-      if (!fs.existsSync(confServerPath)) {
-        logger.error('Server configuration not found', { path: confServerPath });
-        throw new Error(`Server configuration not found: ${confServerPath}`);
-      }
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        if (!fs.existsSync(confServerPath)) {
+          logger.error('Server configuration not found', { path: confServerPath });
+          throw new Error(`Server configuration not found: ${confServerPath}`);
+        }
 
-      const { db } = loadConfServerJson(confServerPath, { resolve: true })[host][path];
+        const { db } = loadConfServerJson(confServerPath, { resolve: true })[host][path];
 
-      try {
-        await DataBaseProvider.load({ apis: ['instance', 'cron'], host, path, db });
+        const maxRetries = 5;
+        const retryDelay = 3000;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            await DataBaseProvider.load({ apis: ['instance', 'cron'], host, path, db });
+            break;
+          } catch (err) {
+            if (attempt === maxRetries) {
+              logger.error('Failed to connect to database after retries', { attempts: maxRetries, error: err.message });
+              throw err;
+            }
+            logger.warn('Database connection failed, retrying...', { attempt, maxRetries, error: err.message });
+            await timer(retryDelay);
+          }
+        }
 
-        /** @type {import('../api/instance/instance.model.js').InstanceModel} */
-        const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
+        try {
+          /** @type {import('../api/instance/instance.model.js').InstanceModel} */
+          const Instance = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Instance;
 
-        await Instance.deleteMany();
-        logger.info('Cleared existing instance metadata');
+          await Instance.deleteMany();
+          logger.info('Cleared existing instance metadata');
 
-        for (const _deployId of deployList) {
-          const deployId = _deployId.trim();
-          if (!deployId) continue;
+          for (const _deployId of deployList) {
+            const deployId = _deployId.trim();
+            if (!deployId) continue;
 
-          logger.info('Processing deployment for metadata', { deployId });
+            logger.info('Processing deployment for metadata', { deployId });
 
-          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-          if (!fs.existsSync(confServerPath)) {
-            logger.warn('Configuration not found for deployment', { deployId, path: confServerPath });
-            continue;
-          }
+            const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+            if (!fs.existsSync(confServerPath)) {
+              logger.warn('Configuration not found for deployment', { deployId, path: confServerPath });
+              continue;
+            }
 
-          const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath, { resolve: true }));
-          const router = await Underpost.deploy.routerFactory(deployId, env);
-          const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);
+            const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath, { resolve: true }));
+            const router = await Underpost.deploy.routerFactory(deployId, env);
+            const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);
 
-          for (const host of Object.keys(confServer)) {
-            for (const { path, port } of pathPortAssignmentData[host]) {
-              if (!confServer[host][path]) continue;
+            for (const host of Object.keys(confServer)) {
+              for (const { path, port } of pathPortAssignmentData[host]) {
+                if (!confServer[host][path]) continue;
 
-              const { client, runtime, apis, peer } = confServer[host][path];
+                const { client, runtime, apis, peer } = confServer[host][path];
 
-              // Save main instance
-              {
-                const body = {
-                  deployId,
-                  host,
-                  path,
-                  port,
-                  client,
-                  runtime,
-                  apis,
-                };
+                // Save main instance
+                {
+                  const body = {
+                    deployId,
+                    host,
+                    path,
+                    port,
+                    client,
+                    runtime,
+                    apis,
+                  };
+
+                  logger.info('Saving instance metadata', body);
+                  await new Instance(body).save();
+                }
 
-                logger.info('Saving instance metadata', body);
-                await new Instance(body).save();
+                // Save peer instance if exists
+                if (peer) {
+                  const body = {
+                    deployId,
+                    host,
+                    path: path === '/' ? '/peer' : `${path}/peer`,
+                    port: port + 1,
+                    runtime: 'nodejs',
+                  };
+
+                  logger.info('Saving peer instance metadata', body);
+                  await new Instance(body).save();
+                }
               }
+            }
 
-              // Save peer instance if exists
-              if (peer) {
+            // Process additional instances
+            const confInstancesPath = `./engine-private/conf/${deployId}/conf.instances.json`;
+            if (fs.existsSync(confInstancesPath)) {
+              const confInstances = JSON.parse(fs.readFileSync(confInstancesPath, 'utf8'));
+              for (const instance of confInstances) {
+                const { id, host, path, fromPort, metadata } = instance;
+                const { runtime } = metadata;
                 const body = {
                   deployId,
                   host,
-                  path: path === '/' ? '/peer' : `${path}/peer`,
-                  port: port + 1,
-                  runtime: 'nodejs',
+                  path,
+                  port: fromPort,
+                  client: id,
+                  runtime,
                 };
-
-                logger.info('Saving peer instance metadata', body);
+                logger.info('Saving additional instance metadata', body);
                 await new Instance(body).save();
               }
             }
           }
-
-          // Process additional instances
-          const confInstancesPath = `./engine-private/conf/${deployId}/conf.instances.json`;
-          if (fs.existsSync(confInstancesPath)) {
-            const confInstances = JSON.parse(fs.readFileSync(confInstancesPath, 'utf8'));
-            for (const instance of confInstances) {
-              const { id, host, path, fromPort, metadata } = instance;
-              const { runtime } = metadata;
-              const body = {
-                deployId,
-                host,
-                path,
-                port: fromPort,
-                client: id,
-                runtime,
-              };
-              logger.info('Saving additional instance metadata', body);
-              await new Instance(body).save();
-            }
-          }
+        } catch (error) {
+          logger.error('Failed to create instance metadata', { error: error.message });
+          throw error;
         }
-      } catch (error) {
-        logger.error('Failed to create instance metadata', { error: error.message });
-        throw error;
-      }
 
-      try {
-        const cronDeployPath = './engine-private/deploy/dd.cron';
-        if (!fs.existsSync(cronDeployPath)) {
-          logger.warn('Cron deploy file not found', { path: cronDeployPath });
-          return;
-        }
+        try {
+          const cronDeployPath = './engine-private/deploy/dd.cron';
+          if (!fs.existsSync(cronDeployPath)) {
+            logger.warn('Cron deploy file not found', { path: cronDeployPath });
+            return;
+          }
 
-        const cronDeployId = fs.readFileSync(cronDeployPath, 'utf8').trim();
-        const confCronPath = `./engine-private/conf/${cronDeployId}/conf.cron.json`;
+          const cronDeployId = fs.readFileSync(cronDeployPath, 'utf8').trim();
+          const confCronPath = `./engine-private/conf/${cronDeployId}/conf.cron.json`;
 
-        if (!fs.existsSync(confCronPath)) {
-          logger.warn('Cron configuration not found', { path: confCronPath });
-          return;
-        }
+          if (!fs.existsSync(confCronPath)) {
+            logger.warn('Cron configuration not found', { path: confCronPath });
+            return;
+          }
 
-        const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
+          const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
 
-        await DataBaseProvider.load({ apis: ['cron'], host, path, db });
+          await DataBaseProvider.load({ apis: ['cron'], host, path, db });
 
-        /** @type {import('../api/cron/cron.model.js').CronModel} */
-        const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
+          /** @type {import('../api/cron/cron.model.js').CronModel} */
+          const Cron = DataBaseProvider.instance[`${host}${path}`].mongoose.models.Cron;
 
-        await Cron.deleteMany();
-        logger.info('Cleared existing cron metadata');
+          await Cron.deleteMany();
+          logger.info('Cleared existing cron metadata');
 
-        for (const jobId of Object.keys(confCron.jobs)) {
-          const body = {
-            jobId,
-            deployId: Underpost.cron.getRelatedDeployIdList(jobId),
-            expression: confCron.jobs[jobId].expression,
-            enabled: confCron.jobs[jobId].enabled,
-          };
-          logger.info('Saving cron metadata', body);
-          await new Cron(body).save();
+          for (const jobId of Object.keys(confCron.jobs)) {
+            const body = {
+              jobId,
+              deployId: Underpost.cron.getRelatedDeployIdList(jobId),
+              expression: confCron.jobs[jobId].expression,
+              enabled: confCron.jobs[jobId].enabled,
+            };
+            logger.info('Saving cron metadata', body);
+            await new Cron(body).save();
+          }
+        } catch (error) {
+          logger.error('Failed to create cron metadata', { error: error.message });
         }
+
+        await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
+        logger.info('Cluster metadata creation completed');
       } catch (error) {
-        logger.error('Failed to create cron metadata', { error: error.message });
+        logger.error('Cluster metadata creation failed', { error: error.message });
+        throw error;
+      } finally {
+        if (ephemeral && isInsideContainer()) {
+          Underpost.repo.cleanupPrivateEngineRepo();
+          Underpost.env.clean();
+        }
       }
-
-      await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
-      logger.info('Cluster metadata creation completed');
     },
 
     /**
@@ -1307,201 +1175,223 @@ class UnderpostDB {
         dryRun: false,
       },
     ) {
-      loadCronDeployEnv();
-      if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
-
-      logger.info('Starting File collection cleanup', { deployList, options });
-
-      // Load file.ref.json to know which models reference File
-      const fileRefPath = './src/api/file/file.ref.json';
-      if (!fs.existsSync(fileRefPath)) {
-        logger.error('file.ref.json not found', { path: fileRefPath });
-        return;
-      }
+      const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
+      const { ephemeral } = Underpost.repo.privateEngineRepoFactory(firstDeployId || undefined);
+      try {
+        loadCronDeployEnv();
+        if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
 
-      const fileRefData = JSON.parse(fs.readFileSync(fileRefPath, 'utf8'));
-      logger.info('Loaded file reference configuration', { apis: fileRefData.length });
+        logger.info('Starting File collection cleanup', { deployList, options });
 
-      // Filter hosts and paths if specified
-      const filterHosts = options.hosts ? options.hosts.split(',').map((h) => h.trim()) : [];
-      const filterPaths = options.paths ? options.paths.split(',').map((p) => p.trim()) : [];
+        // Load file.ref.json to know which models reference File
+        const fileRefPath = './src/api/file/file.ref.json';
+        if (!fs.existsSync(fileRefPath)) {
+          logger.error('file.ref.json not found', { path: fileRefPath });
+          return;
+        }
 
-      // Track all connections to close them at the end
-      const connectionsToClose = [];
+        const fileRefData = JSON.parse(fs.readFileSync(fileRefPath, 'utf8'));
+        logger.info('Loaded file reference configuration', { apis: fileRefData.length });
 
-      for (const _deployId of deployList.split(',')) {
-        const deployId = _deployId.trim();
-        if (!deployId) continue;
+        // Filter hosts and paths if specified
+        const filterHosts = options.hosts ? options.hosts.split(',').map((h) => h.trim()) : [];
+        const filterPaths = options.paths ? options.paths.split(',').map((p) => p.trim()) : [];
 
-        logger.info('Processing deployment for File cleanup', { deployId });
+        // Track all connections to close them at the end
+        const connectionsToClose = [];
 
-        // Load server configuration
-        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
-        if (!fs.existsSync(confServerPath)) {
-          logger.error('Configuration file not found', { path: confServerPath });
-          continue;
-        }
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
 
-        const confServer = loadConfServerJson(confServerPath, { resolve: true });
+          logger.info('Processing deployment for File cleanup', { deployId });
 
-        // Process each host+path combination
-        for (const host of Object.keys(confServer)) {
-          if (filterHosts.length > 0 && !filterHosts.includes(host)) continue;
+          // Load server configuration
+          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+          if (!fs.existsSync(confServerPath)) {
+            logger.error('Configuration file not found', { path: confServerPath });
+            continue;
+          }
 
-          for (const path of Object.keys(confServer[host])) {
-            if (filterPaths.length > 0 && !filterPaths.includes(path)) continue;
+          const confServer = loadConfServerJson(confServerPath, { resolve: true });
 
-            const { db, apis } = confServer[host][path];
-            if (!db || !apis) continue;
+          // Process each host+path combination
+          for (const host of Object.keys(confServer)) {
+            if (filterHosts.length > 0 && !filterHosts.includes(host)) continue;
 
-            // Check if 'file' api is in the apis list
-            if (!apis.includes('file')) {
-              logger.info('Skipping - no file api in configuration', { host, path });
-              continue;
-            }
+            for (const path of Object.keys(confServer[host])) {
+              if (filterPaths.length > 0 && !filterPaths.includes(path)) continue;
 
-            // logger.info('Processing host+path with file api', { host, path, db: db.name });
+              const { db, apis } = confServer[host][path];
+              if (!db || !apis) continue;
 
-            try {
-              // Connect to database
-              const dbProvider = await DataBaseProvider.load({ apis, host, path, db });
-              if (!dbProvider || !dbProvider.models) {
-                logger.error('Failed to load database provider', { host, path });
+              // Check if 'file' api is in the apis list
+              if (!apis.includes('file')) {
+                logger.info('Skipping - no file api in configuration', { host, path });
                 continue;
               }
 
-              const { models } = dbProvider;
+              // logger.info('Processing host+path with file api', { host, path, db: db.name });
+
+              try {
+                // Connect to database with retry
+                let dbProvider;
+                for (let attempt = 1; attempt <= 3; attempt++) {
+                  try {
+                    dbProvider = await DataBaseProvider.load({ apis, host, path, db });
+                    break;
+                  } catch (err) {
+                    if (attempt === 3) throw err;
+                    logger.warn('Database connection failed, retrying...', { attempt, host, path, error: err.message });
+                    await timer(3000);
+                  }
+                }
+                if (!dbProvider || !dbProvider.models) {
+                  logger.error('Failed to load database provider', { host, path });
+                  continue;
+                }
 
-              // Track this connection for cleanup
-              connectionsToClose.push({ host, path, dbProvider });
+                const { models } = dbProvider;
 
-              // Check if File model exists
-              if (!models.File) {
-                logger.warn('File model not loaded', { host, path });
-                continue;
-              }
+                // Track this connection for cleanup
+                connectionsToClose.push({ host, path, dbProvider });
 
-              // Get all File documents
-              const allFiles = await models.File.find({}, '_id').lean();
-              logger.info('Found File documents', { count: allFiles.length, host, path });
+                // Check if File model exists
+                if (!models.File) {
+                  logger.warn('File model not loaded', { host, path });
+                  continue;
+                }
 
-              if (allFiles.length === 0) continue;
+                // Get all File documents
+                const allFiles = await models.File.find({}, '_id').lean();
+                logger.info('Found File documents', { count: allFiles.length, host, path });
 
-              // Track which File IDs are referenced
-              const referencedFileIds = new Set();
+                if (allFiles.length === 0) continue;
 
-              // Check each API from file.ref.json
-              for (const refConfig of fileRefData) {
-                const { api, model: modelFields } = refConfig;
+                // Track which File IDs are referenced
+                const referencedFileIds = new Set();
 
-                // Check if this API is loaded in current context
-                const modelName = api
-                  .split('-')
-                  .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
-                  .join('');
-                const Model = models[modelName];
+                // Check each API from file.ref.json
+                for (const refConfig of fileRefData) {
+                  const { api, model: modelFields } = refConfig;
 
-                if (!Model) {
-                  logger.debug('Model not loaded in current context', { api, modelName, host, path });
-                  continue;
-                }
+                  // Check if this API is loaded in current context
+                  const modelName = api
+                    .split('-')
+                    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+                    .join('');
+                  const Model = models[modelName];
 
-                logger.info('Checking references in model', { api, modelName });
+                  if (!Model) {
+                    logger.debug('Model not loaded in current context', { api, modelName, host, path });
+                    continue;
+                  }
 
-                // Helper function to recursively check field references
-                const checkFieldReferences = async (fieldPath, fieldConfig) => {
-                  for (const [fieldName, fieldValue] of Object.entries(fieldConfig)) {
-                    const currentPath = fieldPath ? `${fieldPath}.${fieldName}` : fieldName;
+                  logger.info('Checking references in model', { api, modelName });
 
-                    if (fieldValue === true) {
-                      // This is a File reference field
-                      const query = {};
-                      query[currentPath] = { $exists: true, $ne: null };
+                  // Helper function to recursively check field references
+                  const checkFieldReferences = async (fieldPath, fieldConfig) => {
+                    for (const [fieldName, fieldValue] of Object.entries(fieldConfig)) {
+                      const currentPath = fieldPath ? `${fieldPath}.${fieldName}` : fieldName;
 
-                      const docs = await Model.find(query, currentPath).lean();
+                      if (fieldValue === true) {
+                        // This is a File reference field
+                        const query = {};
+                        query[currentPath] = { $exists: true, $ne: null };
 
-                      for (const doc of docs) {
-                        // Navigate to the nested field
-                        const parts = currentPath.split('.');
-                        let value = doc;
-                        for (const part of parts) {
-                          value = value?.[part];
-                        }
+                        const docs = await Model.find(query, currentPath).lean();
+
+                        for (const doc of docs) {
+                          // Navigate to the nested field
+                          const parts = currentPath.split('.');
+                          let value = doc;
+                          for (const part of parts) {
+                            value = value?.[part];
+                          }
 
-                        if (value) {
-                          if (Array.isArray(value)) {
-                            value.forEach((id) => id && referencedFileIds.add(id.toString()));
-                          } else {
-                            referencedFileIds.add(value.toString());
+                          if (value) {
+                            if (Array.isArray(value)) {
+                              value.forEach((id) => id && referencedFileIds.add(id.toString()));
+                            } else {
+                              referencedFileIds.add(value.toString());
+                            }
                           }
                         }
-                      }
 
-                      logger.info('Found references', {
-                        model: modelName,
-                        field: currentPath,
-                        count: docs.length,
-                      });
-                    } else if (typeof fieldValue === 'object') {
-                      // Nested object, recurse
-                      await checkFieldReferences(currentPath, fieldValue);
+                        logger.info('Found references', {
+                          model: modelName,
+                          field: currentPath,
+                          count: docs.length,
+                        });
+                      } else if (typeof fieldValue === 'object') {
+                        // Nested object, recurse
+                        await checkFieldReferences(currentPath, fieldValue);
+                      }
                     }
-                  }
-                };
+                  };
 
-                await checkFieldReferences('', modelFields);
-              }
-
-              logger.info('Total referenced File IDs', { count: referencedFileIds.size, host, path });
+                  await checkFieldReferences('', modelFields);
+                }
 
-              // Find orphaned files
-              const orphanedFiles = allFiles.filter((file) => !referencedFileIds.has(file._id.toString()));
+                logger.info('Total referenced File IDs', { count: referencedFileIds.size, host, path });
 
-              if (orphanedFiles.length === 0) {
-                logger.info('No orphaned files found', { host, path });
-              } else {
-                logger.info('Found orphaned files', { count: orphanedFiles.length, host, path });
+                // Find orphaned files
+                const orphanedFiles = allFiles.filter((file) => !referencedFileIds.has(file._id.toString()));
 
-                if (options.dryRun) {
-                  logger.info('Dry run - would delete files', {
-                    count: orphanedFiles.length,
-                    ids: orphanedFiles.map((f) => f._id.toString()),
-                  });
+                if (orphanedFiles.length === 0) {
+                  logger.info('No orphaned files found', { host, path });
                 } else {
-                  const orphanedIds = orphanedFiles.map((f) => f._id);
-                  const deleteResult = await models.File.deleteMany({ _id: { $in: orphanedIds } });
-                  logger.info('Deleted orphaned files', {
-                    deletedCount: deleteResult.deletedCount,
-                    host,
-                    path,
-                  });
+                  logger.info('Found orphaned files', { count: orphanedFiles.length, host, path });
+
+                  if (options.dryRun) {
+                    logger.info('Dry run - would delete files', {
+                      count: orphanedFiles.length,
+                      ids: orphanedFiles.map((f) => f._id.toString()),
+                    });
+                  } else {
+                    const orphanedIds = orphanedFiles.map((f) => f._id);
+                    const deleteResult = await models.File.deleteMany({ _id: { $in: orphanedIds } });
+                    logger.info('Deleted orphaned files', {
+                      deletedCount: deleteResult.deletedCount,
+                      host,
+                      path,
+                    });
+                  }
                 }
+              } catch (error) {
+                logger.error('Error processing host+path', {
+                  host,
+                  path,
+                  error: error.message,
+                });
               }
-            } catch (error) {
-              logger.error('Error processing host+path', {
-                host,
-                path,
-                error: error.message,
-              });
             }
           }
         }
-      }
 
-      // Close all connections
-      logger.info('Closing all database connections', { count: connectionsToClose.length });
-      for (const { host, path, dbProvider } of connectionsToClose) {
-        try {
-          if (dbProvider && dbProvider.close) {
-            await dbProvider.close();
-            logger.info('Connection closed', { host, path });
+        // Close all connections
+        logger.info('Closing all database connections', { count: connectionsToClose.length });
+        for (const { host, path, dbProvider } of connectionsToClose) {
+          try {
+            if (dbProvider && dbProvider.close) {
+              await dbProvider.close();
+              logger.info('Connection closed', { host, path });
+            }
+          } catch (error) {
+            logger.error('Error closing connection', { host, path, error: error.message });
           }
-        } catch (error) {
-          logger.error('Error closing connection', { host, path, error: error.message });
         }
-      }
 
-      logger.info('File collection cleanup completed');
+        logger.info('File collection cleanup completed');
+      } catch (error) {
+        logger.error('File collection cleanup failed', { error: error.message });
+        throw error;
+      } finally {
+        if (ephemeral && isInsideContainer()) {
+          Underpost.repo.cleanupPrivateEngineRepo();
+          Underpost.env.clean();
+        }
+      }
     },
 
     /**
@@ -1535,68 +1425,79 @@ class UnderpostDB {
         crons: false,
       },
     ) {
-      loadCronDeployEnv();
-      deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
-      host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
-      path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
-
-      logger.info('Starting cluster metadata backup operation', {
-        deployId,
-        host,
-        path,
-        options,
-      });
-
-      if (options.generate === true) {
-        logger.info('Generating cluster metadata');
-        await Underpost.db.clusterMetadataFactory(deployId, host, path);
-      }
+      const { ephemeral } = Underpost.repo.privateEngineRepoFactory(deployId || undefined);
+      try {
+        loadCronDeployEnv();
+        deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
+        host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
+        path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
+
+        logger.info('Starting cluster metadata backup operation', {
+          deployId,
+          host,
+          path,
+          options,
+        });
 
-      if (options.instances === true) {
-        const outputPath = './engine-private/instances';
-        if (!fs.existsSync(outputPath)) {
-          fs.mkdirSync(outputPath, { recursive: true });
+        if (options.generate === true) {
+          logger.info('Generating cluster metadata');
+          await Underpost.db.clusterMetadataFactory(deployId, host, path);
         }
-        const collection = 'instances';
 
-        if (options.export === true) {
-          logger.info('Exporting instances collection', { outputPath });
-          shellExec(
-            `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
-          );
-        }
+        if (options.instances === true) {
+          const outputPath = './engine-private/instances';
+          if (!fs.existsSync(outputPath)) {
+            fs.mkdirSync(outputPath, { recursive: true });
+          }
+          const collection = 'instances';
 
-        if (options.import === true) {
-          logger.info('Importing instances collection', { outputPath });
-          shellExec(
-            `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
-          );
-        }
-      }
+          if (options.export === true) {
+            logger.info('Exporting instances collection', { outputPath });
+            shellExec(
+              `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
 
-      if (options.crons === true) {
-        const outputPath = './engine-private/crons';
-        if (!fs.existsSync(outputPath)) {
-          fs.mkdirSync(outputPath, { recursive: true });
+          if (options.import === true) {
+            logger.info('Importing instances collection', { outputPath });
+            shellExec(
+              `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
         }
-        const collection = 'crons';
 
-        if (options.export === true) {
-          logger.info('Exporting crons collection', { outputPath });
-          shellExec(
-            `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
-          );
+        if (options.crons === true) {
+          const outputPath = './engine-private/crons';
+          if (!fs.existsSync(outputPath)) {
+            fs.mkdirSync(outputPath, { recursive: true });
+          }
+          const collection = 'crons';
+
+          if (options.export === true) {
+            logger.info('Exporting crons collection', { outputPath });
+            shellExec(
+              `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
+
+          if (options.import === true) {
+            logger.info('Importing crons collection', { outputPath });
+            shellExec(
+              `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
         }
 
-        if (options.import === true) {
-          logger.info('Importing crons collection', { outputPath });
-          shellExec(
-            `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
-          );
+        logger.info('Cluster metadata backup operation completed');
+      } catch (error) {
+        logger.error('Cluster metadata backup operation failed', { error: error.message });
+        throw error;
+      } finally {
+        if (ephemeral && isInsideContainer()) {
+          Underpost.repo.cleanupPrivateEngineRepo();
+          Underpost.env.clean();
         }
       }
-
-      logger.info('Cluster metadata backup operation completed');
     },
   };
 }