underpost 2.90.4 → 2.92.0

package/src/cli/deploy.js

@@ -387,6 +387,8 @@ spec:
      * @param {boolean} options.etcHosts - Whether to display the /etc/hosts file.
      * @param {boolean} options.disableUpdateUnderpostConfig - Whether to disable Underpost config updates.
      * @param {string} [options.namespace] - Kubernetes namespace for the deployment.
+     * @param {string} [options.kindType] - Type of Kubernetes resource to retrieve information for.
+     * @param {number} [options.port] - Port number for exposing the deployment.
      * @returns {Promise<void>} - Promise that resolves when the deployment process is complete.
      * @memberof UnderpostDeploy
      */
@@ -416,6 +418,8 @@ spec:
         etcHosts: false,
         disableUpdateUnderpostConfig: false,
         namespace: '',
+        kindType: '',
+        port: 0,
       },
     ) {
       const namespace = options.namespace ? options.namespace : 'default';
@@ -495,13 +499,20 @@ EOF`);
         const deployId = _deployId.trim();
         if (!deployId) continue;
         if (options.expose === true) {
-          const svc = UnderpostDeploy.API.get(deployId, 'svc')[0];
-          const port = parseInt(svc[`PORT(S)`].split('/TCP')[0]);
+          const kindType = options.kindType ? options.kindType : 'svc';
+          const svc = UnderpostDeploy.API.get(deployId, kindType)[0];
+          const port = options.port
+            ? options.port
+            : kindType !== 'svc'
+              ? 80
+              : parseInt(svc[`PORT(S)`].split('/TCP')[0]);
           logger.info(deployId, {
             svc,
             port,
           });
-          shellExec(`sudo kubectl port-forward -n default svc/${svc.NAME} ${port}:${port}`, { async: true });
+          shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${svc.NAME} ${port}:${port}`, {
+            async: true,
+          });
           continue;
         }
 
@@ -657,15 +668,9 @@ EOF`);
       for (const pod of pods) {
         const { NAME } = pod;
         if (ignoresNames && ignoresNames.find((t) => NAME.trim().toLowerCase().match(t.trim().toLowerCase()))) continue;
-        if (
-          shellExec(`sudo kubectl exec -i ${NAME} -- sh -c "${cmd}"`, { stdout: true }).match(
-            `${deployId}-${env}-running-deployment`,
-          )
-        ) {
-          readyPods.push(pod);
-        } else {
-          notReadyPods.push(pod);
-        }
+        const out = shellExec(`sudo kubectl exec -i ${NAME} -- sh -c "${cmd}"`, { stdout: true, silent: true });
+        const ready = out.match(`${deployId}-${env}-running-deployment`);
+        ready ? readyPods.push(pod) : notReadyPods.push(pod);
       }
       return {
         ready: pods.length > 0 && notReadyPods.length === 0,

package/src/cli/env.js

@@ -63,7 +63,7 @@ class UnderpostRootEnv {
       const exeRootPath = `${getNpmRootPath()}/underpost`;
       const envPath = `${exeRootPath}/.env`;
       if (!fs.existsSync(envPath)) {
-        logger.error(`Unable to find underpost root environment`);
+        logger.warn(`Empty environment variables`);
         return undefined;
       }
       const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));
@@ -80,7 +80,7 @@ class UnderpostRootEnv {
       const exeRootPath = `${getNpmRootPath()}/underpost`;
       const envPath = `${exeRootPath}/.env`;
       if (!fs.existsSync(envPath)) {
-        logger.error(`Unable to find underpost root environment`);
+        logger.warn(`Empty environment variables`);
         return {};
       }
       const env = dotenv.parse(fs.readFileSync(envPath, 'utf8'));

package/src/cli/index.js

@@ -29,6 +29,11 @@ program
   .option('--deploy-id <deploy-id>', 'Crete deploy ID conf env files')
   .option('--sub-conf <sub-conf>', 'Create sub conf env files')
   .option('--cluster', 'Create deploy ID cluster files and sync to current cluster')
+  .option('--build-repos', 'Create deploy ID repositories')
+  .option('--build', 'Build the deployment to pwa-microservices-template (requires --deploy-id)')
+  .option('--clean-template', 'Clean the build directory (pwa-microservices-template)')
+  .option('--sync-conf', 'Sync configuration to private repositories (requires --deploy-id)')
+  .option('--purge', 'Remove deploy ID conf and all related repositories (requires --deploy-id)')
   .option('--dev', 'Sets the development cli context')
   .description('Initializes a new Underpost project, service, or configuration.')
   .action(Underpost.repo.new);
@@ -238,9 +243,66 @@ program
 
 program
   .command('ip')
+  .argument('[ips]', 'Optional args comma-separated list of IP to process.')
   .option('--copy', 'Copies the IP addresses to the clipboard.')
+  .option('--ban-ingress-add', 'Adds IP addresses to banned ingress list.')
+  .option('--ban-ingress-remove', 'Removes IP addresses from banned ingress list.')
+  .option('--ban-ingress-list', 'Lists all banned ingress IP addresses.')
+  .option('--ban-ingress-clear', 'Clears all banned ingress IP addresses.')
+  .option('--ban-egress-add', 'Adds IP addresses to banned egress list.')
+  .option('--ban-egress-remove', 'Removes IP addresses from banned egress list.')
+  .option('--ban-egress-list', 'Lists all banned egress IP addresses.')
+  .option('--ban-egress-clear', 'Clears all banned egress IP addresses.')
+  .option('--ban-both-add', 'Adds IP addresses to both banned ingress and egress lists.')
+  .option('--ban-both-remove', 'Removes IP addresses from both banned ingress and egress lists.')
   .description('Displays the current public machine IP addresses.')
-  .action(async (options) => {
+  .action(async (ips = '', options) => {
+    const ipList = ips
+      ? ips
+          .split(',')
+          .map((i) => i.trim())
+          .filter(Boolean)
+      : [];
+
+    if (options.banIngressAdd) {
+      return ipList.forEach((ip) => Dns.banIngress(ip));
+    }
+    if (options.banIngressRemove) {
+      return ipList.forEach((ip) => Dns.unbanIngress(ip));
+    }
+    if (options.banIngressList) {
+      return Dns.listBannedIngress();
+    }
+    if (options.banIngressClear) {
+      return Dns.clearBannedIngress();
+    }
+
+    if (options.banEgressAdd) {
+      return ipList.forEach((ip) => Dns.banEgress(ip));
+    }
+    if (options.banEgressRemove) {
+      return ipList.forEach((ip) => Dns.unbanEgress(ip));
+    }
+    if (options.banEgressList) {
+      return Dns.listBannedEgress();
+    }
+    if (options.banEgressClear) {
+      return Dns.clearBannedEgress();
+    }
+
+    if (options.banBothAdd) {
+      return ipList.forEach((ip) => {
+        Dns.banIngress(ip);
+        Dns.banEgress(ip);
+      });
+    }
+    if (options.banBothRemove) {
+      return ipList.forEach((ip) => {
+        Dns.unbanIngress(ip);
+        Dns.unbanEgress(ip);
+      });
+    }
+
     const ip = await Dns.getPublicIp();
     if (options.copy) return pbcopy(ip);
     return console.log(ip);
@@ -331,6 +393,8 @@ program
   .option('--restore-hosts', 'Restores default `/etc/hosts` entries.')
   .option('--disable-update-underpost-config', 'Disables updates to Underpost configuration during deployment.')
   .option('--namespace <namespace>', 'Kubernetes namespace for deployment operations (defaults to "default").')
+  .option('--kind-type <kind-type>', 'Specifies the Kind cluster type for deployment operations.')
+  .option('--port <port>', 'Sets up port forwarding from local to remote ports.')
   .description('Manages application deployments, defaulting to deploying development pods.')
   .action(Underpost.deploy.callback);
 
@@ -394,16 +458,27 @@ program
   .argument('<deploy-list>', 'A comma-separated list of deployment IDs (e.g., "default-a,default-b").')
   .option('--import', 'Imports container backups from specified repositories.')
   .option('--export', 'Exports container backups to specified repositories.')
-  .option('--pod-name <pod-name>', 'Optional: Specifies the pod context for database operations.')
-  .option('--collections <collections>', 'A comma-separated list of database collections to operate on.')
+  .option(
+    '--pod-name <pod-name>',
+    'Comma-separated list of pod names or patterns (supports wildcards like "mariadb-*").',
+  )
+  .option('--node-name <node-name>', 'Comma-separated list of node names to filter pods by their node placement.')
+  .option('--label-selector <selector>', 'Kubernetes label selector for filtering pods (e.g., "app=mariadb").')
+  .option('--all-pods', 'Target all matching pods instead of just the first one.')
+  .option('--primary-pod', 'Automatically detect and use MongoDB primary pod (MongoDB only).')
+  .option('--stats', 'Display database statistics (collection/table names with document/row counts).')
+  .option('--collections <collections>', 'Comma-separated list of database collections to operate on.')
   .option('--out-path <out-path>', 'Specifies a custom output path for backups.')
-  .option('--drop', 'Drops the specified databases or collections.')
-  .option('--preserveUUID', 'Preserves UUIDs during database operations.')
-  .option('--git', 'Uploads database backups to GitHub.')
-  .option('--hosts <hosts>', 'A comma-separated list of database hosts.')
-  .option('--paths <paths>', 'A comma-separated list of paths for database files.')
-  .option('--ns <ns-name>', 'Optional: Specifies the namespace context for database operations.')
-  .description('Manages database operations, including import, export, and collection management.')
+  .option('--drop', 'Drops the specified databases or collections before importing.')
+  .option('--preserveUUID', 'Preserves UUIDs during database import operations.')
+  .option('--git', 'Enables Git integration for backup version control (clone, pull, commit, push to GitHub).')
+  .option('--hosts <hosts>', 'Comma-separated list of database hosts to filter operations.')
+  .option('--paths <paths>', 'Comma-separated list of paths to filter database operations.')
+  .option('--ns <ns-name>', 'Kubernetes namespace context for database operations (defaults to "default").')
+  .option('--dry-run', 'Simulates operations without executing them (useful for testing).')
+  .description(
+    'Manages database operations with support for MariaDB and MongoDB, including import/export, multi-pod targeting, and Git integration.',
+  )
   .action(Underpost.db.callback);
 
 program
@@ -498,8 +573,22 @@ program
 // 'ssh' command: SSH management
 program
   .command('ssh')
+  .option('--deploy-id <deploy-id>', 'Sets deploy id context for ssh operations.')
   .option('--generate', 'Generates new ssh credential and stores it in current private keys file storage.')
-  .description('Import and start ssh server and client based on current default deployment ID.')
+  .option('--user <user>', 'Sets custom ssh user')
+  .option('--password <password>', 'Sets custom ssh password')
+  .option('--host <host>', 'Sets custom ssh host')
+  .option('--port <port>', 'Sets custom ssh port')
+  .option('--filter <filter>', 'Filters ssh user credentials from current private keys file storage.')
+  .option('--groups <groups>', 'Sets comma-separated ssh user groups for the ssh user credential.')
+  .option('--user-add', 'Adds a new ssh user credential to current private keys file storage.')
+  .option('--user-remove', 'Removes an existing ssh user credential from current private keys file storage.')
+  .option('--user-ls', 'Lists all ssh user credentials from current private keys file storage.')
+  .option('--start', 'Starts an SSH session with the specified credentials.')
+  .option('--reset', 'Resets ssh configuration and deletes all stored credentials.')
+  .option('--keys-list', 'Lists all ssh keys from current private keys file storage.')
+  .option('--hosts-list', 'Lists all ssh hosts from current private keys file storage.')
+  .option('--disable-password', 'Disables password authentication for the SSH session.')
   .action(Underpost.ssh.callback);
 
 // 'run' command: Run a script

package/src/cli/repository.js

@@ -234,21 +234,145 @@ class UnderpostRepository {
     /**
      * Initializes a new Underpost repository, optionally setting up a deploy ID or sub-configuration.
      * @param {string} [projectName=''] - The name of the project to create.
-     * @param {object} [options={ deployId: '', subConf: '', cluster: false, dev: false }] - Initialization options.
+     * @param {object} [options] - Initialization options.
      * @param {string} [options.deployId=''] - The deployment ID to set up.
      * @param {string} [options.subConf=''] - The sub-configuration to create.
      * @param {boolean} [options.cluster=false] - If true, sets up a clustered configuration.
      * @param {boolean} [options.dev=false] - If true, uses development settings.
+     * @param {boolean} [options.buildRepos=false] - If true, creates the deployment repositories (engine-*, engine-*-private, engine-*-cron-backups).
+     * @param {boolean} [options.purge=false] - If true, removes the deploy ID conf and all related repositories (requires deployId).
+     * @param {boolean} [options.cleanTemplate=false] - If true, cleans the pwa-microservices-template build directory.
+     * @param {boolean} [options.build=false] - If true, builds the deployment to pwa-microservices-template (requires deployId).
+     * @param {boolean} [options.syncConf=false] - If true, syncs configuration to private repositories (requires deployId).
      * @returns {Promise<boolean>} A promise that resolves when the initialization is complete.
      * @memberof UnderpostRepository
      */
-    new(projectName, options = { deployId: '', subConf: '', cluster: false, dev: false }) {
+    new(
+      projectName,
+      options = {
+        deployId: '',
+        subConf: '',
+        cluster: false,
+        dev: false,
+        buildRepos: false,
+        purge: false,
+        cleanTemplate: false,
+        build: false,
+        syncConf: false,
+      },
+    ) {
       return new Promise(async (resolve, reject) => {
         try {
           await logger.setUpInfo();
           actionInitLog();
+
+          // Handle cleanTemplate operation
+          if (options.cleanTemplate) {
+            logger.info('Cleaning build directory');
+            const basePath = '../pwa-microservices-template';
+            shellExec(`cd ${basePath} && git reset`);
+            shellExec(`cd ${basePath} && git checkout .`);
+            shellExec(`cd ${basePath} && git clean -f -d`);
+            logger.info('Build directory cleaned successfully');
+            return resolve(true);
+          }
+
           if (options.deployId) {
-            Config.deployIdFactory(options.deployId, options);
+            let deployId = options.deployId;
+            if (!deployId.startsWith('dd-')) deployId = `dd-${deployId}`;
+
+            // Handle purge operation
+            if (options.purge) {
+              logger.info(`Purging deploy ID: ${deployId}`);
+
+              const suffix = deployId.split('dd-')[1];
+              const repoName = `engine-${suffix}`;
+              const privateRepoName = `engine-${suffix}-private`;
+              const cronRepoName = `engine-${suffix}-cron-backups`;
+              const confFolder = `./engine-private/conf/${deployId}`;
+
+              // Remove conf folder
+              if (fs.existsSync(confFolder)) {
+                fs.removeSync(confFolder);
+                logger.info(`Removed conf folder: ${confFolder}`);
+              } else {
+                logger.warn(`Conf folder not found: ${confFolder}`);
+              }
+
+              // Remove repositories
+              const repos = [
+                { path: `../${repoName}`, name: repoName },
+                { path: `../${privateRepoName}`, name: privateRepoName },
+                { path: `../${cronRepoName}`, name: cronRepoName },
+              ];
+
+              for (const repo of repos) {
+                if (fs.existsSync(repo.path)) {
+                  fs.removeSync(repo.path);
+                  logger.info(`Removed repository: ${repo.path}`);
+                } else {
+                  logger.warn(`Repository not found: ${repo.path}`);
+                }
+              }
+
+              logger.info(`Successfully purged deploy ID: ${deployId}`);
+              return resolve(true);
+            }
+
+            // Handle sync-conf operation
+            if (options.syncConf) {
+              logger.info(`Syncing configuration for deploy ID: ${deployId}`);
+              shellExec(`node bin/build ${deployId} conf`);
+              logger.info('Configuration synced successfully');
+              return resolve(true);
+            }
+
+            // Handle build operation
+            if (options.build) {
+              logger.info(`Building deployment for deploy ID: ${deployId}`);
+              shellExec(`node bin/build ${deployId}`);
+              logger.info('Build completed successfully');
+              return resolve(true);
+            }
+
+            // Normal deploy ID factory operation
+            const { deployId: normalizedDeployId } = Config.deployIdFactory(deployId, options);
+
+            if (options.buildRepos) {
+              const suffix = normalizedDeployId.split('dd-')[1];
+              const repoName = `engine-${suffix}`;
+              const privateRepoName = `engine-${suffix}-private`;
+              const cronRepoName = `engine-${suffix}-cron-backups`;
+              const repos = [
+                { path: `../${repoName}`, name: repoName },
+                { path: `../${privateRepoName}`, name: privateRepoName },
+                { path: `../${cronRepoName}`, name: cronRepoName },
+              ];
+
+              const username = process.env.GITHUB_USERNAME;
+              const token = process.env.GITHUB_TOKEN;
+
+              if (!username) {
+                logger.error('GITHUB_USERNAME environment variable not set');
+                return reject(false);
+              }
+
+              for (const repo of repos) {
+                if (!fs.existsSync(repo.path)) {
+                  fs.mkdirSync(repo.path, { recursive: true });
+                  logger.info(`Created repository directory: ${repo.path}`);
+                }
+
+                // Initialize git repository
+                shellExec(`cd ${repo.path} && git init`, { disableLog: false });
+                logger.info(`Initialized git repository in: ${repo.path}`);
+
+                // Add remote origin
+                const remoteUrl = `https://${token ? `${token}@` : ''}github.com/${username}/${repo.name}.git`;
+                shellExec(`cd ${repo.path} && git remote add origin ${remoteUrl}`, { disableLog: false });
+                logger.info(`Added remote origin for: ${repo.name}`);
+              }
+            }
             return resolve(true);
           }
           if (projectName) {

package/src/cli/run.js

@@ -19,6 +19,7 @@ import UnderpostTest from './test.js';
 import fs from 'fs-extra';
 import { range, setPad, timer } from '../client/components/core/CommonJs.js';
 import UnderpostDeploy from './deploy.js';
+import UnderpostDB from './db.js';
 import UnderpostRootEnv from './env.js';
 import UnderpostRepository from './repository.js';
 import os from 'os';
@@ -250,10 +251,29 @@ class UnderpostRun {
         );
         shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --valkey --pull-image`);
       }
-      shellExec(`${baseCommand} deploy --expose --disable-update-underpost-config mongo`, { async: true });
-      shellExec(`${baseCommand} deploy --expose --disable-update-underpost-config valkey`, { async: true });
+
       {
-        const hostListenResult = UnderpostDeploy.API.etcHostFactory(mongoHosts);
+        // Detect MongoDB primary pod using centralized method
+        let primaryMongoHost = 'mongodb-0.mongodb-service';
+        try {
+          const primaryPodName = UnderpostDB.API.getMongoPrimaryPodName({
+            namespace: options.namespace,
+            podName: 'mongodb-0',
+          });
+          // shellExec(`${baseCommand} deploy --expose --disable-update-underpost-config mongo`, { async: true });
+          shellExec(`kubectl port-forward -n ${options.namespace} pod/${primaryPodName} 27017:27017`, { async: true });
+          shellExec(
+            `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config valkey`,
+            { async: true },
+          );
+        } catch (error) {
+          logger.warn('Failed to detect MongoDB primary pod, using default', {
+            error: error.message,
+            default: primaryMongoHost,
+          });
+        }
+
+        const hostListenResult = UnderpostDeploy.API.etcHostFactory([primaryMongoHost]);
         logger.info(hostListenResult.renderHosts);
       }
     },
@@ -268,7 +288,7 @@ class UnderpostRun {
     metadata: async (path, options = UnderpostRun.DEFAULT_OPTION) => {
       const ports = '6379,27017';
       shellExec(`node bin run kill '${ports}'`);
-      shellExec(`node bin run dev-cluster --dev --expose`, { async: true });
+      shellExec(`node bin run dev-cluster --dev --expose --namespace ${options.namespace}`, { async: true });
       console.log('Loading fordward services...');
       await timer(5000);
       shellExec(`node bin metadata --generate ${path}`);
@@ -408,8 +428,8 @@ class UnderpostRun {
      * @param {Object} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
-    clean: (path, options = UnderpostRun.DEFAULT_OPTION) => {
-      shellCd(path ?? `/home/dd/engine`);
+    clean: (path = '', options = UnderpostRun.DEFAULT_OPTION) => {
+      shellCd(path ? path : `/home/dd/engine`);
       shellExec(`node bin/deploy clean-core-repo`);
     },
     /**
@@ -422,16 +442,25 @@ class UnderpostRun {
     pull: (path, options = UnderpostRun.DEFAULT_OPTION) => {
       if (!fs.existsSync(`/home/dd`) || !fs.existsSync(`/home/dd/engine`)) {
         fs.mkdirSync(`/home/dd`, { recursive: true });
-        shellExec(`cd /home/dd && underpost clone ${process.env.GITHUB_USERNAME}/engine`);
+        shellExec(`cd /home/dd && underpost clone ${process.env.GITHUB_USERNAME}/engine`, {
+          silent: true,
+        });
       } else {
         shellExec(`underpost run clean`);
-        shellExec(`cd /home/dd/engine && underpost pull . ${process.env.GITHUB_USERNAME}/engine`);
+        shellExec(`cd /home/dd/engine && underpost pull . ${process.env.GITHUB_USERNAME}/engine`, {
+          silent: true,
+        });
       }
       if (!fs.existsSync(`/home/dd/engine/engine-private`))
-        shellExec(`cd /home/dd/engine && underpost clone ${process.env.GITHUB_USERNAME}/engine-private`);
+        shellExec(`cd /home/dd/engine && underpost clone ${process.env.GITHUB_USERNAME}/engine-private`, {
+          silent: true,
+        });
       else
         shellExec(
           `cd /home/dd/engine/engine-private && underpost pull . ${process.env.GITHUB_USERNAME}/engine-private`,
+          {
+            silent: true,
+          },
         );
     },
     /**
@@ -975,7 +1004,7 @@ EOF
       const successInstance = await UnderpostTest.API.statusMonitor('adminer', 'Running', 'pods', 1000, 60 * 10);
 
       if (successInstance) {
-        shellExec(`underpost deploy --expose adminer`);
+        shellExec(`underpost deploy --expose adminer --namespace ${options.namespace}`);
       }
     },
 
@@ -1174,7 +1203,7 @@ EOF
         envObj.DEV_PROXY_PORT_OFFSET = options.devProxyPortOffset;
         writeEnv(envPath, envObj);
       }
-      shellExec(`node bin run dev-cluster --expose`, { async: true });
+      shellExec(`node bin run dev-cluster --expose --namespace ${options.namespace}`, { async: true });
       {
         const cmd = `npm run dev-api ${deployId} ${subConf} ${host} ${_path} ${clientHostPort}${options.tls ? ' tls' : ''}`;
         options.terminal ? openTerminal(cmd) : shellExec(cmd, { async: true });