underpost 3.2.14 → 3.2.21

package/src/cli/monitor.js

@@ -10,11 +10,19 @@ import {
   loadConfServerJson,
   loadCronDeployEnv,
   etcHostFactory,
+  deployRangePortFactory,
 } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
 import { timer } from '../client/components/core/CommonJs.js';
+import {
+  RUNTIME_STATUS,
+  INTERNAL_STATUS_PATH,
+  normalizeContainerStatus,
+  deployStatusPort,
+} from '../server/runtime-status.js';
 import axios from 'axios';
 import fs from 'fs-extra';
+import net from 'node:net';
 import { shellExec } from '../server/process.js';
 import Underpost from '../index.js';
 
@@ -328,26 +336,121 @@ class UnderpostMonitor {
       };
     },
     /**
-     * Monitors the ready status of a deployment.
+     * Resolves a free ephemeral TCP port on the loopback interface, used as the
+     * local end of the `kubectl port-forward` tunnel so it never collides with
+     * host-local services.
+     * @returns {Promise<number>}
+     * @memberof UnderpostMonitor
+     */
+    findFreePort() {
+      return new Promise((resolve) => {
+        const srv = net.createServer();
+        srv.once('error', () => resolve(20000 + Math.floor(Math.random() * 20000)));
+        srv.listen(0, '127.0.0.1', () => {
+          const { port } = srv.address();
+          srv.close(() => resolve(port));
+        });
+      });
+    },
+    /**
+     * Resolves the deployment's internal status port (Phase-2 transport target).
+     *
+     * Canonical value is `fromPort - 1` from the deployment router — the exact
+     * port `buildManifest` injects into the pod (UNDERPOST_INTERNAL_PORT) and
+     * uses for the probes — so the tunnel target always matches the in-pod bind.
+     * `UNDERPOST_INTERNAL_PORT` overrides; ambient resolution is the last resort.
+     *
+     * @param {string} deployId
+     * @param {string} env
+     * @returns {Promise<number>}
+     * @memberof UnderpostMonitor
+     */
+    async deployInternalPort(deployId, env) {
+      const override = parseInt(process.env.UNDERPOST_INTERNAL_PORT);
+      if (!Number.isNaN(override)) return override;
+      try {
+        const router = await Underpost.deploy.routerFactory(deployId, env);
+        const { fromPort } = deployRangePortFactory(router);
+        if (Number.isFinite(fromPort) && fromPort > 0) return fromPort - 1;
+      } catch (_) {
+        /* fall through to ambient resolution */
+      }
+      return deployStatusPort(deployId, env) ?? 3000;
+    },
+    /**
+     * Reads Phase-2 runtime readiness from a single pod over HTTP, tunneling
+     * through `kubectl port-forward` to the in-pod internal status endpoint.
+     *
+     * Transport failures (port-forward down, connection refused, HTTP error)
+     * are reported as `{ ok: false }` and must never be interpreted as success
+     * by callers — they are retried, not promoted. A reachable endpoint returns
+     * `{ ok: true, status }` with the normalized runtime contract value.
+     *
+     * @param {string} podName
+     * @param {string} namespace
+     * @param {number} internalPort
+     * @returns {Promise<{ok: boolean, status?: (string|null), transportError?: string}>}
+     * @memberof UnderpostMonitor
+     */
+    async readRuntimeStatus(podName, namespace, internalPort) {
+      // The local side of the tunnel MUST be an ephemeral free port: pinning it
+      // to internalPort collides with any host-local service on that number
+      // (e.g. a dev runtime on the same machine as the cluster), which makes
+      // port-forward fail to bind and every read return a false transport error.
+      const override = parseInt(process.env.UNDERPOST_PF_LOCAL_PORT);
+      const localPort = Number.isNaN(override) ? await Underpost.monitor.findFreePort() : override;
+      const url = `http://127.0.0.1:${localPort}${INTERNAL_STATUS_PATH}`;
+      let portForward;
+      try {
+        // `exec` collapses the shell so the tracked child PID is the
+        // sudo/kubectl process, letting the SIGTERM teardown reach the tunnel.
+        portForward = shellExec(
+          `exec sudo kubectl port-forward pod/${podName} ${localPort}:${internalPort} -n ${namespace}`,
+          { async: true, silent: true, disableLog: true, silentOnError: true },
+        );
+      } catch (_) {
+        portForward = undefined;
+      }
+      try {
+        let lastError;
+        const attempts = parseInt(process.env.UNDERPOST_PF_ATTEMPTS) || 20;
+        for (let attempt = 0; attempt < attempts; attempt++) {
+          try {
+            const res = await axios.get(url, { timeout: 2500 });
+            const raw = res?.data?.status ?? null;
+            return { ok: true, status: normalizeContainerStatus(raw) ?? raw, payload: res.data };
+          } catch (error) {
+            lastError = error;
+            await timer(350);
+          }
+        }
+        return { ok: false, transportError: lastError?.code || lastError?.message || 'transport_failed' };
+      } finally {
+        if (portForward && typeof portForward.kill === 'function') {
+          try {
+            portForward.kill('SIGTERM');
+          } catch (_) {
+            /* tunnel already gone */
+          }
+        }
+      }
+    },
+    /**
+     * Monitors a deployment to terminal readiness using a deterministic
+     * two-phase state machine.
      *
-     * Ready signal:
-     *   The orchestrator gate is the Kubernetes pod Ready condition. When the
-     *   container's `readinessProbe` succeeds, kubelet flips
-     *   `status.conditions[Ready]` to True and `checkDeploymentReadyStatus`
-     *   returns the pod in `readyPods`. This is the only required signal — see
-     *   `src/client/public/nexodev/docs/references/Deploy custom instance to K8S.md`.
+     *   Phase 1 (Kubernetes): pod `Ready` condition via `checkDeploymentReadyStatus`.
+     *   Phase 2 (Runtime):    `running-deployment` from the in-pod internal
+     *                         status endpoint, read over HTTP (`readRuntimeStatus`).
      *
-     * Container-status:
-     *   `underpost config get container-status` is read from each pod for both
-     *   the display column and as a second ready gate alongside the K8S Ready
-     *   condition. Both must be satisfied before the monitor exits:
-     *     1. K8S readinessProbe (TCP socket) — ensures the port is bound.
-     *     2. container-status == `<deploy>-<env>-running-deployment` — ensures
-     *        the application has completed its own startup sequence.
-     *   Early-abort on `error` container-status remains in effect: a failing
-     *   runtime keeps its pod alive (not Ready) with `container-status=error`,
-     *   so this `exec`-read surfaces the failure and the monitor aborts —
-     *   failing the CD runner instead of waiting out the full timeout.
+     * Contract:
+     *   - Success requires BOTH phases; runtime readiness is never declared
+     *     before Kubernetes readiness.
+     *   - An explicit runtime `error` (or a fatal pod status) transitions
+     *     immediately to `failed` (throw → CD exit 1).
+     *   - Transport failures never count as success and never advance state.
+     *   - `timeout` is a distinct terminal state from `failed`.
+     *   - Every transition emits a structured, secret-free event.
      *
      * @param {string} deployId - Deployment ID for which the ready status is being monitored.
      * @param {string} env - Environment for which the ready status is being monitored.
@@ -358,32 +461,29 @@ class UnderpostMonitor {
      * @memberof UnderpostMonitor
      */
     async monitorReadyRunner(deployId, env, targetTraffic, ignorePods = [], namespace = 'default') {
-      const delayMs = 1000;
-      const maxIterations = 3000;
+      const delayMs = parseInt(process.env.UNDERPOST_MONITOR_DELAY_MS) || 1000;
+      const maxIterations = parseInt(process.env.UNDERPOST_MONITOR_MAX_ITERATIONS) || 3000;
       const deploymentId = `${deployId}-${env}-${targetTraffic}`;
-      const expectedContainerStatus = `${deployId}-${env}-running-deployment`;
       const tag = `[${deploymentId}]`;
-      const containerStatusDefault = 'waiting for status';
-
-      logger.info('Deployment init', { deployId, env, targetTraffic, namespace });
-
-      const podStatusCache = new Map();
+      const expectedStatus = RUNTIME_STATUS.RUNNING;
+      const internalPort = await Underpost.monitor.deployInternalPort(deployId, env);
+      const podErrorStates = ['error', 'crashloopbackoff', 'oomkilled', 'imagepullbackoff', 'errimagepull'];
+
+      const emit = (state, status) =>
+        logger.info('deploy-monitor', {
+          deployId: deploymentId,
+          phase: state.startsWith('runtime') ? 'runtime' : 'kubernetes',
+          state,
+          status: status ?? null,
+          timestamp: new Date().toISOString(),
+        });
+
+      logger.info('Deployment init', { deployId, env, targetTraffic, namespace, internalPort });
+      emit('pending');
+
+      const runtimeStatusCache = new Map();
       const advancedPods = new Set();
 
-      const readContainerStatus = (podName) => {
-        try {
-          const raw = shellExec(
-            `sudo kubectl exec ${podName} -n ${namespace} -- sh -c 'underpost config get container-status --plain'`,
-            { silent: true, disableLog: true, stdout: true, silentOnError: true },
-          );
-          const val = raw ? raw.toString().trim() : '';
-          return val && val !== 'undefined' ? val : containerStatusDefault;
-        } catch (_) {
-          // exec failed (e.g. pod not yet running) — preserve last known value
-          return podStatusCache.get(podName) || containerStatusDefault;
-        }
-      };
-
       for (let i = 0; i < maxIterations; i++) {
         const result = await Underpost.monitor.checkDeploymentReadyStatus(
           deployId,
@@ -392,39 +492,54 @@ class UnderpostMonitor {
           ignorePods,
           namespace,
         );
-
         const allPods = [...result.readyPods, ...result.notReadyPods];
 
+        if (allPods.length === 0) {
+          emit('pending');
+          await timer(delayMs);
+          continue;
+        }
+        emit('pod_scheduled');
+
+        // Phase 1 fatal: a Kubernetes-level pod failure is terminal (failed,
+        // not timeout) — fail the CD runner immediately instead of waiting out
+        // the full window.
         for (const pod of allPods) {
-          if (!pod?.NAME) continue;
           const podStatus = (pod.STATUS || '').toLowerCase().trim();
-          if (
-            ['error', 'crashloopbackoff', 'oomkilled', 'imagepullbackoff', 'errimagepull'].find((s) =>
-              podStatus.match(s),
-            )
-          )
+          if (podErrorStates.find((s) => podStatus.includes(s)))
             throw new Error(`Pod ${pod.NAME} has error pod status: ${pod.STATUS}`);
-          const status = readContainerStatus(pod.NAME);
-          if (status === 'error') throw new Error(`Pod ${pod.NAME} has error container-status`);
-          if (advancedPods.has(pod.NAME) && status === containerStatusDefault)
-            throw new Error(`Pod ${pod.NAME} container-status regressed to default — pod likely restarted`);
-          if (status !== containerStatusDefault) advancedPods.add(pod.NAME);
-          podStatusCache.set(pod.NAME, status);
         }
 
-        const allPodsK8sReady = allPods.length > 0 && result.notReadyPods.length === 0;
+        const allPodsK8sReady = result.notReadyPods.length === 0;
+        if (allPodsK8sReady) emit('pod_ready');
 
-        const allPodsStatusReady =
-          allPods.length > 0 && allPods.every((pod) => podStatusCache.get(pod.NAME) === expectedContainerStatus);
+        // Phase 2: runtime readiness over HTTP. Transport failures neither
+        // advance state nor count as success; explicit `error` is terminal.
+        let allRuntimeRead = true;
+        for (const pod of allPods) {
+          if (!pod?.NAME) continue;
+          const read = await Underpost.monitor.readRuntimeStatus(pod.NAME, namespace, internalPort);
+          if (!read.ok) {
+            allRuntimeRead = false;
+            emit('runtime_booting', `transport:${read.transportError}`);
+            continue;
+          }
+          const status = read.status;
+          if (status === RUNTIME_STATUS.ERROR) throw new Error(`Pod ${pod.NAME} reported runtime status=error`);
+          if (advancedPods.has(pod.NAME) && (!status || status === RUNTIME_STATUS.BUILD))
+            throw new Error(`Pod ${pod.NAME} runtime status regressed (${status ?? 'empty'}) — pod likely restarted`);
+          if (status && status !== RUNTIME_STATUS.BUILD) advancedPods.add(pod.NAME);
+          runtimeStatusCache.set(pod.NAME, status);
+          emit('runtime_booting', status);
+        }
+
+        const allRuntimeReady =
+          allRuntimeRead && allPods.every((pod) => runtimeStatusCache.get(pod.NAME) === expectedStatus);
 
-        // Print snapshot for every pod — annotate when container-status hasn't caught
-        // up to the K8S Ready condition yet.
         for (const pod of allPods) {
-          const status = podStatusCache.get(pod.NAME) || containerStatusDefault;
+          const status = runtimeStatusCache.get(pod.NAME) || 'waiting for status';
           const podStatus = pod.STATUS || 'Unknown';
-          const statusMatchesExpected = status === expectedContainerStatus;
-          const statusDisplay = statusMatchesExpected ? status : `${status} (pending)`;
-
+          const statusDisplay = status === expectedStatus ? status : `${status} (pending)`;
           console.log(
             'Target pod:',
             pod.NAME[pod.NAME.includes('green') ? 'bgGreen' : 'bgBlue'].bold.black,
@@ -435,22 +550,19 @@ class UnderpostMonitor {
           );
         }
 
-        // Both K8S readinessProbe AND container-status must be satisfied before
-        // declaring the deployment ready. The TCP probe ensures the port is bound;
-        // container-status == running-deployment ensures the application has
-        // completed its own startup sequence so traffic is not switched prematurely.
-        if (allPodsK8sReady && allPodsStatusReady) {
-          logger.info(`${tag} | All pods Ready (K8S readinessProbe satisfied)`);
+        // Terminal success requires BOTH phases. runtime_ready cannot precede
+        // Kubernetes readiness.
+        if (allPodsK8sReady && allRuntimeReady) {
+          emit('runtime_ready', expectedStatus);
+          logger.info(`${tag} | Deployment ready (K8S Ready + runtime ${expectedStatus})`);
           return result;
         }
 
         await timer(delayMs);
-
-        if ((i + 1) % 10 === 0) {
-          logger.info(`${tag} | In progress... iteration ${i + 1}`);
-        }
+        if ((i + 1) % 10 === 0) logger.info(`${tag} | In progress... iteration ${i + 1}`);
       }
 
+      emit('timeout');
       logger.error(`${tag} | Deployment timeout after ${maxIterations} iterations`);
       throw new Error(
         `monitorReadyRunner timeout: ${deploymentId} did not become Ready within ${maxIterations}*${delayMs}ms`,

package/src/cli/release.js

@@ -264,18 +264,29 @@ const ISOLATED_ENV = 'env -i HOME="$HOME" PATH="$PATH" USER="$USER" LOGNAME="$LO
  *
  * @returns {boolean} true when the template started cleanly, false otherwise.
  */
-async function buildAndTestTemplate() {
+async function buildAndTestTemplate(opts = {}) {
   killDevServers();
   Underpost.repo.clean({ paths: ['/home/dd/engine', '/home/dd/engine/engine-private '] });
   shellExec(`node bin pull . ${process.env.GITHUB_USERNAME}/engine`);
+  fs.removeSync(TEMPLATE_PATH);
   shellExec(`npm run build:template`);
   shellExec(`node bin run shared-dir ${TEMPLATE_PATH}`);
 
+  const upsertEnvVar = (content, key, value) => {
+    const re = new RegExp(`^(${key}=).*`, 'm');
+    if (re.test(content)) return content.replace(re, `$1${value}`);
+    return `${content.trimEnd()}\n${key}=${value}\n`;
+  };
+
   const dhcpHostIp = Dns.getLocalIPv4Address();
   logger.info(`DHCP host IP for template test: ${dhcpHostIp}`);
   let envContent = fs.readFileSync(`${TEMPLATE_PATH}/.env.example`, 'utf8');
   if (dhcpHostIp) envContent = envContent.replace(/127\.0\.0\.1/g, dhcpHostIp);
-  envContent = envContent.replace(/^ENABLE_FILE_LOGS=.*/m, 'ENABLE_FILE_LOGS=true');
+  envContent = upsertEnvVar(envContent, 'ENABLE_FILE_LOGS', 'true');
+  if (opts.mongoHost) envContent = upsertEnvVar(envContent, 'DB_HOST', opts.mongoHost);
+  if (opts.mongoUser) envContent = upsertEnvVar(envContent, 'DB_USER', opts.mongoUser);
+  if (opts.mongoPassword) envContent = upsertEnvVar(envContent, 'DB_PASSWORD', opts.mongoPassword);
+  if (opts.valkeyHost) envContent = upsertEnvVar(envContent, 'VALKEY_HOST', opts.valkeyHost);
   // fs.writeFileSync(`${TEMPLATE_PATH}/.env`, envContent, 'utf8');
   fs.writeFileSync(`${TEMPLATE_PATH}/.env.example`, envContent, 'utf8');
   shellExec(`cd ${TEMPLATE_PATH} && npm install`);
@@ -337,7 +348,12 @@ class UnderpostRelease {
      *
      * @method build
      * @param {string} [newVersion] - The new version string to set. Defaults to current version if not provided.
-     * @param {{dryRun?: boolean}} [options] - Commander options. `--dry-run` previews changes.
+     * @param {{dryRun?: boolean, mongoHost?: string, mongoUser?: string, mongoPassword?: string, valkeyHost?: string}} [options] - Commander options.
+     *   `--dry-run` previews changes without writing files.
+     *   `--mongo-host` overrides `DB_HOST` in the template `.env.example` smoke test.
+     *   `--mongo-user` overrides `DB_USER` in the template `.env.example` smoke test.
+     *   `--mongo-password` overrides `DB_PASSWORD` in the template `.env.example` smoke test.
+     *   `--valkey-host` overrides `VALKEY_HOST` in the template `.env.example` smoke test.
      * @memberof UnderpostRelease
      */
     async build(newVersion, options = {}) {
@@ -352,7 +368,7 @@ class UnderpostRelease {
       logger.info(`Release build — bumping ${version} → ${newVersion}${dryRun ? ' (dry-run)' : ''}`);
 
       if (!dryRun) {
-        const templateOk = await buildAndTestTemplate();
+        const templateOk = await buildAndTestTemplate(options);
         if (!templateOk) return;
       }
 
@@ -390,8 +406,7 @@ class UnderpostRelease {
       shellExec(`node bin/deploy cli-docs ${version} ${newVersion}`);
       shellExec(`node bin/deploy update-dependencies`);
       shellExec(`node bin/build dd`);
-      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd production`);
-      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd development`);
+      shellExec(`node bin run build-cluster-deployment-manifests`);
       shellExec(`node bin new --default-conf --conf-workflow-id template`);
       shellExec(`sudo rm -rf ./engine-private/conf/dd-default`);
       shellExec(`node bin new --deploy-id dd-default`);

package/src/cli/repository.js

@@ -133,10 +133,21 @@ class UnderpostRepository {
         p: undefined,
         bc: '',
         isRemoteRepo: '',
+        hasChanges: false,
       },
     ) {
       if (!repoPath) repoPath = '.';
 
+      if (options.hasChanges) {
+        const status = shellExec(`cd ${repoPath} && git status --porcelain`, {
+          stdout: true,
+          silent: true,
+          disableLog: true,
+        }).trim();
+        process.stdout.write(status ? '1' : '');
+        return;
+      }
+
       if (options.isRemoteRepo) {
         const accessible = Underpost.repo.isRemoteRepo(options.isRemoteRepo);
         console.log(accessible);
@@ -608,7 +619,8 @@ class UnderpostRepository {
             const npmRoot = getNpmRootPath();
             const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
             const destFolder = `./${projectName}`;
-            logger.info('build app', { destFolder });
+            const deployId = projectName.startsWith('dd-') ? projectName : `dd-${projectName}`;
+            logger.info('build app', { destFolder, deployId });
             if (fs.existsSync(destFolder)) fs.removeSync(destFolder);
             fs.mkdirSync(destFolder, { recursive: true });
             if (!options.dev) {
@@ -621,8 +633,9 @@ class UnderpostRepository {
               UnderpostRepository.API.initLocalRepo({ path: destFolder });
               shellExec(`cd ${destFolder} && git add . && git commit -m "Base template implementation"`);
             }
-            shellExec(`cd ${destFolder} && npm run build`);
-            shellExec(`cd ${destFolder} && npm run dev`);
+            shellExec(`cd ${destFolder} && node bin new --deploy-id ${deployId} --default-conf`);
+            shellExec(`cd ${destFolder} && node bin client ${deployId}`);
+            shellExec(`cd ${destFolder} && DEPLOY_ID=${deployId} npm run dev`);
           }
           return resolve(true);
         } catch (error) {
@@ -1380,8 +1393,9 @@ Prevent build private config repo.`,
       const gitEmail = process.env.GITHUB_EMAIL || `development@underpost.net`;
 
       if (!fs.existsSync(`${repoPath}/.git`)) {
-        shellExec(`cd "${repoPath}" && git init`);
+        shellExec(`mkdir -p "${repoPath}" && git init "${repoPath}"`);
       }
+
       shellExec(`cd "${repoPath}" && git config user.name '${gitUsername}'`);
       shellExec(`cd "${repoPath}" && git config user.email '${gitEmail}'`);
       shellExec(`cd "${repoPath}" && git config core.filemode false`);

package/src/cli/run.js

@@ -265,6 +265,16 @@ class UnderpostRun {
       logger.info(hostListenResult.renderHosts);
     },
 
+    /**
+     * @method etc-hosts
+     * @description Modifies the `/etc/hosts` file to add entries for local access to services,
+     * based on the provided path input.
+     * @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
+     */
+    'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
+      etcHostFactory(path.split(','));
+    },
+
     /**
      * @method ipfs-expose
      * @description Exposes IPFS Cluster services on specified ports for local access.
@@ -2685,6 +2695,18 @@ EOF`;
       }
     },
 
+    /**
+     * @method build-cluster-deployment-manifests
+     * @description Builds deployment manifests for both production and development environments using `node bin deploy --build-manifest`, syncing them, and setting replicas to 1 for the `dd` deployment.
+     * @param {string} path - Unused.
+     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @memberof UnderpostRun
+     */
+    'build-cluster-deployment-manifests': (path = '', options = DEFAULT_OPTION) => {
+      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd development`);
+      shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd production --cert`);
+    },
+
     /**
      * @method monitor-ui
      * @description Installs and enables the Cockpit KVM Dashboard (cockpit, cockpit-machines, libvirt)

package/src/db/mongo/MongooseDB.js

@@ -86,7 +86,8 @@ class MongooseDBService {
 
     const user = config.user || process.env.DB_USER || '';
     const password = config.password || process.env.DB_PASSWORD || '';
-    const directConnection = hosts.length === 1;
+    const hasExplicitReplicaSet = !!(config.replicaSet || process.env.DB_REPLICA_SET);
+    const directConnection = hosts.length === 1 && !hasExplicitReplicaSet;
     const replicaSet = directConnection
       ? ''
       : config.replicaSet || process.env.DB_REPLICA_SET || MONGODB_DEFAULT_REPLICA_SET;

package/src/index.js

@@ -44,7 +44,7 @@ class Underpost {
    * @type {String}
    * @memberof Underpost
    */
-  static version = 'v3.2.14';
+  static version = 'v3.2.21';
 
   /**
    * Required Node.js major version

package/src/server/conf.js

@@ -1086,13 +1086,9 @@ const buildPortProxyRouter = (
 
   if (Object.keys(router).length === 0) return router;
 
-  if (options.devProxyContext === true && process.env.NODE_ENV === 'development') {
-    const confDevApiServer = JSON.parse(
-      fs.readFileSync(
-        `./engine-private/conf/${process.argv[3]}/conf.server.dev.${process.argv[4]}-dev-api.json`,
-        'utf8',
-      ),
-    );
+  const devApiConfPath = `./engine-private/conf/${process.argv[3]}/conf.server.dev.${process.argv[4]}-dev-api.json`;
+  if (options.devProxyContext === true && process.env.NODE_ENV === 'development' && fs.existsSync(devApiConfPath)) {
+    const confDevApiServer = JSON.parse(fs.readFileSync(devApiConfPath, 'utf8'));
     let devApiHosts = [];
     let origins = [];
     for (const _host of Object.keys(confDevApiServer))
@@ -1524,11 +1520,12 @@ const buildCliDoc = (program, oldVersion, newVersion) => {
     if (name === 'help') continue;
     const cmdHelp = parseHelp(help(name));
     details +=
-      `\n### \`underpost ${name}\`\n\n` +
+      `\n### underpost ${name}\n\n` +
       (cmdHelp.description ? `${cmdHelp.description.replace(/\s+/g, ' ')}\n\n` : '') +
       `**Usage:** \`${cmdHelp.usage}\`\n` +
       detailSection(cmdHelp.sections, 'Arguments', ['Argument', 'Description']) +
-      detailSection(cmdHelp.sections, 'Options', ['Option', 'Description']);
+      detailSection(cmdHelp.sections, 'Options', ['Option', 'Description']) +
+      `\n---\n`;
   }
 
   const md = `${index}${details}`.replaceAll(oldVersion, newVersion);
@@ -2029,6 +2026,32 @@ const buildTemplate = async ({ srcPath = './', toPath = '../pwa-microservices-te
   );
 };
 
+const updatePrivateTemplateRepo = async () => {
+  const templatePath = '/home/dd/pwa-microservices-template';
+  shellExec(`sudo rm -rf ${templatePath}
+cd /home/dd/engine && npm run build:template
+cd /home/dd
+underpost clone --bare underpostnet/pwa-microservices-template-private
+sudo rm -rf ${templatePath}/.git
+mv ./pwa-microservices-template-private.git ${templatePath}/.git
+cd ${templatePath}
+npm install --omit=dev --ignore-scripts
+git init
+git config user.name 'underpostnet'
+git config user.email 'development@underpost.net'
+git add .`);
+  const hasChanges = shellExec(`node bin cmt ${templatePath} --has-changes`, {
+    stdout: true,
+    silent: true,
+    disableLog: true,
+  }).trim();
+  if (hasChanges === '1') {
+    shellExec(
+      `cd ${templatePath} && git commit -m 'Update template' && underpost push . underpostnet/pwa-microservices-template-private`,
+    );
+  }
+};
+
 export {
   Config,
   loadConf,
@@ -2080,4 +2103,5 @@ export {
   syncPrivateConf,
   syncDeployIdSources,
   buildTemplate,
+  updatePrivateTemplateRepo,
 };