underpost 3.2.12 → 3.2.14

package/src/cli/monitor.js

@@ -12,6 +12,7 @@ import {
   etcHostFactory,
 } from '../server/conf.js';
 import { loggerFactory } from '../server/logger.js';
+import { timer } from '../client/components/core/CommonJs.js';
 import axios from 'axios';
 import fs from 'fs-extra';
 import { shellExec } from '../server/process.js';
@@ -93,13 +94,13 @@ class UnderpostMonitor {
       }
 
       if (options.readyDeployment) {
-        for (const version of options.versions.split(',')) {
-          (async () => {
-            await Underpost.deploy.monitorReadyRunner(deployId, env, version, [], options.namespace, 'underpost');
+        await Promise.all(
+          options.versions.split(',').map(async (version) => {
+            await Underpost.monitor.monitorReadyRunner(deployId, env, version, [], options.namespace);
             if (options.promote)
               Underpost.deploy.switchTraffic(deployId, env, version, options.replicas, options.namespace, options);
-          })();
-        }
+          }),
+        );
         return;
       }
 
@@ -227,7 +228,7 @@ class UnderpostMonitor {
                     monitorPodName = undefined;
                   }
                   const checkDeploymentReadyStatus = async () => {
-                    const { ready, notReadyPods, readyPods } = await Underpost.deploy.checkDeploymentReadyStatus(
+                    const { ready, notReadyPods, readyPods } = await Underpost.monitor.checkDeploymentReadyStatus(
                       deployId,
                       env,
                       traffic,
@@ -272,6 +273,189 @@ class UnderpostMonitor {
       };
       return new Promise((...args) => monitorCallBack(...args));
     },
+    /**
+     * Checks the status of a deployment.
+     * @param {string} deployId - Deployment ID for which the status is being checked.
+     * @param {string} env - Environment for which the status is being checked.
+     * @param {string} traffic - Current traffic status for the deployment.
+     * @param {Array<string>} ignoresNames - List of pod names to ignore.
+     * @param {string} [namespace='default'] - Kubernetes namespace for the deployment.
+     * @returns {object} - Object containing the status of the deployment.
+     * @memberof UnderpostMonitor
+     */
+    async checkDeploymentReadyStatus(deployId, env, traffic, ignoresNames = [], namespace = 'default') {
+      const pods = Underpost.kubectl.get(`${deployId}-${env}-${traffic}`, 'pods', namespace);
+      const readyPods = [];
+      const notReadyPods = [];
+
+      // Readiness signal: the pod's Kubernetes `Ready` condition driven by the
+      // container's readinessProbe (TCP socket, HTTP get, or exec). Set by kubelet
+      // when the probe passes. A failed or crashing runtime never becomes Ready —
+      // kubelet surfaces CrashLoopBackOff and this gate stays closed.
+      for (const pod of pods) {
+        const { NAME } = pod;
+        if (ignoresNames && ignoresNames.find((t) => NAME.trim().toLowerCase().match(t.trim().toLowerCase()))) continue;
+
+        let podJson = null;
+        try {
+          // Pod may not exist yet (between deployment apply and pod
+          // scheduling). silentOnError lets the monitor loop continue
+          // instead of aborting on the transient NotFound exit.
+          const raw = shellExec(`sudo kubectl get pod ${NAME} -n ${namespace} -o json`, {
+            silent: true,
+            disableLog: true,
+            stdout: true,
+            silentOnError: true,
+          });
+          podJson = raw ? JSON.parse(raw) : null;
+        } catch (_) {
+          podJson = null;
+        }
+        const conditions = podJson?.status?.conditions || [];
+        const readyCondition = conditions.find((c) => c.type === 'Ready');
+        const k8sReady = readyCondition?.status === 'True';
+
+        pod.out = JSON.stringify({ k8sReady, condition: readyCondition ?? null });
+
+        if (k8sReady) readyPods.push(pod);
+        else notReadyPods.push(pod);
+      }
+      const consideredCount = readyPods.length + notReadyPods.length;
+      return {
+        ready: consideredCount > 0 && notReadyPods.length === 0,
+        notReadyPods,
+        readyPods,
+      };
+    },
+    /**
+     * Monitors the ready status of a deployment.
+     *
+     * Ready signal:
+     *   The orchestrator gate is the Kubernetes pod Ready condition. When the
+     *   container's `readinessProbe` succeeds, kubelet flips
+     *   `status.conditions[Ready]` to True and `checkDeploymentReadyStatus`
+     *   returns the pod in `readyPods`. This is the only required signal — see
+     *   `src/client/public/nexodev/docs/references/Deploy custom instance to K8S.md`.
+     *
+     * Container-status:
+     *   `underpost config get container-status` is read from each pod for both
+     *   the display column and as a second ready gate alongside the K8S Ready
+     *   condition. Both must be satisfied before the monitor exits:
+     *     1. K8S readinessProbe (TCP socket) — ensures the port is bound.
+     *     2. container-status == `<deploy>-<env>-running-deployment` — ensures
+     *        the application has completed its own startup sequence.
+     *   Early-abort on `error` container-status remains in effect: a failing
+     *   runtime keeps its pod alive (not Ready) with `container-status=error`,
+     *   so this `exec`-read surfaces the failure and the monitor aborts —
+     *   failing the CD runner instead of waiting out the full timeout.
+     *
+     * @param {string} deployId - Deployment ID for which the ready status is being monitored.
+     * @param {string} env - Environment for which the ready status is being monitored.
+     * @param {string} targetTraffic - Target traffic status for the deployment.
+     * @param {Array<string>} ignorePods - List of pod names to ignore.
+     * @param {string} [namespace='default'] - Kubernetes namespace for the deployment.
+     * @returns {object} - Object containing the ready status of the deployment.
+     * @memberof UnderpostMonitor
+     */
+    async monitorReadyRunner(deployId, env, targetTraffic, ignorePods = [], namespace = 'default') {
+      const delayMs = 1000;
+      const maxIterations = 3000;
+      const deploymentId = `${deployId}-${env}-${targetTraffic}`;
+      const expectedContainerStatus = `${deployId}-${env}-running-deployment`;
+      const tag = `[${deploymentId}]`;
+      const containerStatusDefault = 'waiting for status';
+
+      logger.info('Deployment init', { deployId, env, targetTraffic, namespace });
+
+      const podStatusCache = new Map();
+      const advancedPods = new Set();
+
+      const readContainerStatus = (podName) => {
+        try {
+          const raw = shellExec(
+            `sudo kubectl exec ${podName} -n ${namespace} -- sh -c 'underpost config get container-status --plain'`,
+            { silent: true, disableLog: true, stdout: true, silentOnError: true },
+          );
+          const val = raw ? raw.toString().trim() : '';
+          return val && val !== 'undefined' ? val : containerStatusDefault;
+        } catch (_) {
+          // exec failed (e.g. pod not yet running) — preserve last known value
+          return podStatusCache.get(podName) || containerStatusDefault;
+        }
+      };
+
+      for (let i = 0; i < maxIterations; i++) {
+        const result = await Underpost.monitor.checkDeploymentReadyStatus(
+          deployId,
+          env,
+          targetTraffic,
+          ignorePods,
+          namespace,
+        );
+
+        const allPods = [...result.readyPods, ...result.notReadyPods];
+
+        for (const pod of allPods) {
+          if (!pod?.NAME) continue;
+          const podStatus = (pod.STATUS || '').toLowerCase().trim();
+          if (
+            ['error', 'crashloopbackoff', 'oomkilled', 'imagepullbackoff', 'errimagepull'].find((s) =>
+              podStatus.match(s),
+            )
+          )
+            throw new Error(`Pod ${pod.NAME} has error pod status: ${pod.STATUS}`);
+          const status = readContainerStatus(pod.NAME);
+          if (status === 'error') throw new Error(`Pod ${pod.NAME} has error container-status`);
+          if (advancedPods.has(pod.NAME) && status === containerStatusDefault)
+            throw new Error(`Pod ${pod.NAME} container-status regressed to default — pod likely restarted`);
+          if (status !== containerStatusDefault) advancedPods.add(pod.NAME);
+          podStatusCache.set(pod.NAME, status);
+        }
+
+        const allPodsK8sReady = allPods.length > 0 && result.notReadyPods.length === 0;
+
+        const allPodsStatusReady =
+          allPods.length > 0 && allPods.every((pod) => podStatusCache.get(pod.NAME) === expectedContainerStatus);
+
+        // Print snapshot for every pod — annotate when container-status hasn't caught
+        // up to the K8S Ready condition yet.
+        for (const pod of allPods) {
+          const status = podStatusCache.get(pod.NAME) || containerStatusDefault;
+          const podStatus = pod.STATUS || 'Unknown';
+          const statusMatchesExpected = status === expectedContainerStatus;
+          const statusDisplay = statusMatchesExpected ? status : `${status} (pending)`;
+
+          console.log(
+            'Target pod:',
+            pod.NAME[pod.NAME.includes('green') ? 'bgGreen' : 'bgBlue'].bold.black,
+            '| Pod status:',
+            podStatus.bold.yellow,
+            '| Runtime status:',
+            statusDisplay.bold.cyan,
+          );
+        }
+
+        // Both K8S readinessProbe AND container-status must be satisfied before
+        // declaring the deployment ready. The TCP probe ensures the port is bound;
+        // container-status == running-deployment ensures the application has
+        // completed its own startup sequence so traffic is not switched prematurely.
+        if (allPodsK8sReady && allPodsStatusReady) {
+          logger.info(`${tag} | All pods Ready (K8S readinessProbe satisfied)`);
+          return result;
+        }
+
+        await timer(delayMs);
+
+        if ((i + 1) % 10 === 0) {
+          logger.info(`${tag} | In progress... iteration ${i + 1}`);
+        }
+      }
+
+      logger.error(`${tag} | Deployment timeout after ${maxIterations} iterations`);
+      throw new Error(
+        `monitorReadyRunner timeout: ${deploymentId} did not become Ready within ${maxIterations}*${delayMs}ms`,
+      );
+    },
   };
 }
 

package/src/cli/release.js

@@ -268,7 +268,7 @@ async function buildAndTestTemplate() {
   killDevServers();
   Underpost.repo.clean({ paths: ['/home/dd/engine', '/home/dd/engine/engine-private '] });
   shellExec(`node bin pull . ${process.env.GITHUB_USERNAME}/engine`);
-  shellExec(`npm run update:template`);
+  shellExec(`npm run build:template`);
   shellExec(`node bin run shared-dir ${TEMPLATE_PATH}`);
 
   const dhcpHostIp = Dns.getLocalIPv4Address();
@@ -392,7 +392,7 @@ class UnderpostRelease {
       shellExec(`node bin/build dd`);
       shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd production`);
       shellExec(`node bin deploy --build-manifest --sync --info-router --replicas 1 dd development`);
-      shellExec(`node bin/deploy build-default-confs`);
+      shellExec(`node bin new --default-conf --conf-workflow-id template`);
       shellExec(`sudo rm -rf ./engine-private/conf/dd-default`);
       shellExec(`node bin new --deploy-id dd-default`);
       console.log(fs.existsSync(`./engine-private/conf/dd-default`));
@@ -460,7 +460,7 @@ class UnderpostRelease {
      * Runs the pwa-microservices-template update and push flow locally.
      *
      * Always removes and re-clones pwa-microservices-template, then:
-     * 1. Runs update:template (node bin/build.template) to sync engine sources.
+     * 1. Runs build:template (node bin/build.template) to sync engine sources.
      * 2. Installs dependencies and builds the template.
      * 3. Commits and pushes to the pwa-microservices-template remote repository.
      *
@@ -488,7 +488,7 @@ class UnderpostRelease {
       shellExec(`sudo rm -rf /home/dd/pwa-microservices-template`);
       shellExec(`node engine/bin clone ${githubOrg}/pwa-microservices-template`);
       shellCd('/home/dd/engine');
-      shellExec(`npm run update:template`);
+      shellExec(`npm run build:template`);
       shellExec(`cd ../pwa-microservices-template && npm install && npm run build`);
       shellCd('/home/dd/pwa-microservices-template');
       shellExec(`git add .`);
@@ -520,7 +520,7 @@ class UnderpostRelease {
       shellExec(
         `node bin secret underpost --create-from-file /home/dd/engine/engine-private/conf/dd-cron/.env.production`,
       );
-      shellExec(`node bin/build dd conf`);
+      shellExec(`node bin/build dd --conf`);
       shellExec(`git add . && cd ./engine-private && git add .`);
       shellExec(`node bin cmt . ci package-pwa-microservices-template 'New release v:${version}'`);
       shellExec(`node bin cmt ./engine-private ci package-pwa-microservices-template`);

package/src/cli/repository.js

@@ -556,7 +556,7 @@ class UnderpostRepository {
             // Handle sync-conf operation
             if (options.syncConf) {
               logger.info(`Syncing configuration for deploy ID: ${deployId}`);
-              shellExec(`node bin/build ${deployId} conf`);
+              shellExec(`node bin/build ${deployId} --conf`);
               logger.info('Configuration synced successfully');
               return resolve(true);
             }
@@ -852,6 +852,7 @@ class UnderpostRepository {
               }
             }
             await buildClient({
+              deployId: resolvedDeployId,
               buildZip: options.buildZip || false,
               split: options.split || '',
               fullBuild: options.liteBuild ? false : true,
@@ -862,7 +863,12 @@ class UnderpostRepository {
                 logger.warn('Skip replica client build: replica folder not found', { replicaDeployId });
                 continue;
               }
-              await Underpost.repo.client(replicaDeployId);
+              await Underpost.repo.client(replicaDeployId, '', '', '', {
+                buildZip: options.buildZip || false,
+                split: options.split || '',
+                liteBuild: options.liteBuild || false,
+                iconsBuild: options.iconsBuild || false,
+              });
             }
 
             return resolve(true);
@@ -937,7 +943,7 @@ Prevent build private config repo.`,
           deployVersion: packageJsonDeploy.version,
         };
       }
-      shellExec(`node bin/build ${deployId} conf`);
+      shellExec(`node bin/build ${deployId} --conf`);
       return {
         validVersion: true,
         engineVersion: packageJsonEngine.version,
@@ -1653,6 +1659,77 @@ Prevent build private config repo.`,
       }
       return fallback;
     },
+
+    /**
+     * Performs a shallow sparse Git checkout of a single subdirectory from any
+     * GitHub repository into a local target directory.
+     *
+     * Uses `--depth 1 --no-checkout` + `git sparse-checkout` so only the
+     * requested path is fetched — no full clone of the remote repo.
+     * Skips the clone entirely when `<targetDir>/<subPath>` already exists on
+     * disk (idempotent).
+     *
+     * Requires `GITHUB_TOKEN` to be set in the environment for authenticated
+     * access to private repositories.
+     *
+     * @param {string} subPath - The subdirectory path within the remote repo to
+     *   check out (e.g. `'conf/dd-prototype'`, `'src/api/payments'`).
+     * @param {object} [options]
+     * @param {string} [options.repoOwner='underpostnet'] - GitHub organisation or
+     *   user that owns the repository.
+     * @param {string} [options.repoName='engine-private'] - Name of the
+     *   repository on GitHub.
+     * @param {string} [options.targetDir='./engine-private'] - Local directory
+     *   where the repo will be cloned.
+     * @returns {boolean} `true` when the checkout was performed, `false` when it
+     *   was skipped because the target path already existed.
+     * @memberof UnderpostRepository
+     */
+    sparseCheckoutDirectory(
+      subPath,
+      options = { repoOwner: 'underpostnet', repoName: 'engine-private', targetDir: './engine-private' },
+    ) {
+      const { repoOwner = 'underpostnet', repoName = 'engine-private', targetDir = './engine-private' } = options;
+      const localPath = `${targetDir}/${subPath}`;
+      if (fs.existsSync(localPath)) {
+        logger.info('[sparseCheckoutDirectory] path already present, skipping', localPath);
+        return false;
+      }
+      const authUrl = `https://${process.env.GITHUB_TOKEN}@github.com/${repoOwner}/${repoName}.git`;
+      shellExec(`git clone --depth 1 --no-checkout ${authUrl} ${targetDir}`, { disableLog: true });
+      shellExec(`cd ${targetDir} && git sparse-checkout set ${subPath} && git checkout`, { disableLog: true });
+      logger.info('[sparseCheckoutDirectory] sparse checkout complete', localPath);
+      return true;
+    },
+
+    /**
+     * Ensures a deploy's public source repo (e.g. `engine-prototype`) is present
+     * next to the engine and reset to a pristine HEAD, so catalog `sourceMoves`
+     * can (re)pull custom sources even after a previous build moved them out of
+     * the source tree.
+     *
+     * Clones `../<repoName>` when missing; otherwise restores a clean checkout
+     * (`git checkout .` brings back any moved-out tracked files) and pulls latest.
+     * Mirrors the sibling-repo handling used by `syncPrivateConf`.
+     *
+     * @param {string} repoName - Public source repo name (e.g. `engine-prototype`).
+     * @returns {boolean} `true` when the repo is available on disk.
+     * @memberof UnderpostRepository
+     */
+    pullSourceRepo(repoName) {
+      const username = process.env.GITHUB_USERNAME;
+      if (!username || !repoName) return false;
+      const repoPath = `../${repoName}`;
+      const gitUri = `${username}/${repoName}`;
+      if (!fs.existsSync(repoPath)) {
+        shellExec(`cd .. && underpost clone ${gitUri}`, { silent: true });
+      } else {
+        shellExec(`cd ${repoPath} && git checkout . && git clean -f -d && underpost pull . ${gitUri}`, {
+          silent: true,
+        });
+      }
+      return fs.existsSync(repoPath);
+    },
   };
 }