underpost 3.2.28 → 3.2.70

package/src/cli/secrets.js

@@ -13,6 +13,58 @@ import { loggerFactory } from '../server/logger.js';
 
 const logger = loggerFactory(import.meta);
 
+// Shell/runtime-critical and Kubernetes-injected env keys that must never be persisted as
+// application secrets nor injected into a pod via `envFrom`. An injected PATH (or HOME, etc.)
+// overrides the container image's own and breaks coreutils/sudo resolution inside the pod
+// ("rm: command not found"). Single source of truth for both container-env capture and the
+// `underpost-config` secret built from an env file.
+const RESERVED_ENV_KEYS = new Set([
+  'HOME',
+  'HOSTNAME',
+  'PATH',
+  'TERM',
+  'SHLVL',
+  'PWD',
+  '_',
+  'LANG',
+  'LANGUAGE',
+  'LC_ALL',
+  'container',
+  'SHELL',
+  'USER',
+  'LOGNAME',
+  'MAIL',
+  'OLDPWD',
+  'LESSOPEN',
+  'LESSCLOSE',
+  'LS_COLORS',
+  'DISPLAY',
+  'COLORTERM',
+  'EDITOR',
+  'VISUAL',
+  'TERM_PROGRAM',
+  'TERM_PROGRAM_VERSION',
+  'SSH_AUTH_SOCK',
+  'SSH_CLIENT',
+  'SSH_CONNECTION',
+  'SSH_TTY',
+  'XDG_SESSION_ID',
+  'XDG_RUNTIME_DIR',
+  'XDG_DATA_DIRS',
+  'XDG_CONFIG_DIRS',
+  'DBUS_SESSION_BUS_ADDRESS',
+  'GPG_AGENT_INFO',
+  'WINDOWID',
+  'DESKTOP_SESSION',
+  'SESSION_MANAGER',
+  'XAUTHORITY',
+  'WAYLAND_DISPLAY',
+  'which_declare',
+]);
+const RESERVED_ENV_KEY_PREFIXES = ['KUBERNETES_', 'npm_', 'NODE_'];
+const isReservedEnvKey = (key) =>
+  RESERVED_ENV_KEYS.has(key) || RESERVED_ENV_KEY_PREFIXES.some((prefix) => key.startsWith(prefix));
+
 /**
  * @class UnderpostSecret
  * @description Manages the secrets of the application.
@@ -49,67 +101,49 @@ class UnderpostSecret {
        */
       createFromContainerEnv() {
         Underpost.env.clean();
-        const systemKeys = new Set([
-          'HOME',
-          'HOSTNAME',
-          'PATH',
-          'TERM',
-          'SHLVL',
-          'PWD',
-          '_',
-          'LANG',
-          'LANGUAGE',
-          'LC_ALL',
-          'container',
-          'SHELL',
-          'USER',
-          'LOGNAME',
-          'MAIL',
-          'OLDPWD',
-          'LESSOPEN',
-          'LESSCLOSE',
-          'LS_COLORS',
-          'DISPLAY',
-          'COLORTERM',
-          'EDITOR',
-          'VISUAL',
-          'TERM_PROGRAM',
-          'TERM_PROGRAM_VERSION',
-          'SSH_AUTH_SOCK',
-          'SSH_CLIENT',
-          'SSH_CONNECTION',
-          'SSH_TTY',
-          'XDG_SESSION_ID',
-          'XDG_RUNTIME_DIR',
-          'XDG_DATA_DIRS',
-          'XDG_CONFIG_DIRS',
-          'DBUS_SESSION_BUS_ADDRESS',
-          'GPG_AGENT_INFO',
-          'WINDOWID',
-          'DESKTOP_SESSION',
-          'SESSION_MANAGER',
-          'XAUTHORITY',
-          'WAYLAND_DISPLAY',
-          'which_declare',
-        ]);
-        const systemKeyPrefixes = ['KUBERNETES_', 'npm_', 'NODE_'];
         for (const [key, value] of Object.entries(process.env)) {
-          if (systemKeys.has(key)) continue;
-          if (systemKeyPrefixes.some((prefix) => key.startsWith(prefix))) continue;
+          if (isReservedEnvKey(key)) continue;
           Underpost.env.set(key, value);
         }
       },
     },
 
+    /**
+     * @method sanitizeSecretEnvFile
+     * @description Strips shell/runtime-critical and Kubernetes-injected keys (PATH, HOME, …) from
+     * raw `.env` file content so the resulting `underpost-config` secret can be safely injected via
+     * `envFrom` without clobbering the container image's own PATH. Blank lines and comments are
+     * preserved. Uses the same {@link RESERVED_ENV_KEYS} blocklist as container-env capture.
+     * @param {string} envFileContent - Raw contents of a `.env.<env>` file.
+     * @returns {string} Filtered env-file content.
+     * @memberof UnderpostSecret
+     */
+    sanitizeSecretEnvFile(envFileContent) {
+      return envFileContent
+        .split('\n')
+        .filter((line) => {
+          const trimmed = line.trimStart();
+          if (!trimmed || trimmed.startsWith('#')) return true;
+          const key = line.slice(0, line.indexOf('=')).trim();
+          return !key || !isReservedEnvKey(key);
+        })
+        .join('\n');
+    },
+
     /**
      * Removes all filesystem traces of secrets after deployment startup.
      * Centralizes the defense-in-depth cleanup performed
+     * @param {object} options - Options for cleaning the environment.
+     * @param {Array<string>} [options.keepKeys=[]] - List of keys to keep in the environment file. If provided, only these keys will be retained.
      * @memberof UnderpostSecret
      */
-    globalSecretClean() {
+    globalSecretClean(options = { keepKeys: [] }) {
+      const { keepKeys } = options;
       loadConf('clean');
       Underpost.repo.cleanupPrivateEngineRepo();
-      Underpost.env.clean();
+      Underpost.env.clean({
+        keepKeys: keepKeys.length > 0 ? keepKeys : ['container-status', 'start-container-status'],
+      });
     },
   };
 }

package/src/cli/ssh.js

@@ -496,6 +496,50 @@ EOF`);
       if (options.status) shellExec('service sshd status');
     },
 
+    /**
+     * Synchronously copies a local directory to a remote host over key-only SSH,
+     * streaming it as a tar archive (no intermediate file) and fixing ownership.
+     * Mirrors the kind-node `tar | docker cp` provisioning pattern but targets a
+     * real node via SSH, so node-local hostPath volumes can be materialized on the
+     * node where the pod will actually run.
+     *
+     * Idempotent and re-runnable: `mkdir -p` + `tar -x` overwrite in place. Throws
+     * on any SSH/tar failure so an empty-volume deploy is never produced silently.
+     *
+     * @function copyDirToNode
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP (key-only SSH reachable).
+     * @param {string} params.localDir - Local source directory.
+     * @param {string} params.remoteDir - Destination directory on the node.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {string} [params.user='root'] - SSH user (key-only).
+     * @param {string} [params.keyPath='./engine-private/deploy/id_rsa'] - Private key path.
+     * @param {string} [params.owner='1000:1000'] - chown target on the node (empty to skip).
+     * @param {string} [params.mode='755'] - chmod mode on the node (empty to skip).
+     * @returns {void}
+     */
+    copyDirToNode: ({
+      host,
+      localDir,
+      remoteDir,
+      port = 22,
+      user = 'root',
+      keyPath = './engine-private/deploy/id_rsa',
+      owner = '1000:1000',
+      mode = '755',
+    }) => {
+      if (!host) throw new Error('copyDirToNode requires a host');
+      if (!localDir || !fs.existsSync(localDir)) throw new Error(`copyDirToNode: local dir not found: ${localDir}`);
+      if (!remoteDir) throw new Error('copyDirToNode requires a remoteDir');
+      shellExec(`chmod 600 ${keyPath}`, { silent: true, silentOnError: true, disableLog: true });
+      const sshOpts = `-i ${keyPath} -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p ${port}`;
+      shellExec(`ssh ${sshOpts} ${user}@${host} 'mkdir -p ${remoteDir}'`);
+      shellExec(`tar -C ${localDir} -c . | ssh ${sshOpts} ${user}@${host} 'tar -C ${remoteDir} -x'`);
+      const fixups = `${owner ? `chown -R ${owner} ${remoteDir}; ` : ''}${mode ? `chmod -R ${mode} ${remoteDir}` : ''}`.trim();
+      if (fixups) shellExec(`ssh ${sshOpts} ${user}@${host} '${fixups}'`);
+    },
+
     /**
      * Generic SSH remote command runner that SSH execution logic.
      * Executes arbitrary shell commands on a remote server via SSH with proper credential handling.

package/src/client.build.js

@@ -15,5 +15,3 @@ const logger = loggerFactory(import.meta);
 await logger.setUpInfo();
 
 await clientLiveBuild();
-
-// ProcessController.init(logger);

package/src/db/mongo/MongoBootstrap.js

@@ -290,19 +290,16 @@ class MongoBootstrap {
    * @returns {Promise<number>} Number of pods that failed to become ready (0 = all good).
    */
   static async waitForPods(namespace, replicaCount) {
-    const { default: Underpost } = await import('../../index.js');
-    const results = await Promise.all(
-      Array.from({ length: replicaCount }, async (_, i) => {
-        const podName = `${MONGODB_STATEFULSET_NAME}-${i}`;
-        const ready = await Underpost.test.statusMonitor(podName, 'Running', 'pods', 1000, 60 * 10);
-        return { index: i, ready };
-      }),
-    );
-    const failed = results.filter((r) => !r.ready).map((r) => `${MONGODB_STATEFULSET_NAME}-${r.index}`);
-    if (failed.length > 0) {
-      logger.error('MongoDB pods did not become ready', { failed });
+    let failedCount = 0;
+    for (let i = 0; i < replicaCount; i++) {
+      const podName = `${MONGODB_STATEFULSET_NAME}-${i}`;
+      const result = shellExec(`kubectl wait --for=condition=Ready pod/${podName} -n ${namespace} --timeout=60s`);
+      if (result.code !== 0) {
+        logger.error(`Pod ${podName} did not become ready`);
+        failedCount++;
+      }
     }
-    return failed.length;
+    return failedCount;
   }
 
   /**
@@ -620,6 +617,9 @@ class MongoBootstrap {
       process.env.DB_PASSWORD ||
       readTrimmedFile('./engine-private/mongodb-password');
 
+    // Ensure the pod is ready before querying
+    shellExec(`kubectl wait --for=condition=Ready pod/${podName} -n ${namespace} --timeout=60s`);
+
     const evalExpr = 'rs.status().members.filter(m=>m.stateStr=="PRIMARY").map(m=>m.name)';
 
     const cli = disableAuth ? 'mongo' : 'mongosh';

package/src/index.js

@@ -45,7 +45,7 @@ class Underpost {
    * @type {String}
    * @memberof Underpost
    */
-  static version = 'v3.2.28';
+  static version = 'v3.2.70';
 
   /**
    * Required Node.js major version

package/src/server/catalog.js

@@ -24,6 +24,8 @@ const EMPTY_CATALOG = {
   privateConfPaths: [],
   templatePaths: [],
   stripPaths: [],
+  moves: [],
+  copies: [],
   keywords: [],
   description: '',
 };

package/src/server/conf.js

@@ -476,9 +476,6 @@ const loadConf = (deployId = DEFAULT_DEPLOY_ID, subConf) => {
     fs.removeSync(`${path}/.env.production`);
     fs.removeSync(`${path}/.env.development`);
     fs.removeSync(`${path}/.env.test`);
-    if (fs.existsSync(`${path}/typedoc.json`)) shellExec(`git checkout ${path}/typedoc.json`);
-    shellExec(`git checkout ${path}/package.json`);
-    shellExec(`git checkout ${path}/package-lock.json`);
     return;
   }
   const folder = getConfFolder(deployId);

package/src/server/runtime-status.js

@@ -40,6 +40,7 @@ const RUNTIME_STATUS = {
 };
 
 const CONTAINER_STATUS_KEY = 'container-status';
+const START_CONTAINER_STATUS_KEY = 'start-container-status';
 const INTERNAL_STATUS_PATH = '/_internal/status';
 const INTERNAL_READY_PATH = '/_internal/ready';
 const INTERNAL_HEALTH_PATH = '/_internal/health';
@@ -118,6 +119,20 @@ const normalizeContainerStatus = (raw) => {
 const getRuntimeStatus = () =>
   normalizeContainerStatus(Underpost.env.get(CONTAINER_STATUS_KEY, undefined, { disableLog: true }));
 
+/**
+ * Reads the start-container-status env key — an insulated marker set by the
+ * start pipeline after it completes the running phase. Unlike container-status
+ * (which external scripts / backup failures may clobber), this key is written
+ * once and survives globalSecretClean. Used exclusively by the readinessProbe
+ * endpoint so K8s pod readiness is never derailed by lifecycle noise.
+ * @memberof RuntimeStatus
+ * @returns {string|undefined}
+ */
+const getStartContainerStatus = () => {
+  const raw = Underpost.env.get(START_CONTAINER_STATUS_KEY, undefined, { disableLog: true });
+  return raw && typeof raw === 'string' && raw.trim() ? raw.trim() : undefined;
+};
+
 /**
  * Minimal, secret-free payload served by the internal status endpoint and used
  * by the monitor for failure classification and observability.
@@ -188,7 +203,7 @@ const startInternalStatusServer = (port = resolveInternalStatusPort()) => {
       case INTERNAL_HEALTH_PATH:
         return sendJson(200, { status: 'ok' });
       case INTERNAL_READY_PATH:
-        return getRuntimeStatus() === RUNTIME_STATUS.RUNNING
+        return getStartContainerStatus()
           ? sendJson(200, { status: RUNTIME_STATUS.RUNNING })
           : sendJson(503, { status: getRuntimeStatus() ?? null });
       case INTERNAL_STATUS_PATH:
@@ -220,6 +235,7 @@ const stopInternalStatusServer = () =>
 export {
   RUNTIME_STATUS,
   CONTAINER_STATUS_KEY,
+  START_CONTAINER_STATUS_KEY,
   INTERNAL_STATUS_PATH,
   INTERNAL_READY_PATH,
   INTERNAL_HEALTH_PATH,
@@ -228,6 +244,7 @@ export {
   containerStatusValue,
   normalizeContainerStatus,
   getRuntimeStatus,
+  getStartContainerStatus,
   runtimeStatusPayload,
   setRuntimeStatus,
   startInternalStatusServer,

package/src/server/start.js

@@ -8,7 +8,14 @@ import fs from 'fs-extra';
 import { awaitDeployMonitor } from './conf.js';
 import { actionInitLog, loggerFactory } from './logger.js';
 import { shellCd, shellExec } from './process.js';
-import { RUNTIME_STATUS, setRuntimeStatus, startInternalStatusServer, deployStatusPort } from './runtime-status.js';
+import {
+  RUNTIME_STATUS,
+  START_CONTAINER_STATUS_KEY,
+  setRuntimeStatus,
+  startInternalStatusServer,
+  deployStatusPort,
+  containerStatusValue,
+} from './runtime-status.js';
 import Underpost from '../index.js';
 const logger = loggerFactory(import.meta);
 
@@ -241,7 +248,10 @@ class UnderpostStartUp {
       const result = await awaitDeployMonitor(true);
       if (result === true) {
         if (env === 'production' && Underpost.env.isInsideContainer()) Underpost.secret.globalSecretClean();
-        setRuntimeStatus(deployId, env, RUNTIME_STATUS.RUNNING);
+        setTimeout(() => {
+          setRuntimeStatus(deployId, env, RUNTIME_STATUS.RUNNING);
+          Underpost.env.set(START_CONTAINER_STATUS_KEY, containerStatusValue(deployId, env, RUNTIME_STATUS.RUNNING));
+        });
       } else {
         setRuntimeStatus(deployId, env, RUNTIME_STATUS.ERROR);
       }

package/src/server.js

@@ -3,18 +3,22 @@
 // https://nodejs.org/api
 // https://expressjs.com/en/4x/api.html
 
+import dotenv from 'dotenv';
 import { loggerFactory } from './server/logger.js';
 import { buildClient } from './client-builder/client-build.js';
 import { buildRuntime } from './server/runtime.js';
 import { ProcessController } from './server/process.js';
 import { Config } from './server/conf.js';
+
+dotenv.config();
+
 await Config.build();
 
 const logger = loggerFactory(import.meta);
 
 await logger.setUpInfo();
 
-await buildClient();
+if (process.env.NODE_ENV === 'development') await buildClient();
 
 await buildRuntime();
 

package/test/deploy-monitor.test.js

@@ -69,12 +69,10 @@ describe('Deploy monitor — two-phase state machine (e2e, real HTTP transport)'
     process.env.npm_config_prefix = tmpPrefix;
 
     Underpost.env.set('container-status', 'init');
+    Underpost.env.set('start-container-status', '');
     const npmRoot = shellExec('npm root -g', { stdout: true, silent: true, disableLog: true }).trim();
     envFile = path.join(npmRoot, 'underpost', '.env');
 
-    // Real in-pod internal status server: serves container-status from the same
-    // env file the runtime writes. Bound in this test process; the monitor's
-    // port-forward tunnel (localPort == INTERNAL_PORT) resolves straight to it.
     process.env.UNDERPOST_INTERNAL_PORT = String(INTERNAL_PORT);
     startInternalStatusServer(INTERNAL_PORT);
 
@@ -148,10 +146,11 @@ try {
   beforeEach(() => {
     // Deploy in flight: app not yet reporting running.
     Underpost.env.set('container-status', INIT_STATUS);
+    // start-container-status is unset (empty) — the readinessProbe
+    // sees 503 until the start pipeline completes and stamps this key.
+    Underpost.env.set('start-container-status', '');
   });
 
-  // Spawns the real monitorReadyRunner with the fake cluster on PATH; resolves
-  // with its exit code. `overrides` inject deterministic timing / target port.
   const spawnMonitor = (overrides = {}) =>
     new Promise((resolve) => {
       const envVars = {
@@ -160,9 +159,6 @@ try {
         POD_NAME,
         npm_config_prefix: tmpPrefix,
         UNDERPOST_INTERNAL_PORT: String(INTERNAL_PORT),
-        // Pin the tunnel's local port so the no-op fake port-forward + the real
-        // internal server (bound to INTERNAL_PORT in this process) resolve to the
-        // same address the monitor's HTTP GET targets.
         UNDERPOST_PF_LOCAL_PORT: String(INTERNAL_PORT),
         UNDERPOST_MONITOR_DELAY_MS: '100',
         UNDERPOST_MONITOR_MAX_ITERATIONS: '60',
@@ -183,12 +179,15 @@ try {
 
   it('success (default exec transport): both phases satisfied → monitor exits 0', async () => {
     Underpost.env.set('container-status', RUNNING_STATUS);
+    // start-container-status not needed for monitor (reads container-status);
+    // this test exercises the default exec transport path.
     const code = await spawnMonitor({ FAKE_POD_READY: 'True' });
     expect(code).to.equal(0);
   });
 
   it('success (opt-in http transport): both phases satisfied → monitor exits 0', async () => {
     Underpost.env.set('container-status', RUNNING_STATUS);
+    Underpost.env.set('start-container-status', RUNNING_STATUS);
     const code = await spawnMonitor({ FAKE_POD_READY: 'True', MON_TRANSPORT: 'http' });
     expect(code).to.equal(0);
   });
@@ -202,13 +201,13 @@ try {
   it('readiness mismatch: runtime running but pod not Ready → never succeeds (exits 1)', async () => {
     // Phase 2 satisfied, Phase 1 not: success requires BOTH, so it must time out.
     Underpost.env.set('container-status', RUNNING_STATUS);
+    // start-container-status is NOT set — the readinessProbe would fail,
+    // but the monitor only checks container-status via exec/http transport.
     const code = await spawnMonitor({ FAKE_POD_READY: 'False', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
     expect(code).to.equal(1);
   });
 
   it('transport failure (http): endpoint unreachable is never success (exits 1)', async () => {
-    // Opt into http and point the monitor at a port with no internal server; the
-    // HTTP read always fails, so runtime readiness is never confirmed → timeout.
     Underpost.env.set('container-status', RUNNING_STATUS);
     const code = await spawnMonitor({
       MON_TRANSPORT: 'http',
@@ -221,6 +220,8 @@ try {
 
   it('timeout: runtime stuck initializing → monitor exits 1', async () => {
     Underpost.env.set('container-status', INIT_STATUS);
+    // start-container-status remains unset — readinessProbe returns 503
+    // (correct, since the runtime is still initializing).
     const code = await spawnMonitor({ FAKE_POD_READY: 'True', UNDERPOST_MONITOR_MAX_ITERATIONS: '4' });
     expect(code).to.equal(1);
   });
@@ -235,6 +236,21 @@ try {
     expect(await monitorExit).to.equal(1);
   });
 
+  it('readinessProbe: start-container-status set → returns 200 (probe passes)', async () => {
+    Underpost.env.set('start-container-status', RUNNING_STATUS);
+    const res = await fetch(`http://127.0.0.1:${INTERNAL_PORT}/_internal/ready`);
+    expect(res.status).to.equal(200);
+    const body = await res.json();
+    expect(body.status).to.equal('running-deployment');
+  });
+
+  it('readinessProbe: start-container-status unset → returns 503 (probe fails)', async () => {
+    // Unset from beforeEach re-stamp; clear it.
+    Underpost.env.set('start-container-status', '');
+    const res = await fetch(`http://127.0.0.1:${INTERNAL_PORT}/_internal/ready`);
+    expect(res.status).to.equal(503);
+  });
+
   // Custom instances (cyberia-*) gate on K8s Ready and read status via exec;
   // their runtime never stamps `running-deployment` (stays `initializing`).
   it('instance (kubernetes gate + exec): K8s Ready with initializing status → exits 0', async () => {