underpost 3.2.21 → 3.2.28

package/src/cli/repository.js

@@ -8,6 +8,7 @@ import dotenv from 'dotenv';
 import { commitData } from '../client/components/core/CommonJs.js';
 import { pbcopy, shellCd, shellExec } from '../server/process.js';
 import { actionInitLog, loggerFactory } from '../server/logger.js';
+import path from 'path';
 import fs from 'fs-extra';
 import {
   getNpmRootPath,
@@ -21,7 +22,7 @@ import {
   buildReplicaId,
   writeEnv,
 } from '../server/conf.js';
-import { buildClient, unzipClientBuild, mergeClientBuildZip } from '../server/client-build.js';
+import { buildClient, unzipClientBuild, mergeClientBuildZip } from '../client-builder/client-build.js';
 import { DefaultConf } from '../../conf.js';
 import Underpost from '../index.js';
 
@@ -1482,18 +1483,63 @@ Prevent build private config repo.`,
     },
 
     /**
-     * Runtime-type → in-pod site-root directory resolver.
-     * Maps each known runtime to the filesystem path where its repository lives inside the container.
-     * @param {string} runtime - The runtime identifier (e.g. 'wp').
-     * @param {string} host - The virtual-host name.
-     * @returns {string|null} Absolute path inside the pod, or null if the runtime has no known mapping.
+     * Resolves the in-pod site-root directory where a conf route's repository lives.
+     *
+     * General-purpose resolution, independent of any single runtime:
+     *   1. An explicit `directory` (the conf-declared document root) always wins.
+     *   2. Otherwise the runtime's base directory is used (e.g. `wp` → `/opt/lampp/htdocs/wp/<host>`).
+     *   3. A subdirectory route (e.g. `/wp`) appends `<subDir>` to the resolved root.
+     *
+     * This mirrors {@link WpService.createApp}'s `vhostDir`/`wpDir` layout so backups target the
+     * exact directory provisioning created, including subdirectory installs and custom directories.
+     *
+     * @param {object} opts
+     * @param {string} opts.runtime - The runtime identifier (e.g. 'wp').
+     * @param {string} opts.host - The virtual-host name.
+     * @param {string} [opts.routePath='/'] - The conf route path the repository is mounted under.
+     * @param {string} [opts.directory] - Explicit document root from conf; overrides the runtime base.
+     * @returns {string|null} Absolute path inside the pod, or null when neither a directory nor a
+     *   known runtime base resolves.
      * @memberof UnderpostRepository
      */
-    runtimeSiteRoot(runtime, host) {
-      const runtimePaths = {
+    runtimeSiteRoot({ runtime, host, routePath = '/', directory } = {}) {
+      const runtimeBase = {
         wp: `/opt/lampp/htdocs/wp/${host}`,
       };
-      return runtimePaths[runtime] || null;
+      const vhostDir = directory || runtimeBase[runtime];
+      if (!vhostDir) return null;
+      const subDir = routePath && routePath !== '/' ? routePath.replace(/^\/+/, '').replace(/\/+$/, '') : '';
+      return subDir ? `${vhostDir}/${subDir}` : vhostDir;
+    },
+
+    /**
+     * Probes a running pod for the first candidate directory that is a git repository
+     * (i.e. contains a `.git` entry). Used to locate a site root before backing it up so a
+     * missing/unprovisioned directory is detected up-front instead of producing an opaque
+     * shell `exit 1` from a failed `cd`.
+     *
+     * @param {object}   opts
+     * @param {string}   opts.podName   - Target pod name.
+     * @param {string}   opts.namespace - Kubernetes namespace.
+     * @param {string[]} opts.candidates - Absolute paths to probe, most-specific first.
+     * @returns {string|null} The first candidate that is a git repo, or null if none match.
+     * @memberof UnderpostRepository
+     */
+    podRepoDir({ podName, namespace, candidates }) {
+      const probe = candidates.map((dir) => `if [ -d '${dir}/.git' ]; then echo '${dir}'; exit 0; fi`).join('; ');
+      let out = '';
+      try {
+        out = Underpost.kubectl.exec({ podName, namespace, command: `${probe}; echo ''` }) || '';
+      } catch (err) {
+        logger.warn(`podRepoDir: probe failed in pod ${podName}`, err.message);
+        return null;
+      }
+      return (
+        out
+          .split('\n')
+          .map((line) => line.trim())
+          .find(Boolean) || null
+      );
     },
 
     /**
@@ -1542,12 +1588,28 @@ Prevent build private config repo.`,
           const entry = confServer[host][routePath];
           if (!entry.repository) continue;
 
-          const siteRoot = Underpost.repo.runtimeSiteRoot(entry.runtime, host);
-          if (!siteRoot) {
+          const siteRootArgs = { runtime: entry.runtime, host, directory: entry.directory };
+          const repoRoot = Underpost.repo.runtimeSiteRoot({ ...siteRootArgs, routePath });
+          if (!repoRoot) {
             logger.warn(`backupPodRepositories: no site-root mapping for runtime '${entry.runtime}' (${host})`);
             continue;
           }
 
+          // Probe the pod for the actual git repo so an unprovisioned/missing site root is
+          // detected and skipped cleanly instead of failing the in-pod `cd` with exit 1.
+          // Fall back to the vhost dir (root route) to cover conf/path drift.
+          const vhostRoot = Underpost.repo.runtimeSiteRoot({ ...siteRootArgs, routePath: '/' });
+          const candidates = [...new Set([repoRoot, vhostRoot].filter(Boolean))];
+          const siteRoot = Underpost.repo.podRepoDir({ podName, namespace, candidates });
+          if (!siteRoot) {
+            logger.warn(
+              `backupPodRepositories: no git repository found in pod ${podName} for ${host} (checked ${candidates.join(
+                ', ',
+              )}) — site may not be provisioned yet; skipping`,
+            );
+            continue;
+          }
+
           const repoName = entry.repository.split('/').pop().split('.')[0];
 
           // Build the backup script — secrets are injected as env vars in the exec,
@@ -1738,6 +1800,8 @@ Prevent build private config repo.`,
       if (!fs.existsSync(repoPath)) {
         shellExec(`cd .. && underpost clone ${gitUri}`, { silent: true });
       } else {
+        const repoAbsPath = path.resolve(repoPath);
+        shellExec(`git config --global --add safe.directory '${repoAbsPath}'`);
         shellExec(`cd ${repoPath} && git checkout . && git clean -f -d && underpost pull . ${gitUri}`, {
           silent: true,
         });

package/src/cli/run.js

@@ -120,6 +120,7 @@ const logger = loggerFactory(import.meta);
  * @property {boolean} skipFullBuild - Whether to skip the full client bundle build during deployment (supported by: sync, template-deploy).
  * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
  * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
+ * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
  * @memberof UnderpostRun
  */
 const DEFAULT_OPTION = {
@@ -188,6 +189,7 @@ const DEFAULT_OPTION = {
   skipFullBuild: false,
   pullBundle: false,
   remove: false,
+  test: false,
 };
 
 /**
@@ -223,7 +225,7 @@ class UnderpostRun {
         shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''}`);
 
         shellExec(
-          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb4 --service-host ${mongoHosts.join(
+          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --service-host ${mongoHosts.join(
             ',',
           )} --pull-image`,
         );
@@ -371,6 +373,214 @@ class UnderpostRun {
       );
     },
 
+    /**
+     * @method node-move
+     * @description Abstract runner that relocates any schedulable Kubernetes workload
+     * (Deployment, StatefulSet, DaemonSet, ReplicaSet, Job, CronJob, ReplicationController)
+     * onto a target node by patching its pod-template `nodeSelector` and rolling it out.
+     * Resource-kind agnostic: it resolves the kind dynamically and applies the right
+     * patch path, so it works for `sts`, `deployment`, etc. without bespoke logic.
+     *
+     * Selection grammar via `path`:
+     *   - `<kind>/<name>`  -> a single resource (e.g. `deployment/dd-core-production-blue`)
+     *   - `<kind>`         -> every resource of that kind in the namespace (e.g. `statefulset`)
+     *   - ``               -> all movable workloads (deployment, statefulset, daemonset) in the namespace
+     *
+     * Placement:
+     *   - default: built-in `kubernetes.io/hostname=<node>` (no node mutation required)
+     *   - `--labels k=v,...`: label the target node with those pairs and use them as the
+     *     nodeSelector (matches the "label node + nodeSelector" pattern), enabling reusable
+     *     workload pools instead of pinning to a single hostname.
+     *
+     * Flags: `--node-name <node>` (target), `--namespace <ns>`, `--dry-run` (preview only),
+     *        `--remove` (clear the nodeSelector / unpin placement).
+     *
+     * Caveats: Services/ConfigMaps and bare Pods are not schedulable controllers and are
+     * skipped (move the owning controller). StatefulSets bound to node-local PVs may stay
+     * Pending after a move until their volume is available on the target node.
+     * @param {string} path - Resource selector (`kind/name`, `kind`, or empty).
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @memberof UnderpostRun
+     * @returns {Array<{ref:string,kind:string,status:string,node?:string}>} Per-resource outcome.
+     */
+    'node-move': (path = '', options = DEFAULT_OPTION) => {
+      const node = options.nodeName;
+      const ns = options.namespace || 'default';
+      const dryRun = options.dryRun === true;
+      const remove = options.remove === true;
+
+      if (!remove && !node) {
+        throw new Error('node-move requires --node-name <target-node> (or --remove to clear placement)');
+      }
+
+      const normalizeKind = (k) =>
+        ({
+          deploy: 'deployment',
+          deployments: 'deployment',
+          deployment: 'deployment',
+          sts: 'statefulset',
+          statefulsets: 'statefulset',
+          statefulset: 'statefulset',
+          ds: 'daemonset',
+          daemonsets: 'daemonset',
+          daemonset: 'daemonset',
+          rs: 'replicaset',
+          replicasets: 'replicaset',
+          replicaset: 'replicaset',
+          rc: 'replicationcontroller',
+          replicationcontroller: 'replicationcontroller',
+          job: 'job',
+          jobs: 'job',
+          cj: 'cronjob',
+          cronjob: 'cronjob',
+          cronjobs: 'cronjob',
+          po: 'pod',
+          pod: 'pod',
+          pods: 'pod',
+          svc: 'service',
+          service: 'service',
+          services: 'service',
+        })[k] || k;
+
+      // Kinds that own a pod template we can patch; rolloutKinds additionally
+      // support `kubectl rollout restart` to reschedule existing pods now.
+      const templated = ['deployment', 'statefulset', 'daemonset', 'replicaset', 'job', 'cronjob', 'replicationcontroller'];
+      const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
+      const templateSelectorPath = (kind) =>
+        kind === 'cronjob'
+          ? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
+          : ['spec', 'template', 'spec', 'nodeSelector'];
+
+      // Resolve the desired nodeSelector. Custom --labels enables reusable pools;
+      // otherwise pin by the always-present hostname label.
+      let selector = { 'kubernetes.io/hostname': node };
+      if (!remove && options.labels) {
+        selector = {};
+        for (const pair of `${options.labels}`.split(',').map((s) => s.trim()).filter(Boolean)) {
+          const eq = pair.indexOf('=');
+          if (eq < 0) continue;
+          selector[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim();
+        }
+      }
+
+      // Verify the target node exists, and apply custom labels to it if provided.
+      if (!remove) {
+        const found = shellExec(`kubectl get node ${node} -o name`, {
+          silent: true,
+          stdout: true,
+          silentOnError: true,
+        }).trim();
+        if (!found) throw new Error(`Target node not found: ${node}`);
+        if (options.labels) {
+          const labelArgs = Object.entries(selector)
+            .map(([k, v]) => `${k}=${v}`)
+            .join(' ');
+          const labelCmd = `kubectl label node ${node} ${labelArgs} --overwrite`;
+          if (dryRun) logger.info(`[dry-run] ${labelCmd}`);
+          else shellExec(labelCmd);
+        }
+      }
+
+      const kubectlNames = (kind) =>
+        (
+          shellExec(`kubectl get ${kind} -n ${ns} -o name`, { silent: true, stdout: true, silentOnError: true }).trim() ||
+          ''
+        )
+          .split('\n')
+          .map((s) => s.trim())
+          .filter(Boolean);
+
+      // Build the list of "kind/name" targets from the selection grammar.
+      let targets = [];
+      if (!path) {
+        for (const kind of ['deployment', 'statefulset', 'daemonset']) targets.push(...kubectlNames(kind));
+      } else if (path.includes('/')) {
+        targets = [path];
+      } else {
+        targets = kubectlNames(path);
+      }
+
+      if (targets.length === 0) {
+        logger.warn('node-move: no matching resources found', { path, namespace: ns });
+        return [];
+      }
+
+      // Merge-patch body that sets (or clears, when --remove) the nodeSelector.
+      const buildPatch = (kind) => {
+        const keys = templateSelectorPath(kind);
+        let obj = remove ? null : selector;
+        for (let i = keys.length - 1; i >= 0; i--) obj = { [keys[i]]: obj };
+        return JSON.stringify(obj);
+      };
+
+      const results = [];
+      for (const ref of targets) {
+        const slash = ref.indexOf('/');
+        const rawKind = slash >= 0 ? ref.slice(0, slash) : path && !path.includes('/') ? path : '';
+        const name = slash >= 0 ? ref.slice(slash + 1) : ref;
+        const kind = normalizeKind(`${rawKind}`.split('.')[0].toLowerCase());
+
+        if (!templated.includes(kind)) {
+          logger.warn(`node-move: ${kind}/${name} is not a schedulable controller; skipping`, {
+            hint: 'move its owning controller (deployment/statefulset/daemonset) instead',
+          });
+          results.push({ ref, kind, status: 'skipped' });
+          continue;
+        }
+
+        // Idempotency: skip the patch + rollout if the resource is already where
+        // we want it. Compares the live pod-template nodeSelector against the
+        // desired placement so a repeated run does not trigger an unnecessary
+        // rollout restart.
+        const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
+        const jsonpath = (expr) =>
+          shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
+            silent: true,
+            stdout: true,
+            silentOnError: true,
+            disableLog: true,
+          }).trim();
+
+        if (remove) {
+          const current = jsonpath(`{.${basePath}.nodeSelector}`);
+          if (!current || current === 'map[]') {
+            logger.info(`node-move: ${kind}/${name} already has no nodeSelector; nothing to clear`);
+            results.push({ ref, kind, status: 'already-cleared' });
+            continue;
+          }
+        } else {
+          const alreadyOnNode = Object.entries(selector).every(([k, v]) => {
+            const esc = k.replace(/\./g, '\\.');
+            return jsonpath(`{.${basePath}.nodeSelector.${esc}}`) === v;
+          });
+          if (alreadyOnNode) {
+            logger.info(`node-move: ${kind}/${name} already pinned to ${node}; skipping`, { namespace: ns });
+            results.push({ ref, kind, status: 'already-on-node', node });
+            continue;
+          }
+        }
+
+        const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
+        const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
+        if (dryRun) {
+          logger.info(`[dry-run] ${patchCmd}`);
+          if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
+          results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
+          continue;
+        }
+
+        shellExec(patchCmd);
+        if (rolloutKinds.includes(kind)) shellExec(restartCmd);
+        logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
+          namespace: ns,
+        });
+        results.push({ ref, kind, status: remove ? 'cleared' : 'moved', node: remove ? undefined : node });
+      }
+
+      logger.info('node-move complete', { namespace: ns, node: remove ? null : node, count: results.length });
+      return results;
+    },
+
     /**
      * @method dev-hosts-expose
      * @description Deploys a specified service in development mode with `/etc/hosts` modification for local access.
@@ -1007,8 +1217,19 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
             // pathRewritePolicy,
           });
         if (options.tls) {
-          shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
-          proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+          if (options.test) {
+            const sslDir = `./engine-private/ssl/${_host}`;
+            const nameSafe = _host.replace(/[^a-zA-Z0-9_.-]/g, '_');
+            fs.mkdirpSync(sslDir);
+            shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${_host}"`);
+            shellExec(`kubectl delete secret ${_host} -n ${options.namespace} --ignore-not-found`);
+            shellExec(
+              `kubectl create secret tls ${_host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
+            );
+          } else {
+            shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
+            proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+          }
         }
         // console.log(proxyYaml);
         shellExec(`kubectl delete HTTPProxy ${_host} --namespace ${options.namespace} --ignore-not-found`);
@@ -1077,6 +1298,7 @@ EOF
         // Examples images:
         // `underpost/underpost-engine:${Underpost.version}`
         // `localhost/rockylinux9-underpost:${Underpost.version}`
+        if (options.imageName) _image = options.imageName;
         if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
 
         if (_image && !_image.startsWith('localhost'))
@@ -1172,12 +1394,16 @@ EOF
 `,
           { disableLog: true },
         );
+        // Custom instances run a bare binary (no `underpost start` / internal
+        // HTTP endpoint): Kubernetes readiness is the running signal and
+        // container-status is read via exec. See `Deploy custom instance to K8S.md`.
         const { ready, readyPods } = await Underpost.monitor.monitorReadyRunner(
           _deployId,
           env,
           targetTraffic,
           ignorePods,
           options.namespace,
+          { readyGate: 'kubernetes', statusTransport: 'exec' },
         );
 
         if (!ready) {
@@ -1187,7 +1413,7 @@ EOF
         shellExec(
           `${baseCommand} run${baseClusterCommand} --namespace ${options.namespace}` +
             `${options.nodeName ? ` --node-name ${options.nodeName}` : ''}` +
-            `${options.tls ? ` --tls` : ''}` +
+            `${options.tls ? ` --tls ${options.test ? '--test' : ''}` : ''}` +
             ` instance-promote '${path}'`,
         );
       }
@@ -1425,8 +1651,24 @@ gpgcheck=1
 gpgkey=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.33/rpm/repodata/repomd.xml.key
 EOF`);
       shellExec(`sudo dnf -y install cri-o`);
-      // crictl is in the kubernetes repo but excluded by default — install it explicitly
-      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes`);
+      // Add the Kubernetes repo so cri-tools (crictl CLI) is available.
+      // The repo has exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni, so we
+      // use --disableexcludes=kubernetes to override and install cri-tools.
+      shellExec(`cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
+[kubernetes]
+name=Kubernetes
+baseurl=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/repodata/repomd.xml.key
+exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
+EOF`);
+      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes 2>/dev/null || {
+        CRICTL_VERSION="v1.31.0"
+        ARCH="amd64"
+        log "Kubernetes repo not available, downloading crictl binary directly..."
+        curl -sL "https://github.com/kubernetes-sigs/cri-tools/releases/download/\${CRICTL_VERSION}/crictl-\${CRICTL_VERSION}-\${ARCH}.tar.gz" | sudo tar -C /usr/local/bin -xz
+      }`);
       // Ensure CRI-O uses systemd cgroup driver (matches kubelet default)
       shellExec(`sudo sed -i 's/^#\?cgroup_manager =.*/cgroup_manager = "systemd"/' /etc/crio/crio.conf`, {
         silentOnError: true,
@@ -2757,6 +2999,33 @@ EOF`;
 
       logger.info(`[setup-shared-dir] Shared directory setup complete: ${dir}`);
     },
+    /**
+     * @method shared-dir-add-user
+     * @description Add a user to an existing shared directory without changing
+     * file owners. Grants recursive group/ACL access so the user can read and
+     * write throughout the shared workspace while preserving existing ownership.
+     *
+     * @param {string} path - Shared directory (defaults to `/home/dd/engine`).
+     * @param {Object} options - Underpost runner options.
+     *   Key fields:
+     *     - options.user  (default: 'admin')
+     *     - options.group (default: 'engine-dev')
+     *
+     * @memberof UnderpostRun
+     */ 'shared-dir-add-user': (path = '/home/dd/engine', options = DEFAULT_OPTION) => {
+      const dir = path || '/home/dd/engine';
+      const user = options.user || 'admin';
+
+      logger.info(`[shared-dir-add-user] dir=${dir} user=${user}`);
+
+      // Give the user direct access without changing owners or group ownership.
+      shellExec(`sudo setfacl -R -m u:${user}:rwx ${dir}`);
+
+      // Make future files/directories inherit the same user ACL.
+      shellExec(`sudo find ${dir} -type d -exec setfacl -d -m u:${user}:rwx {} \\;`);
+
+      logger.info(`[shared-dir-add-user] User '${user}' added to shared directory: ${dir}`);
+    },
   };
 
   static API = {

package/src/cli/ssh.js

@@ -545,6 +545,196 @@ EOF
       return shellExec(sshScript, { stdout: true });
     },
 
+    /**
+     * Waits until a TCP SSH port becomes reachable on a host.
+     * @async
+     * @function waitForSshPort
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {number} [params.timeoutMs=600000] - Maximum wait window.
+     * @param {number} [params.intervalMs=3000] - Poll interval.
+     * @returns {Promise<boolean>} True once the port accepts connections, false on timeout.
+     */
+    waitForSshPort: async ({ host, port = 22, timeoutMs = 10 * 60 * 1000, intervalMs = 3000 }) => {
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+        const probe = shellExec(
+          `timeout 5 bash -c '</dev/tcp/${host}/${port}' >/dev/null 2>&1 && echo open || echo closed`,
+          { silent: true, stdout: true, silentOnError: true, disableLog: true },
+        );
+        if (`${probe}`.trim() === 'open') return true;
+        await new Promise((r) => setTimeout(r, intervalMs));
+      }
+      logger.warn(`SSH port ${host}:${port} not reachable within timeout`);
+      return false;
+    },
+
+    /**
+     * Waits until a host's SSH port stops accepting connections (e.g. while it
+     * reboots). Used to detect a reboot edge before waiting for the port to come
+     * back up, so callers don't latch onto the pre-reboot (ephemeral) sshd.
+     * @async
+     * @function waitForSshPortClosed
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {number} [params.timeoutMs=180000] - Maximum wait window.
+     * @param {number} [params.intervalMs=3000] - Poll interval.
+     * @returns {Promise<boolean>} True once the port is closed, false on timeout.
+     */
+    waitForSshPortClosed: async ({ host, port = 22, timeoutMs = 3 * 60 * 1000, intervalMs = 3000 }) => {
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+        const probe = shellExec(
+          `timeout 5 bash -c '</dev/tcp/${host}/${port}' >/dev/null 2>&1 && echo open || echo closed`,
+          { silent: true, stdout: true, silentOnError: true, disableLog: true },
+        );
+        if (`${probe}`.trim() === 'closed') return true;
+        await new Promise((r) => setTimeout(r, intervalMs));
+      }
+      return false;
+    },
+
+    /**
+     * Orchestrates a non-interactive, key-only SSH session against a freshly
+     * provisioned host: waits for the port, attempts key-based auth, runs a
+     * remote command batch, and returns a structured result. Used by the
+     * commissioning flow once the ephemeral runtime reports SSH readiness.
+     * @async
+     * @function sshExecBatch
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {string} params.command - Remote command batch to execute.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {string} [params.user='root'] - SSH user (key-only).
+     * @param {string} [params.keyPath] - Private key path (defaults to engine deploy key).
+     * @param {number} [params.connectTimeoutSec=15] - Per-attempt SSH connect timeout.
+     * @param {number} [params.retries=3] - Auth/exec retry attempts.
+     * @param {number} [params.retryDelayMs=5000] - Base backoff between retries.
+     * @param {number} [params.waitForPortMs=0] - When > 0, wait for the port first.
+     * @returns {Promise<{ok: boolean, code: number, stdout: string, stderr: string, attempts: number}>}
+     */
+    sshExecBatch: async ({
+      host,
+      command,
+      port = 22,
+      user = 'root',
+      keyPath = './engine-private/deploy/id_rsa',
+      connectTimeoutSec = 15,
+      retries = 3,
+      retryDelayMs = 5000,
+      waitForPortMs = 0,
+    }) => {
+      if (!host) throw new Error('sshExecBatch requires a host');
+      if (!command) throw new Error('sshExecBatch requires a command');
+
+      if (waitForPortMs > 0) {
+        const reachable = await Underpost.ssh.waitForSshPort({ host, port, timeoutMs: waitForPortMs });
+        if (!reachable) return { ok: false, code: 255, stdout: '', stderr: 'ssh port unreachable', attempts: 0 };
+      }
+
+      shellExec(`chmod 600 ${keyPath}`, { silent: true, silentOnError: true, disableLog: true });
+
+      const sshOpts = [
+        `-i ${keyPath}`,
+        `-o BatchMode=yes`,
+        `-o PreferredAuthentications=publickey`,
+        `-o PubkeyAuthentication=yes`,
+        `-o PasswordAuthentication=no`,
+        `-o StrictHostKeyChecking=no`,
+        `-o UserKnownHostsFile=/dev/null`,
+        `-o ConnectTimeout=${connectTimeoutSec}`,
+        // Tolerate a freshly-booted node whose network briefly flaps (e.g. while
+        // NetworkManager applies a static profile): retry the TCP connect and
+        // keep the session alive across short stalls.
+        `-o ConnectionAttempts=3`,
+        `-o ServerAliveInterval=10`,
+        `-o ServerAliveCountMax=6`,
+        `-p ${port}`,
+      ].join(' ');
+
+      let last = { ok: false, code: 255, stdout: '', stderr: '', attempts: 0 };
+      for (let attempt = 1; attempt <= retries; attempt++) {
+        const result = shellExec(`ssh ${sshOpts} ${user}@${host} bash -s <<'UNDERPOST_SSH_BATCH_EOF'\n${command}\nUNDERPOST_SSH_BATCH_EOF`, {
+          stdout: false,
+          silentOnError: true,
+          disableLog: true,
+        });
+        last = {
+          ok: result.code === 0,
+          code: result.code,
+          stdout: result.stdout || '',
+          stderr: result.stderr || '',
+          attempts: attempt,
+        };
+        if (last.ok) {
+          logger.info(`sshExecBatch succeeded on ${user}@${host}:${port} (attempt ${attempt})`);
+          return last;
+        }
+        logger.warn(`sshExecBatch attempt ${attempt}/${retries} failed on ${user}@${host}:${port}`, {
+          code: last.code,
+          stderr: last.stderr.slice(-400),
+        });
+        if (attempt < retries) await new Promise((r) => setTimeout(r, retryDelayMs * attempt));
+      }
+      return last;
+    },
+
+    /**
+     * Transfers a local script to a remote host and runs it over key-only SSH.
+     * The script is base64-encoded so no shell-quoting/escaping is needed, then
+     * decoded, made executable, and executed with the given arguments. Reuses
+     * sshExecBatch for the actual transport, retries, and structured result.
+     * @async
+     * @function sshRunScript
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {string} params.scriptPath - Local path to the script to run.
+     * @param {string} [params.args=''] - Arguments appended to the remote invocation.
+     * @param {object} [params.env={}] - Environment variables exported for the remote run (e.g. secrets). Passed inline to the command, never echoed.
+     * @param {string} [params.remotePath='/tmp/underpost-remote-script.sh'] - Remote path to write the script.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {string} [params.user='root'] - SSH user (key-only).
+     * @param {string} [params.keyPath] - Private key path (defaults to engine deploy key).
+     * @param {number} [params.retries=3] - Retry attempts.
+     * @param {number} [params.waitForPortMs=0] - When > 0, wait for the port first.
+     * @returns {Promise<{ok: boolean, code: number, stdout: string, stderr: string, attempts: number}>}
+     */
+    sshRunScript: async ({
+      host,
+      scriptPath,
+      args = '',
+      env = {},
+      remotePath = '/tmp/underpost-remote-script.sh',
+      port = 22,
+      user = 'root',
+      keyPath = './engine-private/deploy/id_rsa',
+      retries = 3,
+      waitForPortMs = 0,
+    }) => {
+      if (!fs.existsSync(scriptPath)) throw new Error(`sshRunScript: script not found: ${scriptPath}`);
+      const b64 = Buffer.from(fs.readFileSync(scriptPath, 'utf8'), 'utf8').toString('base64');
+      // Inline env assignments (single-quote escaped) so secrets are exported for
+      // the remote run without appearing as logged CLI args.
+      const sq = (v) => `'${String(v).replace(/'/g, "'\\''")}'`;
+      const envPrefix = Object.entries(env)
+        .filter(([, v]) => v !== undefined && v !== null && `${v}` !== '')
+        .map(([k, v]) => `${k}=${sq(v)}`)
+        .join(' ');
+      const command = [
+        'set -e',
+        `echo '${b64}' | base64 -d > ${remotePath}`,
+        `chmod +x ${remotePath}`,
+        `${envPrefix ? `${envPrefix} ` : ''}bash ${remotePath} ${args}`,
+      ].join('\n');
+      return Underpost.ssh.sshExecBatch({ host, port, user, keyPath, retries, waitForPortMs, command });
+    },
+
     /**
      * Loads saved SSH credentials from config and sets them in the UnderpostRootEnv API.
      * @async