underpost 3.2.22 → 3.2.30

package/src/cli/run.js

@@ -56,6 +56,7 @@ const logger = loggerFactory(import.meta);
  * @property {boolean} dev - Whether to run in development mode.
  * @property {string} podName - The name of the pod to run.
  * @property {string} nodeName - The name of the node to run.
+ * @property {string} sshKeyPath - Private key path for node SSH operations, forwarded to volume shipping over SSH.
  * @property {number} port - Custom port to use.
  * @property {string} volumeHostPath - The host path for the volume.
  * @property {string} volumeMountPath - The mount path for the volume.
@@ -127,6 +128,7 @@ const DEFAULT_OPTION = {
   dev: false,
   podName: '',
   nodeName: '',
+  sshKeyPath: '',
   port: 0,
   volumeHostPath: '',
   volumeMountPath: '',
@@ -373,6 +375,228 @@ class UnderpostRun {
       );
     },
 
+    /**
+     * @method node-move
+     * @description Abstract runner that relocates any schedulable Kubernetes workload
+     * (Deployment, StatefulSet, DaemonSet, ReplicaSet, Job, CronJob, ReplicationController)
+     * onto a target node by patching its pod-template `nodeSelector` and rolling it out.
+     * Resource-kind agnostic: it resolves the kind dynamically and applies the right
+     * patch path, so it works for `sts`, `deployment`, etc. without bespoke logic.
+     *
+     * Selection grammar via `path`:
+     *   - `<kind>/<name>`  -> a single resource (e.g. `deployment/dd-core-production-blue`)
+     *   - `<kind>`         -> every resource of that kind in the namespace (e.g. `statefulset`)
+     *   - ``               -> all movable workloads (deployment, statefulset, daemonset) in the namespace
+     *
+     * Placement:
+     *   - default: built-in `kubernetes.io/hostname=<node>` (no node mutation required)
+     *   - `--labels k=v,...`: label the target node with those pairs and use them as the
+     *     nodeSelector (matches the "label node + nodeSelector" pattern), enabling reusable
+     *     workload pools instead of pinning to a single hostname.
+     *
+     * Flags: `--node-name <node>` (target), `--namespace <ns>`, `--dry-run` (preview only),
+     *        `--remove` (clear the nodeSelector / unpin placement).
+     *
+     * Caveats: Services/ConfigMaps and bare Pods are not schedulable controllers and are
+     * skipped (move the owning controller). StatefulSets bound to node-local PVs may stay
+     * Pending after a move until their volume is available on the target node.
+     * @param {string} path - Resource selector (`kind/name`, `kind`, or empty).
+     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @memberof UnderpostRun
+     * @returns {Array<{ref:string,kind:string,status:string,node?:string}>} Per-resource outcome.
+     */
+    'node-move': (path = '', options = DEFAULT_OPTION) => {
+      const node = options.nodeName;
+      const ns = options.namespace || 'default';
+      const dryRun = options.dryRun === true;
+      const remove = options.remove === true;
+
+      if (!remove && !node) {
+        throw new Error('node-move requires --node-name <target-node> (or --remove to clear placement)');
+      }
+
+      const normalizeKind = (k) =>
+        ({
+          deploy: 'deployment',
+          deployments: 'deployment',
+          deployment: 'deployment',
+          sts: 'statefulset',
+          statefulsets: 'statefulset',
+          statefulset: 'statefulset',
+          ds: 'daemonset',
+          daemonsets: 'daemonset',
+          daemonset: 'daemonset',
+          rs: 'replicaset',
+          replicasets: 'replicaset',
+          replicaset: 'replicaset',
+          rc: 'replicationcontroller',
+          replicationcontroller: 'replicationcontroller',
+          job: 'job',
+          jobs: 'job',
+          cj: 'cronjob',
+          cronjob: 'cronjob',
+          cronjobs: 'cronjob',
+          po: 'pod',
+          pod: 'pod',
+          pods: 'pod',
+          svc: 'service',
+          service: 'service',
+          services: 'service',
+        })[k] || k;
+
+      // Kinds that own a pod template we can patch; rolloutKinds additionally
+      // support `kubectl rollout restart` to reschedule existing pods now.
+      const templated = [
+        'deployment',
+        'statefulset',
+        'daemonset',
+        'replicaset',
+        'job',
+        'cronjob',
+        'replicationcontroller',
+      ];
+      const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
+      const templateSelectorPath = (kind) =>
+        kind === 'cronjob'
+          ? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
+          : ['spec', 'template', 'spec', 'nodeSelector'];
+
+      // Resolve the desired nodeSelector. Custom --labels enables reusable pools;
+      // otherwise pin by the always-present hostname label.
+      let selector = { 'kubernetes.io/hostname': node };
+      if (!remove && options.labels) {
+        selector = {};
+        for (const pair of `${options.labels}`
+          .split(',')
+          .map((s) => s.trim())
+          .filter(Boolean)) {
+          const eq = pair.indexOf('=');
+          if (eq < 0) continue;
+          selector[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim();
+        }
+      }
+
+      // Verify the target node exists, and apply custom labels to it if provided.
+      if (!remove) {
+        const found = shellExec(`kubectl get node ${node} -o name`, {
+          silent: true,
+          stdout: true,
+          silentOnError: true,
+        }).trim();
+        if (!found) throw new Error(`Target node not found: ${node}`);
+        if (options.labels) {
+          const labelArgs = Object.entries(selector)
+            .map(([k, v]) => `${k}=${v}`)
+            .join(' ');
+          const labelCmd = `kubectl label node ${node} ${labelArgs} --overwrite`;
+          if (dryRun) logger.info(`[dry-run] ${labelCmd}`);
+          else shellExec(labelCmd);
+        }
+      }
+
+      const kubectlNames = (kind) =>
+        (
+          shellExec(`kubectl get ${kind} -n ${ns} -o name`, {
+            silent: true,
+            stdout: true,
+            silentOnError: true,
+          }).trim() || ''
+        )
+          .split('\n')
+          .map((s) => s.trim())
+          .filter(Boolean);
+
+      // Build the list of "kind/name" targets from the selection grammar.
+      let targets = [];
+      if (!path) {
+        for (const kind of ['deployment', 'statefulset', 'daemonset']) targets.push(...kubectlNames(kind));
+      } else if (path.includes('/')) {
+        targets = [path];
+      } else {
+        targets = kubectlNames(path);
+      }
+
+      if (targets.length === 0) {
+        logger.warn('node-move: no matching resources found', { path, namespace: ns });
+        return [];
+      }
+
+      // Merge-patch body that sets (or clears, when --remove) the nodeSelector.
+      const buildPatch = (kind) => {
+        const keys = templateSelectorPath(kind);
+        let obj = remove ? null : selector;
+        for (let i = keys.length - 1; i >= 0; i--) obj = { [keys[i]]: obj };
+        return JSON.stringify(obj);
+      };
+
+      const results = [];
+      for (const ref of targets) {
+        const slash = ref.indexOf('/');
+        const rawKind = slash >= 0 ? ref.slice(0, slash) : path && !path.includes('/') ? path : '';
+        const name = slash >= 0 ? ref.slice(slash + 1) : ref;
+        const kind = normalizeKind(`${rawKind}`.split('.')[0].toLowerCase());
+
+        if (!templated.includes(kind)) {
+          logger.warn(`node-move: ${kind}/${name} is not a schedulable controller; skipping`, {
+            hint: 'move its owning controller (deployment/statefulset/daemonset) instead',
+          });
+          results.push({ ref, kind, status: 'skipped' });
+          continue;
+        }
+
+        // Idempotency: skip the patch + rollout if the resource is already where
+        // we want it. Compares the live pod-template nodeSelector against the
+        // desired placement so a repeated run does not trigger an unnecessary
+        // rollout restart.
+        const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
+        const jsonpath = (expr) =>
+          shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
+            silent: true,
+            stdout: true,
+            silentOnError: true,
+            disableLog: true,
+          }).trim();
+
+        if (remove) {
+          const current = jsonpath(`{.${basePath}.nodeSelector}`);
+          if (!current || current === 'map[]') {
+            logger.info(`node-move: ${kind}/${name} already has no nodeSelector; nothing to clear`);
+            results.push({ ref, kind, status: 'already-cleared' });
+            continue;
+          }
+        } else {
+          const alreadyOnNode = Object.entries(selector).every(([k, v]) => {
+            const esc = k.replace(/\./g, '\\.');
+            return jsonpath(`{.${basePath}.nodeSelector.${esc}}`) === v;
+          });
+          if (alreadyOnNode) {
+            logger.info(`node-move: ${kind}/${name} already pinned to ${node}; skipping`, { namespace: ns });
+            results.push({ ref, kind, status: 'already-on-node', node });
+            continue;
+          }
+        }
+
+        const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
+        const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
+        if (dryRun) {
+          logger.info(`[dry-run] ${patchCmd}`);
+          if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
+          results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
+          continue;
+        }
+
+        shellExec(patchCmd);
+        if (rolloutKinds.includes(kind)) shellExec(restartCmd);
+        logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
+          namespace: ns,
+        });
+        results.push({ ref, kind, status: remove ? 'cleared' : 'moved', node: remove ? undefined : node });
+      }
+
+      logger.info('node-move complete', { namespace: ns, node: remove ? null : node, count: results.length });
+      return results;
+    },
+
     /**
      * @method dev-hosts-expose
      * @description Deploys a specified service in development mode with `/etc/hosts` modification for local access.
@@ -405,7 +629,9 @@ class UnderpostRun {
      * @memberof UnderpostRun
      */
     'cluster-build': (path, options = DEFAULT_OPTION) => {
-      const nodeOptions = options.nodeName ? ` --node-name ${options.nodeName}` : '';
+      const nodeOptions =
+        (options.nodeName ? ` --node-name ${options.nodeName}` : '') +
+        (options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '');
       shellExec(`node bin run clean`);
       shellExec(`node bin run --dev sync-replica template-deploy${nodeOptions}`);
       shellExec(`node bin run sync-replica template-deploy${nodeOptions}`);
@@ -701,7 +927,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       replicas = replicas ? replicas : defaultPath[1];
       versions = versions ? versions.replaceAll('+', ',') : defaultPath[2];
       image = image ? image : defaultPath[3];
-      node = node ? node : defaultPath[4];
+      node = node ? node : options.nodeName ? options.nodeName : defaultPath[4];
       shellExec(`${baseCommand} cluster --ns-use ${options.namespace}`);
 
       if (image && !image.startsWith('localhost'))
@@ -742,13 +968,14 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
       const skipFullBuildFlag = options.skipFullBuild ? ' --skip-full-build' : '';
       const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
       const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
+      const sshKeyPathFlag = options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '';
 
       shellExec(
         `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
           image ? ` --image ${image}` : ''
         }${versions ? ` --versions ${versions}` : ''}${
           options.namespace ? ` --namespace ${options.namespace}` : ''
-        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag} ${deployId} ${env}`,
+        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag} ${deployId} ${env}`,
       );
 
       if (isDeployRunnerContext(path, options)) {
@@ -757,9 +984,9 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
           `${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
         );
         shellExec(
-          `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
+          `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
             options.namespace ? ` --namespace ${options.namespace}` : ''
-          }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}`,
+          }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}`,
         );
         if (!targetTraffic)
           targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
@@ -1119,9 +1346,16 @@ EOF
               deployId: _deployId,
               env,
               version: targetTraffic,
-              nodeName: options.nodeName,
+              nodeName: Underpost.deploy.resolveDeployNode({
+                node: options.nodeName,
+                kind: options.kind,
+                kubeadm: options.kubeadm,
+                k3s: options.k3s,
+                env,
+              }),
               clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
               gitClean: options.gitClean || false,
+              sshKeyPath: options.sshKeyPath || '',
             });
         // Regenerate the parent deploy's gRPC ClusterIP service pointing to the
         // parent's current traffic colour and apply it before the instance pod starts so
@@ -1443,8 +1677,24 @@ gpgcheck=1
 gpgkey=https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.33/rpm/repodata/repomd.xml.key
 EOF`);
       shellExec(`sudo dnf -y install cri-o`);
-      // crictl is in the kubernetes repo but excluded by default — install it explicitly
-      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes`);
+      // Add the Kubernetes repo so cri-tools (crictl CLI) is available.
+      // The repo has exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni, so we
+      // use --disableexcludes=kubernetes to override and install cri-tools.
+      shellExec(`cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
+[kubernetes]
+name=Kubernetes
+baseurl=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://pkgs.k8s.io/core:/stable:/v1.36/rpm/repodata/repomd.xml.key
+exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
+EOF`);
+      shellExec(`sudo yum install -y cri-tools --disableexcludes=kubernetes 2>/dev/null || {
+        CRICTL_VERSION="v1.31.0"
+        ARCH="amd64"
+        log "Kubernetes repo not available, downloading crictl binary directly..."
+        curl -sL "https://github.com/kubernetes-sigs/cri-tools/releases/download/\${CRICTL_VERSION}/crictl-\${CRICTL_VERSION}-\${ARCH}.tar.gz" | sudo tar -C /usr/local/bin -xz
+      }`);
       // Ensure CRI-O uses systemd cgroup driver (matches kubelet default)
       shellExec(`sudo sed -i 's/^#\?cgroup_manager =.*/cgroup_manager = "systemd"/' /etc/crio/crio.conf`, {
         silentOnError: true,
@@ -2775,6 +3025,33 @@ EOF`;
 
       logger.info(`[setup-shared-dir] Shared directory setup complete: ${dir}`);
     },
+    /**
+     * @method shared-dir-add-user
+     * @description Add a user to an existing shared directory without changing
+     * file owners. Grants recursive group/ACL access so the user can read and
+     * write throughout the shared workspace while preserving existing ownership.
+     *
+     * @param {string} path - Shared directory (defaults to `/home/dd/engine`).
+     * @param {Object} options - Underpost runner options.
+     *   Key fields:
+     *     - options.user  (default: 'admin')
+     *     - options.group (default: 'engine-dev')
+     *
+     * @memberof UnderpostRun
+     */ 'shared-dir-add-user': (path = '/home/dd/engine', options = DEFAULT_OPTION) => {
+      const dir = path || '/home/dd/engine';
+      const user = options.user || 'admin';
+
+      logger.info(`[shared-dir-add-user] dir=${dir} user=${user}`);
+
+      // Give the user direct access without changing owners or group ownership.
+      shellExec(`sudo setfacl -R -m u:${user}:rwx ${dir}`);
+
+      // Make future files/directories inherit the same user ACL.
+      shellExec(`sudo find ${dir} -type d -exec setfacl -d -m u:${user}:rwx {} \\;`);
+
+      logger.info(`[shared-dir-add-user] User '${user}' added to shared directory: ${dir}`);
+    },
   };
 
   static API = {

package/src/cli/secrets.js

@@ -104,12 +104,17 @@ class UnderpostSecret {
     /**
      * Removes all filesystem traces of secrets after deployment startup.
      * Centralizes the defense-in-depth cleanup performed
+     * @param {object} options - Options for cleaning the environment.
+     * @param {Array<string>} [options.keepKeys=[]] - List of keys to keep in the environment file. If provided, only these keys will be retained.
      * @memberof UnderpostSecret
      */
-    globalSecretClean() {
+    globalSecretClean(options = { keepKeys: [] }) {
+      const { keepKeys } = options;
       loadConf('clean');
       Underpost.repo.cleanupPrivateEngineRepo();
-      Underpost.env.clean();
+      Underpost.env.clean({
+        keepKeys: keepKeys.length > 0 ? keepKeys : ['container-status', 'start-container-status'],
+      });
     },
   };
 }

package/src/cli/ssh.js

@@ -496,6 +496,50 @@ EOF`);
       if (options.status) shellExec('service sshd status');
     },
 
+    /**
+     * Synchronously copies a local directory to a remote host over key-only SSH,
+     * streaming it as a tar archive (no intermediate file) and fixing ownership.
+     * Mirrors the kind-node `tar | docker cp` provisioning pattern but targets a
+     * real node via SSH, so node-local hostPath volumes can be materialized on the
+     * node where the pod will actually run.
+     *
+     * Idempotent and re-runnable: `mkdir -p` + `tar -x` overwrite in place. Throws
+     * on any SSH/tar failure so an empty-volume deploy is never produced silently.
+     *
+     * @function copyDirToNode
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP (key-only SSH reachable).
+     * @param {string} params.localDir - Local source directory.
+     * @param {string} params.remoteDir - Destination directory on the node.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {string} [params.user='root'] - SSH user (key-only).
+     * @param {string} [params.keyPath='./engine-private/deploy/id_rsa'] - Private key path.
+     * @param {string} [params.owner='1000:1000'] - chown target on the node (empty to skip).
+     * @param {string} [params.mode='755'] - chmod mode on the node (empty to skip).
+     * @returns {void}
+     */
+    copyDirToNode: ({
+      host,
+      localDir,
+      remoteDir,
+      port = 22,
+      user = 'root',
+      keyPath = './engine-private/deploy/id_rsa',
+      owner = '1000:1000',
+      mode = '755',
+    }) => {
+      if (!host) throw new Error('copyDirToNode requires a host');
+      if (!localDir || !fs.existsSync(localDir)) throw new Error(`copyDirToNode: local dir not found: ${localDir}`);
+      if (!remoteDir) throw new Error('copyDirToNode requires a remoteDir');
+      shellExec(`chmod 600 ${keyPath}`, { silent: true, silentOnError: true, disableLog: true });
+      const sshOpts = `-i ${keyPath} -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p ${port}`;
+      shellExec(`ssh ${sshOpts} ${user}@${host} 'mkdir -p ${remoteDir}'`);
+      shellExec(`tar -C ${localDir} -c . | ssh ${sshOpts} ${user}@${host} 'tar -C ${remoteDir} -x'`);
+      const fixups = `${owner ? `chown -R ${owner} ${remoteDir}; ` : ''}${mode ? `chmod -R ${mode} ${remoteDir}` : ''}`.trim();
+      if (fixups) shellExec(`ssh ${sshOpts} ${user}@${host} '${fixups}'`);
+    },
+
     /**
      * Generic SSH remote command runner that SSH execution logic.
      * Executes arbitrary shell commands on a remote server via SSH with proper credential handling.
@@ -545,6 +589,196 @@ EOF
       return shellExec(sshScript, { stdout: true });
     },
 
+    /**
+     * Waits until a TCP SSH port becomes reachable on a host.
+     * @async
+     * @function waitForSshPort
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {number} [params.timeoutMs=600000] - Maximum wait window.
+     * @param {number} [params.intervalMs=3000] - Poll interval.
+     * @returns {Promise<boolean>} True once the port accepts connections, false on timeout.
+     */
+    waitForSshPort: async ({ host, port = 22, timeoutMs = 10 * 60 * 1000, intervalMs = 3000 }) => {
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+        const probe = shellExec(
+          `timeout 5 bash -c '</dev/tcp/${host}/${port}' >/dev/null 2>&1 && echo open || echo closed`,
+          { silent: true, stdout: true, silentOnError: true, disableLog: true },
+        );
+        if (`${probe}`.trim() === 'open') return true;
+        await new Promise((r) => setTimeout(r, intervalMs));
+      }
+      logger.warn(`SSH port ${host}:${port} not reachable within timeout`);
+      return false;
+    },
+
+    /**
+     * Waits until a host's SSH port stops accepting connections (e.g. while it
+     * reboots). Used to detect a reboot edge before waiting for the port to come
+     * back up, so callers don't latch onto the pre-reboot (ephemeral) sshd.
+     * @async
+     * @function waitForSshPortClosed
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {number} [params.timeoutMs=180000] - Maximum wait window.
+     * @param {number} [params.intervalMs=3000] - Poll interval.
+     * @returns {Promise<boolean>} True once the port is closed, false on timeout.
+     */
+    waitForSshPortClosed: async ({ host, port = 22, timeoutMs = 3 * 60 * 1000, intervalMs = 3000 }) => {
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+        const probe = shellExec(
+          `timeout 5 bash -c '</dev/tcp/${host}/${port}' >/dev/null 2>&1 && echo open || echo closed`,
+          { silent: true, stdout: true, silentOnError: true, disableLog: true },
+        );
+        if (`${probe}`.trim() === 'closed') return true;
+        await new Promise((r) => setTimeout(r, intervalMs));
+      }
+      return false;
+    },
+
+    /**
+     * Orchestrates a non-interactive, key-only SSH session against a freshly
+     * provisioned host: waits for the port, attempts key-based auth, runs a
+     * remote command batch, and returns a structured result. Used by the
+     * commissioning flow once the ephemeral runtime reports SSH readiness.
+     * @async
+     * @function sshExecBatch
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {string} params.command - Remote command batch to execute.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {string} [params.user='root'] - SSH user (key-only).
+     * @param {string} [params.keyPath] - Private key path (defaults to engine deploy key).
+     * @param {number} [params.connectTimeoutSec=15] - Per-attempt SSH connect timeout.
+     * @param {number} [params.retries=3] - Auth/exec retry attempts.
+     * @param {number} [params.retryDelayMs=5000] - Base backoff between retries.
+     * @param {number} [params.waitForPortMs=0] - When > 0, wait for the port first.
+     * @returns {Promise<{ok: boolean, code: number, stdout: string, stderr: string, attempts: number}>}
+     */
+    sshExecBatch: async ({
+      host,
+      command,
+      port = 22,
+      user = 'root',
+      keyPath = './engine-private/deploy/id_rsa',
+      connectTimeoutSec = 15,
+      retries = 3,
+      retryDelayMs = 5000,
+      waitForPortMs = 0,
+    }) => {
+      if (!host) throw new Error('sshExecBatch requires a host');
+      if (!command) throw new Error('sshExecBatch requires a command');
+
+      if (waitForPortMs > 0) {
+        const reachable = await Underpost.ssh.waitForSshPort({ host, port, timeoutMs: waitForPortMs });
+        if (!reachable) return { ok: false, code: 255, stdout: '', stderr: 'ssh port unreachable', attempts: 0 };
+      }
+
+      shellExec(`chmod 600 ${keyPath}`, { silent: true, silentOnError: true, disableLog: true });
+
+      const sshOpts = [
+        `-i ${keyPath}`,
+        `-o BatchMode=yes`,
+        `-o PreferredAuthentications=publickey`,
+        `-o PubkeyAuthentication=yes`,
+        `-o PasswordAuthentication=no`,
+        `-o StrictHostKeyChecking=no`,
+        `-o UserKnownHostsFile=/dev/null`,
+        `-o ConnectTimeout=${connectTimeoutSec}`,
+        // Tolerate a freshly-booted node whose network briefly flaps (e.g. while
+        // NetworkManager applies a static profile): retry the TCP connect and
+        // keep the session alive across short stalls.
+        `-o ConnectionAttempts=3`,
+        `-o ServerAliveInterval=10`,
+        `-o ServerAliveCountMax=6`,
+        `-p ${port}`,
+      ].join(' ');
+
+      let last = { ok: false, code: 255, stdout: '', stderr: '', attempts: 0 };
+      for (let attempt = 1; attempt <= retries; attempt++) {
+        const result = shellExec(`ssh ${sshOpts} ${user}@${host} bash -s <<'UNDERPOST_SSH_BATCH_EOF'\n${command}\nUNDERPOST_SSH_BATCH_EOF`, {
+          stdout: false,
+          silentOnError: true,
+          disableLog: true,
+        });
+        last = {
+          ok: result.code === 0,
+          code: result.code,
+          stdout: result.stdout || '',
+          stderr: result.stderr || '',
+          attempts: attempt,
+        };
+        if (last.ok) {
+          logger.info(`sshExecBatch succeeded on ${user}@${host}:${port} (attempt ${attempt})`);
+          return last;
+        }
+        logger.warn(`sshExecBatch attempt ${attempt}/${retries} failed on ${user}@${host}:${port}`, {
+          code: last.code,
+          stderr: last.stderr.slice(-400),
+        });
+        if (attempt < retries) await new Promise((r) => setTimeout(r, retryDelayMs * attempt));
+      }
+      return last;
+    },
+
+    /**
+     * Transfers a local script to a remote host and runs it over key-only SSH.
+     * The script is base64-encoded so no shell-quoting/escaping is needed, then
+     * decoded, made executable, and executed with the given arguments. Reuses
+     * sshExecBatch for the actual transport, retries, and structured result.
+     * @async
+     * @function sshRunScript
+     * @memberof UnderpostSSH
+     * @param {object} params
+     * @param {string} params.host - Target host/IP.
+     * @param {string} params.scriptPath - Local path to the script to run.
+     * @param {string} [params.args=''] - Arguments appended to the remote invocation.
+     * @param {object} [params.env={}] - Environment variables exported for the remote run (e.g. secrets). Passed inline to the command, never echoed.
+     * @param {string} [params.remotePath='/tmp/underpost-remote-script.sh'] - Remote path to write the script.
+     * @param {number} [params.port=22] - SSH port.
+     * @param {string} [params.user='root'] - SSH user (key-only).
+     * @param {string} [params.keyPath] - Private key path (defaults to engine deploy key).
+     * @param {number} [params.retries=3] - Retry attempts.
+     * @param {number} [params.waitForPortMs=0] - When > 0, wait for the port first.
+     * @returns {Promise<{ok: boolean, code: number, stdout: string, stderr: string, attempts: number}>}
+     */
+    sshRunScript: async ({
+      host,
+      scriptPath,
+      args = '',
+      env = {},
+      remotePath = '/tmp/underpost-remote-script.sh',
+      port = 22,
+      user = 'root',
+      keyPath = './engine-private/deploy/id_rsa',
+      retries = 3,
+      waitForPortMs = 0,
+    }) => {
+      if (!fs.existsSync(scriptPath)) throw new Error(`sshRunScript: script not found: ${scriptPath}`);
+      const b64 = Buffer.from(fs.readFileSync(scriptPath, 'utf8'), 'utf8').toString('base64');
+      // Inline env assignments (single-quote escaped) so secrets are exported for
+      // the remote run without appearing as logged CLI args.
+      const sq = (v) => `'${String(v).replace(/'/g, "'\\''")}'`;
+      const envPrefix = Object.entries(env)
+        .filter(([, v]) => v !== undefined && v !== null && `${v}` !== '')
+        .map(([k, v]) => `${k}=${sq(v)}`)
+        .join(' ');
+      const command = [
+        'set -e',
+        `echo '${b64}' | base64 -d > ${remotePath}`,
+        `chmod +x ${remotePath}`,
+        `${envPrefix ? `${envPrefix} ` : ''}bash ${remotePath} ${args}`,
+      ].join('\n');
+      return Underpost.ssh.sshExecBatch({ host, port, user, keyPath, retries, waitForPortMs, command });
+    },
+
     /**
      * Loads saved SSH credentials from config and sets them in the UnderpostRootEnv API.
      * @async

package/src/cli/static.js

@@ -6,10 +6,10 @@
 import fs from 'fs-extra';
 import path from 'path';
 import express from 'express';
-import { ssrFactory } from '../server/ssr.js';
+import { ssrFactory } from '../client-builder/ssr.js';
 import { shellExec } from '../server/process.js';
 import Underpost from '../index.js';
-import { JSONweb } from '../server/client-formatted.js';
+import { JSONweb } from '../client-builder/client-formatted.js';
 import { loggerFactory, loggerMiddleware } from '../server/logger.js';
 const logger = loggerFactory(import.meta);
 /**