underpost 3.2.30 → 3.2.80

package/src/cli/run.js

@@ -14,7 +14,9 @@ import {
   etcHostFactory,
   getNpmRootPath,
   isDeployRunnerContext,
+  loadConfInstances,
   loadConfServerJson,
+  selectConfInstances,
   loadReplicas,
   writeEnv,
 } from '../server/conf.js';
@@ -22,7 +24,7 @@ import { actionInitLog, loggerFactory } from '../server/logger.js';
 
 import fs from 'fs-extra';
 import net from 'net';
-import { range, setPad, timer } from '../client/components/core/CommonJs.js';
+import { range, s4, setPad, timer } from '../client/components/core/CommonJs.js';
 
 import os from 'os';
 import Underpost from '../index.js';
@@ -49,9 +51,36 @@ const waitForPort = (port, host = '127.0.0.1', { maxAttempts = 30, interval = 20
 
 const logger = loggerFactory(import.meta);
 
+/**
+ * @method instanceProxyRoutesFactory
+ * @description Renders the HTTPProxy route block for every instance sharing a host.
+ * Routes are emitted longest-prefix first so a specific instance path (`/FOREST`)
+ * is never shadowed by the default instance's catch-all (`/`).
+ * @param {string} deployId - Parent deployment identifier.
+ * @param {Array<object>} instances - Expanded instance entries bound to one host.
+ * @param {string} env - `development` | `production`.
+ * @param {Object<string,string>} trafficById - Instance id → traffic colour.
+ * @returns {string} Concatenated route YAML.
+ */
+const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =>
+  [...instances]
+    .sort((a, b) => (b.path || '/').length - (a.path || '/').length)
+    .map((instance) =>
+      Underpost.deploy.deploymentYamlServiceFactory({
+        path: instance.path,
+        port: env === 'development' && instance.fromDebugPort ? instance.fromDebugPort : instance.fromPort,
+        deployId: `${deployId}-${instance.id}`,
+        env,
+        deploymentVersions: [trafficById[instance.id] || 'blue'],
+        pathRewritePolicy: instance.pathRewritePolicy,
+      }),
+    )
+    .join('');
+
 /**
  * @constant DEFAULT_OPTION
  * @description Default options for the UnderpostRun class.
+ * @typedef {Object} UnderpostRunDefaultOptions
  * @type {Object}
  * @property {boolean} dev - Whether to run in development mode.
  * @property {string} podName - The name of the pod to run.
@@ -122,6 +151,7 @@ const logger = loggerFactory(import.meta);
  * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
  * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
  * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
+ * @property {string} branch - The Git branch to use for operations (e.g., for template-deploy, ssh-deploy).
  * @memberof UnderpostRun
  */
 const DEFAULT_OPTION = {
@@ -192,6 +222,7 @@ const DEFAULT_OPTION = {
   pullBundle: false,
   remove: false,
   test: false,
+  branch: '',
 };
 
 /**
@@ -215,7 +246,7 @@ class UnderpostRun {
      * @method dev-cluster
      * @description Resets and deploys a full development cluster including MongoDB, Valkey, exposes services, and updates `/etc/hosts` for local access.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dev-cluster': (path, options = DEFAULT_OPTION) => {
@@ -297,7 +328,7 @@ class UnderpostRun {
      * @method metadata
      * @description Generates metadata for the specified path after exposing the development cluster.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     metadata: async (path, options = DEFAULT_OPTION) => {
@@ -322,7 +353,7 @@ class UnderpostRun {
      * @method svc-ls
      * @description Lists systemd services and installed packages, optionally filtering by the provided path.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional filter for services and packages).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'svc-ls': (path, options = DEFAULT_OPTION) => {
@@ -342,7 +373,7 @@ class UnderpostRun {
      * @method svc-rm
      * @description Removes a systemd service by stopping it, disabling it, uninstalling the package, and deleting related files.
      * @param {string} path - The input value, identifier, or path for the operation (used as the service name).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'svc-rm': (path, options = DEFAULT_OPTION) => {
@@ -357,7 +388,7 @@ class UnderpostRun {
      * @method ssh-deploy-info
      * @description Retrieves deployment status and pod information from a remote server via SSH.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
@@ -401,7 +432,7 @@ class UnderpostRun {
      * skipped (move the owning controller). StatefulSets bound to node-local PVs may stay
      * Pending after a move until their volume is available on the target node.
      * @param {string} path - Resource selector (`kind/name`, `kind`, or empty).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      * @returns {Array<{ref:string,kind:string,status:string,node?:string}>} Per-resource outcome.
      */
@@ -601,7 +632,7 @@ class UnderpostRun {
      * @method dev-hosts-expose
      * @description Deploys a specified service in development mode with `/etc/hosts` modification for local access.
      * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID to deploy).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dev-hosts-expose': (path, options = DEFAULT_OPTION) => {
@@ -614,7 +645,7 @@ class UnderpostRun {
      * @method dev-hosts-restore
      * @description Restores the `/etc/hosts` file to its original state after modifications made during development deployments.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dev-hosts-restore': (path, options = DEFAULT_OPTION) => {
@@ -625,7 +656,7 @@ class UnderpostRun {
      * @method cluster-build
      * @description Build configuration for cluster deployment.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'cluster-build': (path, options = DEFAULT_OPTION) => {
@@ -649,7 +680,7 @@ class UnderpostRun {
      * and optionally triggers engine-<conf-id> CI with sync/init which in turn dispatches the CD workflow
      * after the build chain completes (template → ghpkg → engine-<conf-id> → CD).
      * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'template-deploy': (path = '', options = DEFAULT_OPTION) => {
@@ -663,13 +694,10 @@ class UnderpostRun {
       shellExec(`${baseCommand} run pull`);
       shellExec(`${baseCommand} run shared-dir`);
 
-      // Capture last N commit messages for propagation.
-      // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
-      const fromN =
-        options.fromNCommit && parseInt(options.fromNCommit) > 0
-          ? parseInt(options.fromNCommit)
-          : Underpost.repo.getUnpushedCount('.').count;
-      const message = shellExec(`node bin cmt --changelog ${fromN} --changelog-no-hash`, {
+      // Capture the sanitized message from the last N commits (--from-n-commit, default 1) for
+      // propagation to pwa-microservices-template and every engine-* repo.
+      const fromN = parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
+      const sanitizedMessage = shellExec(`node bin cmt --changelog-msg --from-n-commit ${fromN} --changelog-no-hash`, {
         silent: true,
         stdout: true,
       }).trim();
@@ -681,8 +709,6 @@ class UnderpostRun {
       );
       shellCd('/home/dd/engine');
 
-      const sanitizedMessage = Underpost.repo.sanitizeChangelogMessage(message);
-
       // Push engine repo so workflow YAML changes reach GitHub
       shellExec(`git reset`);
       shellExec(`${baseCommand} push . ${options.force ? '-f ' : ''}${process.env.GITHUB_USERNAME}/engine`);
@@ -718,10 +744,11 @@ class UnderpostRun {
       if (deployConfId) inputs.deploy_conf_id = deployConfId;
       if (deployType) inputs.deploy_type = deployType;
 
+      // Omit `ref` so dispatchWorkflow auto-detects the repo's default branch
+      // (a fork may default to `main` rather than the monorepo's `master`).
       Underpost.repo.dispatchWorkflow({
         repo,
         workflowFile: 'npmpkg.ci.yml',
-        ref: 'master',
         inputs,
       });
     },
@@ -730,7 +757,7 @@ class UnderpostRun {
      * @method template-deploy-local
      * @description Similar to `template-deploy` but runs the workflow locally without dispatching GitHub Actions. It pulls the latest changes, pushes to GitHub, builds the template, and optionally triggers a local release with CI push.
      * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'template-deploy-local': async (path, options = DEFAULT_OPTION) => {
@@ -744,17 +771,12 @@ class UnderpostRun {
       shellExec(`${baseCommand} run pull`);
       shellExec(`${baseCommand} run shared-dir`);
 
-      // Capture last N commit messages from the engine repo.
-      // When --from-n-commit is not set, auto-detect unpushed commit count (same as --unpush flag).
-      const fromN =
-        options.fromNCommit && parseInt(options.fromNCommit) > 0
-          ? parseInt(options.fromNCommit)
-          : Underpost.repo.getUnpushedCount('.').count;
-      const rawMessage = shellExec(`node bin cmt --changelog ${fromN} --changelog-no-hash`, {
+      // Capture the sanitized message from the last N commits (--from-n-commit, default 1).
+      const fromN = parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
+      const sanitizedMessage = shellExec(`node bin cmt --changelog-msg --from-n-commit ${fromN} --changelog-no-hash`, {
         silent: true,
         stdout: true,
       }).trim();
-      const sanitizedMessage = Underpost.repo.sanitizeChangelogMessage(rawMessage);
 
       const { triggerCmd } = path
         ? await Underpost.release.ci(path, sanitizedMessage, options)
@@ -766,15 +788,14 @@ class UnderpostRun {
      * @description Dispatches the Docker image CI workflow (`docker-image[.<runtime>].ci.yml`) via `workflow_dispatch`.
      * Repository resolution is delegated to `Underpost.repo.resolveInstanceRepo(path)`.
      * @param {string} path - Optional runtime / workflow suffix (e.g. `cyberia-server`, `cyberia-client`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'docker-image': (path, options = DEFAULT_OPTION) => {
-      const repo = Underpost.repo.resolveInstanceRepo(path);
+      const repo = Underpost.repo.resolveInstanceRepo(path, !options.test);
       Underpost.repo.dispatchWorkflow({
         repo,
-        workflowFile: `docker-image${path ? `.${path}` : ''}.ci.yml`,
-        ref: 'master',
+        workflowFile: `docker-image${path ? `.${path}` : ''}${options.dev ? '.dev' : ''}.ci.yml`,
         inputs: {},
       });
     },
@@ -782,7 +803,7 @@ class UnderpostRun {
      * @method clean
      * @description Changes directory to the provided path (defaulting to `/home/dd/engine`) and runs `node bin/deploy clean-core-repo`.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional directory path).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     clean: (path = '', options = DEFAULT_OPTION) => {
@@ -793,7 +814,7 @@ class UnderpostRun {
      * @method pull
      * @description Clones or pulls updates for the `engine` and `engine-private` repositories into `/home/dd/engine` and `/home/dd/engine/engine-private`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     pull: (path, options = DEFAULT_OPTION) => {
@@ -820,7 +841,7 @@ class UnderpostRun {
      * @method release-deploy
      * @description Executes deployment (`underpost run deploy`) for all deployment IDs listed in `./engine-private/deploy/dd.router`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'release-deploy': (path, options = DEFAULT_OPTION) => {
@@ -836,7 +857,7 @@ class UnderpostRun {
      * @method ssh-deploy
      * @description Dispatches the corresponding CD workflow for SSH-based deployment, replacing empty commits with workflow_dispatch.
      * @param {string} path - The deployment identifier (e.g., 'engine-core', 'sync-engine-core', 'init-engine-core').
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ssh-deploy': (path, options = DEFAULT_OPTION) => {
@@ -851,11 +872,13 @@ class UnderpostRun {
         job = 'init';
         confId = path.replace(/^init-/, '');
       }
-
+      const repo = Underpost.repo.resolveInstanceRepo(confId, !options.test);
+      // Omit `ref` so dispatchWorkflow auto-detects the target repo's default
+      // branch (getDefaultBranch): the monorepo is `master` but instance repos
+      // like engine-cyberia default to `main` — hardcoding either 422s.
       Underpost.repo.dispatchWorkflow({
-        repo: `${process.env.GITHUB_USERNAME}/engine`,
+        repo,
         workflowFile: `${confId}.cd.yml`,
-        ref: 'master',
         inputs: { job },
       });
     },
@@ -864,7 +887,7 @@ class UnderpostRun {
      * @description Opens a Visual Studio Code (VS Code) session for the specified path using `node ${underpostRoot}/bin/zed ${path}`,
      * or installs Zed and sublime-text IDE if `path` is 'install'.
      * @param {string} path - The input value, identifier, or path for the operation (used as the path to the directory to open in the IDE).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     ide: (path = '', options = DEFAULT_OPTION) => {
@@ -891,7 +914,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method crypto-policy
      * @description Sets the system's crypto policies to `DEFAULT:SHA1` using `update-crypto-policies` command.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'crypto-policy': (path, options = DEFAULT_OPTION) => {
@@ -906,7 +929,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * `deployment.yaml`. Useful when you want to force `Always` so the kubelet re-pulls a mutable tag on every rollout. Example:
      *   `node bin run sync dd-core --kubeadm --image-pull-policy Always`
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string containing deploy parameters).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     sync: async (path, options = DEFAULT_OPTION) => {
@@ -1000,7 +1023,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method stop
      * @description Stops a deployment by deleting the corresponding Kubernetes deployment and service resources.
      * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     stop: async (path = '', options = DEFAULT_OPTION) => {
@@ -1024,7 +1047,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-stop
      * @description Stops a remote deployment via SSH by executing the appropriate Underpost command on the remote server.
      * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
@@ -1051,7 +1074,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-db-rollback
      * @description Performs a database rollback on remote deployment via SSH.
      * @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @param {string} options.deployId - The deployment identifier
      * @param {string} options.user - The SSH user for credential lookup
      * @param {boolean} options.dev - Development mode flag
@@ -1078,7 +1101,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-db
      * @description Imports/restores a database on remote deployment via SSH.
      * @param {string} path - The deployment ID for database import
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @param {string} options.deployId - The deployment identifier
      * @param {string} options.user - The SSH user for credential lookup
      * @param {boolean} options.dev - Development mode flag
@@ -1103,7 +1126,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method ssh-deploy-db-status
      * @description Retrieves database status/stats for a deployment (or all deployments from dd.router) via SSH.
      * @param {string} path - Comma-separated deployId(s) or 'dd' to use the dd.router list.
-     * @param {Object} options - Runner options (uses options.deployId for SSH host lookup).
+     * @param {UnderpostRunDefaultOptions} options - Runner options (uses options.deployId for SSH host lookup).
      * @param {string} options.deployId - Deployment identifier used for SSH config lookup.
      * @param {string} options.user - SSH user for credential lookup.
      * @param {boolean} options.dev - Development mode flag.
@@ -1149,7 +1172,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method tz
      * @description Sets the system timezone using `timedatectl set-timezone` command.
      * @param {string} path - The input value, identifier, or path for the operation (used as the timezone string).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     tz: (path, options = DEFAULT_OPTION) => {
@@ -1170,7 +1193,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
      * @method get-proxy
      * @description Retrieves and logs the HTTPProxy resources in the specified namespace using `kubectl get HTTPProxy`.
      * @param {string} path - The input value, identifier, or path for the operation (used as an optional filter for the HTTPProxy resources).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'get-proxy': async (path = '', options = DEFAULT_OPTION) => {
@@ -1192,66 +1215,54 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
 
     'instance-promote': async (path, options = DEFAULT_OPTION) => {
       const env = options.dev ? 'development' : 'production';
-      const baseCommand = options.dev ? 'node bin' : 'underpost';
-      const baseClusterCommand = options.dev ? ' --dev' : '';
       let [deployId, id] = path.split(',');
-      const confInstances = JSON.parse(
-        fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
-      );
+      const confInstances = loadConfInstances(deployId);
+      const promoted = selectConfInstances(confInstances, id);
       let promotedTraffic = '';
-      for (const instance of confInstances) {
-        let {
-          id: _id,
-          host: _host,
-          path: _path,
-          image: _image,
-          fromPort: _fromPort,
-          toPort: _toPort,
-          fromDebugPort: _fromDebugPort,
-          toDebugPort: _toDebugPort,
-          cmd: _cmd,
-          volumes: _volumes,
-          metadata: _metadata,
-        } = instance;
-        if (id !== _id) continue;
-        const _deployId = `${deployId}-${_id}`;
-        // Use debug ports in development when defined, fall back to production ports.
-        if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
-        if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
-        const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
-          hostTest: _host,
+
+      // A Contour HTTPProxy is named after its host, so every instance sharing a
+      // host shares one object. Rebuilding it from the promoted instance alone
+      // would drop its siblings' routes, so each host is rendered from the full
+      // set: promoted instances flip colour, the rest keep their live colour.
+      const promotedIds = new Set(promoted.map((instance) => instance.id));
+      const hosts = [...new Set(promoted.map((instance) => instance.host))];
+      const affected = confInstances.filter((instance) => hosts.includes(instance.host));
+      const trafficById = {};
+      for (const instance of affected) {
+        const currentTraffic = Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
+          hostTest: instance.host,
           namespace: options.namespace,
+          env,
         });
-        const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
-        promotedTraffic = targetTraffic;
+        if (!promotedIds.has(instance.id)) {
+          trafficById[instance.id] = currentTraffic || 'blue';
+          continue;
+        }
+        trafficById[instance.id] = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
+        promotedTraffic = trafficById[instance.id];
+      }
+
+      for (const host of hosts) {
+        const hostInstances = affected.filter((instance) => instance.host === host);
         let proxyYaml =
-          Underpost.deploy.baseProxyYamlFactory({ host: _host, env: options.tls ? 'production' : env, options }) +
-          Underpost.deploy.deploymentYamlServiceFactory({
-            path: _path,
-            port: _fromPort,
-            // serviceId: deployId,
-            deployId: _deployId,
-            env,
-            deploymentVersions: [targetTraffic],
-            // pathRewritePolicy,
-          });
+          Underpost.deploy.baseProxyYamlFactory({ host, env: options.tls ? 'production' : env, options }) +
+          instanceProxyRoutesFactory({ deployId, instances: hostInstances, env, trafficById });
         if (options.tls) {
           if (options.test) {
-            const sslDir = `./engine-private/ssl/${_host}`;
-            const nameSafe = _host.replace(/[^a-zA-Z0-9_.-]/g, '_');
+            const sslDir = `./engine-private/ssl/${host}`;
+            const nameSafe = host.replace(/[^a-zA-Z0-9_.-]/g, '_');
             fs.mkdirpSync(sslDir);
-            shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${_host}"`);
-            shellExec(`kubectl delete secret ${_host} -n ${options.namespace} --ignore-not-found`);
+            shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${host}"`);
+            shellExec(`kubectl delete secret ${host} -n ${options.namespace} --ignore-not-found`);
             shellExec(
-              `kubectl create secret tls ${_host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
+              `kubectl create secret tls ${host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
             );
           } else {
-            shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
-            proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
+            shellExec(`sudo kubectl delete Certificate ${host} -n ${options.namespace} --ignore-not-found`);
+            proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host });
           }
         }
-        // console.log(proxyYaml);
-        shellExec(`kubectl delete HTTPProxy ${_host} --namespace ${options.namespace} --ignore-not-found`);
+        shellExec(`kubectl delete HTTPProxy ${host} --namespace ${options.namespace} --ignore-not-found`);
         shellExec(
           `kubectl apply -f - -n ${options.namespace} <<EOF
 ${proxyYaml}
@@ -1277,7 +1288,7 @@ EOF
     /**
      * @method instance
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string containing workflow parameters).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     instance: async (path = '', options = DEFAULT_OPTION) => {
@@ -1286,9 +1297,7 @@ EOF
       const baseClusterCommand = options.dev ? ' --dev' : '';
       let [deployId, id, replicas] = path.split(',');
       if (!replicas) replicas = options.replicas;
-      const confInstances = JSON.parse(
-        fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
-      );
+      const confInstances = selectConfInstances(loadConfInstances(deployId), id);
       const etcHosts = [];
       for (const instance of confInstances) {
         let {
@@ -1307,7 +1316,6 @@ EOF
           readinessProbe: _readinessProbe,
           livenessProbe: _livenessProbe,
         } = instance;
-        if (id !== _id) continue;
         const _deployId = `${deployId}-${_id}`;
         // Use debug ports in development when defined, fall back to production ports.
         if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
@@ -1331,6 +1339,7 @@ EOF
         const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
           hostTest: _host,
           namespace: options.namespace,
+          env,
         });
 
         const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
@@ -1436,25 +1445,54 @@ EOF
           logger.error(`Deployment ${deployId} did not become ready in time.`);
           return;
         }
-        shellExec(
-          `${baseCommand} run${baseClusterCommand} --namespace ${options.namespace}` +
-            `${options.nodeName ? ` --node-name ${options.nodeName}` : ''}` +
-            `${options.tls ? ` --tls ${options.test ? '--test' : ''}` : ''}` +
-            ` instance-promote '${path}'`,
-        );
       }
+      // Promote (switch the HTTPProxy to the new colour) ONLY after EVERY
+      // instance in the family is deployed and ready — never per-instance inside
+      // the loop. A per-variant switch would flip server.cyberiaonline.com to the
+      // default's new deployment while -forest/-test are still being created,
+      // pointing routes at services that don't exist yet. instance-promote
+      // rebuilds the whole host proxy (all variant routes) in one shot, so a
+      // single call with the family id promotes the family atomically.
+      if (!options.expose) await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, options);
       if (options.etcHosts) {
         const hostListenResult = etcHostFactory(etcHosts);
         logger.info(hostListenResult.renderHosts);
       }
     },
 
+    /**
+     * @method deploy-key
+     * @description Copies the deploy key for a specific user and deployId to a temporary location on the local machine.
+     * @param {string} path - The input value, identifier, or path for the operation (not used in this method).
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
+     * @param {string} options.user - The user for which to copy the deploy key.
+     * @param {string} options.deployId - The deployment identifier associated with the deploy key.
+     * @memberof UnderpostRun
+     */
+    'deploy-key': (path, options = DEFAULT_OPTION) => {
+      const prefix = 'dd-key';
+      if (options.reset) {
+        shellExec(`rm -rf /home/dd/tmp/${prefix}_*`);
+        return;
+      }
+      if (!options.user || !options.deployId) {
+        logger.error('Both --user and --deploy-id options are required to copy the deploy key.');
+        return;
+      }
+      const targetPath = `/home/dd/tmp/${prefix}_${s4()}${s4()}`;
+      fs.mkdirSync('/home/dd/tmp', { recursive: true });
+      fs.copyFileSync(`./engine-private/conf/${options.deployId}/users/${options.user}/id_rsa`, targetPath);
+      logger.info(`Copied deploy key to ${targetPath}`);
+      if (options.copy) pbcopy(targetPath);
+    },
+
     /**
      * @method instance-build-manifest
      * @description Builds a Kubernetes Deployment + Service manifest for a specific instance entry
-     * from `conf.instances.json` and writes it to a file.
-     * Traffic colour is automatically chosen as the opposite of the current live colour (blue/green),
-     * defaulting to `blue` when no deployment is running yet.
+     * from `conf.instances.json` and writes it to a file. This is a purely local
+     * artifact generator: it never probes a live cluster. Traffic colour defaults
+     * to the canonical initial `blue` and can be overridden with `--traffic`; the
+     * real blue/green swap is resolved at deploy time (`deploy --sync`).
      *
      * If `--build` is supplied the image is built from the project Dockerfile and loaded into the
      * cluster before the manifest is written (kind by default; `--kubeadm` / `--k3s` override).
@@ -1465,33 +1503,49 @@ EOF
      *   `<projectPath>/manifests/<env>/deployment.yaml`.
      *   In production, files are also copied to `<projectPath>/Dockerfile` and
      *   `<projectPath>/deployment.yaml`.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'instance-build-manifest': (path, options = DEFAULT_OPTION) => {
       const env = options.dev ? 'development' : 'production';
       let [deployId, id, projectPath] = path.split(',');
       const rootPath = projectPath ? projectPath : '.';
+
+      const confInstances = loadConfInstances(deployId);
+      // Targeting a template id builds every world in the family. The fan-out
+      // re-enters this runner with `instanceOnly` because the default world
+      // keeps the template id verbatim — without it, selecting `mmo-server`
+      // would match the family again and recurse forever. Only that default
+      // world publishes to the project root, so the repo keeps exactly one
+      // canonical Dockerfile/deployment.yaml pair.
+      const selected = options.instanceOnly
+        ? confInstances.filter((instance) => instance.id === id)
+        : selectConfInstances(confInstances, id);
+      if (selected.length === 0) {
+        logger.error(`Instance with id '${id}' not found in conf.instances.json for deployId '${deployId}'`);
+        return;
+      }
+      if (!options.instanceOnly && (selected.length > 1 || selected[0].id !== id)) {
+        for (const instance of selected)
+          UnderpostRun.RUNNERS['instance-build-manifest'](
+            [deployId, instance.id, projectPath].filter((v) => v !== undefined).join(','),
+            { ...options, instanceOnly: true },
+          );
+        return;
+      }
+
       const envManifestPath = `${rootPath}/manifests/deployments/${id}-${env}`;
       const outputPath = `${envManifestPath}/deployment.yaml`;
       const dockerfileManifestPath = `${envManifestPath}/Dockerfile`;
 
       fs.mkdirpSync(envManifestPath);
 
-      const confInstances = JSON.parse(
-        fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
-      );
-
-      const instance = confInstances.find((i) => i.id === id);
-      if (!instance) {
-        logger.error(`Instance with id '${id}' not found in conf.instances.json for deployId '${deployId}'`);
-        return;
-      }
+      const instance = selected[0];
+      const isDefaultInstance = instance.id === instance.templateId || !instance.templateId;
 
       let {
         id: _id,
         host: _host,
-        path: _path,
         image: _image,
         fromPort: _fromPort,
         toPort: _toPort,
@@ -1544,7 +1598,7 @@ EOF
           path: projectPath,
           imageName: _image,
           podmanSave: true,
-          imagePath: projectPath,
+          imageOutPath: projectPath,
           kind: isKind,
           kubeadm: !!options.kubeadm,
           k3s: !!options.k3s,
@@ -1557,15 +1611,8 @@ EOF
         });
       }
 
-      // Determine target traffic: opposite of current, or 'blue' if nothing is running yet.
-      const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
-        hostTest: _host,
-        namespace: options.namespace,
-      });
-      const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
-
-      // Resolve {{grpc-service-dns}} using the parent deploy's current (or default) traffic.
-      const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
+      const targetTraffic = options.traffic || 'blue';
+      const parentTraffic = targetTraffic;
       const resolvedCmd = _cmd[env].map((c) =>
         c.replaceAll(
           '{{grpc-service-dns}}',
@@ -1613,11 +1660,122 @@ EOF
         image: _image,
       });
 
-      if (env === 'production') {
+      // --- Sibling manifests (pv-pvc, proxy, grpc-service) ------------------
+      // Emit the same apply-able set the parent deploy writes to build/<env>,
+      // scoped to this instance so the project repo and engine-private ship
+      // more than just deployment.yaml. Content matches what `run instance`
+      // creates dynamically at deploy time (deployVolume / instance-promote /
+      // the parent gRPC ClusterIP), so a static `kubectl apply` is equivalent.
+      const pvDataNode = Underpost.deploy.resolveDeployNode({
+        node: options.nodeName,
+        kind: options.kind,
+        kubeadm: options.kubeadm,
+        k3s: options.k3s,
+        env,
+      });
+
+      // pv-pvc.yaml — one PV+PVC per instance volume; names mirror deployVolume.
+      let pvPvcYaml = '';
+      for (const volume of _volumes || []) {
+        if (!volume.claimName) continue;
+        const pvcId = `${volume.claimName}-${_deployId}-${env}-${targetTraffic}`;
+        const pvId = `${volume.claimName.replace('pvc-', 'pv-')}-${_deployId}-${env}-${targetTraffic}`;
+        pvPvcYaml += `---\n${Underpost.deploy.persistentVolumeFactory({
+          pvcId,
+          namespace: options.namespace,
+          hostPath: `/home/dd/engine/volume/${pvId}`,
+          nodeName: pvDataNode,
+        })}\n`;
+      }
+
+      // proxy.yaml — this instance's OWN route only (its sub-path → its own
+      // service), so each variant's build dir carries a distinct, instance-scoped
+      // fragment rather than an identical copy of the whole host proxy. The
+      // complete host HTTPProxy — every variant's route aggregated onto the one
+      // fqdn, each pointing at its live colour — is assembled and applied by
+      // `instance-promote` at deploy time, and only once EVERY variant is ready.
+      const proxyYaml =
+        Underpost.deploy.baseProxyYamlFactory({ host: _host, env, options }) +
+        instanceProxyRoutesFactory({
+          deployId,
+          instances: [instance],
+          env,
+          trafficById: { [instance.id]: targetTraffic },
+        });
+
+      // grpc-service.yaml — the parent deploy's gRPC ClusterIP (shared; the
+      // instance cmd resolves {{grpc-service-dns}} to it). Reuse the parent's
+      // generated manifest when present rather than regenerating it here.
+      const parentGrpcServicePath = `./engine-private/conf/${deployId}/build/${env}/grpc-service.yaml`;
+      const grpcServiceYaml = fs.existsSync(parentGrpcServicePath)
+        ? fs.readFileSync(parentGrpcServicePath, 'utf8')
+        : '';
+
+      // Write the sibling set next to deployment.yaml (project) and into the
+      // engine-private per-instance build dir (mirrors instances/<id>/ layout).
+      const instanceBuildDir = `./engine-private/conf/${deployId}/instances/${_id}/build/${env}`;
+      fs.mkdirpSync(instanceBuildDir);
+      fs.writeFileSync(`${instanceBuildDir}/deployment.yaml`, deploymentYaml, 'utf8');
+      const siblingManifests = {
+        'pv-pvc.yaml': pvPvcYaml,
+        'proxy.yaml': proxyYaml,
+        'grpc-service.yaml': grpcServiceYaml,
+      };
+      for (const [name, content] of Object.entries(siblingManifests)) {
+        if (!content) continue;
+        fs.writeFileSync(`${envManifestPath}/${name}`, content, 'utf8');
+        fs.writeFileSync(`${instanceBuildDir}/${name}`, content, 'utf8');
+      }
+      logger.info('[instance-build-manifest] Sibling manifests written', {
+        project: envManifestPath,
+        enginePrivate: instanceBuildDir,
+        pvPvc: !!pvPvcYaml,
+        proxy: !!proxyYaml,
+        grpcService: !!grpcServiceYaml,
+      });
+
+      // --- Per-instance env files -----------------------------------------
+      // Each env file is seeded from the template instance's env file for the
+      // same mode so operator-owned keys (API keys, memory limits, service URLs)
+      // are inherited verbatim; only the keys the instance declares under
+      // `multiInstance.env` — plus the derived container id — are (re)written.
+      //
+      // A derived instance's env dir is generated in full: both development.env
+      // and production.env are written on every build, so a deploy in either
+      // environment always finds the env file its `cmd` sources, no matter which
+      // mode this build ran. The default/template instance owns the committed
+      // source files, so only its current-mode file is idempotently refreshed.
+      if (instance.templateId) {
+        const instanceEnvDir = `./engine-private/conf/${deployId}/instances/${_id}/env`;
+        fs.mkdirpSync(instanceEnvDir);
+        const envsToWrite = isDefaultInstance ? [env] : ['development', 'production'];
+        for (const targetEnv of envsToWrite) {
+          const templateEnvPath = `./engine-private/conf/${deployId}/instances/${instance.templateId}/env/${targetEnv}.env`;
+          const baseEnv = fs.existsSync(templateEnvPath) ? dotenv.parse(fs.readFileSync(templateEnvPath, 'utf8')) : {};
+          writeEnv(`${instanceEnvDir}/${targetEnv}.env`, {
+            ...baseEnv,
+            ...(instance.instanceEnv?.[targetEnv] || {}),
+            CONTAINER_DEPLOY_ID: `${_deployId}-${targetEnv}`,
+          });
+        }
+        logger.info('[instance-build-manifest] Instance env written', {
+          dir: instanceEnvDir,
+          instanceCode: instance.instanceCode,
+          envs: envsToWrite,
+          keys: Object.keys(instance.instanceEnv?.[env] || {}),
+        });
+      }
+
+      if (env === 'production' && isDefaultInstance) {
         if (fs.existsSync(dockerfileManifestPath)) {
           fs.copyFileSync(dockerfileManifestPath, `${rootPath}/Dockerfile`);
         }
         fs.copyFileSync(outputPath, `${rootPath}/deployment.yaml`);
+        // Sibling manifests alongside deployment.yaml at the project root.
+        for (const name of ['pv-pvc.yaml', 'proxy.yaml', 'grpc-service.yaml']) {
+          const src = `${envManifestPath}/${name}`;
+          if (fs.existsSync(src)) fs.copyFileSync(src, `${rootPath}/${name}`);
+        }
         logger.info('[instance-build-manifest] Production artifacts copied to project root', {
           rootPath,
           dockerfile: `${rootPath}/Dockerfile`,
@@ -1629,6 +1787,25 @@ EOF
           fs.copyFileSync(ciSrc, `${rootPath}/.github/workflows/docker-image.${_runtime}.ci.yml`);
           logger.info(`[instance-build-manifest] CI workflow copied`, { src: ciSrc });
         }
+
+        // Ship the development variant alongside production so the instance repo
+        // is self-contained: the dev Dockerfile (built by the -dev CI workflow
+        // into underpost/<runtime>-dev, consumed by the development compose
+        // stack) and its dispatchable workflow. Both are optional — synced only
+        // when the source-of-truth files exist in the engine repo.
+        if (_runtime) {
+          const devDockerfileSrc = `src/runtime/${_runtime}/Dockerfile.dev`;
+          if (fs.existsSync(devDockerfileSrc)) {
+            fs.copyFileSync(devDockerfileSrc, `${rootPath}/Dockerfile.dev`);
+            logger.info('[instance-build-manifest] Dev Dockerfile copied', { src: devDockerfileSrc });
+          }
+          const devCiSrc = `./.github/workflows/docker-image.${_runtime}.dev.ci.yml`;
+          if (fs.existsSync(devCiSrc)) {
+            if (!fs.existsSync(`${rootPath}/.github/workflows`)) fs.mkdirpSync(`${rootPath}/.github/workflows`);
+            fs.copyFileSync(devCiSrc, `${rootPath}/.github/workflows/docker-image.${_runtime}.dev.ci.yml`);
+            logger.info(`[instance-build-manifest] Dev CI workflow copied`, { src: devCiSrc });
+          }
+        }
       }
     },
 
@@ -1636,7 +1813,7 @@ EOF
      * @method ls-deployments
      * @description Retrieves and logs a table of Kubernetes deployments using `Underpost.deploy.get`.
      * @param {string} path - The input value, identifier, or path for the operation (used as an optional deployment name filter).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ls-deployments': async (path, options = DEFAULT_OPTION) => {
@@ -1647,7 +1824,7 @@ EOF
      * @method host-update
      * @description Executes the `rocky-setup.sh` script to update the host system configuration.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'host-update': async (path, options = DEFAULT_OPTION) => {
@@ -1663,7 +1840,7 @@ EOF
      * the systemd cgroup driver, enables the `crio` service, and writes `/etc/crictl.yaml`
      * so that `crictl` targets the CRI-O socket by default.
      * @param {string} path - Unused.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @memberof UnderpostRun
      */
     'install-crio': (path, options = DEFAULT_OPTION) => {
@@ -1714,7 +1891,7 @@ EOF`);
      * @method dd-container
      * @description Deploys a development or debug container tasks jobs, setting up necessary volumes and images, and running specified commands within the container.
      * @param {string} path - The input value, identifier, or path for the operation (used as the command to run inside the container).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'dd-container': async (path = '', options = DEFAULT_OPTION) => {
@@ -1741,7 +1918,11 @@ EOF`);
       }
 
       if (!currentImage)
-        shellExec(`${baseCommand} image${baseClusterCommand} --pull-base ${options.dev ? '--kind' : '--kubeadm'}`);
+        shellExec(
+          `${baseCommand} image${baseClusterCommand} --pull-base --build --path ${
+            options.dev ? '.' : options.underpostRoot
+          } ${options.dev ? '--kind' : '--kubeadm'}`,
+        );
       // shellExec(`kubectl delete pod ${podName} --ignore-not-found`);
 
       const payload = {
@@ -1769,7 +1950,7 @@ EOF`);
      * @method ip-info
      * @description Executes the `ip-info.sh` script to display IP-related information for the specified path.
      * @param {string} path - The input value, identifier, or path for the operation (used as an argument to the script).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'ip-info': (path, options = DEFAULT_OPTION) => {
@@ -1782,7 +1963,7 @@ EOF`);
      * @method db-client
      * @description Deploys and exposes the Adminer database client application (using `adminer:4.7.6-standalone` image) on the cluster.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'db-client': async (path, options = DEFAULT_OPTION) => {
@@ -1809,7 +1990,7 @@ EOF`);
      * @method git-conf
      * @description Configures Git global and local user name and email settings based on the provided `path` (formatted as `username,email`), or defaults to environment variables.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string: `username,email`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'git-conf': (path = '', options = DEFAULT_OPTION) => {
@@ -1857,7 +2038,7 @@ EOF`);
      * TLS config, deletes stale Certificate resources, then reapplies the proxy and secret.yaml
      * (cert-manager Certificate resources) for each affected deployment.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string: `deployId,env,replicas`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     promote: async (path, options = DEFAULT_OPTION) => {
@@ -1911,10 +2092,31 @@ EOF`);
      * @method metrics
      * @description Deploys Prometheus and Grafana for metrics monitoring, targeting the hosts defined in the deployment configuration files.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     metrics: async (path, options = DEFAULT_OPTION) => {
+      if (path === 'server') {
+        shellExec(
+          `kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/high-availability-1.21+.yaml`,
+        );
+        await timer(2000);
+
+        shellExec(`kubectl patch deployment metrics-server -n kube-system \
+  --type='json' \
+  -p='[
+    {
+      "op":"add",
+      "path":"/spec/template/spec/containers/0/args/-",
+      "value":"--kubelet-insecure-tls"
+    }
+  ]'`);
+        shellExec(`kubectl scale deployment metrics-server \
+  -n kube-system \
+  --replicas=1`);
+
+        return;
+      }
       const deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',');
       let hosts = [];
       for (const deployId of deployList) {
@@ -1928,7 +2130,7 @@ EOF`);
      * @method cluster
      * @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB, MariaDB, Valkey, Contour (Ingress), and Cert-Manager, and deploys all services.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     cluster: async (path = '', options = DEFAULT_OPTION) => {
@@ -1990,7 +2192,7 @@ EOF`);
      * @method deploy
      * @description Deploys a specified service (identified by `path`) using blue/green strategy, monitors its status, and switches traffic upon readiness.
      * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID to deploy).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     deploy: async (path, options = DEFAULT_OPTION) => {
@@ -2015,7 +2217,7 @@ EOF`);
      * @method disk-clean
      * @description Executes the `disk-clean-sh` script to perform disk cleanup operations.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'disk-clean': async (path, options = DEFAULT_OPTION) => {
@@ -2028,7 +2230,7 @@ EOF`);
      * @method disk-devices
      * @description Executes the `disk-devices.sh` script to display information about disk devices.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'disk-devices': async (path = '/', options = DEFAULT_OPTION) => {
@@ -2041,7 +2243,7 @@ EOF`);
      * @method disk-usage
      * @description Displays disk usage statistics using the `du` command, sorted by size.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'disk-usage': async (path = '/', options = DEFAULT_OPTION) => {
@@ -2056,7 +2258,7 @@ EOF`);
      * @method dev
      * @description Starts development servers for client, API, and proxy based on provided parameters (deployId, host, path, clientHostPort).
      * @param {string} path - The input value, identifier, or path for the operation (formatted as `deployId,subConf,host,path,clientHostPort`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     dev: async (path = '', options = DEFAULT_OPTION) => {
@@ -2116,7 +2318,7 @@ EOF`);
      * @method service
      * @description Deploys and exposes specific services (like `mongo-express-service`) on the cluster, updating deployment configurations and monitoring status.
      * @param {string} path - The input value, identifier, or path for the operation (formatted as `deployId,serviceId,host,path,replicas,image,node`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     service: async (path = '', options = DEFAULT_OPTION) => {
@@ -2212,7 +2414,7 @@ EOF`);
      * @method etc-hosts
      * @description Generates and logs the contents for the `/etc/hosts` file based on provided hosts or deployment configurations.
      * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated list of hosts).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'etc-hosts': async (path = '', options = DEFAULT_OPTION) => {
@@ -2229,7 +2431,7 @@ EOF`);
      * @method sh
      * @description Enables remote control for the Kitty terminal emulator.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     sh: async (path = '', options = DEFAULT_OPTION) => {
@@ -2249,7 +2451,7 @@ EOF`);
      * @method log
      * @description Searches and highlights keywords in a specified log file, optionally showing surrounding lines.
      * @param {string} path - The input value, identifier, or path for the operation (formatted as `filePath,keywords,lines`).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     log: async (path, options = DEFAULT_OPTION) => {
@@ -2266,7 +2468,7 @@ EOF`);
      * @method ps
      * @description Displays running processes that match a specified path or keyword.
      * @param {string} path - The input value, identifier, or path for the operation (used as a keyword to filter processes).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     ps: async (path = '', options = DEFAULT_OPTION) => {
@@ -2291,7 +2493,7 @@ EOF`);
      * @method pid-info
      * @description Displays detailed information about a process by PID, including service details, command line, executable path, working directory, environment variables, and parent process tree.
      * @param {string} path - The PID of the process to inspect.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'pid-info': (path, options = DEFAULT_OPTION) => {
@@ -2331,7 +2533,7 @@ EOF`);
      * @method background
      * @description Runs a custom command in the background using nohup, logging output to `/var/log/<id>.log` and saving the PID to `/var/run/<id>.pid`.
      * @param {string} path - The command to run in the background (e.g. 'npm run prod:container dd-cyberia-r3').
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     background: (path, options = DEFAULT_OPTION) => {
@@ -2351,7 +2553,7 @@ EOF`);
      * @method ports
      * @description Set on ~/.bashrc alias: ports <port> Command to list listening ports that match the given keyword.
      * @param {string} path - The input value, identifier, or path for the operation (used as a keyword to filter listening ports).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     ports: async (path = '', options = DEFAULT_OPTION) => {
@@ -2363,7 +2565,7 @@ EOF`);
      * @method deploy-test
      * @description Deploys a test deployment (`dd-test`) in either development or production mode, setting up necessary secrets and starting the deployment.
      * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'deploy-test': async (path, options = DEFAULT_OPTION) => {
@@ -2388,7 +2590,7 @@ EOF`);
      * @method tf-vae-test
      * @description Creates and runs a job pod (`tf-vae-test`) that installs TensorFlow dependencies, clones the TensorFlow docs, and runs the CVAE tutorial script, with a terminal monitor attached.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'tf-vae-test': async (path, options = DEFAULT_OPTION) => {
@@ -2488,7 +2690,7 @@ EOF`);
      * @method spark-template
      * @description Creates a new Spark template project using `sbt new` in `/home/dd/spark-template`, initializes a Git repository, and runs `replace_params.sh` and `build.sh`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'spark-template': (path, options = DEFAULT_OPTION) => {
@@ -2516,7 +2718,7 @@ EOF`);
      * @method pull-rocky-image
      * @description Pulls the base `rockylinux:9` image from Docker Hub via Podman.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'pull-rocky-image': (path, options = DEFAULT_OPTION) => {
@@ -2526,7 +2728,7 @@ EOF`);
      * @method rmi
      * @description Forces the removal of all local Podman images (`podman rmi $(podman images -qa) --force`).
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     rmi: (path, options = DEFAULT_OPTION) => {
@@ -2536,7 +2738,7 @@ EOF`);
      * @method kill
      * @description Kills processes listening on the specified port(s). If the `path` contains a `+`, it treats it as a range of ports to kill.
      * @param {string} path - The input value, identifier, or path for the operation (used as the port number).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     kill: (path = '', options = DEFAULT_OPTION) => {
@@ -2568,7 +2770,7 @@ EOF`);
      * constraints (lowercase, uppercase, digit, special char, min 8 chars). Logs the plain password
      * to the console or, when `--copy` is set, copies it to the clipboard via pbcopy.
      * @param {string} path - Optional password length (default: 16).
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @param {boolean} options.copy - When true, copies to clipboard instead of logging.
      * @memberof UnderpostRun
      */
@@ -2602,7 +2804,7 @@ EOF`);
      * @method secret
      * @description Creates an Underpost secret named 'underpost' from a file, defaulting to `/home/dd/engine/engine-private/conf/dd-cron/.env.production` if no path is provided.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional path to the secret file).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     secret: (path, options = DEFAULT_OPTION) => {
@@ -2615,7 +2817,7 @@ EOF`);
      * @method underpost-config
      * @description Calls `Underpost.deploy.configMap` to create a Kubernetes ConfigMap, defaulting to the 'production' environment.
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional configuration name/environment).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'underpost-config': (path = '', options = DEFAULT_OPTION) => {
@@ -2625,7 +2827,7 @@ EOF`);
      * @method gpu-env
      * @description Sets up a dedicated GPU development environment cluster, resetting and then setting up the cluster with `--dedicated-gpu` and monitoring the pods.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'gpu-env': (path, options = DEFAULT_OPTION) => {
@@ -2638,7 +2840,7 @@ EOF`);
      * @method tf-gpu-test
      * @description Deletes existing `tf-gpu-test-script` ConfigMap and `tf-gpu-test-pod`, and applies the test manifest from `manifests/deployment/tensorflow/tf-gpu-test.yaml`.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'tf-gpu-test': (path, options = DEFAULT_OPTION) => {
@@ -2652,7 +2854,7 @@ EOF`);
      * @method deploy-job
      * @description Creates and applies a custom Kubernetes Pod manifest (Job) for running arbitrary commands inside a container image (defaulting to a TensorFlow/NVIDIA image).
      * @param {string} path - The input value, identifier, or path for the operation (used as the optional script path or job argument).
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      */
     'deploy-job': async (path, options = DEFAULT_OPTION) => {
@@ -2787,7 +2989,7 @@ EOF`;
      *   Only files matching `<host>-<route>.zip.part*` or `<host>-<route>.zip` for each non-skipped route are uploaded.
      * @param {string} path - Optional `fsPath.splitOption` string.
      *   Examples: `build` (default split 8), `build.16` (split 16 MB), `build.none-split` (no split flag).
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @param {string} [options.deployId] - Override deploy ID.
      * @param {boolean} [options.dev] - Use development environment; defaults to production.
      * @memberof UnderpostRun
@@ -2875,7 +3077,7 @@ EOF`;
      *   so that multi-path deployments are handled correctly.
      * @param {string} path - Optional comma-separated host name(s) to restrict processing (e.g. 'underpost.net' or 'a.com,b.com').
      *   If omitted, all hosts from `engine-private/conf/<deployId>/conf.server.json` are used.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @param {string} [options.deployId] - Deploy ID for storage lookup (defaults to 'dd-default').
      * @param {boolean} [options.dev] - Use development environment; defaults to production.
      * @memberof UnderpostRun
@@ -2967,7 +3169,7 @@ EOF`;
      * @method build-cluster-deployment-manifests
      * @description Builds deployment manifests for both production and development environments using `node bin deploy --build-manifest`, syncing them, and setting replicas to 1 for the `dd` deployment.
      * @param {string} path - Unused.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      * @memberof UnderpostRun
      */
     'build-cluster-deployment-manifests': (path = '', options = DEFAULT_OPTION) => {
@@ -2980,7 +3182,7 @@ EOF`;
      * @description Installs and enables the Cockpit KVM Dashboard (cockpit, cockpit-machines, libvirt)
      * and opens the cockpit firewall service. With `--remove`, closes the firewall service instead.
      * @param {string} path - Unused.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      *   `options.remove` — when true, removes the cockpit firewall rule instead of adding it.
      * @memberof UnderpostRun
      */
@@ -3004,7 +3206,7 @@ EOF`;
      * Use `reload-shared-dir` for subsequent permission repairs without recreating the group.
      * @param {string} path - Target directory to set up (defaults to `/home/dd/engine`).
      *   Customise via the `path` argument or leave empty to use the default.
-     * @param {Object} options - The default underpost runner options for customizing workflow.
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow.
      *   Key fields: `options.user` (default `'admin'`), `options.group` (default `'engine-dev'`).
      * @memberof UnderpostRun
      */
@@ -3032,7 +3234,7 @@ EOF`;
      * write throughout the shared workspace while preserving existing ownership.
      *
      * @param {string} path - Shared directory (defaults to `/home/dd/engine`).
-     * @param {Object} options - Underpost runner options.
+     * @param {UnderpostRunDefaultOptions} options - Underpost runner options.
      *   Key fields:
      *     - options.user  (default: 'admin')
      *     - options.group (default: 'engine-dev')
@@ -3080,7 +3282,7 @@ EOF`;
      * @description Executes a specified runner function from the UnderpostRun class with the provided path and options.
      * @param {string} runner - The name of the runner to execute.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      * @returns {Promise<any>} The result of the runner execution.
      */
@@ -3093,7 +3295,7 @@ EOF`;
      * @description Initiates the execution of a specified CLI command (runner) with the given input value (`path`) and processed options.
      * @param {string} runner - The name of the runner to execute.
      * @param {string} path - The input value, identifier, or path for the operation.
-     * @param {Object} options - The default underpost runner options for customizing workflow
+     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
      * @memberof UnderpostRun
      * @returns {Promise<any>} The result of the callback execution.
      */