underpost 3.2.80 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.github/workflows/ghpkg.ci.yml +7 -1
  2. package/.github/workflows/pwa-microservices-template-page.cd.yml +1 -16
  3. package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
  4. package/.github/workflows/release.cd.yml +1 -9
  5. package/CHANGELOG.md +291 -1
  6. package/CLI-HELP.md +174 -23
  7. package/README.md +5 -2
  8. package/bin/build.js +7 -5
  9. package/bin/deploy.js +19 -17
  10. package/deploy/lib/logging.sh +96 -0
  11. package/deploy/pwa-microservices-template/deploy.sh +72 -0
  12. package/deploy/release/deploy.sh +62 -0
  13. package/docker-compose.yml +1 -1
  14. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +5 -1
  15. package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
  16. package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +52 -0
  17. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  18. package/manifests/deployment/playwright/deployment.yaml +1 -1
  19. package/manifests/mongodb/kustomization.yaml +4 -1
  20. package/manifests/mongodb/statefulset.yaml +4 -0
  21. package/manifests/mongodb/storage-class.yaml +9 -2
  22. package/package.json +19 -19
  23. package/scripts/audit-selinux.sh +64 -0
  24. package/scripts/coverall-test.sh +24 -0
  25. package/scripts/gpu-diag.sh +0 -0
  26. package/scripts/ip-info.sh +0 -0
  27. package/scripts/k3s-node-setup.sh +18 -15
  28. package/scripts/kubeadm-node-setup.sh +12 -23
  29. package/scripts/link-local-underpost-cli.sh +0 -0
  30. package/scripts/lxd-vm-setup.sh +0 -0
  31. package/scripts/maas-nat-firewalld.sh +0 -0
  32. package/scripts/nat-iptables.sh +12 -4
  33. package/scripts/rhel-grpc-setup.sh +0 -0
  34. package/scripts/rocky-kickstart.sh +25 -9
  35. package/scripts/test-monitor.sh +4 -3
  36. package/src/cli/baremetal.js +1 -2
  37. package/src/cli/cloud-init.js +1 -1
  38. package/src/cli/cluster.js +786 -96
  39. package/src/cli/db.js +11 -4
  40. package/src/cli/deploy.js +1698 -177
  41. package/src/cli/docker-compose.js +19 -178
  42. package/src/cli/env.js +1 -1
  43. package/src/cli/image.js +15 -7
  44. package/src/cli/index.js +245 -44
  45. package/src/cli/ipfs.js +82 -11
  46. package/src/cli/lxd.js +1 -1
  47. package/src/cli/monitor.js +2 -2
  48. package/src/cli/release.js +57 -22
  49. package/src/cli/repository.js +12 -10
  50. package/src/cli/run.js +2195 -427
  51. package/src/cli/secrets.js +969 -0
  52. package/src/cli/ssh.js +206 -105
  53. package/src/cli/system.js +26 -13
  54. package/src/cli/test.js +1 -1
  55. package/src/cli/vultr.js +583 -0
  56. package/src/cli/wireguard.js +2125 -0
  57. package/src/client-builder/client-build.js +102 -13
  58. package/src/client-builder/ssr.js +27 -73
  59. package/src/db/mongo/MongoBootstrap.js +295 -54
  60. package/src/db/mongo/MongooseDB.js +51 -32
  61. package/src/index.js +25 -1
  62. package/src/projects/underpost/catalog-underpost.js +4 -1
  63. package/src/server/backup.js +1 -1
  64. package/src/server/conf.js +1216 -168
  65. package/src/server/cri.js +70 -0
  66. package/src/server/cron.js +249 -51
  67. package/src/server/dns.js +100 -6
  68. package/src/server/environment.js +98 -0
  69. package/src/server/forward-proxy.js +549 -0
  70. package/src/server/middlewares.js +56 -1
  71. package/src/server/process.js +0 -1
  72. package/src/server/selinux.js +185 -0
  73. package/src/server/systemd.js +205 -0
  74. package/src/server/underpost-compression.js +186 -0
  75. package/src/server/underpost-gateway.js +1083 -0
  76. package/src/server/underpost-ingress.js +380 -0
  77. package/test/cluster-instances.test.js +435 -0
  78. package/test/deploy-node-placement.test.js +45 -0
  79. package/test/instance-traffic-plan.test.js +710 -0
  80. package/test/selinux.test.js +71 -0
  81. package/test/sops-secret-store.test.js +612 -0
  82. package/test/underpost-gateway.test.js +510 -0
  83. package/test/underpost-ingress.test.js +305 -0
  84. package/test/wireguard-edge.test.js +1177 -0
package/src/cli/run.js CHANGED
@@ -4,79 +4,74 @@
4
4
  * @namespace UnderpostRun
5
5
  */
6
6
 
7
- import { daemonProcess, getTerminalPid, pbcopy, shellCd, shellExec } from '../server/process.js';
8
- import crypto from 'crypto';
7
+ import { daemonProcess, pbcopy, shellCd, shellExec } from '../server/process.js';
9
8
  import {
10
9
  awaitDeployMonitor,
11
10
  buildKindPorts,
12
- Config,
13
- cronDeployIdResolve,
11
+ clusterContextFactory,
12
+ clusterTypeFactory,
13
+ dispatchBuildInstanceEnv,
14
+ deployHostsFactory,
14
15
  etcHostFactory,
15
- getNpmRootPath,
16
+ exposePartialMatchesFactory,
17
+ exposePathPartsFactory,
18
+ exposePortListFactory,
19
+ exposePortPlanFactory,
20
+ exposeTcpPortsFactory,
21
+ gatewayApiEnabledFactory,
22
+ generateSecurePassword,
23
+ instanceHttpRouteRulesFactory,
24
+ instanceInterceptStatusesFactory,
25
+ instancePortFactory,
26
+ instanceProxyRoutesFactory,
27
+ instanceStatusPageEntriesFactory,
16
28
  isDeployRunnerContext,
17
29
  loadConfInstances,
30
+ loadProjectInstanceEnvBuilder,
18
31
  loadConfServerJson,
19
- selectConfInstances,
20
32
  loadReplicas,
21
- writeEnv,
33
+ resolveDeployList,
34
+ resolveEnvScoped,
35
+ selectConfInstances,
36
+ waitForPort,
37
+ clusterInstancesFactory,
38
+ deployTrafficEntriesFactory,
39
+ curlStatusChainFactory,
40
+ hostIngressFactsFactory,
41
+ hostRenderInstancesFactory,
42
+ instanceTrafficPlanFactory,
43
+ trafficTableRowsFactory,
44
+ isTrafficServingFactory,
45
+ nextTrafficFactory,
46
+ stopPlanFactory,
47
+ trafficFromRoutingInfoFactory,
22
48
  } from '../server/conf.js';
49
+ import { cronDeployIdResolve } from '../server/cron.js';
50
+ import { getNpmRootPath, writeEnv } from '../server/environment.js';
23
51
  import { actionInitLog, loggerFactory } from '../server/logger.js';
24
52
 
25
53
  import fs from 'fs-extra';
26
- import net from 'net';
27
54
  import { range, s4, setPad, timer } from '../client/components/core/CommonJs.js';
28
55
 
29
56
  import os from 'os';
30
57
  import Underpost from '../index.js';
31
58
  import dotenv from 'dotenv';
32
59
  import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
33
-
34
- const waitForPort = (port, host = '127.0.0.1', { maxAttempts = 30, interval = 2000 } = {}) =>
35
- new Promise((resolve, reject) => {
36
- let attempts = 0;
37
- const tryConnect = () => {
38
- attempts++;
39
- const socket = net.createConnection({ port, host }, () => {
40
- socket.destroy();
41
- resolve();
42
- });
43
- socket.on('error', () => {
44
- socket.destroy();
45
- if (attempts >= maxAttempts) return reject(new Error(`Port ${port} not ready after ${maxAttempts} attempts`));
46
- setTimeout(tryConnect, interval);
47
- });
48
- };
49
- tryConnect();
50
- });
51
-
60
+ import {
61
+ UNDERPOST_GATEWAY,
62
+ assertStaticAssets,
63
+ gatewayFallbackProbeRunner,
64
+ hostServerConfFactory,
65
+ installGatewayConf,
66
+ instanceFallbackChecksFactory,
67
+ placeInstanceStaticAssets,
68
+ pwaFallbackChecksFactory,
69
+ readHostInstanceRegistry,
70
+ writeHostInstanceRegistry,
71
+ writeHostServerConf,
72
+ } from '../server/underpost-gateway.js';
52
73
  const logger = loggerFactory(import.meta);
53
74
 
54
- /**
55
- * @method instanceProxyRoutesFactory
56
- * @description Renders the HTTPProxy route block for every instance sharing a host.
57
- * Routes are emitted longest-prefix first so a specific instance path (`/FOREST`)
58
- * is never shadowed by the default instance's catch-all (`/`).
59
- * @param {string} deployId - Parent deployment identifier.
60
- * @param {Array<object>} instances - Expanded instance entries bound to one host.
61
- * @param {string} env - `development` | `production`.
62
- * @param {Object<string,string>} trafficById - Instance id → traffic colour.
63
- * @returns {string} Concatenated route YAML.
64
- */
65
- const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =>
66
- [...instances]
67
- .sort((a, b) => (b.path || '/').length - (a.path || '/').length)
68
- .map((instance) =>
69
- Underpost.deploy.deploymentYamlServiceFactory({
70
- path: instance.path,
71
- port: env === 'development' && instance.fromDebugPort ? instance.fromDebugPort : instance.fromPort,
72
- deployId: `${deployId}-${instance.id}`,
73
- env,
74
- deploymentVersions: [trafficById[instance.id] || 'blue'],
75
- pathRewritePolicy: instance.pathRewritePolicy,
76
- }),
77
- )
78
- .join('');
79
-
80
75
  /**
81
76
  * @constant DEFAULT_OPTION
82
77
  * @description Default options for the UnderpostRun class.
@@ -85,8 +80,12 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
85
80
  * @property {boolean} dev - Whether to run in development mode.
86
81
  * @property {string} podName - The name of the pod to run.
87
82
  * @property {string} nodeName - The name of the node to run.
83
+ * @property {string} ingressNode - Dedicated node for the host-network public ingress; never inherited from nodeName.
88
84
  * @property {string} sshKeyPath - Private key path for node SSH operations, forwarded to volume shipping over SSH.
89
85
  * @property {number} port - Custom port to use.
86
+ * @property {string} exposeContainerPorts - Comma-separated Service/container destination ports.
87
+ * @property {string} exposeHostPorts - Comma-separated host listening ports.
88
+ * @property {boolean} localProxy - Start the development path proxy after exposing matched resources.
90
89
  * @property {string} volumeHostPath - The host path for the volume.
91
90
  * @property {string} volumeMountPath - The mount path for the volume.
92
91
  * @property {string} imageName - The name of the image to run.
@@ -101,6 +100,11 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
101
100
  * @property {boolean} force - Whether to force the operation.
102
101
  * @property {boolean} reset - Whether to reset the operation.
103
102
  * @property {boolean} tls - Whether to use TLS.
103
+ * @property {boolean} gatewayApi - Apply the Gateway API stack (Gateway + HTTPRoute) instead of the Contour HTTPProxy. Both manifest sets are always generated.
104
+ * @property {boolean} disableGatewayApi - Fall back to the Contour HTTPProxy stack in runners where the Gateway API is the default (`cluster`).
105
+ * @property {string} gatewayClass - GatewayClass name baked into generated Gateway manifests.
106
+ * @property {boolean} disableHttp3 - Omit QUIC/HTTP3 listener config and the Alt-Svc advertisement.
107
+ * @property {number} quicPort - UDP port advertised for QUIC/HTTP3.
104
108
  * @property {string} cmd - The command to run in the container.
105
109
  * @property {string} tty - The TTY option for the container.
106
110
  * @property {string} stdin - The stdin option for the container.
@@ -122,7 +126,7 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
122
126
  * @property {string} confServerPath - The configuration server path.
123
127
  * @property {string} underpostRoot - The root path of the Underpost installation.
124
128
  * @property {string} cmdCronJobs - Pre-script commands to run before cron job execution.
125
- * @property {string} deployIdCronJobs - The deployment ID for cron jobs.
129
+ * @property {string} deployIdCronJobs - Cron deploy-id passed to `cron --setup-start`; unset resolves dd.cron, `none` skips cron setup.
126
130
  * @property {string} timezone - The timezone to set.
127
131
  * @property {boolean} kubeadm - Whether to run in kubeadm mode.
128
132
  * @property {boolean} kind - Whether to run in kind mode.
@@ -151,6 +155,18 @@ const instanceProxyRoutesFactory = ({ deployId, instances, env, trafficById }) =
151
155
  * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
152
156
  * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
153
157
  * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
158
+ * @property {string} hostAliases - Pod `/etc/hosts` entries, as semicolon-separated `ip=host1,host2` groups.
159
+ * @property {string} args - Comma-separated arguments forwarded to the runner's own command.
160
+ * @property {boolean} cert - Issue cert-manager certificates; set implicitly by `tls` in the promote workflow.
161
+ * @property {boolean} instanceOnly - Act on the one instance id given, without expanding its variant family.
162
+ * @property {string} labels - Comma-separated `key=value` pairs applied to the created resources.
163
+ * @property {string} npmRoot - Resolved npm global root, cached on the options once looked up.
164
+ * @property {object} on - Lifecycle hooks (`{ init }`) a programmatic caller supplies; unused from the CLI.
165
+ * @property {string} traffic - Blue/green colour to bake into generated manifests (default: `blue`).
166
+ * @property {boolean} gatewayBootstrapComplete - Internal marker: a parent orchestration already proved the static gateway fallback.
167
+ * @property {boolean} noBackendCheckpoint - Internal marker: this promote is the deliberate no-backend fallback checkpoint, so it must not wait for the target colour's endpoints.
168
+ * @property {Object<string,string>} targetTrafficById - Internal instance id → explicitly pre-routed traffic colour.
169
+ * @property {string} volumeType - hostPath volume type (`DirectoryOrCreate`, `FileOrCreate`, or `dev` for the latter).
154
170
  * @property {string} branch - The Git branch to use for operations (e.g., for template-deploy, ssh-deploy).
155
171
  * @memberof UnderpostRun
156
172
  */
@@ -158,8 +174,12 @@ const DEFAULT_OPTION = {
158
174
  dev: false,
159
175
  podName: '',
160
176
  nodeName: '',
177
+ ingressNode: '',
161
178
  sshKeyPath: '',
162
179
  port: 0,
180
+ exposeContainerPorts: '',
181
+ exposeHostPorts: '',
182
+ localProxy: false,
163
183
  volumeHostPath: '',
164
184
  volumeMountPath: '',
165
185
  imageName: '',
@@ -174,6 +194,11 @@ const DEFAULT_OPTION = {
174
194
  force: false,
175
195
  reset: false,
176
196
  tls: false,
197
+ gatewayApi: false,
198
+ disableGatewayApi: false,
199
+ gatewayClass: '',
200
+ disableHttp3: false,
201
+ quicPort: 0,
177
202
  cmd: '',
178
203
  tty: '',
179
204
  stdin: '',
@@ -223,6 +248,17 @@ const DEFAULT_OPTION = {
223
248
  remove: false,
224
249
  test: false,
225
250
  branch: '',
251
+ args: '',
252
+ cert: false,
253
+ instanceOnly: false,
254
+ gatewayBootstrapComplete: false,
255
+ noBackendCheckpoint: false,
256
+ targetTrafficById: {},
257
+ labels: '',
258
+ npmRoot: '',
259
+ on: undefined,
260
+ traffic: '',
261
+ volumeType: '',
226
262
  };
227
263
 
228
264
  /**
@@ -234,6 +270,34 @@ const DEFAULT_OPTION = {
234
270
  * runners for executing specific commands.
235
271
  * @memberof UnderpostRun
236
272
  */
273
+
274
+ // Secrets `sops-setup` onboards when no explicit list is passed: the full self-hosted data tier.
275
+ // `mongodb-keyfile` is listed alongside `mongodb-secret` because the MongoDB StatefulSet mounts
276
+ // it as a volume for intra-replica-set auth and will not start without it, so onboarding the
277
+ // credentials alone would leave Mongo broken.
278
+ const SOPS_SETUP_DEFAULT_SECRETS = ['postgres-secret', 'mariadb-secret', 'mongodb-secret', 'mongodb-keyfile'];
279
+
280
+ /**
281
+ * Produces a value for a Secret data key that has no origin seed file and no `--args` override.
282
+ * Key-aware because the data tier does not want one shape of secret: a replica-set keyfile is a
283
+ * long base64 blob, a username is an identifier, and everything else is a password.
284
+ * @param {string} key - Secret data key (e.g. 'password', 'username', 'mongodb-keyfile').
285
+ * @returns {string} Generated value.
286
+ * @memberof UnderpostRun
287
+ */
288
+ const generateSeedValue = (key) => {
289
+ if (key === 'username') return 'admin';
290
+ // MongoDB keyfile: 6-1024 base64 characters shared by every replica-set member. Newlines are
291
+ // stripped so the value round-trips identically through YAML and through
292
+ // MongoBootstrap.readCredential, which strips them too.
293
+ if (key === 'mongodb-keyfile')
294
+ return shellExec(`openssl rand -base64 756`, { stdout: true, silent: true, disableLog: true }).replace(
295
+ /\r?\n/g,
296
+ '',
297
+ );
298
+ return generateSecurePassword(24);
299
+ };
300
+
237
301
  class UnderpostRun {
238
302
  /**
239
303
  * @static
@@ -242,6 +306,180 @@ class UnderpostRun {
242
306
  * @memberof UnderpostRun
243
307
  */
244
308
  static RUNNERS = {
309
+ /**
310
+ * @method status
311
+ * @description Reports deployment traffic, routing, Pods, expanded instances, and host capacity.
312
+ * @param {string} path - Deploy id, comma-separated ids, or `dd`; empty uses the router/configured projects.
313
+ * @param {UnderpostRunDefaultOptions} options - Namespace, environment (`--dev`), cluster, and node options.
314
+ * @returns {Promise<{deployments: object[], machine: object}>} Structured status report.
315
+ * @memberof UnderpostRun
316
+ */
317
+ status: async (path = '', options = DEFAULT_OPTION) => {
318
+ options = {
319
+ ...options,
320
+ gatewayApi: gatewayApiEnabledFactory(options),
321
+ namespace: options.namespace || 'default',
322
+ };
323
+ if (!/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(options.namespace))
324
+ throw new Error(`Invalid Kubernetes namespace: ${options.namespace}`);
325
+ if (options.nodeName && !/^[a-zA-Z0-9._-]+$/.test(options.nodeName))
326
+ throw new Error(`Invalid Kubernetes node name: ${options.nodeName}`);
327
+ const env = options.dev ? 'development' : 'production';
328
+ const requestedDeploys = `${path || options.deployId || ''}`.trim();
329
+ const routerPath = './engine-private/deploy/dd.router';
330
+ const confRoot = './engine-private/conf';
331
+ const deployIds = [
332
+ ...new Set(
333
+ requestedDeploys
334
+ ? resolveDeployList(requestedDeploys)
335
+ : fs.existsSync(routerPath)
336
+ ? resolveDeployList('dd')
337
+ : fs.existsSync(confRoot)
338
+ ? fs
339
+ .readdirSync(confRoot)
340
+ .filter(
341
+ (deployId) =>
342
+ fs.existsSync(`${confRoot}/${deployId}/conf.server.json`) ||
343
+ fs.existsSync(`${confRoot}/${deployId}/conf.instances.json`),
344
+ )
345
+ .sort()
346
+ : [],
347
+ ),
348
+ ];
349
+ if (deployIds.length === 0) throw new Error('No deployments found for status');
350
+ if (deployIds.some((deployId) => !/^[a-zA-Z0-9._-]+$/.test(deployId)))
351
+ throw new Error(`Invalid deployment status path: ${requestedDeploys}`);
352
+
353
+ const deployments = [];
354
+ for (const deployId of deployIds) {
355
+ const instances = [];
356
+ if (fs.existsSync(`${confRoot}/${deployId}/conf.instances.json`)) {
357
+ for (const instance of loadConfInstances(deployId)) {
358
+ const instanceDeployId = `${deployId}-${instance.id}`;
359
+ instances.push({
360
+ id: instance.id,
361
+ host: instance.host,
362
+ path: instance.path,
363
+ fromPort: instance.fromPort,
364
+ toPort: instance.toPort,
365
+ fromDebugPort: instance.fromDebugPort,
366
+ toDebugPort: instance.toDebugPort,
367
+ traffic: Underpost.deploy.getCurrentTraffic(instanceDeployId, {
368
+ namespace: options.namespace,
369
+ hostTest: instance.host,
370
+ env,
371
+ gatewayApi: options.gatewayApi,
372
+ }),
373
+ });
374
+ }
375
+ }
376
+ const deployment = {
377
+ deployId,
378
+ env,
379
+ traffic: Underpost.deploy.getCurrentTraffic(deployId, {
380
+ namespace: options.namespace,
381
+ env,
382
+ gatewayApi: options.gatewayApi,
383
+ }),
384
+ router: await Underpost.deploy.routerFactory(deployId, env),
385
+ pods: Underpost.kubectl.get(deployId, 'pods', options.namespace),
386
+ instances,
387
+ };
388
+ deployments.push(deployment);
389
+ logger.info('', deployment);
390
+ }
391
+
392
+ const interfaceName = Underpost.dns.getDefaultNetworkInterface();
393
+ const machine = {
394
+ hostname: os.hostname(),
395
+ arch: Underpost.baremetal.getHostArch(),
396
+ clusterType: clusterTypeFactory(options),
397
+ ipv4Public: await Underpost.dns.getPublicIp(),
398
+ ipv4Local: Underpost.dns.getLocalIPv4Address(),
399
+ resources: Underpost.cluster.getResourcesCapacity(options.nodeName),
400
+ defaultInterfaceName: interfaceName,
401
+ defaultInterfaceInfo: os.networkInterfaces()[interfaceName],
402
+ };
403
+ logger.info('Machine', machine);
404
+ return { deployments, machine };
405
+ },
406
+
407
+ /**
408
+ * @method expose
409
+ * @description Port-forwards every Service whose name partially matches path, falling back to matching Pods.
410
+ * Works through the active kubeconfig for Kind, k3s, and kubeadm clusters.
411
+ * Comma-separated path fragments determine resource index order; host and
412
+ * container port lists are paired against that same order.
413
+ * @param {string} path - One or more comma-separated literal Service/Pod name fragments.
414
+ * @param {UnderpostRunDefaultOptions} options - Namespace, cluster type, and optional port overrides.
415
+ * @returns {Array<{kindType: string, name: string, localPort: number, remotePort: number}>} Forward plan.
416
+ * @memberof UnderpostRun
417
+ */
418
+ expose: (path, options = DEFAULT_OPTION) => {
419
+ const namespace = options.namespace || 'default';
420
+ const clusterType = clusterTypeFactory(options);
421
+ const pathParts = exposePathPartsFactory(path || options.podName);
422
+ if (!/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(namespace))
423
+ throw new Error(`Invalid Kubernetes namespace: ${namespace}`);
424
+ let kindType = 'svc';
425
+ let resources = exposePartialMatchesFactory(Underpost.kubectl.get('', kindType, namespace), pathParts);
426
+
427
+ if (resources.length === 0) {
428
+ kindType = 'pod';
429
+ resources = exposePartialMatchesFactory(Underpost.kubectl.get('', 'pods', namespace), pathParts);
430
+ }
431
+ if (resources.length === 0)
432
+ throw new Error(`No Service or Pod partially matching '${pathParts.join(',')}' in namespace '${namespace}'`);
433
+
434
+ const containerPorts = exposePortListFactory(options.exposeContainerPorts, '--expose-container-ports');
435
+ const hostPorts = exposePortListFactory(options.exposeHostPorts, '--expose-host-ports');
436
+ const portsOf = (resource) => {
437
+ let declaredPorts = exposeTcpPortsFactory(resource);
438
+ if (kindType === 'pod' && declaredPorts.length === 0) {
439
+ const podJson = shellExec(`sudo kubectl get pod ${resource.NAME} -n ${namespace} -o json`, {
440
+ stdout: true,
441
+ silent: true,
442
+ });
443
+ const pod = JSON.parse(podJson);
444
+ declaredPorts = (pod.spec?.containers || [])
445
+ .flatMap((container) => container.ports || [])
446
+ .map(({ containerPort }) => parseInt(containerPort))
447
+ .filter((port) => Number.isInteger(port) && port > 0);
448
+ }
449
+ return declaredPorts;
450
+ };
451
+ const plan = exposePortPlanFactory({ resources, kindType, containerPorts, hostPorts, portsOf });
452
+
453
+ logger.info('[expose] Kubernetes port-forward plan', {
454
+ clusterType,
455
+ namespace,
456
+ matches: pathParts,
457
+ plan,
458
+ });
459
+ for (const { kindType, name, localPort, remotePort } of plan)
460
+ shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${name} ${localPort}:${remotePort}`, {
461
+ async: true,
462
+ });
463
+
464
+ if (options.localProxy) {
465
+ const deployId = options.deployId || pathParts[0];
466
+ const env = options.dev ? 'development' : 'production';
467
+ const envFile = `./engine-private/conf/${deployId}/.env.${env}`;
468
+ let basePort = plan[0].localPort - 1;
469
+ if (fs.existsSync(envFile)) {
470
+ const portMatch = fs.readFileSync(envFile, 'utf8').match(/^PORT=(\d+)/m);
471
+ if (portMatch) basePort = parseInt(portMatch[1]);
472
+ }
473
+ const tlsFlag = options.tls ? ' tls' : '';
474
+ shellExec(
475
+ `NODE_ENV=${env} PORT=${basePort} DEV_PROXY_PORT_OFFSET=0 node src/proxy proxy ${deployId} ${env}${tlsFlag}`,
476
+ { async: true },
477
+ );
478
+ }
479
+
480
+ return plan;
481
+ },
482
+
245
483
  /**
246
484
  * @method dev-cluster
247
485
  * @description Resets and deploys a full development cluster including MongoDB, Valkey, exposes services, and updates `/etc/hosts` for local access.
@@ -253,26 +491,21 @@ class UnderpostRun {
253
491
  const baseCommand = options.dev ? 'node bin' : 'underpost';
254
492
  const mongoHosts = ['mongodb-0.mongodb-service'];
255
493
  let primaryMongoHost = 'mongodb-0.mongodb-service';
256
- if (!options.expose) {
257
- shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --reset`);
258
- shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''}`);
494
+ const clusterType = clusterTypeFactory(options);
495
+ const clusterFlag = ` --${clusterType}`;
496
+ const clusterInitFlag = clusterType === 'kind' ? '' : clusterFlag;
497
+ const clusterOptions = `${options.dev ? ' --dev' : ''}${clusterInitFlag} --namespace ${options.namespace}`;
498
+ if (!options.expose && !options.remove) {
499
+ shellExec(`${baseCommand} cluster${clusterOptions} --reset`);
500
+ shellExec(`${baseCommand} cluster${clusterOptions}`);
259
501
 
260
502
  shellExec(
261
- `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --service-host ${mongoHosts.join(
262
- ',',
263
- )} --pull-image`,
503
+ `${baseCommand} cluster${clusterOptions} --mongodb --service-host ${mongoHosts.join(',')} --pull-image`,
264
504
  );
265
- shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --valkey --pull-image`);
505
+ shellExec(`${baseCommand} cluster${clusterOptions} --valkey --pull-image`);
266
506
  }
267
- if (options.k3s) {
268
- if (options.remove) {
269
- shellExec(`${baseCommand} lxd --delete-expose k3s-control:27017`);
270
- shellExec(`${baseCommand} lxd --delete-expose k3s-control:6379`);
271
- } else {
272
- shellExec(`${baseCommand} lxd --expose k3s-control:27017 --node-port 32017`);
273
- shellExec(`${baseCommand} lxd --expose k3s-control:6379 --node-port 32079`);
274
- }
275
- shellExec(`lxc config device show k3s-control`);
507
+ if (options.remove) {
508
+ shellExec(`${baseCommand} run kill '6379,27017'`);
276
509
  } else {
277
510
  try {
278
511
  const primaryPodName =
@@ -281,49 +514,26 @@ class UnderpostRun {
281
514
  podName: 'mongodb-0',
282
515
  disableAuth: options.dev,
283
516
  }) || 'mongodb-0';
284
- shellExec(
285
- `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config mongo`,
286
- { async: true },
287
- );
288
- shellExec(
289
- `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config valkey`,
290
- { async: true },
291
- );
517
+ primaryMongoHost = `${primaryPodName}.mongodb-service`;
292
518
  } catch (error) {
293
519
  logger.warn('Failed to detect MongoDB primary pod, using default', {
294
520
  error: error.message,
295
521
  default: primaryMongoHost,
296
522
  });
297
523
  }
524
+ shellExec(
525
+ `${baseCommand} run expose mongodb-service --namespace ${options.namespace}${clusterFlag} --expose-container-ports 27017 --expose-host-ports 27017`,
526
+ { async: true },
527
+ );
528
+ shellExec(
529
+ `${baseCommand} run expose valkey-service --namespace ${options.namespace}${clusterFlag} --expose-container-ports 6379 --expose-host-ports 6379`,
530
+ { async: true },
531
+ );
298
532
  }
299
533
  const hostListenResult = etcHostFactory([primaryMongoHost]);
300
534
  logger.info(hostListenResult.renderHosts);
301
535
  },
302
536
 
303
- /**
304
- * @method etc-hosts
305
- * @description Modifies the `/etc/hosts` file to add entries for local access to services,
306
- * based on the provided path input.
307
- * @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
308
- */
309
- 'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
310
- etcHostFactory(path.split(','));
311
- },
312
-
313
- /**
314
- * @method ipfs-expose
315
- * @description Exposes IPFS Cluster services on specified ports for local access.
316
- * @type {Function}
317
- * @memberof UnderpostRun
318
- */
319
- 'ipfs-expose': (path, options = DEFAULT_OPTION) => {
320
- const ports = [5001, 9094, 8080];
321
- for (const port of ports)
322
- shellExec(`node bin deploy --expose ipfs-cluster --expose-port ${port} --disable-update-underpost-config`, {
323
- async: true,
324
- });
325
- },
326
-
327
537
  /**
328
538
  * @method metadata
329
539
  * @description Generates metadata for the specified path after exposing the development cluster.
@@ -336,19 +546,30 @@ class UnderpostRun {
336
546
  shellExec(`node bin run kill '${ports}'`);
337
547
  shellExec(`node bin run dev-cluster --dev --expose --namespace ${options.namespace}`, { async: true });
338
548
  logger.info('Waiting for port-forward services to be ready...');
339
- try {
340
- await Promise.all([waitForPort(27017), waitForPort(6379)]);
341
- logger.info('Port-forward services are ready');
342
- } catch (err) {
343
- logger.error('Port-forward services failed to become ready', { error: err.message });
549
+ const ready = await Promise.all([27017, 6379].map((port) => waitForPort({ port })));
550
+ if (ready.some((reachable) => !reachable)) {
344
551
  shellExec(`node bin run kill '${ports}'`);
345
- throw err;
552
+ throw new Error('Port-forward services failed to become ready');
346
553
  }
554
+ logger.info('Port-forward services are ready');
347
555
  shellExec(`node bin metadata --generate ${path}`);
348
556
  shellExec(`node bin db --dev --clean-fs-collection dd`);
349
557
  shellExec(`node bin run kill '${ports}'`);
350
558
  },
351
559
 
560
+ /**
561
+ * @method ipfs-expose
562
+ * @description Exposes every declared TCP port on the matching IPFS Cluster Service.
563
+ * @type {Function}
564
+ * @memberof UnderpostRun
565
+ */
566
+ 'ipfs-expose': (path, options = DEFAULT_OPTION) => {
567
+ // 5001 Kubo RPC API / WebUI: Kubo/IPFS HTTP API + IPFS WebUI (e.g., http://localhost:5001/webui).
568
+ // 9094 IPFS Cluster HTTP API: IPFS Cluster REST API consumed by ipfs-cluster-ctl and WebUI clients (e.g., http://localhost:9094/).
569
+ // 8080 (or 8081) IPFS Gateway: Public HTTP Gateway for accessing pinned content by CID (e.g., http://localhost:8080/ipfs/Qm...).
570
+ shellExec(`node bin run expose ipfs --expose-host-ports 5001,9094,8080`);
571
+ },
572
+
352
573
  /**
353
574
  * @method svc-ls
354
575
  * @description Lists systemd services and installed packages, optionally filtering by the provided path.
@@ -384,28 +605,6 @@ class UnderpostRun {
384
605
  shellExec(`sudo rm -f /etc/yum.repos.d/${path}*.repo`);
385
606
  },
386
607
 
387
- /**
388
- * @method ssh-deploy-info
389
- * @description Retrieves deployment status and pod information from a remote server via SSH.
390
- * @param {string} path - The input value, identifier, or path for the operation.
391
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
392
- * @memberof UnderpostRun
393
- */
394
- 'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
395
- const env = options.dev ? 'development' : 'production';
396
- await Underpost.ssh.sshRemoteRunner(
397
- `node bin deploy ${path ? path : 'dd'} ${env} --status && kubectl get pods -A`,
398
- {
399
- deployId: options.deployId,
400
- user: options.user,
401
- dev: options.dev,
402
- remote: true,
403
- useSudo: true,
404
- cd: '/home/dd/engine',
405
- },
406
- );
407
- },
408
-
409
608
  /**
410
609
  * @method node-move
411
610
  * @description Abstract runner that relocates any schedulable Kubernetes workload
@@ -475,8 +674,8 @@ class UnderpostRun {
475
674
  services: 'service',
476
675
  })[k] || k;
477
676
 
478
- // Kinds that own a pod template we can patch; rolloutKinds additionally
479
- // support `kubectl rollout restart` to reschedule existing pods now.
677
+ // Kinds that own a pod template we can patch. Changing that template is
678
+ // itself the controller's rollout trigger.
480
679
  const templated = [
481
680
  'deployment',
482
681
  'statefulset',
@@ -486,7 +685,6 @@ class UnderpostRun {
486
685
  'cronjob',
487
686
  'replicationcontroller',
488
687
  ];
489
- const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
490
688
  const templateSelectorPath = (kind) =>
491
689
  kind === 'cronjob'
492
690
  ? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
@@ -575,10 +773,9 @@ class UnderpostRun {
575
773
  continue;
576
774
  }
577
775
 
578
- // Idempotency: skip the patch + rollout if the resource is already where
579
- // we want it. Compares the live pod-template nodeSelector against the
580
- // desired placement so a repeated run does not trigger an unnecessary
581
- // rollout restart.
776
+ // Idempotency: skip the patch if the resource is already where we want
777
+ // it. Compares the live pod-template nodeSelector against the desired
778
+ // placement so a repeated run does not trigger an unnecessary rollout.
582
779
  const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
583
780
  const jsonpath = (expr) =>
584
781
  shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
@@ -608,16 +805,17 @@ class UnderpostRun {
608
805
  }
609
806
 
610
807
  const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
611
- const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
612
808
  if (dryRun) {
613
809
  logger.info(`[dry-run] ${patchCmd}`);
614
- if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
615
810
  results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
616
811
  continue;
617
812
  }
618
813
 
619
814
  shellExec(patchCmd);
620
- if (rolloutKinds.includes(kind)) shellExec(restartCmd);
815
+ // nodeSelector is part of the pod template, so this patch already
816
+ // creates a new controller revision. A rollout restart here creates a
817
+ // second, immediately superseding revision and can strand the previous
818
+ // Ready replica in "pending termination" while the newest pod starts.
621
819
  logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
622
820
  namespace: ns,
623
821
  });
@@ -635,10 +833,12 @@ class UnderpostRun {
635
833
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
636
834
  * @memberof UnderpostRun
637
835
  */
638
- 'dev-hosts-expose': (path, options = DEFAULT_OPTION) => {
639
- shellExec(
640
- `node bin deploy ${path} development --disable-update-deployment --disable-update-proxy --kubeadm --etc-hosts`,
641
- );
836
+ 'dev-hosts-expose': async (path, options = DEFAULT_OPTION) => {
837
+ shellExec(`node bin deploy ${path} development --disable-update-deployment --disable-update-proxy --kubeadm`);
838
+ // /etc/hosts is written here, not by `deploy`: that command has no
839
+ // --etc-hosts option, and the `etc-hosts` runner already resolves a
840
+ // deploy's hosts from its conf.server.json.
841
+ await UnderpostRun.RUNNERS['etc-hosts']('', { ...options, deployId: path });
642
842
  },
643
843
 
644
844
  /**
@@ -649,7 +849,10 @@ class UnderpostRun {
649
849
  * @memberof UnderpostRun
650
850
  */
651
851
  'dev-hosts-restore': (path, options = DEFAULT_OPTION) => {
652
- shellExec(`node bin deploy --restore-hosts`);
852
+ // Rewrite /etc/hosts with the loopback block alone, dropping the deploy
853
+ // host entries `dev-hosts-expose` (and the `cluster` runner) added.
854
+ const hostListenResult = etcHostFactory([]);
855
+ logger.info(hostListenResult.renderHosts);
653
856
  },
654
857
 
655
858
  /**
@@ -935,6 +1138,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
935
1138
  sync: async (path, options = DEFAULT_OPTION) => {
936
1139
  // Dev usage: node bin run --dev --build sync dd-default
937
1140
  const env = options.dev ? 'development' : 'production';
1141
+ options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
938
1142
  const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
939
1143
  const baseClusterCommand = options.dev ? ' --dev' : '';
940
1144
  const clusterFlag = options.k3s ? ' --k3s' : options.kind ? ' --kind' : ' --kubeadm';
@@ -967,16 +1171,41 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
967
1171
  if (!validVersion) throw new Error('Version mismatch');
968
1172
  }
969
1173
  if (options.timezone !== 'none') shellExec(`${baseCommand} run${baseClusterCommand} tz`);
970
- if (options.deployIdCronJobs !== 'none')
971
- shellExec(`node bin cron${baseClusterCommand}${clusterFlag} --setup-start --git --apply`);
1174
+ if (options.deployIdCronJobs !== 'none') {
1175
+ // --deploy-id-cron-jobs overrides the cron deploy-id; unset falls back to
1176
+ // dd.cron, the same source `cron --setup-start` resolves against on its own.
1177
+ const cronDeployId = options.deployIdCronJobs || cronDeployIdResolve() || '';
1178
+ if (cronDeployId && !/^[a-zA-Z0-9._,-]+$/.test(cronDeployId))
1179
+ throw new Error(`Invalid cron deploy-id: ${cronDeployId}`);
1180
+ shellExec(
1181
+ `node bin cron${cronDeployId ? ` ${cronDeployId}` : ''}${baseClusterCommand}${clusterFlag} --setup-start --git --apply`,
1182
+ );
1183
+ }
972
1184
  }
973
1185
 
974
1186
  const currentTraffic = isDeployRunnerContext(path, options)
975
- ? Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace })
1187
+ ? Underpost.deploy.getCurrentTraffic(deployId, {
1188
+ namespace: options.namespace,
1189
+ env,
1190
+ gatewayApi: options.gatewayApi,
1191
+ })
976
1192
  : '';
977
1193
  let targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'green';
978
1194
  if (targetTraffic) versions = versions ? versions : targetTraffic;
979
1195
 
1196
+ // The routed colour is only live traffic while it still has a ready
1197
+ // endpoint. Everything downstream that would take the host offline to prove
1198
+ // the maintenance fallback is conditional on this being false: a first
1199
+ // bring-up has nothing to interrupt, a re-deploy does.
1200
+ const serving = isTrafficServingFactory({
1201
+ liveTraffic: currentTraffic,
1202
+ hasReadyEndpoints: (colour) =>
1203
+ Underpost.deploy.serviceHasReadyEndpoints({
1204
+ service: `${deployId}-${env}-${colour}-service`,
1205
+ namespace: options.namespace || 'default',
1206
+ }),
1207
+ });
1208
+
980
1209
  const ignorePods =
981
1210
  isDeployRunnerContext(path, options) && targetTraffic
982
1211
  ? Underpost.kubectl.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace).map((p) => p.NAME)
@@ -992,16 +1221,69 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
992
1221
  const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
993
1222
  const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
994
1223
  const sshKeyPathFlag = options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '';
1224
+ const gatewayApiFlags = Underpost.deploy.gatewayApiFlagsFactory(options);
1225
+
1226
+ // A direct sync owns the same gateway-first contract as the full cluster
1227
+ // runner. Build the host-side SSR documents before generating routes unless
1228
+ // the caller explicitly selected a pre-built bundle workflow.
1229
+ if (isDeployRunnerContext(path, options) && !options.skipFullBuild)
1230
+ shellExec(`${baseCommand} client ${deployId} ${env} --ssr`);
995
1231
 
996
1232
  shellExec(
997
1233
  `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
998
1234
  image ? ` --image ${image}` : ''
999
1235
  }${versions ? ` --versions ${versions}` : ''}${
1000
1236
  options.namespace ? ` --namespace ${options.namespace}` : ''
1001
- }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag} ${deployId} ${env}`,
1237
+ }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags} ${deployId} ${env}`,
1002
1238
  );
1003
1239
 
1004
1240
  if (isDeployRunnerContext(path, options)) {
1241
+ if (options.gatewayApi) {
1242
+ const namespace = options.namespace || 'default';
1243
+ const gatewayRoot = Underpost.deploy.underpostGatewayRootFactory(options);
1244
+ shellExec(`kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${namespace} --timeout=5m`);
1245
+ const staticAssets = Underpost.deploy.syncStaticAssets(deployId, env, {
1246
+ ...options,
1247
+ namespace,
1248
+ // Prefer the currently serving workload as the document source. On
1249
+ // an initial deploy there is none, so sync falls back to the checkout.
1250
+ versions: currentTraffic || versions || targetTraffic,
1251
+ });
1252
+ assertStaticAssets({ records: staticAssets, hostRoot: gatewayRoot, label: 'sync' });
1253
+
1254
+ if (serving)
1255
+ logger.info('[sync] Live colour serving; holding traffic until the target colour is Ready', {
1256
+ deployId,
1257
+ live: currentTraffic,
1258
+ target: targetTraffic,
1259
+ });
1260
+ else {
1261
+ // Remove any inactive-colour workload left by an earlier cycle. Without
1262
+ // this cleanup the probe could hit a stale Ready pod and never exercise
1263
+ // the configured unavailable-backend fallback.
1264
+ shellExec(
1265
+ `kubectl delete service ${deployId}-${env}-${targetTraffic}-service -n ${namespace} --ignore-not-found`,
1266
+ );
1267
+ shellExec(
1268
+ `kubectl delete deployment ${deployId}-${env}-${targetTraffic} -n ${namespace} --ignore-not-found`,
1269
+ );
1270
+ // Publish the target-colour route while its Service is deliberately
1271
+ // absent. Site paths reach underpost-gateway and must return the
1272
+ // configured maintenance body before deployment.yaml is submitted.
1273
+ shellExec(
1274
+ `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-deployment ${deployId} ${env} --versions ${versions}${
1275
+ options.namespace ? ` --namespace ${options.namespace}` : ''
1276
+ }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags}`,
1277
+ );
1278
+ await gatewayFallbackProbeRunner({
1279
+ gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
1280
+ checks: pwaFallbackChecksFactory(deployId),
1281
+ options,
1282
+ label: 'sync',
1283
+ });
1284
+ }
1285
+ }
1286
+
1005
1287
  // Backup app/services repositories with repo-backup configured
1006
1288
  shellExec(
1007
1289
  `${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
@@ -1009,163 +1291,119 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
1009
1291
  shellExec(
1010
1292
  `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
1011
1293
  options.namespace ? ` --namespace ${options.namespace}` : ''
1012
- }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}`,
1294
+ }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags}`,
1013
1295
  );
1014
1296
  if (!targetTraffic)
1015
- targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
1297
+ targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
1298
+ namespace: options.namespace,
1299
+ env,
1300
+ gatewayApi: options.gatewayApi,
1301
+ });
1016
1302
  await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
1017
1303
  Underpost.deploy.switchTraffic(deployId, env, targetTraffic, replicas, options.namespace, options);
1018
1304
  } else
1019
- logger.info('current traffic', Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }));
1305
+ logger.info(
1306
+ 'current traffic',
1307
+ Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }),
1308
+ );
1020
1309
  },
1021
1310
 
1022
1311
  /**
1023
1312
  * @method stop
1024
- * @description Stops a deployment by deleting the corresponding Kubernetes deployment and service resources.
1025
- * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
1313
+ * @description Deletes colour-suffixed Deployments and their Services, leaving routing untouched.
1314
+ *
1315
+ * Four ways to say what to stop, resolved by {@link ServerConfBuilder.stopPlanFactory}:
1316
+ *
1317
+ * ```bash
1318
+ * # Literal: exactly this Deployment, flags ignored
1319
+ * node bin run stop dd-cyberia-mmo-server-forest-development-blue
1320
+ *
1321
+ * # The deploy's PWA workload, inactive colour
1322
+ * node bin run stop --deploy-id dd-cyberia
1323
+ *
1324
+ * # ...plus every variant of each instance family, inactive colour
1325
+ * node bin run stop --deploy-id dd-cyberia --instance-id mmo-client,mmo-server
1326
+ *
1327
+ * # Explicit colours; both, where they exist
1328
+ * node bin run stop --deploy-id dd-cyberia --traffic blue,green
1329
+ * ```
1330
+ *
1331
+ * The default colour is the blue/green partner of whatever each target is
1332
+ * serving, so a stop is safe against a live host unless `--traffic` names the
1333
+ * serving colour outright — which is warned about, not refused.
1334
+ *
1335
+ * That colour is read through the same routing stack that published it: the
1336
+ * Gateway API HTTPRoute by default, the Contour HTTPProxy under
1337
+ * `--disable-gateway-api`. Reading the wrong kind finds no colour, degrades to
1338
+ * "blue", and stops whichever Deployment happens to carry that name. It is
1339
+ * resolved per target too — an instance's colour lives under its own
1340
+ * `<deployId>-<instanceId>` prefix, and on a shared host the parent's answer
1341
+ * belongs to a different variant.
1342
+ * @param {string} [path] - Literal comma-separated Deployment names; when set, every flag is ignored.
1026
1343
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1027
1344
  * @memberof UnderpostRun
1028
1345
  */
1029
1346
  stop: async (path = '', options = DEFAULT_OPTION) => {
1030
- let currentTraffic = Underpost.deploy.getCurrentTraffic(options.deployId, {
1031
- namespace: options.namespace,
1032
- hostTest: options.hosts,
1033
- });
1034
1347
  const env = options.dev ? 'development' : 'production';
1348
+ const namespace = options.namespace || 'default';
1349
+ const gatewayApi = gatewayApiEnabledFactory(options);
1350
+
1351
+ // Memoized because the plan and the serving check ask the same question,
1352
+ // and `--traffic` skips the plan's lookup entirely — which is exactly when
1353
+ // the check has to make its own.
1354
+ const liveTraffic = {};
1355
+ const liveTrafficOf = (target) => {
1356
+ if (liveTraffic[target.id] === undefined)
1357
+ liveTraffic[target.id] =
1358
+ Underpost.deploy.getCurrentTraffic(target.id, {
1359
+ hostTest: target.host || options.hosts,
1360
+ namespace,
1361
+ env,
1362
+ gatewayApi,
1363
+ }) || '';
1364
+ return liveTraffic[target.id];
1365
+ };
1035
1366
 
1036
- if (!path.match('current')) currentTraffic === 'blue' ? (currentTraffic = 'green') : (currentTraffic = 'blue');
1037
- const [_deployId] = path.split(',');
1038
- const deploymentId = `${_deployId ? _deployId : options.deployId}${
1039
- options.instanceId ? `-${options.instanceId}` : ''
1040
- }-${env}-${currentTraffic}`;
1041
-
1042
- shellExec(`kubectl delete deployment ${deploymentId} -n ${options.namespace}`);
1043
- shellExec(`kubectl delete svc ${deploymentId}-service -n ${options.namespace}`);
1044
- },
1045
-
1046
- /**
1047
- * @method ssh-deploy-stop
1048
- * @description Stops a remote deployment via SSH by executing the appropriate Underpost command on the remote server.
1049
- * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
1050
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1051
- * @memberof UnderpostRun
1052
- */
1053
- 'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
1054
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1055
- const baseClusterCommand = options.dev ? ' --dev' : '';
1056
-
1057
- const remoteCommand = [
1058
- `${baseCommand} run${baseClusterCommand} stop${path ? ` ${path}` : ''}`,
1059
- ` --deploy-id ${options.deployId}${options.instanceId ? ` --instance-id ${options.instanceId}` : ''}`,
1060
- ` --namespace ${options.namespace}${options.hosts ? ` --hosts ${options.hosts}` : ''}`,
1061
- ].join('');
1062
-
1063
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1064
- deployId: options.deployId,
1065
- user: options.user,
1066
- dev: options.dev,
1067
- remote: true,
1068
- useSudo: true,
1069
- cd: '/home/dd/engine',
1070
- });
1071
- },
1072
-
1073
- /**
1074
- * @method ssh-deploy-db-rollback
1075
- * @description Performs a database rollback on remote deployment via SSH.
1076
- * @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
1077
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1078
- * @param {string} options.deployId - The deployment identifier
1079
- * @param {string} options.user - The SSH user for credential lookup
1080
- * @param {boolean} options.dev - Development mode flag
1081
- * @memberof UnderpostRun
1082
- */
1083
- 'ssh-deploy-db-rollback': async (path = '', options = DEFAULT_OPTION) => {
1084
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1085
- let [deployId, nCommitsReset] = path.split(',');
1086
- if (!nCommitsReset) nCommitsReset = 1;
1087
-
1088
- const remoteCommand = `${baseCommand} db ${deployId} --git --kubeadm --primary-pod --force-clone --macro-rollback-export ${nCommitsReset}${options.namespace ? ` --ns ${options.namespace}` : ''}`;
1089
-
1090
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1091
- deployId: options.deployId,
1092
- user: options.user,
1093
- dev: options.dev,
1094
- remote: true,
1095
- useSudo: true,
1096
- cd: '/home/dd/engine',
1097
- });
1098
- },
1099
-
1100
- /**
1101
- * @method ssh-deploy-db
1102
- * @description Imports/restores a database on remote deployment via SSH.
1103
- * @param {string} path - The deployment ID for database import
1104
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1105
- * @param {string} options.deployId - The deployment identifier
1106
- * @param {string} options.user - The SSH user for credential lookup
1107
- * @param {boolean} options.dev - Development mode flag
1108
- * @memberof UnderpostRun
1109
- */
1110
- 'ssh-deploy-db': async (path, options = DEFAULT_OPTION) => {
1111
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1112
-
1113
- const remoteCommand = `${baseCommand} db ${path} --import --drop --preserveUUID --git --kubeadm --primary-pod --force-clone${options.namespace ? ` --ns ${options.namespace}` : ''}`;
1114
-
1115
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1367
+ const { deployments, error } = stopPlanFactory({
1368
+ path,
1116
1369
  deployId: options.deployId,
1117
- user: options.user,
1118
- dev: options.dev,
1119
- remote: true,
1120
- useSudo: true,
1121
- cd: '/home/dd/engine',
1370
+ instanceId: options.instanceId,
1371
+ traffic: options.traffic,
1372
+ env,
1373
+ instancesFor: (instanceId) => selectConfInstances(loadConfInstances(options.deployId), instanceId),
1374
+ liveTrafficOf,
1122
1375
  });
1123
- },
1124
-
1125
- /**
1126
- * @method ssh-deploy-db-status
1127
- * @description Retrieves database status/stats for a deployment (or all deployments from dd.router) via SSH.
1128
- * @param {string} path - Comma-separated deployId(s) or 'dd' to use the dd.router list.
1129
- * @param {UnderpostRunDefaultOptions} options - Runner options (uses options.deployId for SSH host lookup).
1130
- * @param {string} options.deployId - Deployment identifier used for SSH config lookup.
1131
- * @param {string} options.user - SSH user for credential lookup.
1132
- * @param {boolean} options.dev - Development mode flag.
1133
- * @param {string} [options.namespace] - Kubernetes namespace to pass to the db check.
1134
- * @memberof UnderpostRun
1135
- */
1136
- 'ssh-deploy-db-status': async (path = '', options = DEFAULT_OPTION) => {
1137
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1138
-
1139
- let deployList = [];
1140
- if (!path || path === 'dd') {
1141
- if (!fs.existsSync('./engine-private/deploy/dd.router')) {
1142
- logger.warn('dd.router not found; nothing to run');
1143
- return;
1144
- }
1145
- deployList = fs
1146
- .readFileSync('./engine-private/deploy/dd.router', 'utf8')
1147
- .split(',')
1148
- .map((d) => d.trim())
1149
- .filter(Boolean);
1150
- } else {
1151
- deployList = path
1152
- .split(',')
1153
- .map((d) => d.trim())
1154
- .filter(Boolean);
1376
+ if (error) {
1377
+ logger.error(error);
1378
+ return [];
1155
1379
  }
1156
1380
 
1157
- for (const deployId of deployList) {
1158
- const remoteCommand = `${baseCommand} db ${deployId} --stats --kubeadm --primary-pod${options.namespace ? ` --ns ${options.namespace}` : ''}`;
1381
+ // Read once: it decides what is reported as actually stopped, and a target
1382
+ // that is already gone is a no-op worth naming rather than a silent delete.
1383
+ const deployedNames = Underpost.kubectl.get('', 'deployment', namespace).map((entry) => entry.NAME);
1384
+ const stopped = [];
1385
+ for (const target of deployments) {
1386
+ const existed = deployedNames.includes(target.deployment);
1387
+ if (existed) stopped.push(target.deployment);
1388
+ // The Service is deleted either way: an orphan left by a half-finished
1389
+ // cycle outlives its Deployment and would keep resolving to no endpoints.
1390
+ shellExec(`kubectl delete deployment ${target.deployment} -n ${namespace} --ignore-not-found`);
1391
+ shellExec(`kubectl delete svc ${target.deployment}-service -n ${namespace} --ignore-not-found`);
1392
+ }
1159
1393
 
1160
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1161
- deployId: options.deployId,
1162
- user: options.user,
1163
- dev: options.dev,
1164
- remote: true,
1165
- useSudo: true,
1166
- cd: '/home/dd/engine',
1394
+ const serving = deployments.filter(
1395
+ (target) => target.kind !== 'literal' && target.colour === liveTrafficOf(target),
1396
+ );
1397
+ if (serving.length > 0)
1398
+ logger.warn('Stopped the serving colour; those routes stay published and now have no backend', {
1399
+ deployments: serving.map((target) => target.deployment),
1167
1400
  });
1168
- }
1401
+ logger.info('Stop complete', {
1402
+ namespace,
1403
+ stopped,
1404
+ absent: deployments.filter((target) => !stopped.includes(target.deployment)).map((t) => t.deployment),
1405
+ });
1406
+ return stopped;
1169
1407
  },
1170
1408
 
1171
1409
  /**
@@ -1190,31 +1428,326 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
1190
1428
  },
1191
1429
 
1192
1430
  /**
1193
- * @method get-proxy
1194
- * @description Retrieves and logs the HTTPProxy resources in the specified namespace using `kubectl get HTTPProxy`.
1195
- * @param {string} path - The input value, identifier, or path for the operation (used as an optional filter for the HTTPProxy resources).
1431
+ * @method get-traffic
1432
+ * @description Prints the live blue/green colour of every routable
1433
+ * deployment, of both kinds, as a table.
1434
+ * @param {string} [path] - Comma-separated hosts to report on; empty reports every host.
1196
1435
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1197
1436
  * @memberof UnderpostRun
1198
1437
  */
1199
- 'get-proxy': async (path = '', options = DEFAULT_OPTION) => {
1200
- console.log(
1201
- shellExec(`kubectl get HTTPProxy -n ${options.namespace} ${path} -o yaml`, {
1202
- silent: true,
1438
+ 'get-traffic': async (path = '', options = DEFAULT_OPTION) => {
1439
+ options = {
1440
+ ...options,
1441
+ gatewayApi: gatewayApiEnabledFactory(options),
1442
+ namespace: options.namespace || 'default',
1443
+ };
1444
+ // A report answers "what is live", so it cannot pick one environment from a
1445
+ // flag and call the other absent: a cluster running `development` would be
1446
+ // reported as entirely unrouted. Both are scanned and the environment is a
1447
+ // column; `--dev` narrows to development when only that is wanted.
1448
+ const envs = options.dev ? ['development'] : options.test ? ['development', 'production'] : ['production'];
1449
+ const hosts = `${path || ''}`
1450
+ .split(',')
1451
+ .map((host) => host.trim())
1452
+ .filter(Boolean);
1453
+
1454
+ const confRoot = './engine-private/conf';
1455
+ const deployList = options.deployId
1456
+ ? resolveDeployList(options.deployId)
1457
+ : fs.existsSync(confRoot)
1458
+ ? fs
1459
+ .readdirSync(confRoot)
1460
+ .filter((deployId) => fs.existsSync(`${confRoot}/${deployId}/conf.server.json`))
1461
+ .sort()
1462
+ : [];
1463
+
1464
+ const deployments = Underpost.kubectl.get('', 'deployment', options.namespace);
1465
+ const deployedNames = deployments.map((entry) => entry.NAME);
1466
+
1467
+ // Four cluster-wide reads, correlated once: which kind describes a host,
1468
+ // whether its listener terminates TLS, and whether QUIC is enabled on it.
1469
+ // A missing CRD is an empty list, so a single-stack cluster reads the same
1470
+ // way as one running both.
1471
+ const listResources = (kind) => {
1472
+ const raw = shellExec(`kubectl get ${kind} -A -o json`, {
1203
1473
  stdout: true,
1204
- })
1205
- .replaceAll(`blue`, `blue`.bgBlue.bold.black)
1206
- .replaceAll('green', 'green'.bgGreen.bold.black)
1207
- .replaceAll('Error', 'Error'.bold.red)
1208
- .replaceAll('error', 'error'.bold.red)
1209
- .replaceAll('ERROR', 'ERROR'.bold.red)
1210
- .replaceAll('Invalid', 'Invalid'.bold.red)
1211
- .replaceAll('invalid', 'invalid'.bold.red)
1212
- .replaceAll('INVALID', 'INVALID'.bold.red),
1474
+ silent: true,
1475
+ silentOnError: true,
1476
+ });
1477
+ try {
1478
+ const parsed = JSON.parse(`${raw || ''}`);
1479
+ return Array.isArray(parsed?.items) ? parsed.items : [];
1480
+ } catch {
1481
+ return [];
1482
+ }
1483
+ };
1484
+ const ingressFacts = hostIngressFactsFactory({
1485
+ httpRoutes: listResources('httproute'),
1486
+ httpProxies: listResources('httpproxy'),
1487
+ gateways: listResources('gateway'),
1488
+ clientTrafficPolicies: listResources('clienttrafficpolicy'),
1489
+ });
1490
+ const trafficServiceSelectors = Object.fromEntries(
1491
+ listResources('service')
1492
+ .filter((service) => service?.metadata?.labels?.['underpost.net/traffic-service'] === 'true')
1493
+ .map((service) => [
1494
+ `${service?.metadata?.namespace || 'default'}/${service?.metadata?.name}`,
1495
+ service?.spec?.selector?.app || '',
1496
+ ]),
1497
+ );
1498
+
1499
+ // One read per host, reused across every deployment and environment that
1500
+ // shares it — the colour match is pure, only the fetch is expensive.
1501
+ const routingInfo = {};
1502
+ const hostRoutingInfo = (host) => {
1503
+ if (routingInfo[host] === undefined)
1504
+ routingInfo[host] = Underpost.deploy.readHostRoutingInfo({ host, options });
1505
+ return routingInfo[host];
1506
+ };
1507
+ const trafficState = {};
1508
+ const liveTrafficStateOf = (entry, env) => {
1509
+ const key = `${entry.id}/${env}`;
1510
+ if (trafficState[key]) return trafficState[key];
1511
+ const stableService = Underpost.deploy.trafficServiceNameFactory({ deployId: entry.id, env });
1512
+ const selector = trafficServiceSelectors[`${options.namespace}/${stableService}`] || '';
1513
+ const stableTraffic = trafficFromRoutingInfoFactory({ info: selector, deployId: entry.id, env });
1514
+ if (stableTraffic)
1515
+ return (trafficState[key] = {
1516
+ colour: stableTraffic,
1517
+ service: stableService,
1518
+ });
1519
+ const legacyTraffic = trafficFromRoutingInfoFactory({
1520
+ info: hostRoutingInfo(entry.host),
1521
+ deployId: entry.id,
1522
+ env,
1523
+ });
1524
+ return (trafficState[key] = {
1525
+ colour: legacyTraffic,
1526
+ service: legacyTraffic ? `${entry.deployment}-${legacyTraffic}-service` : '',
1527
+ });
1528
+ };
1529
+ const servingState = {};
1530
+ const servesTraffic = (entry, env) => {
1531
+ const service = liveTrafficStateOf(entry, env).service;
1532
+ if (!service) return false;
1533
+ const key = `${options.namespace}/${service}`;
1534
+ if (servingState[key] === undefined)
1535
+ servingState[key] = Underpost.deploy.serviceHasReadyEndpoints({
1536
+ service,
1537
+ namespace: options.namespace,
1538
+ });
1539
+ return servingState[key];
1540
+ };
1541
+
1542
+ const rows = envs.flatMap((env) =>
1543
+ trafficTableRowsFactory({
1544
+ entries: deployList
1545
+ .flatMap((deployId) => deployTrafficEntriesFactory({ deployId, env }))
1546
+ .map((entry) => ({ ...entry, env })),
1547
+ hosts,
1548
+ liveTrafficOf: (entry) => liveTrafficStateOf(entry, env).colour,
1549
+ servesTraffic: (entry) => servesTraffic(entry, env),
1550
+ }),
1213
1551
  );
1552
+
1553
+ if (rows.length === 0) {
1554
+ logger.warn('No configured hosts matched the requested traffic report', {
1555
+ hosts,
1556
+ envs,
1557
+ deployList,
1558
+ namespace: options.namespace,
1559
+ });
1560
+ return rows;
1561
+ }
1562
+
1563
+ // Probe the exact public URL represented by each host/path row. PWA rows
1564
+ // can carry several configured paths in one cell, while instance rows
1565
+ // carry one; cache by URL so shared rows never repeat network work.
1566
+ const shellArg = (value) => `'${`${value}`.replaceAll("'", "'\\''")}'`;
1567
+ const probeCache = new Map();
1568
+ const probePath = (host, routePath, tls) => {
1569
+ const normalizedPath = `${routePath || '/'}`.startsWith('/') ? `${routePath || '/'}` : `/${routePath}`;
1570
+ let url;
1571
+ try {
1572
+ url = new URL(normalizedPath, `${tls ? 'https' : 'http'}://${host}`).href;
1573
+ } catch {
1574
+ return { path: normalizedPath, url: '', statuses: ['000'] };
1575
+ }
1576
+ if (!probeCache.has(url)) {
1577
+ // -L follows the real redirect chain, -v supplies one response line
1578
+ // per hop, -i keeps full response headers available, and -s removes
1579
+ // only the progress meter. The body is discarded to keep a status
1580
+ // report bounded even when a host returns a large application page.
1581
+ const raw = shellExec(
1582
+ `curl -L -v -i -s --connect-timeout 3 --max-time 12 --max-redirs 10 ` +
1583
+ `-o /dev/null -w '\nUNDERPOST_CURL_FINAL=%{http_code}\n' ${shellArg(url)} 2>&1 || true`,
1584
+ { stdout: true, silent: true, silentOnError: true, disableLog: true },
1585
+ );
1586
+ probeCache.set(url, curlStatusChainFactory(raw));
1587
+ }
1588
+ return { path: normalizedPath, url, statuses: probeCache.get(url) };
1589
+ };
1590
+
1591
+ const deploymentByName = new Map(deployments.map((deployment) => [deployment.NAME, deployment]));
1592
+ const readinessOf = (deployment) => {
1593
+ const replicas = deployment?.READY || '-';
1594
+ const [ready, desired] = `${replicas}`.split('/').map(Number);
1595
+ return {
1596
+ replicas,
1597
+ exists: Boolean(deployment),
1598
+ ready: Number.isFinite(ready) && Number.isFinite(desired) && desired > 0 && ready === desired,
1599
+ };
1600
+ };
1601
+ const deploymentStatusOf = (row, traffic) => {
1602
+ if (!traffic) return null;
1603
+ const deployment = `${row.deployment}-${traffic}`;
1604
+ return {
1605
+ deployment,
1606
+ traffic,
1607
+ ...readinessOf(deploymentByName.get(deployment)),
1608
+ };
1609
+ };
1610
+ const reportRows = rows.map((row) => {
1611
+ const facts = ingressFacts[row.host] || {};
1612
+ const probes = [...new Set(`${row.path || '/'}`.split(/\s+/).filter(Boolean))].map((routePath) =>
1613
+ probePath(row.host, routePath, facts.tls),
1614
+ );
1615
+ const oppositeTraffic = row.traffic ? nextTrafficFactory(row.traffic) : '';
1616
+ const current = deploymentStatusOf(row, row.traffic);
1617
+ const opposite = deploymentStatusOf(row, oppositeTraffic);
1618
+ return {
1619
+ ...row,
1620
+ probes,
1621
+ current: current ? { ...current, serving: row.serving } : null,
1622
+ opposite,
1623
+ };
1624
+ });
1625
+
1626
+ // Padded on the raw values, coloured afterwards: an ANSI escape counts
1627
+ // toward String.length and would skew every column right of it.
1628
+ const columns = ['HOST', 'PATH', 'KIND', 'ROUTE', 'TLS', 'HTTP3', 'CURRENT', 'OPPOSITE'];
1629
+ const deploymentStatus = (status, includeServing = false) => {
1630
+ if (!status) return `unrouted - missing${includeServing ? ' not-serving' : ''}`;
1631
+ return [
1632
+ status.deployment,
1633
+ status.replicas,
1634
+ status.exists ? (status.ready ? 'ready' : 'not-ready') : 'missing',
1635
+ ...(includeServing ? [status.serving ? 'serving' : 'not-serving'] : []),
1636
+ ].join(' ');
1637
+ };
1638
+ const cellOf = (row) => {
1639
+ const facts = ingressFacts[row.host] || {};
1640
+ const pathStatus = row.probes.map((probe) => `${probe.path} [${probe.statuses.join('→')}]`).join(' ');
1641
+ return [
1642
+ row.host,
1643
+ pathStatus,
1644
+ row.kind,
1645
+ facts.route || 'none',
1646
+ facts.tls ? 'yes' : 'no',
1647
+ facts.http3 ? 'yes' : 'no',
1648
+ deploymentStatus(row.current, true),
1649
+ deploymentStatus(row.opposite),
1650
+ ];
1651
+ };
1652
+ const paint = (value, i) => {
1653
+ if (columns[i] === 'PATH')
1654
+ return value.replace(/\b(?:000|[1-5][0-9]{2})\b/g, (status) => {
1655
+ if (/^1/.test(status)) return status.cyan;
1656
+ if (/^2/.test(status)) return status.green;
1657
+ if (/^3/.test(status)) return status.cyan;
1658
+ if (/^4/.test(status)) return status.yellow;
1659
+ return status.red;
1660
+ });
1661
+ if (columns[i] === 'CURRENT' || columns[i] === 'OPPOSITE') {
1662
+ if (value === '-') return value;
1663
+ const [deployment, replicas, status, serving] = value.split(' ');
1664
+ const deploymentDisplay = deployment.replace(
1665
+ /-(blue|green)$/,
1666
+ (_, traffic) => `-${traffic === 'blue' ? traffic.bgBlue.bold.black : traffic.bgGreen.bold.black}`,
1667
+ );
1668
+ const replicasDisplay = readinessOf({ READY: replicas }).ready ? replicas.green : replicas.red;
1669
+ const statusDisplay = status === 'ready' ? status.green : status.red;
1670
+ const servingDisplay = serving ? (serving === 'serving' ? serving.green : serving.red) : '';
1671
+ return [deploymentDisplay, replicasDisplay, statusDisplay, servingDisplay].filter(Boolean).join(' ');
1672
+ }
1673
+ // TLS and HTTP/3 being off is a normal development state, not a fault, so
1674
+ // only the affirmative is highlighted.
1675
+ if (columns[i] === 'TLS' || columns[i] === 'HTTP3') return value === 'yes' ? value.green : value;
1676
+ if (columns[i] === 'ROUTE') return value === 'none' ? value.red : value;
1677
+ return value;
1678
+ };
1679
+ // A shared table renderer, called once per environment when both are
1680
+ // scanned: mixing development and production rows into one table is what
1681
+ // made an unrouted production host look like a duplicate of the same,
1682
+ // live development host.
1683
+ const printTable = (cells, heading) => {
1684
+ const widths = columns.map((column, i) => Math.max(column.length, ...cells.map((cell) => `${cell[i]}`.length)));
1685
+ const line = (values, painted) =>
1686
+ values
1687
+ .map(
1688
+ (value, i) => (painted ? paint(`${value}`, i) : `${value}`) + ' '.repeat(widths[i] - `${value}`.length),
1689
+ )
1690
+ .join(' ');
1691
+ console.log(heading ? `\n${heading.bold}\n${line(columns, false).bold}` : `\n${line(columns, false).bold}`);
1692
+ console.log(widths.map((width) => '-'.repeat(width)).join(' '));
1693
+ for (const cell of cells) console.log(line(cell, true));
1694
+ };
1695
+ for (const env of envs) {
1696
+ const envCells = reportRows.filter((row) => row.env === env).map(cellOf);
1697
+ if (envCells.length > 0) printTable(envCells, `[${env.toUpperCase()}]`);
1698
+ }
1699
+ console.log('');
1700
+ return reportRows;
1701
+ },
1702
+
1703
+ /**
1704
+ * @method restore-mongo
1705
+ * @description Initializes a MongoDB replica set in the cluster without resetting existing data.
1706
+ * @param {string} path - The input value, identifier, or path for the operation.
1707
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1708
+ * @memberof UnderpostRun
1709
+ */
1710
+ 'restore-mongo': async (path, options = DEFAULT_OPTION) => {
1711
+ await MongoBootstrap.initReplicaSet({
1712
+ namespace: options.namespace || 'default',
1713
+ reset: options.reset || false,
1714
+ clusterType: options.kubeadm ? 'kubeadm' : options.k3s ? 'k3s' : 'kind', // o 'k3s' / 'kubeadm' según corresponda
1715
+ underpostRoot: '.',
1716
+ });
1717
+ },
1718
+
1719
+ /**
1720
+ * @method ingress-refresh
1721
+ * @description Rebuilds the shared HTTPProxy/HTTPRoute host map without
1722
+ * inheriting application workload placement. Supplying a path or
1723
+ * `--ingress-node` is the explicit recovery mechanism for relocating the
1724
+ * public 80/443 listener.
1725
+ * @param {string} [path] - Optional ingress node name.
1726
+ * @param {UnderpostRunDefaultOptions} options - Runner options.
1727
+ * @returns {boolean} True after the ingress is Ready with the refreshed map.
1728
+ * @memberof UnderpostRun
1729
+ */
1730
+ 'ingress-refresh': (path = '', options = DEFAULT_OPTION) => {
1731
+ const namespace = options.namespace || 'default';
1732
+ const ingressNode = options.ingressNode || `${path || ''}`.trim();
1733
+ const updated = Underpost.cluster.refreshUnderpostIngress({
1734
+ namespace,
1735
+ options: { ...options, ingressNode },
1736
+ });
1737
+ if (!updated)
1738
+ throw new Error(
1739
+ `[ingress-refresh] ${UNDERPOST_INGRESS.name} is not installed; install both ingress stacks first`,
1740
+ );
1741
+ logger.info('[ingress-refresh] Shared ingress is operational', {
1742
+ namespace,
1743
+ node: ingressNode || '(preserved)',
1744
+ });
1745
+ return true;
1214
1746
  },
1215
1747
 
1216
1748
  'instance-promote': async (path, options = DEFAULT_OPTION) => {
1217
1749
  const env = options.dev ? 'development' : 'production';
1750
+ options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
1218
1751
  let [deployId, id] = path.split(',');
1219
1752
  const confInstances = loadConfInstances(deployId);
1220
1753
  const promoted = selectConfInstances(confInstances, id);
@@ -1224,56 +1757,214 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
1224
1757
  // host shares one object. Rebuilding it from the promoted instance alone
1225
1758
  // would drop its siblings' routes, so each host is rendered from the full
1226
1759
  // set: promoted instances flip colour, the rest keep their live colour.
1760
+ //
1761
+ // "Full set" is not what the conf currently declares. Each variant sub-path
1762
+ // is its own deployment, and one that was dropped from the conf while its
1763
+ // workload is still up must keep its route — otherwise editing the variant
1764
+ // list to scope a deploy takes the untouched variants offline. The
1765
+ // descriptors last published for the host cover exactly that gap, and a
1766
+ // variant leaves the render only once its Deployment is gone.
1227
1767
  const promotedIds = new Set(promoted.map((instance) => instance.id));
1228
1768
  const hosts = [...new Set(promoted.map((instance) => instance.host))];
1229
- const affected = confInstances.filter((instance) => hosts.includes(instance.host));
1769
+ const namespace = options.namespace || 'default';
1770
+ const gatewayConfDir = Underpost.deploy.gatewayConfDirFactory({ deployId, env });
1771
+ const deployedNames = Underpost.kubectl.get('', 'deployment', namespace).map((entry) => entry.NAME);
1772
+ const instancesByHost = Object.fromEntries(
1773
+ hosts.map((host) => [
1774
+ host,
1775
+ hostRenderInstancesFactory({
1776
+ declared: confInstances.filter((instance) => instance.host === host),
1777
+ preserved: readHostInstanceRegistry({ confDir: gatewayConfDir, host }),
1778
+ isDeployed: (instance) =>
1779
+ deployedNames.some((name) => name.startsWith(`${deployId}-${instance.id}-${env}-`)),
1780
+ }),
1781
+ ]),
1782
+ );
1783
+ const affected = hosts.flatMap((host) => instancesByHost[host]);
1230
1784
  const trafficById = {};
1785
+ const currentTrafficById = {};
1786
+ const bootstrapTrafficById = {};
1231
1787
  for (const instance of affected) {
1232
1788
  const currentTraffic = Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
1233
1789
  hostTest: instance.host,
1234
1790
  namespace: options.namespace,
1235
1791
  env,
1792
+ gatewayApi: options.gatewayApi,
1236
1793
  });
1794
+ currentTrafficById[instance.id] = currentTraffic;
1237
1795
  if (!promotedIds.has(instance.id)) {
1238
1796
  trafficById[instance.id] = currentTraffic || 'blue';
1239
1797
  continue;
1240
1798
  }
1241
- trafficById[instance.id] = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
1799
+ trafficById[instance.id] = nextTrafficFactory(currentTraffic, options.targetTrafficById?.[instance.id]);
1242
1800
  promotedTraffic = trafficById[instance.id];
1243
1801
  }
1244
1802
 
1803
+ // Readiness is mandatory for an actual promotion. The only exception is
1804
+ // the explicit no-backend checkpoint, whose purpose is to prove that an
1805
+ // endpointless selector returns the configured fallback before a workload
1806
+ // exists.
1807
+ if (!options.noBackendCheckpoint)
1808
+ for (const instance of affected.filter((entry) => promotedIds.has(entry.id))) {
1809
+ const podId = `${deployId}-${instance.id}-${env}-${trafficById[instance.id]}`;
1810
+ if (
1811
+ !Underpost.deploy.awaitDeploymentReady({ deployment: podId, namespace }) ||
1812
+ !Underpost.deploy.awaitServiceEndpoints({ service: `${podId}-service`, namespace })
1813
+ )
1814
+ throw new Error(`Refusing to promote ${instance.id} to unready colour ${trafficById[instance.id]}`);
1815
+ }
1816
+
1817
+ // Bootstrap one stable Service per instance on its current ready colour.
1818
+ // Routes and fallback blocks can now migrate without changing traffic;
1819
+ // promoted selectors move to their targets only after the whole host has
1820
+ // converged below.
1821
+ for (const instance of affected) {
1822
+ const instanceDeployId = `${deployId}-${instance.id}`;
1823
+ const currentTraffic = currentTrafficById[instance.id];
1824
+ const currentReady =
1825
+ !!currentTraffic &&
1826
+ Underpost.deploy.serviceHasReadyEndpoints({
1827
+ service: `${instanceDeployId}-${env}-${currentTraffic}-service`,
1828
+ namespace,
1829
+ });
1830
+ const bootstrapTraffic = currentReady ? currentTraffic : trafficById[instance.id];
1831
+ bootstrapTrafficById[instance.id] = bootstrapTraffic;
1832
+ Underpost.deploy.applyTrafficService({
1833
+ deployId: instanceDeployId,
1834
+ env,
1835
+ traffic: bootstrapTraffic,
1836
+ namespace,
1837
+ fromPort: instancePortFactory({ instance, env }),
1838
+ toPort: instancePortFactory({ instance, env, container: true }),
1839
+ });
1840
+ }
1841
+
1842
+ // Instance routes attach to the Gateway the parent deploy owns. A Gateway
1843
+ // per instance host cannot work beside it: `mergeGateways` collapses every
1844
+ // Gateway of the class onto one listener per (port, protocol), so a
1845
+ // hostname-scoped listener and the deploy's hostname-less one contend for
1846
+ // the same port — on 80 the hostname-scoped one is dropped outright, and
1847
+ // on 443 it keeps an SNI filter chain whose route table is left empty.
1848
+ // Either way every path on the instance host answers 404 while the
1849
+ // Gateway reports Programmed and the route reports Accepted.
1850
+ const gatewayName = Underpost.deploy.gatewayNameFactory({ deployId, env });
1245
1851
  for (const host of hosts) {
1246
- const hostInstances = affected.filter((instance) => instance.host === host);
1247
- let proxyYaml =
1248
- Underpost.deploy.baseProxyYamlFactory({ host, env: options.tls ? 'production' : env, options }) +
1249
- instanceProxyRoutesFactory({ deployId, instances: hostInstances, env, trafficById });
1852
+ const hostInstances = instancesByHost[host];
1853
+ const routingEnv = options.tls ? 'production' : env;
1854
+ // Every variant of this host contributes one proxied sub-path to the
1855
+ // shared gateway, whose interception turns each workload's own error
1856
+ // into that variant's declared document.
1857
+ if (options.gatewayApi)
1858
+ writeHostServerConf({
1859
+ confDir: gatewayConfDir,
1860
+ host,
1861
+ conf: hostServerConfFactory({
1862
+ host,
1863
+ namespace: options.namespace || 'default',
1864
+ routes: hostInstances.map((instance) => ({
1865
+ path: instance.path,
1866
+ upstream: `${Underpost.deploy.trafficServiceNameFactory({
1867
+ deployId: `${deployId}-${instance.id}`,
1868
+ env,
1869
+ })}:${instancePortFactory({ instance, env })}`,
1870
+ statuses: instanceInterceptStatusesFactory(instance),
1871
+ stripPrefix: Array.isArray(instance.pathRewritePolicy) && instance.pathRewritePolicy.length > 0,
1872
+ })),
1873
+ }),
1874
+ });
1875
+ // Recorded before the route is published, so a variant dropped from the
1876
+ // conf keeps its descriptor from the render that still carried it.
1877
+ writeHostInstanceRegistry({ confDir: gatewayConfDir, host, instances: hostInstances });
1878
+ // The route below sends every intercepted path to this shared Nginx
1879
+ // service. Load the host block first so an Accepted route can never race
1880
+ // a gateway that still has only its default server configuration.
1881
+ if (options.gatewayApi)
1882
+ installGatewayConf({
1883
+ hostRoot: Underpost.deploy.underpostGatewayRootFactory(options),
1884
+ confSourceDir: gatewayConfDir,
1885
+ namespace: options.namespace,
1886
+ });
1887
+ let proxyYaml = options.gatewayApi
1888
+ ? Underpost.deploy.httpRouteYamlFactory({
1889
+ host,
1890
+ options,
1891
+ parentName: gatewayName,
1892
+ rules: instanceHttpRouteRulesFactory({ deployId, instances: hostInstances, env, trafficById, options }),
1893
+ })
1894
+ : Underpost.deploy.baseProxyYamlFactory({ host, env: routingEnv, options }) +
1895
+ instanceProxyRoutesFactory({ deployId, instances: hostInstances, env, trafficById });
1250
1896
  if (options.tls) {
1251
1897
  if (options.test) {
1252
- const sslDir = `./engine-private/ssl/${host}`;
1253
- const nameSafe = host.replace(/[^a-zA-Z0-9_.-]/g, '_');
1254
- fs.mkdirpSync(sslDir);
1255
- shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${host}"`);
1256
- shellExec(`kubectl delete secret ${host} -n ${options.namespace} --ignore-not-found`);
1257
- shellExec(
1258
- `kubectl create secret tls ${host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
1259
- );
1898
+ Underpost.deploy.selfSignedTlsSecretFactory({
1899
+ host,
1900
+ namespace: options.namespace,
1901
+ underpostRoot: options.underpostRoot || '.',
1902
+ });
1260
1903
  } else {
1261
1904
  shellExec(`sudo kubectl delete Certificate ${host} -n ${options.namespace} --ignore-not-found`);
1262
1905
  proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host });
1263
1906
  }
1264
1907
  }
1265
- shellExec(`kubectl delete HTTPProxy ${host} --namespace ${options.namespace} --ignore-not-found`);
1908
+ if (options.gatewayApi) {
1909
+ // Left by the per-host model this replaces. Both outlive the route
1910
+ // that referenced them and keep contending for the merged listener.
1911
+ for (const name of [`Gateway ${host}`, `ClientTrafficPolicy ${host}-http3`])
1912
+ shellExec(`kubectl delete ${name} --namespace ${options.namespace} --ignore-not-found`, { silent: true });
1913
+ }
1914
+ // The host's route object is replaced in place, never deleted first:
1915
+ // `apply` moves the whole spec in one transition, so the hostname always
1916
+ // has a route. Delete-then-apply left a window with none, and Envoy
1917
+ // answered every request that landed in it 404 — an outage on each
1918
+ // promote, independent of which colour was healthy.
1266
1919
  shellExec(
1267
- `kubectl apply -f - -n ${options.namespace} <<EOF
1920
+ `kubectl apply -f - -n ${options.namespace} <<'EOF'
1268
1921
  ${proxyYaml}
1269
1922
  EOF
1270
1923
  `,
1271
1924
  { disableLog: true },
1272
1925
  );
1273
1926
  }
1927
+ // These hostnames may have just moved between stacks, or appeared for the
1928
+ // first time. A shared edge that has not been told answers them from the
1929
+ // other data plane, which has no route for them — a 404 in front of a
1930
+ // healthy workload. No-op when no shared edge is installed.
1931
+ const sharedIngressUpdated = Underpost.cluster.refreshUnderpostIngress({ namespace, options });
1932
+ if (sharedIngressUpdated) {
1933
+ Underpost.deploy.removeInactiveHostRoutes({ hosts, gatewayApi: options.gatewayApi, namespace });
1934
+ Underpost.cluster.refreshUnderpostIngress({ namespace, options });
1935
+ }
1936
+ if (!options.noBackendCheckpoint)
1937
+ for (const instance of affected.filter((entry) => promotedIds.has(entry.id))) {
1938
+ const instanceDeployId = `${deployId}-${instance.id}`;
1939
+ const targetTraffic = trafficById[instance.id];
1940
+ const bootstrapTraffic = bootstrapTrafficById[instance.id];
1941
+ if (targetTraffic !== bootstrapTraffic)
1942
+ Underpost.deploy.applyTrafficService({
1943
+ deployId: instanceDeployId,
1944
+ env,
1945
+ traffic: targetTraffic,
1946
+ namespace,
1947
+ fromPort: instancePortFactory({ instance, env }),
1948
+ toPort: instancePortFactory({ instance, env, container: true }),
1949
+ });
1950
+ const trafficService = Underpost.deploy.trafficServiceNameFactory({ deployId: instanceDeployId, env });
1951
+ if (!Underpost.deploy.awaitServiceEndpoints({ service: trafficService, namespace })) {
1952
+ if (targetTraffic !== bootstrapTraffic)
1953
+ Underpost.deploy.applyTrafficService({
1954
+ deployId: instanceDeployId,
1955
+ env,
1956
+ traffic: bootstrapTraffic,
1957
+ namespace,
1958
+ fromPort: instancePortFactory({ instance, env }),
1959
+ toPort: instancePortFactory({ instance, env, container: true }),
1960
+ });
1961
+ throw new Error(`Traffic Service ${trafficService} never became ready on ${targetTraffic}`);
1962
+ }
1963
+ }
1274
1964
  // Refresh the gRPC service to ensure it points to the parent deploy's current traffic.
1275
1965
  if (promotedTraffic) {
1276
- const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
1966
+ const parentTraffic =
1967
+ Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }) || 'blue';
1277
1968
  const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
1278
1969
  deployId,
1279
1970
  env,
@@ -1293,11 +1984,75 @@ EOF
1293
1984
  */
1294
1985
  instance: async (path = '', options = DEFAULT_OPTION) => {
1295
1986
  const env = options.dev ? 'development' : 'production';
1987
+ options = {
1988
+ ...options,
1989
+ gatewayApi: gatewayApiEnabledFactory(options),
1990
+ namespace: options.namespace || 'default',
1991
+ };
1296
1992
  const baseCommand = options.dev ? 'node bin' : 'underpost';
1297
1993
  const baseClusterCommand = options.dev ? ' --dev' : '';
1298
1994
  let [deployId, id, replicas] = path.split(',');
1299
1995
  if (!replicas) replicas = options.replicas;
1300
1996
  const confInstances = selectConfInstances(loadConfInstances(deployId), id);
1997
+ const { liveTrafficById, targetTrafficById, serving } = instanceTrafficPlanFactory({
1998
+ instances: confInstances,
1999
+ requestedTraffic: options.traffic,
2000
+ liveTrafficOf: (instance) =>
2001
+ Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
2002
+ hostTest: instance.host,
2003
+ namespace: options.namespace,
2004
+ env,
2005
+ gatewayApi: options.gatewayApi,
2006
+ }),
2007
+ servesTraffic: (instance, colour) =>
2008
+ Underpost.deploy.serviceHasReadyEndpoints({
2009
+ service: `${deployId}-${instance.id}-${env}-${colour}-service`,
2010
+ namespace: options.namespace,
2011
+ }),
2012
+ });
2013
+
2014
+ let prePromoted = false;
2015
+ const fallbackChecks = instanceFallbackChecksFactory(confInstances);
2016
+ // The promote points intercepted statuses at these documents on either
2017
+ // path, so they are placed regardless of which one runs.
2018
+ if (options.gatewayApi && fallbackChecks.length > 0 && !options.expose)
2019
+ placeInstanceStaticAssets({ instances: confInstances, options, label: 'instance' });
2020
+ if (
2021
+ options.gatewayApi &&
2022
+ !options.gatewayBootstrapComplete &&
2023
+ fallbackChecks.length > 0 &&
2024
+ !options.expose &&
2025
+ serving.length === 0
2026
+ ) {
2027
+ // Clear the target colour before routing to it. A previous blue/green
2028
+ // cycle may have left that inactive Deployment Ready, which would turn
2029
+ // this into a stale-app probe instead of a no-backend fallback probe.
2030
+ for (const instance of confInstances) {
2031
+ const podId = `${deployId}-${instance.id}-${env}-${targetTrafficById[instance.id]}`;
2032
+ shellExec(`kubectl delete service ${podId}-service --namespace ${options.namespace} --ignore-not-found`);
2033
+ shellExec(`kubectl delete deployment ${podId} --namespace ${options.namespace} --ignore-not-found`);
2034
+ }
2035
+ // A direct `run instance` must publish and prove its configured static
2036
+ // fallback before it submits the first instance Deployment document.
2037
+ await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, {
2038
+ ...options,
2039
+ targetTrafficById,
2040
+ noBackendCheckpoint: true,
2041
+ });
2042
+ await gatewayFallbackProbeRunner({
2043
+ gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
2044
+ checks: fallbackChecks,
2045
+ options,
2046
+ label: 'instance',
2047
+ });
2048
+ prePromoted = true;
2049
+ } else if (serving.length > 0)
2050
+ logger.info('[instance] Live colour serving; holding traffic until the target colour is Ready', {
2051
+ hosts: [...new Set(serving.map((instance) => instance.host))],
2052
+ live: serving.map((instance) => `${instance.id}:${liveTrafficById[instance.id]}`),
2053
+ target: serving.map((instance) => `${instance.id}:${targetTrafficById[instance.id]}`),
2054
+ });
2055
+
1301
2056
  const etcHosts = [];
1302
2057
  for (const instance of confInstances) {
1303
2058
  let {
@@ -1305,10 +2060,6 @@ EOF
1305
2060
  host: _host,
1306
2061
  path: _path,
1307
2062
  image: _image,
1308
- fromPort: _fromPort,
1309
- toPort: _toPort,
1310
- fromDebugPort: _fromDebugPort,
1311
- toDebugPort: _toDebugPort,
1312
2063
  cmd: _cmd,
1313
2064
  volumes: _volumes,
1314
2065
  metadata: _metadata,
@@ -1317,9 +2068,8 @@ EOF
1317
2068
  livenessProbe: _livenessProbe,
1318
2069
  } = instance;
1319
2070
  const _deployId = `${deployId}-${_id}`;
1320
- // Use debug ports in development when defined, fall back to production ports.
1321
- if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
1322
- if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
2071
+ const _fromPort = instancePortFactory({ instance, env });
2072
+ const _toPort = instancePortFactory({ instance, env, container: true });
1323
2073
  etcHosts.push(_host);
1324
2074
  if (options.expose) continue;
1325
2075
  // Examples images:
@@ -1336,13 +2086,7 @@ EOF
1336
2086
  k3s: options.k3s,
1337
2087
  });
1338
2088
 
1339
- const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
1340
- hostTest: _host,
1341
- namespace: options.namespace,
1342
- env,
1343
- });
1344
-
1345
- const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
2089
+ const targetTraffic = targetTrafficById[instance.id];
1346
2090
  const podId = `${_deployId}-${env}-${targetTraffic}`;
1347
2091
  const ignorePods = Underpost.kubectl.get(podId, 'pods', options.namespace).map((p) => p.NAME);
1348
2092
  Underpost.deploy.configMap(env, options.namespace);
@@ -1362,20 +2106,21 @@ EOF
1362
2106
  k3s: options.k3s,
1363
2107
  env,
1364
2108
  }),
1365
- clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
2109
+ clusterContext: clusterTypeFactory(options),
1366
2110
  gitClean: options.gitClean || false,
1367
2111
  sshKeyPath: options.sshKeyPath || '',
1368
2112
  });
1369
2113
  // Regenerate the parent deploy's gRPC ClusterIP service pointing to the
1370
2114
  // parent's current traffic colour and apply it before the instance pod starts so
1371
2115
  // DNS is resolvable the moment the pod boots.
1372
- const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
2116
+ const parentTraffic =
2117
+ Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }) || 'blue';
1373
2118
  const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
1374
2119
  deployId,
1375
2120
  env,
1376
2121
  confServer: loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`),
1377
2122
  namespace: options.namespace,
1378
- traffic: [targetTraffic],
2123
+ traffic: [parentTraffic],
1379
2124
  host: _host,
1380
2125
  });
1381
2126
  if (grpcServicePath) shellExec(`kubectl apply -f ${grpcServicePath} -n ${options.namespace}`);
@@ -1390,7 +2135,6 @@ EOF
1390
2135
  // Resolve env-scoped lifecycle/probe blocks: each can be either
1391
2136
  // { ...envObj } // shared shape
1392
2137
  // { development: {...}, production: {...} } // env-specific
1393
- const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
1394
2138
 
1395
2139
  // Convention: an instance config may place `imagePullPolicy` inside
1396
2140
  // the env-scoped lifecycle block (alongside postStart/preStop).
@@ -1398,7 +2142,7 @@ EOF
1398
2142
  // strip it from the lifecycle hash so the rendered YAML stays valid.
1399
2143
  // CLI override (`--image-pull-policy`) wins over the conf value.
1400
2144
  const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
1401
- Underpost.deploy.extractInstanceImagePullPolicy(pickEnv(_lifecycle));
2145
+ Underpost.deploy.extractInstanceImagePullPolicy(resolveEnvScoped(_lifecycle, env));
1402
2146
  const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
1403
2147
 
1404
2148
  let deploymentYaml = `---
@@ -1414,16 +2158,31 @@ ${Underpost.deploy
1414
2158
  volumes: _volumes,
1415
2159
  cmd: resolvedCmd,
1416
2160
  lifecycle: lifecycleForManifest,
1417
- readinessProbe: pickEnv(_readinessProbe),
1418
- livenessProbe: pickEnv(_livenessProbe),
2161
+ readinessProbe: Underpost.deploy.requiredReadinessProbeFactory({
2162
+ probe: resolveEnvScoped(_readinessProbe, env),
2163
+ port: _toPort,
2164
+ }),
2165
+ livenessProbe: resolveEnvScoped(_livenessProbe, env),
1419
2166
  containerPort: _toPort,
1420
2167
  imagePullPolicy: instanceImagePullPolicy,
2168
+ // Pin the pod in the manifest submitted for its only rollout. Volumes were
2169
+ // already resolved against this node; leaving the pod unconstrained could
2170
+ // schedule it away from its data and require a second, post-ready move.
2171
+ nodeName: options.nodeName
2172
+ ? Underpost.deploy.resolveDeployNode({
2173
+ node: options.nodeName,
2174
+ kind: options.kind,
2175
+ kubeadm: options.kubeadm,
2176
+ k3s: options.k3s,
2177
+ env,
2178
+ })
2179
+ : '',
1421
2180
  })
1422
2181
  .replace('{{ports}}', buildKindPorts(_fromPort, _toPort))}
1423
2182
  `;
1424
2183
  // console.log(deploymentYaml);
1425
2184
  shellExec(
1426
- `kubectl apply -f - -n ${options.namespace} <<EOF
2185
+ `kubectl apply -f - -n ${options.namespace} <<'EOF'
1427
2186
  ${deploymentYaml}
1428
2187
  EOF
1429
2188
  `,
@@ -1446,14 +2205,12 @@ EOF
1446
2205
  return;
1447
2206
  }
1448
2207
  }
1449
- // Promote (switch the HTTPProxy to the new colour) ONLY after EVERY
1450
- // instance in the family is deployed and ready never per-instance inside
1451
- // the loop. A per-variant switch would flip server.cyberiaonline.com to the
1452
- // default's new deployment while -forest/-test are still being created,
1453
- // pointing routes at services that don't exist yet. instance-promote
1454
- // rebuilds the whole host proxy (all variant routes) in one shot, so a
1455
- // single call with the family id promotes the family atomically.
1456
- if (!options.expose) await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, options);
2208
+ // Cluster-invoked instances inherit a fallback route for the old
2209
+ // colour, so they still promote the family atomically after every variant
2210
+ // is Ready. A direct run already routed the exact target colour before the
2211
+ // Deployment and proved its fallback; that route simply starts proxying the
2212
+ // new endpoints and must not be toggled a second time.
2213
+ if (!options.expose && !prePromoted) await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, options);
1457
2214
  if (options.etcHosts) {
1458
2215
  const hostListenResult = etcHostFactory(etcHosts);
1459
2216
  logger.info(hostListenResult.renderHosts);
@@ -1506,7 +2263,7 @@ EOF
1506
2263
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1507
2264
  * @memberof UnderpostRun
1508
2265
  */
1509
- 'instance-build-manifest': (path, options = DEFAULT_OPTION) => {
2266
+ 'instance-build-manifest': async (path, options = DEFAULT_OPTION) => {
1510
2267
  const env = options.dev ? 'development' : 'production';
1511
2268
  let [deployId, id, projectPath] = path.split(',');
1512
2269
  const rootPath = projectPath ? projectPath : '.';
@@ -1527,7 +2284,7 @@ EOF
1527
2284
  }
1528
2285
  if (!options.instanceOnly && (selected.length > 1 || selected[0].id !== id)) {
1529
2286
  for (const instance of selected)
1530
- UnderpostRun.RUNNERS['instance-build-manifest'](
2287
+ await UnderpostRun.RUNNERS['instance-build-manifest'](
1531
2288
  [deployId, instance.id, projectPath].filter((v) => v !== undefined).join(','),
1532
2289
  { ...options, instanceOnly: true },
1533
2290
  );
@@ -1542,15 +2299,12 @@ EOF
1542
2299
 
1543
2300
  const instance = selected[0];
1544
2301
  const isDefaultInstance = instance.id === instance.templateId || !instance.templateId;
2302
+ const instanceEnvBuilder = await loadProjectInstanceEnvBuilder(deployId);
1545
2303
 
1546
2304
  let {
1547
2305
  id: _id,
1548
2306
  host: _host,
1549
2307
  image: _image,
1550
- fromPort: _fromPort,
1551
- toPort: _toPort,
1552
- fromDebugPort: _fromDebugPort,
1553
- toDebugPort: _toDebugPort,
1554
2308
  cmd: _cmd,
1555
2309
  volumes: _volumes,
1556
2310
  metadata: _metadata,
@@ -1587,9 +2341,8 @@ EOF
1587
2341
 
1588
2342
  const _deployId = `${deployId}-${_id}`;
1589
2343
  if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
1590
- // Use debug ports in development when defined, fall back to production ports.
1591
- if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
1592
- if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
2344
+ const _fromPort = instancePortFactory({ instance, env });
2345
+ const _toPort = instancePortFactory({ instance, env, container: true });
1593
2346
 
1594
2347
  // Build image from projectPath Dockerfile and load into cluster when --build is set.
1595
2348
  if (options.build && projectPath) {
@@ -1622,13 +2375,12 @@ EOF
1622
2375
 
1623
2376
  // Env-aware lifecycle / probe selection. Each block may either be
1624
2377
  // a single object (shared across envs) or `{ development, production }`.
1625
- const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
1626
2378
 
1627
2379
  // Convention: an instance config may place `imagePullPolicy` inside
1628
2380
  // the env-scoped lifecycle block (alongside postStart/preStop).
1629
2381
  // Extract it onto the container spec and strip it from the lifecycle hash.
1630
2382
  const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
1631
- Underpost.deploy.extractInstanceImagePullPolicy(pickEnv(_lifecycle));
2383
+ Underpost.deploy.extractInstanceImagePullPolicy(resolveEnvScoped(_lifecycle, env));
1632
2384
  const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
1633
2385
 
1634
2386
  const deploymentYaml =
@@ -1645,10 +2397,22 @@ EOF
1645
2397
  volumes: _volumes,
1646
2398
  cmd: resolvedCmd,
1647
2399
  lifecycle: lifecycleForManifest,
1648
- readinessProbe: pickEnv(_readinessProbe),
1649
- livenessProbe: pickEnv(_livenessProbe),
2400
+ readinessProbe: Underpost.deploy.requiredReadinessProbeFactory({
2401
+ probe: resolveEnvScoped(_readinessProbe, env),
2402
+ port: _toPort,
2403
+ }),
2404
+ livenessProbe: resolveEnvScoped(_livenessProbe, env),
1650
2405
  containerPort: _toPort,
1651
2406
  imagePullPolicy: instanceImagePullPolicy,
2407
+ nodeName: options.nodeName
2408
+ ? Underpost.deploy.resolveDeployNode({
2409
+ node: options.nodeName,
2410
+ kind: options.kind,
2411
+ kubeadm: options.kubeadm,
2412
+ k3s: options.k3s,
2413
+ env,
2414
+ })
2415
+ : '',
1652
2416
  })
1653
2417
  .replace('{{ports}}', buildKindPorts(_fromPort, _toPort));
1654
2418
 
@@ -1703,6 +2467,36 @@ EOF
1703
2467
  trafficById: { [instance.id]: targetTraffic },
1704
2468
  });
1705
2469
 
2470
+ // A status route is only emitted for a page this project actually ships, so
2471
+ // a rewrite never points at a document that cannot exist. The check is a
2472
+ // read: placing the document into the gateway volume is `deploy
2473
+ // --sync-static`'s job at apply time, and a build must not mutate the host.
2474
+ // `projectPath` is passed through because this runner is given one
2475
+ // explicitly; the sync derives the same root from the instance itself.
2476
+ const statusPageEntries = instanceStatusPageEntriesFactory({
2477
+ instances: [instance],
2478
+ projectPath: rootPath,
2479
+ }).filter((entry) => fs.existsSync(entry.sourcePath));
2480
+
2481
+ // httproute.yaml — this instance's own routes, including the status routes
2482
+ // that reach the static utility instead of this workload. No Gateway is
2483
+ // emitted beside it: the parent deploy owns the one Gateway that
2484
+ // terminates every hostname it serves, and a second, hostname-scoped one
2485
+ // would contend with it for the merged listener rather than add to it.
2486
+ const httpRouteYaml = Underpost.deploy.httpRouteYamlFactory({
2487
+ host: _host,
2488
+ options,
2489
+ parentName: Underpost.deploy.gatewayNameFactory({ deployId, env }),
2490
+ rules: instanceHttpRouteRulesFactory({
2491
+ deployId,
2492
+ instances: [instance],
2493
+ env,
2494
+ trafficById: { [instance.id]: targetTraffic },
2495
+ options,
2496
+ servedStatuses: statusPageEntries.map((page) => `${page.status}`),
2497
+ }),
2498
+ });
2499
+
1706
2500
  // grpc-service.yaml — the parent deploy's gRPC ClusterIP (shared; the
1707
2501
  // instance cmd resolves {{grpc-service-dns}} to it). Reuse the parent's
1708
2502
  // generated manifest when present rather than regenerating it here.
@@ -1718,27 +2512,56 @@ EOF
1718
2512
  fs.writeFileSync(`${instanceBuildDir}/deployment.yaml`, deploymentYaml, 'utf8');
1719
2513
  const siblingManifests = {
1720
2514
  'pv-pvc.yaml': pvPvcYaml,
2515
+ 'traffic-service.yaml': Underpost.deploy.trafficServiceYamlFactory({
2516
+ deployId: _deployId,
2517
+ env,
2518
+ traffic: targetTraffic,
2519
+ namespace: options.namespace,
2520
+ fromPort: _fromPort,
2521
+ toPort: _toPort,
2522
+ }),
1721
2523
  'proxy.yaml': proxyYaml,
2524
+ // No gateway.yaml: the parent deploy owns the Gateway. `writeManifest`
2525
+ // removes the file a previous per-host build left behind.
2526
+ 'gateway.yaml': '',
2527
+ 'httproute.yaml': httpRouteYaml,
1722
2528
  'grpc-service.yaml': grpcServiceYaml,
1723
2529
  };
1724
- for (const [name, content] of Object.entries(siblingManifests)) {
1725
- if (!content) continue;
1726
- fs.writeFileSync(`${envManifestPath}/${name}`, content, 'utf8');
1727
- fs.writeFileSync(`${instanceBuildDir}/${name}`, content, 'utf8');
1728
- }
2530
+ // Written only when they carry objects, and removed otherwise: an empty
2531
+ // manifest makes `kubectl apply` fail with "no objects passed to apply",
2532
+ // so a build that produces none must leave none behind either.
2533
+ for (const [name, content] of Object.entries(siblingManifests))
2534
+ for (const dir of [envManifestPath, instanceBuildDir])
2535
+ Underpost.deploy.writeManifest({ filePath: `${dir}/${name}`, content });
1729
2536
  logger.info('[instance-build-manifest] Sibling manifests written', {
1730
2537
  project: envManifestPath,
1731
2538
  enginePrivate: instanceBuildDir,
1732
2539
  pvPvc: !!pvPvcYaml,
1733
2540
  proxy: !!proxyYaml,
2541
+ httpRoute: !!httpRouteYaml,
2542
+ statusPages: statusPageEntries.length,
1734
2543
  grpcService: !!grpcServiceYaml,
1735
2544
  });
2545
+ const { gatewayClassName, http3, quicPort, altSvc } = Underpost.deploy.gatewayApiConfigFactory(options);
2546
+ logger.info('[instance-build-manifest] Gateway API manifests written', {
2547
+ host: _host,
2548
+ gatewayClass: gatewayClassName,
2549
+ http3,
2550
+ quicPort,
2551
+ altSvc: http3 ? altSvc : null,
2552
+ statusPages: (instance.customStatusPages || []).map((page) => ({
2553
+ status: page.status,
2554
+ route: `${instance.path === '/' ? '' : instance.path}/${page.status}`,
2555
+ hostPath: page.hostPath,
2556
+ })),
2557
+ statusPageResources: statusPageEntries,
2558
+ });
1736
2559
 
1737
2560
  // --- Per-instance env files -----------------------------------------
1738
- // Each env file is seeded from the template instance's env file for the
1739
- // same mode so operator-owned keys (API keys, memory limits, service URLs)
1740
- // are inherited verbatim; only the keys the instance declares under
1741
- // `multiInstance.env` plus the derived container id — are (re)written.
2561
+ // Each env file starts from the template instance's canonical file for the
2562
+ // same mode. Operator-owned keys remain private and are copied verbatim;
2563
+ // deploy-specific builders may then derive application env from the
2564
+ // normalized instance path/code.
1742
2565
  //
1743
2566
  // A derived instance's env dir is generated in full: both development.env
1744
2567
  // and production.env are written on every build, so a deploy in either
@@ -1751,18 +2574,24 @@ EOF
1751
2574
  const envsToWrite = isDefaultInstance ? [env] : ['development', 'production'];
1752
2575
  for (const targetEnv of envsToWrite) {
1753
2576
  const templateEnvPath = `./engine-private/conf/${deployId}/instances/${instance.templateId}/env/${targetEnv}.env`;
1754
- const baseEnv = fs.existsSync(templateEnvPath) ? dotenv.parse(fs.readFileSync(templateEnvPath, 'utf8')) : {};
1755
- writeEnv(`${instanceEnvDir}/${targetEnv}.env`, {
1756
- ...baseEnv,
1757
- ...(instance.instanceEnv?.[targetEnv] || {}),
1758
- CONTAINER_DEPLOY_ID: `${_deployId}-${targetEnv}`,
2577
+ if (!fs.existsSync(templateEnvPath))
2578
+ throw new Error(`[instance-build-manifest] Missing canonical env file: ${templateEnvPath}`);
2579
+ const baseEnv = dotenv.parse(fs.readFileSync(templateEnvPath, 'utf8'));
2580
+ const builtEnv = dispatchBuildInstanceEnv({
2581
+ deployId,
2582
+ instance,
2583
+ environment: targetEnv,
2584
+ baseEnv,
2585
+ containerDeployId: `${_deployId}-${targetEnv}`,
2586
+ builders: instanceEnvBuilder ? { [deployId]: instanceEnvBuilder } : {},
1759
2587
  });
2588
+ writeEnv(`${instanceEnvDir}/${targetEnv}.env`, builtEnv);
1760
2589
  }
1761
2590
  logger.info('[instance-build-manifest] Instance env written', {
1762
2591
  dir: instanceEnvDir,
1763
2592
  instanceCode: instance.instanceCode,
1764
2593
  envs: envsToWrite,
1765
- keys: Object.keys(instance.instanceEnv?.[env] || {}),
2594
+ builder: instanceEnvBuilder?.name || 'canonical-copy',
1766
2595
  });
1767
2596
  }
1768
2597
 
@@ -1772,9 +2601,19 @@ EOF
1772
2601
  }
1773
2602
  fs.copyFileSync(outputPath, `${rootPath}/deployment.yaml`);
1774
2603
  // Sibling manifests alongside deployment.yaml at the project root.
1775
- for (const name of ['pv-pvc.yaml', 'proxy.yaml', 'grpc-service.yaml']) {
2604
+ for (const name of [
2605
+ 'pv-pvc.yaml',
2606
+ 'traffic-service.yaml',
2607
+ 'proxy.yaml',
2608
+ 'gateway.yaml',
2609
+ 'httproute.yaml',
2610
+ 'grpc-service.yaml',
2611
+ ]) {
1776
2612
  const src = `${envManifestPath}/${name}`;
2613
+ // Absence is mirrored too, so the repo never ships a manifest this
2614
+ // build stopped producing.
1777
2615
  if (fs.existsSync(src)) fs.copyFileSync(src, `${rootPath}/${name}`);
2616
+ else fs.removeSync(`${rootPath}/${name}`);
1778
2617
  }
1779
2618
  logger.info('[instance-build-manifest] Production artifacts copied to project root', {
1780
2619
  rootPath,
@@ -1981,9 +2820,7 @@ EOF`);
1981
2820
  shellExec(`kubectl apply -k ${underpostRoot}/manifests/deployment/adminer/. -n ${options.namespace}`);
1982
2821
  const successInstance = await Underpost.test.statusMonitor('adminer', 'Running', 'pods', 1000, 60 * 10);
1983
2822
 
1984
- if (successInstance) {
1985
- shellExec(`underpost deploy --expose adminer --namespace ${options.namespace}`);
1986
- }
2823
+ if (successInstance) return UnderpostRun.RUNNERS.expose(path || 'adminer', options);
1987
2824
  },
1988
2825
 
1989
2826
  /**
@@ -2042,6 +2879,7 @@ EOF`);
2042
2879
  * @memberof UnderpostRun
2043
2880
  */
2044
2881
  promote: async (path, options = DEFAULT_OPTION) => {
2882
+ options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
2045
2883
  let [inputDeployId, inputEnv, inputReplicas] = path.split(',');
2046
2884
  if (!inputEnv) inputEnv = 'production';
2047
2885
  if (!inputReplicas) inputReplicas = 1;
@@ -2069,13 +2907,19 @@ EOF`);
2069
2907
 
2070
2908
  if (inputDeployId === 'dd') {
2071
2909
  for (const deployId of fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',')) {
2072
- const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
2910
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
2911
+ namespace: options.namespace,
2912
+ env: inputEnv,
2913
+ });
2073
2914
  const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
2074
2915
  Underpost.deploy.switchTraffic(deployId, inputEnv, targetTraffic, inputReplicas, options.namespace, options);
2075
2916
  applyCerts(deployId, targetTraffic);
2076
2917
  }
2077
2918
  } else {
2078
- const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, { namespace: options.namespace });
2919
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, {
2920
+ namespace: options.namespace,
2921
+ env: inputEnv,
2922
+ });
2079
2923
  const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
2080
2924
  Underpost.deploy.switchTraffic(
2081
2925
  inputDeployId,
@@ -2128,8 +2972,24 @@ EOF`);
2128
2972
  },
2129
2973
  /**
2130
2974
  * @method cluster
2131
- * @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB, MariaDB, Valkey, Contour (Ingress), and Cert-Manager, and deploys all services.
2132
- * @param {string} path - The input value, identifier, or path for the operation.
2975
+ * @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB,
2976
+ * MariaDB, Valkey, the Gateway API data plane (Envoy Gateway), Contour, and Cert-Manager, and deploys all services.
2977
+ *
2978
+ * Ingress is served by the Gateway API stack (Gateway + HTTPRoute) with QUIC/HTTP3 in **both** environments;
2979
+ * `--disable-gateway-api` falls back to the Contour HTTPProxy stack. Because HTTP/3 has no cleartext transport,
2980
+ * development terminates TLS too: a self-signed certificate per host (mkcert via `scripts/ssl.sh`, whose root CA
2981
+ * the script installs into the system + NSS trust stores) is issued into the secret the Gateway listener
2982
+ * references, and every host is written to `/etc/hosts` so the operator's browser reaches the PWA at
2983
+ * `https://<host>` on the local machine.
2984
+ *
2985
+ * Custom instances are the optional third segment of `path`. They are resolved per deploy against that deploy's
2986
+ * own `conf.instances.json`, so an id only runs where its deploy declares it, and each one is deployed after its
2987
+ * deploy's default workload is serving — an instance reads the parent's world configuration over the parent's
2988
+ * gRPC ClusterIP at boot. Instance hosts share the deploy's environment: the same self-signed certificates and
2989
+ * `/etc/hosts` pass in development, the same cert-manager issuance in production.
2990
+ * @param {string} path - `<runtime-image>,<deploy-list>[,<instance-list>]` — `+`-separated lists, e.g.
2991
+ * `express,dd-cyberia,mmo-server` or `express,dd-cyberia+dd-core,mmo-server+mmo-client`. An instance list entry
2992
+ * may be a template id (`mmo-server`), which selects its whole variant family.
2133
2993
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
2134
2994
  * @memberof UnderpostRun
2135
2995
  */
@@ -2138,30 +2998,39 @@ EOF`);
2138
2998
  const env = options.dev ? 'development' : 'production';
2139
2999
  const baseCommand = options.dev ? 'node bin' : 'underpost';
2140
3000
  const baseClusterCommand = options.dev ? ' --dev' : '';
2141
- const clusterType = options.k3s ? 'k3s' : 'kubeadm';
3001
+ const clusterType = clusterTypeFactory(options, 'kubeadm');
2142
3002
  shellCd(`/home/dd/engine`);
2143
3003
  shellExec(`${baseCommand} cluster${baseClusterCommand} --reset --${clusterType}`);
2144
3004
  await timer(5000);
2145
3005
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType}`);
2146
3006
  await timer(5000);
2147
- let [runtimeImage, deployList] =
3007
+ let [runtimeImage, deployList, instanceListId] =
2148
3008
  path && path.trim() && path.split(',')
2149
3009
  ? path.split(',')
2150
3010
  : [
2151
3011
  'express',
2152
3012
  fs.readFileSync(`${underpostRoot}/engine-private/deploy/dd.router`, 'utf8').replaceAll(',', '+'),
3013
+ '',
2153
3014
  ];
2154
- shellExec(
2155
- `${baseCommand} image${baseClusterCommand} --build ${
2156
- runtimeImage ? ` --pull-base --path ${underpostRoot}/src/runtime/${runtimeImage}` : ''
2157
- } --${clusterType}`,
2158
- );
3015
+ // shellExec(
3016
+ // `${baseCommand} image${baseClusterCommand} --build ${
3017
+ // runtimeImage ? ` --pull-base --path ${underpostRoot}/src/runtime/${runtimeImage}` : ''
3018
+ // } --${clusterType}`,
3019
+ // );
2159
3020
  if (!deployList) {
2160
3021
  deployList = [];
2161
3022
  logger.warn('No deploy list provided');
2162
3023
  } else deployList = deployList.split('+');
2163
3024
  await timer(5000);
2164
- shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mongodb`);
3025
+ // --reset-mongodb wipes the retained hostPath volumes before the rollout.
3026
+ // This workflow already tore the node down and re-imports every database
3027
+ // from its git backup a few lines below, so inheriting the previous
3028
+ // cluster's replica set config is never wanted — that state leaves mongod
3029
+ // parked outside its own config and the bootstrap fighting to recover it.
3030
+ shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mongodb --reset-mongodb`);
3031
+ if (deployList.includes('dd-cyberia'))
3032
+ shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --ipfs --replicas 1`);
3033
+
2165
3034
  if (runtimeImage === 'lampp') {
2166
3035
  await timer(5000);
2167
3036
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mariadb`);
@@ -2174,19 +3043,614 @@ EOF`);
2174
3043
  }
2175
3044
  await timer(5000);
2176
3045
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --valkey`);
3046
+ // Exactly one ingress stack is installed, because the two cannot coexist
3047
+ // on this node: Contour's Envoy DaemonSet declares hostPort 80/443, so the
3048
+ // CNI hostport plugin DNATs everything arriving on those ports straight to
3049
+ // it — before the Gateway API data plane's own listener can see them. With
3050
+ // no HTTPProxy objects to program (this workflow applies HTTPRoutes),
3051
+ // Contour's Envoy has no listeners and refuses the redirected connection,
3052
+ // which looks exactly like a gateway that is not listening at all.
3053
+ const gatewayApi = gatewayApiEnabledFactory(options);
3054
+ const gatewayApiFlags = Underpost.deploy.gatewayApiFlagsFactory({ ...options, gatewayApi });
2177
3055
  await timer(5000);
2178
- shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --contour`);
3056
+ if (gatewayApi) {
3057
+ shellExec(
3058
+ `${baseCommand} cluster${baseClusterCommand} --${clusterType} --gateway-api${
3059
+ options.gatewayClass ? ` --gateway-class ${options.gatewayClass}` : ''
3060
+ }`,
3061
+ );
3062
+ } else shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --contour`);
3063
+ if (gatewayApi)
3064
+ shellExec(
3065
+ `kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${options.namespace || 'default'} --timeout=5m`,
3066
+ );
2179
3067
  if (env === 'production') {
2180
3068
  await timer(5000);
2181
3069
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --cert-manager`);
2182
3070
  }
3071
+
3072
+ const { byDeployId: instancesByDeployId, unmatched: unmatchedInstanceIds } = clusterInstancesFactory(
3073
+ deployList,
3074
+ instanceListId,
3075
+ );
3076
+ if (unmatchedInstanceIds.length > 0)
3077
+ logger.warn('[cluster] No deploy declares these instances; they will not be deployed', {
3078
+ instances: unmatchedInstanceIds,
3079
+ deployList,
3080
+ });
3081
+
3082
+ // Development terminates TLS with a locally trusted certificate instead of
3083
+ // cert-manager: QUIC/HTTP3 has no cleartext transport, so without it the
3084
+ // dev gateway would fall back to an HTTP-only listener. The hosts are
3085
+ // written to /etc/hosts in a single pass — etcHostFactory rewrites the
3086
+ // file, so one call per deploy would drop the previous deploy's entries.
3087
+ // Instance hosts come through the same resolver the Gateway's certificate
3088
+ // list uses, so the two can never disagree about what the deploy serves.
3089
+ const hosts = [...new Set(deployList.flatMap((deployId) => deployHostsFactory(deployId)))];
3090
+ if (env === 'development') {
3091
+ for (const host of hosts)
3092
+ Underpost.deploy.selfSignedTlsSecretFactory({
3093
+ host,
3094
+ namespace: options.namespace || 'default',
3095
+ underpostRoot,
3096
+ });
3097
+ const hostListenResult = etcHostFactory(hosts);
3098
+ logger.info(hostListenResult.renderHosts);
3099
+ }
3100
+ const version = 'v3.3.0';
3101
+ const instanceOptionsFactory = (deployId, instanceId) => ({
3102
+ ...options,
3103
+ ...clusterContextFactory(clusterType),
3104
+ gatewayApi,
3105
+ gatewayBootstrapComplete: true,
3106
+ tls: true,
3107
+ test: env === 'development',
3108
+ etcHosts: false,
3109
+ namespace: options.namespace || 'default',
3110
+ imageName:
3111
+ deployId === 'dd-cyberia' && env === 'development' && instanceId === 'mmo-server'
3112
+ ? `underpost/cyberia-server-dev:${version}`
3113
+ : deployId === 'dd-cyberia' && env === 'development' && instanceId === 'mmo-client'
3114
+ ? `underpost/cyberia-client-dev:${version}`
3115
+ : undefined,
3116
+ });
3117
+ const deployFlagsById = {};
3118
+ const fallbackChecks = new Map();
3119
+
3120
+ // Regenerating the manifests is required, not incidental: the TLS listener
3121
+ // — and with it the QUIC policy and the HTTPRoute set — is only emitted
3122
+ // when the TLS and gateway flags are known at generation time. It takes two
3123
+ // passes because `--build-manifest` returns after writing the manifests, so
3124
+ // the same flags have to be repeated on the apply call.
2183
3125
  for (const deployId of deployList) {
2184
- shellExec(
2185
- `${baseCommand} deploy ${deployId} ${env} --${clusterType}${env === 'production' ? ' --cert' : ''}${
2186
- env === 'development' ? ' --etc-hosts' : ''
2187
- }${options.namespace ? ` --namespace ${options.namespace}` : ''}`,
3126
+ const deployFlags =
3127
+ `--${clusterType}${env === 'production' ? ' --cert' : ' --self-signed'}${gatewayApiFlags}` +
3128
+ `${options.namespace ? ` --namespace ${options.namespace}` : ''}` +
3129
+ (deployId === 'dd-cyberia'
3130
+ ? ` --image 'underpost/engine-cyberia:${version}' \
3131
+ --versions blue \
3132
+ --image-pull-policy Always \
3133
+ --cmd 'cd /home/dd/engine, \
3134
+ underpost clone underpostnet/engine-cyberia, \
3135
+ mkdir -p /home/dd/engine/src/client/public/itemledger \
3136
+ /home/dd/engine/src/client/public/cryptokoyn \
3137
+ /home/dd/engine/src/client/components/cryptokoyn \
3138
+ /home/dd/engine/src/client/components/itemledger \
3139
+ /home/dd/engine/hardhat, \
3140
+ cp -a ./engine-cyberia/src/client/public/itemledger/. /home/dd/engine/src/client/public/itemledger/, \
3141
+ cp -a ./engine-cyberia/src/client/public/cryptokoyn/. /home/dd/engine/src/client/public/cryptokoyn/, \
3142
+ cp -a ./engine-cyberia/src/client/components/cryptokoyn/. /home/dd/engine/src/client/components/cryptokoyn/, \
3143
+ cp -a ./engine-cyberia/src/client/components/itemledger/. /home/dd/engine/src/client/components/itemledger/, \
3144
+ cp -a ./engine-cyberia/src/client/Itemledger.index.js /home/dd/engine/src/client/Itemledger.index.js, \
3145
+ cp -a ./engine-cyberia/src/client/Cryptokoyn.index.js /home/dd/engine/src/client/Cryptokoyn.index.js, \
3146
+ rm -rf ./engine-cyberia, \
3147
+ sudo rm -rf ./engine-private/, \
3148
+ node bin clone underpostnet/engine-cyberia-private, \
3149
+ sudo mv ./engine-cyberia-private ./engine-private, \
3150
+ node bin env dd-cyberia ${env}, \
3151
+ node ./engine-private/itc-scripts/dd-cyberia-0.js, \
3152
+ sudo chown -R dd:dd /home/dd/engine/src/client/public/cyberia, \
3153
+ node bin env dd-cyberia ${env}, \
3154
+ node bin client dd-cyberia ${env}, \
3155
+ node bin start dd-cyberia ${env} --run'`
3156
+ : '');
3157
+ deployFlagsById[deployId] = deployFlags;
3158
+ // SSR status and context documents belong to the ingress bootstrap, so
3159
+ // build them on the host before any workload Deployment is submitted.
3160
+ shellExec(`${baseCommand} client ${deployId} ${env}`);
3161
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --build-manifest ${deployFlags}`);
3162
+ // Seed the static tree before the routes exist, so every status page and
3163
+ // intercepted context the manifests just pointed at resolves from the
3164
+ // first request. This pass places what this checkout built; the pass
3165
+ // after the rollout replaces each document with the container's own,
3166
+ // which is the only place clients built from private sources exist.
3167
+ if (gatewayApi) {
3168
+ const staticAssets = Underpost.deploy.syncStaticAssets(deployId, env, {
3169
+ ...options,
3170
+ ...clusterContextFactory(clusterType),
3171
+ gatewayApi,
3172
+ namespace: options.namespace || 'default',
3173
+ versions: /--versions\s+([^\s]+)/.exec(deployFlags)?.[1] || options.versions || 'blue',
3174
+ });
3175
+ const missingAssets = staticAssets.filter((entry) => !entry.source);
3176
+ if (missingAssets.length > 0)
3177
+ throw new Error(
3178
+ `[cluster] Static gateway bootstrap is missing configured assets for ${deployId}: ` +
3179
+ missingAssets.map((entry) => entry.assetPath).join(', '),
3180
+ );
3181
+
3182
+ // Record the exact documents the no-backend checkpoint must return.
3183
+ // PWA paths use the SSR maintenance view; custom instances have no
3184
+ // maintenance view and reuse their first declared status document.
3185
+ const confServer = loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`);
3186
+ const confSSRPath = `./engine-private/conf/${deployId}/conf.ssr.json`;
3187
+ const confSSR = fs.existsSync(confSSRPath) ? JSON.parse(fs.readFileSync(confSSRPath, 'utf8')) : {};
3188
+ for (const host of Object.keys(confServer))
3189
+ for (const path of Object.keys(confServer[host])) {
3190
+ const maintenance = Underpost.deploy
3191
+ .edgeRouteEntriesFactory({ confServer, confSSR, host, path })
3192
+ .find((entry) => entry.context === 'maintenance');
3193
+ if (maintenance)
3194
+ fallbackChecks.set(`${host}${path}`, {
3195
+ host,
3196
+ path,
3197
+ assetPath: maintenance.assetPath,
3198
+ kind: maintenance.kind,
3199
+ });
3200
+ }
3201
+ const selectedInstances = instancesByDeployId[deployId].ids.flatMap((instanceId) =>
3202
+ selectConfInstances(loadConfInstances(deployId), instanceId),
3203
+ );
3204
+ for (const entry of instanceStatusPageEntriesFactory({ instances: selectedInstances }))
3205
+ if (!fallbackChecks.has(`${entry.host}${entry.path}`))
3206
+ fallbackChecks.set(`${entry.host}${entry.path}`, {
3207
+ host: entry.host,
3208
+ path: entry.path,
3209
+ assetPath: entry.assetPath,
3210
+ kind: `status:${entry.status}`,
3211
+ });
3212
+ }
3213
+ // Apply only the gateway tier. Application Services and Deployments are
3214
+ // deliberately absent so the status fallback can be observed first.
3215
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --disable-update-deployment ${deployFlags}`);
3216
+ // Instance host routes and their custom status pages are part of
3217
+ // the same ingress bootstrap. `instance-promote` is safe here: it only
3218
+ // writes the Nginx host block and routing objects; no instance Deployment
3219
+ // is created until the second phase below.
3220
+ for (const instanceId of instancesByDeployId[deployId].ids) {
3221
+ logger.info('[cluster] Bootstrapping custom instance gateway', { deployId, instanceId, env });
3222
+ await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${instanceId}`, {
3223
+ ...instanceOptionsFactory(deployId, instanceId),
3224
+ noBackendCheckpoint: true,
3225
+ });
3226
+ }
3227
+ }
3228
+
3229
+ // This is the deliberate no-backend checkpoint. The static Nginx pod, all
3230
+ // Gateway listeners, parent HTTPRoutes and selected instance HTTPRoutes
3231
+ // must be live before the first application Deployment YAML is submitted.
3232
+ if (gatewayApi && fallbackChecks.size > 0) {
3233
+ const fallbackResults = await gatewayFallbackProbeRunner({
3234
+ gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
3235
+ checks: [...fallbackChecks.values()],
3236
+ options: { ...options, gatewayApi, namespace: options.namespace || 'default' },
3237
+ label: 'cluster',
3238
+ });
3239
+ logger.info('[cluster] Gateway fallback checkpoint passed; starting application deployments', {
3240
+ hosts,
3241
+ fallbacks: fallbackResults,
3242
+ });
3243
+ }
3244
+
3245
+ for (const deployId of deployList) {
3246
+ const deployFlags = deployFlagsById[deployId];
3247
+ // Preserve the already-operational ingress objects and apply only the
3248
+ // workload manifests. EndpointSlices will update as pods become Ready;
3249
+ // the site route continues to reach underpost-gateway throughout.
3250
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --disable-update-proxy ${deployFlags}`);
3251
+ if (gatewayApi) {
3252
+ const namespace = options.namespace || 'default';
3253
+ const version = /--versions\s+([^\s,]+)/.exec(deployFlags)?.[1] || 'blue';
3254
+ shellExec(`kubectl rollout status deployment/${deployId}-${env}-${version} -n ${namespace} --timeout=15m`);
3255
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --sync-static ${deployFlags}`);
3256
+ }
3257
+
3258
+ // Custom instance pods depend on the parent's gRPC service, so they are
3259
+ // still started after the parent is Ready. Their routes already exist and
3260
+ // keep serving the custom fallback until the atomic promotion completes.
3261
+ for (const instanceId of instancesByDeployId[deployId].ids) {
3262
+ logger.info('[cluster] Deploying custom instance', { deployId, instanceId, env, clusterType });
3263
+ await UnderpostRun.RUNNERS.instance(
3264
+ `${deployId},${instanceId},${options.replicas || 1}`,
3265
+ instanceOptionsFactory(deployId, instanceId),
3266
+ );
3267
+ }
3268
+ }
3269
+ logger.info('[cluster] Ingress stack deployed', {
3270
+ env,
3271
+ stack: gatewayApi ? 'gateway-api' : 'httpproxy',
3272
+ gatewayClass: gatewayApi ? Underpost.deploy.gatewayApiConfigFactory(options).gatewayClassName : null,
3273
+ http3: gatewayApi && options.disableHttp3 !== true,
3274
+ tls: env === 'production' ? 'cert-manager' : 'self-signed',
3275
+ hosts,
3276
+ instances: Object.fromEntries(
3277
+ deployList
3278
+ .filter((deployId) => instancesByDeployId[deployId].ids.length > 0)
3279
+ .map((deployId) => [deployId, instancesByDeployId[deployId].ids]),
3280
+ ),
3281
+ });
3282
+ if (gatewayApi) await UnderpostRun.RUNNERS['gateway-status']('', options);
3283
+ },
3284
+
3285
+ /**
3286
+ * @method gateway-status
3287
+ * @description Reports whether the Gateway API data plane is actually
3288
+ * serving. Applying a Gateway only records intent: the controller
3289
+ * provisions Envoy asynchronously, so a deploy can finish cleanly while
3290
+ * nothing listens on the node — the failure then surfaces much later as a
3291
+ * bare "connection refused" from the browser. This waits for the Gateways to
3292
+ * be Programmed and prints the data plane's pods and services.
3293
+ * @param {string} path - Unused.
3294
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
3295
+ * @memberof UnderpostRun
3296
+ */
3297
+ 'gateway-status': async (path = '', options = DEFAULT_OPTION) => {
3298
+ const namespace = options.namespace || 'default';
3299
+ const dataPlaneNamespace = 'envoy-gateway-system';
3300
+ const capture = (cmd) => shellExec(cmd, { stdout: true, silent: true, silentOnError: true })?.trim?.() || '';
3301
+
3302
+ const programmed = shellExec(
3303
+ `kubectl wait --for=condition=Programmed --timeout=180s gateway --all -n ${namespace}`,
3304
+ { silentOnError: true },
3305
+ );
3306
+ if (programmed?.code !== 0)
3307
+ logger.warn(
3308
+ '[gateway-status] Gateways are not Programmed. The listeners are not being served; ' +
3309
+ `check 'kubectl describe gateway -n ${namespace}' and the controller logs in ${dataPlaneNamespace}.`,
3310
+ );
3311
+ shellExec(`kubectl get gateway -n ${namespace} -o wide`, { silentOnError: true });
3312
+
3313
+ // Per-listener truth. A Gateway reports Programmed while individual
3314
+ // listeners are rejected, so the aggregate condition hides exactly the
3315
+ // case where some hostnames serve and others do not. Prints each
3316
+ // listener's attached route count and any failing condition with its
3317
+ // reason — an unresolved TLS secret or a rejected listener names itself.
3318
+ const listenerStatus = capture(
3319
+ `kubectl get gateway -n ${namespace} -o jsonpath=` +
3320
+ `'{range .items[*]}{.metadata.name}{" "}` +
3321
+ `{range .status.listeners[*]}{.name}=routes:{.attachedRoutes}` +
3322
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}{end}{" "}{end}{"\\n"}{end}'`,
3323
+ );
3324
+ logger.info('[gateway-status] Listeners (name=routes + failing conditions)\n' + (listenerStatus || '(none)'));
3325
+
3326
+ // Envoy Gateway policies attach to a Gateway but configure the merged
3327
+ // listener, so two of them competing for the same listener is resolved by
3328
+ // rejecting one — and the rejection is recorded here, not on the Gateway.
3329
+ const policyStatus = capture(
3330
+ `kubectl get clienttrafficpolicy -n ${namespace} -o jsonpath=` +
3331
+ `'{range .items[*]}{.metadata.name}{range .status.ancestors[*]}` +
3332
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\\n"}{end}'`,
3333
+ );
3334
+ logger.info(
3335
+ '[gateway-status] ClientTrafficPolicies (failing conditions)\n' +
3336
+ (policyStatus
3337
+ .split('\n')
3338
+ .filter((line) => line.includes(' '))
3339
+ .join('\n') || '(none — all policies accepted)'),
3340
+ );
3341
+
3342
+ // Envoy Gateway policies that fail to translate are not visible on the
3343
+ // Gateway or the route: the resource is admitted, its status carries the
3344
+ // rejection, and the xDS snapshot it belonged to can go with it — which
3345
+ // reads downstream as every hostname answering route_not_found.
3346
+ const backendPolicyStatus = capture(
3347
+ `kubectl get backendtrafficpolicy -n ${namespace} -o jsonpath=` +
3348
+ `'{range .items[*]}{.metadata.name}{range .status.ancestors[*]}` +
3349
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\n"}{end}' ` +
3350
+ `2>/dev/null`,
3351
+ );
3352
+ logger.info(
3353
+ '[gateway-status] BackendTrafficPolicies (failing conditions)\n' +
3354
+ (backendPolicyStatus
3355
+ .split('\n')
3356
+ .filter((line) => line.includes(' '))
3357
+ .join('\n') || '(none — all policies accepted)'),
3358
+ );
3359
+
3360
+ // Route-level conditions. A rule that references something unresolvable
3361
+ // can cost the whole route, and then every path on that hostname answers
3362
+ // with a bare gateway 404 while the Gateway and its listeners stay green.
3363
+ const routeStatus = capture(
3364
+ `kubectl get httproute -n ${namespace} -o jsonpath=` +
3365
+ `'{range .items[*]}{.metadata.name}{range .status.parents[*]}` +
3366
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\\n"}{end}'`,
3367
+ )
3368
+ .split('\n')
3369
+ .filter((line) => line.includes(' '));
3370
+ logger.info(
3371
+ '[gateway-status] HTTPRoutes (failing conditions)\n' +
3372
+ (routeStatus.join('\n') || '(none — all routes accepted)'),
3373
+ );
3374
+
3375
+ // The workloads the routes point at. A status code that moves between runs
3376
+ // (500 here, 404 there) is the application's, not the gateway's, and the
3377
+ // gateway config cannot explain it — restarts and unready containers can.
3378
+ const backends = capture(
3379
+ `kubectl get pods -n ${namespace} -o custom-columns=` +
3380
+ `'NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount,STATUS:.status.phase'`,
3381
+ );
3382
+ logger.info('[gateway-status] Workloads behind the routes\n' + (backends || '(none)'));
3383
+
3384
+ // A Programmed Gateway only means Envoy was provisioned — not that it is
3385
+ // reachable from this machine. Which of the two is false decides the fix,
3386
+ // so report the pod's network mode and container ports, the service type
3387
+ // and node ports, and what the host is actually listening on.
3388
+ const dataPlane = capture(
3389
+ `kubectl get pods -n ${dataPlaneNamespace} -o custom-columns=` +
3390
+ `'NAME:.metadata.name,HOST_NETWORK:.spec.hostNetwork,PORTS:.spec.containers[*].ports[*].containerPort'`,
3391
+ );
3392
+ const services = capture(
3393
+ `kubectl get svc -n ${dataPlaneNamespace} -o custom-columns=` +
3394
+ `'NAME:.metadata.name,TYPE:.spec.type,PORTS:.spec.ports[*].port,NODEPORTS:.spec.ports[*].nodePort,TARGETS:.spec.ports[*].targetPort'`,
3395
+ );
3396
+ logger.info('[gateway-status] Data plane\n' + dataPlane + '\n\n' + services);
3397
+
3398
+ // Envoy creates its listener sockets only once the control plane has
3399
+ // pushed a config with routes attached, which lands a little after the
3400
+ // pod reports Ready. Polling that window is the difference between
3401
+ // "connection refused" and a working gateway, so wait for the socket
3402
+ // rather than sampling it once.
3403
+ const listenerFilter = `grep -E ':(80|443|10080|10443) '`;
3404
+ let hostListeners = '';
3405
+ for (let attempt = 0; attempt < 30; attempt++) {
3406
+ hostListeners = capture(`sudo ss -lntupH 2>/dev/null | ${listenerFilter}`);
3407
+ if (/:443\s/.test(hostListeners)) break;
3408
+ await timer(2000);
3409
+ }
3410
+ const servesHttps = /:443\s/.test(hostListeners);
3411
+ logger.info('[gateway-status] Host listeners on 80/443/10080/10443\n' + (hostListeners || '(none)'));
3412
+
3413
+ if (!servesHttps) {
3414
+ logger.warn(
3415
+ '[gateway-status] Nothing is listening on this host port 443, so a browser reaching the ' +
3416
+ 'hostnames through /etc/hosts gets "connection refused". Compare the two tables above: ' +
3417
+ 'HOST_NETWORK=false means the pod is on the pod network (only the ClusterIP/NodePort is ' +
3418
+ 'reachable); HOST_NETWORK=true with PORTS 10080/10443 means the privileged-port remap is ' +
3419
+ 'still active. Until it is resolved, forward the merged service to expose it locally:\n' +
3420
+ ` kubectl port-forward -n ${dataPlaneNamespace} svc/<envoy-service> 443:443 80:80 --address 127.0.0.1`,
3421
+ );
3422
+ return { programmed: programmed?.code === 0, servesHttps, dataPlane, services, hostListeners, probes: [] };
3423
+ }
3424
+
3425
+ // An open socket still is not proof the hostname routes anywhere. Probe
3426
+ // each Gateway hostname exactly as the browser would — through /etc/hosts,
3427
+ // validating the certificate against the trust store `scripts/ssl.sh`
3428
+ // populated — so the workflow ends on an observed response, not an
3429
+ // inference. Each host is probed twice: over loopback, and pinned to the
3430
+ // node IP. The two answers separate a loopback-specific block from a data
3431
+ // plane that is not reachable at all.
3432
+ const nodeIp = capture(
3433
+ `kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'`,
3434
+ );
3435
+ // The `server` header is reported alongside the status because the code
3436
+ // alone cannot say who produced it: `500 (envoy)` is a gateway or upstream
3437
+ // failure, while `500` from the workload's own server is an application
3438
+ // error, and the two lead to completely different fixes.
3439
+ const probeUrl = (url, resolveArgs = '') => {
3440
+ const raw = capture(
3441
+ `curl -sS -o /dev/null -D - -w 'HTTP_CODE=%{http_code}' --max-time 10 ${resolveArgs}${url} 2>&1 | tr -d '\\r'`,
3442
+ );
3443
+ const status = /HTTP_CODE=([0-9]{3})/.exec(raw)?.[1];
3444
+ if (!status) return raw.split('\n').find((line) => line.startsWith('curl:')) || 'no-response';
3445
+ const server = /^server:\s*(.+)$/im.exec(raw)?.[1]?.trim();
3446
+ return server ? `${status} (${server})` : status;
3447
+ };
3448
+ const probe = (host, resolveTo) =>
3449
+ probeUrl(`https://${host}`, resolveTo ? `--resolve ${host}:443:${resolveTo} ` : '');
3450
+ // Port 80 is the same Envoy process, same host, same listener set — only
3451
+ // the port differs. It separates "this gateway is unreachable" from
3452
+ // "something specifically rejects 443".
3453
+ const probeHttp = (host) => probeUrl(`http://${host}`);
3454
+ // curl writes `000` when it never got a response, so a bare three-digit
3455
+ // match would read a failed connection as success.
3456
+ const answered = (status) => /^[1-5][0-9]{2}\b/.test(status);
3457
+ // The routes, not the listeners, are where the hostnames live: a
3458
+ // consolidated Gateway serves every hostname from one hostname-less
3459
+ // listener and picks the certificate by SNI, so reading the listeners
3460
+ // would leave nothing to probe.
3461
+ const hosts = [
3462
+ ...new Set(
3463
+ (path
3464
+ ? path.split(',')
3465
+ : capture(`kubectl get httproute -n ${namespace} -o jsonpath='{.items[*].spec.hostnames[*]}'`).split(/\s+/)
3466
+ )
3467
+ .map((host) => host.trim())
3468
+ .filter(Boolean),
3469
+ ),
3470
+ ];
3471
+ const probes = hosts.map((host) => ({
3472
+ host,
3473
+ http: probeHttp(host),
3474
+ loopback: probe(host),
3475
+ ...(nodeIp ? { nodeIp: probe(host, nodeIp) } : {}),
3476
+ }));
3477
+ logger.info('[gateway-status] HTTPS probe', { nodeIp: nodeIp || '(unknown)', probes });
3478
+
3479
+ // A gateway answer alone cannot say whether the gateway or the workload
3480
+ // produced the code: Envoy relays an upstream response unchanged. So when
3481
+ // a hostname fails, ask its backend the *same* question from inside the
3482
+ // cluster, bypassing Envoy.
3483
+ //
3484
+ // Same question is the whole point: these workloads route by virtual host,
3485
+ // so a probe carrying `Host: <service-name>` exercises a different branch
3486
+ // than the browser did and its answer means nothing. Each probe therefore
3487
+ // replays the route it came from — the Gateway hostname, the rule path,
3488
+ // and the rewrite the rule would have applied — against the rule's own
3489
+ // backend. Only then do the two columns compare.
3490
+ const failing = probes.filter((entry) => /^[45]/.test(entry.loopback) || /^[45]/.test(entry.http));
3491
+ if (failing.length > 0) {
3492
+ const routes = JSON.parse(
3493
+ capture(`kubectl get httproute -n ${namespace} -o json`) || '{"items":[]}',
3494
+ ).items.filter((route) => failing.some((entry) => route.spec?.hostnames?.includes(entry.host)));
3495
+ // Exec into the static utility rather than spawning a probe pod: it is
3496
+ // installed with the gateway stack, sits in this namespace, and its
3497
+ // BusyBox shell already carries wget — no image pull, no pod churn.
3498
+ const probePod = capture(
3499
+ `kubectl get pods -n ${namespace} -l app=${UNDERPOST_GATEWAY.name} -o jsonpath='{.items[0].metadata.name}'`,
3500
+ );
3501
+ if (!probePod || routes.length === 0) {
3502
+ logger.warn(
3503
+ '[gateway-status] Cannot probe backends directly: ' +
3504
+ (probePod
3505
+ ? 'no HTTPRoute matched a failing hostname'
3506
+ : `no ${UNDERPOST_GATEWAY.name} pod in ${namespace}`),
3507
+ );
3508
+ } else {
3509
+ const backendProbes = [];
3510
+ for (const route of routes) {
3511
+ const host = route.spec.hostnames[0];
3512
+ for (const rule of route.spec.rules || []) {
3513
+ const backend = (rule.backendRefs || [])[0];
3514
+ if (!backend) continue;
3515
+ const match = rule.matches?.[0]?.path?.value || '/';
3516
+ const rewrite = (rule.filters || []).find((filter) => filter.type === 'URLRewrite')?.urlRewrite?.path;
3517
+ const target =
3518
+ rewrite?.type === 'ReplaceFullPath'
3519
+ ? rewrite.replaceFullPath
3520
+ : rewrite?.type === 'ReplacePrefixMatch'
3521
+ ? rewrite.replacePrefixMatch
3522
+ : match;
3523
+ const raw = capture(
3524
+ `kubectl exec -n ${namespace} ${probePod} -- wget -S -O /dev/null -T 5 ` +
3525
+ `--header 'Host: ${host}' http://${backend.name}:${backend.port}${target} 2>&1 || true`,
3526
+ );
3527
+ const code = /HTTP\/[0-9.]+\s+([0-9]{3})/.exec(raw)?.[1];
3528
+ const failure = raw.split('\n').find((line) => line.includes('wget:'));
3529
+ backendProbes.push({
3530
+ request: `${host}${match}`,
3531
+ backend: `${backend.name}:${backend.port}${target}`,
3532
+ direct: code || failure?.trim() || 'no-response',
3533
+ gateway: failing.find((entry) => entry.host === host)?.http,
3534
+ });
3535
+ }
3536
+ }
3537
+ logger.info('[gateway-status] Backend probe (same Host and path, bypassing Envoy)', { backendProbes });
3538
+ // Only the rule the browser actually hit is comparable, so judge on
3539
+ // the root rule rather than on every rule of the route.
3540
+ const rootProbes = backendProbes.filter((entry) => entry.request.endsWith('/'));
3541
+ const appFault = rootProbes.filter((entry) => `${entry.direct}` === `${entry.gateway}`);
3542
+ const gatewayFault = rootProbes.filter(
3543
+ (entry) => /^[0-9]{3}$/.test(entry.direct) && `${entry.direct}` !== `${entry.gateway}`,
3544
+ );
3545
+ if (gatewayFault.length > 0) {
3546
+ logger.warn(
3547
+ '[gateway-status] These backends answer differently without Envoy in the path, so the gateway is ' +
3548
+ 'not relaying the workload response — the routing layer is the fault:\n ' +
3549
+ gatewayFault
3550
+ .map((entry) => `${entry.request} -> backend ${entry.direct}, gateway ${entry.gateway}`)
3551
+ .join('\n '),
3552
+ );
3553
+ // The access log is the only artifact that says *why*. Its
3554
+ // response_flags column separates an Envoy local reply (NR no route,
3555
+ // UF upstream connect failure, UH no healthy upstream, DPE protocol
3556
+ // error) from a relayed upstream status, which no amount of probing
3557
+ // from outside can distinguish.
3558
+ // Selector, not owning-gateway labels: under `mergeGateways` the data
3559
+ // plane belongs to the GatewayClass, so per-Gateway labels are absent
3560
+ // and a selector built from them silently matches nothing.
3561
+ const dataPlaneLog = capture(
3562
+ `kubectl logs -n ${dataPlaneNamespace} ` +
3563
+ `-l app.kubernetes.io/name=envoy,app.kubernetes.io/component=proxy ` +
3564
+ `-c envoy --tail=60 --prefix 2>/dev/null | tail -60`,
3565
+ );
3566
+ logger.info(
3567
+ '[gateway-status] Data plane access log (response_flags names the reason)\n' +
3568
+ (dataPlaneLog || `(none — check: kubectl get pods -n ${dataPlaneNamespace} --show-labels)`),
3569
+ );
3570
+ // A translation the control plane rejected after admitting the
3571
+ // resource surfaces only here, never on the object's own status.
3572
+ const controlPlaneLog = capture(
3573
+ `kubectl logs -n ${dataPlaneNamespace} deployment/envoy-gateway --tail=40 2>/dev/null ` +
3574
+ `| grep -Ei 'error|warn|reject|invalid' | tail -20`,
3575
+ );
3576
+ logger.info('[gateway-status] Control plane errors\n' + (controlPlaneLog || '(none in the last 40 lines)'));
3577
+ }
3578
+ if (appFault.length > 0)
3579
+ logger.warn(
3580
+ '[gateway-status] These backends return the same code without Envoy in the path, so the gateway is ' +
3581
+ 'relaying an application response — the routing layer is not the fault:\n ' +
3582
+ appFault.map((entry) => `${entry.request} -> ${entry.direct}`).join('\n ') +
3583
+ `\n Read the workload log: kubectl logs -n ${namespace} <pod>`,
3584
+ );
3585
+ }
3586
+ }
3587
+
3588
+ const unreachable = probes.filter((entry) => !answered(entry.loopback));
3589
+ if (unreachable.length > 0) {
3590
+ // Narrow it here rather than leaving the operator with a bare refusal.
3591
+ // A socket bound on 0.0.0.0 that resets a loopback connection is either
3592
+ // not in this network namespace, or something is rejecting the packet.
3593
+ const listenerPid = /pid=(\d+)/.exec(hostListeners)?.[1];
3594
+ const hostNetns = capture(`sudo readlink /proc/1/ns/net`);
3595
+ const listenerNetns = listenerPid ? capture(`sudo readlink /proc/${listenerPid}/ns/net`) : '';
3596
+ // Anything that can answer a SYN with ICMP port-unreachable, from either
3597
+ // rule engine. firewalld on RHEL 9 keeps its rules in its own `inet
3598
+ // firewalld` nftables table, which `iptables-save` cannot see at all —
3599
+ // grepping only iptables hides half the packet path.
3600
+ const rejectRules = capture(
3601
+ `sudo iptables-save 2>/dev/null | grep -E '(REJECT|DROP)' | grep -E '(dport|dpt:) ?(443|https)\\b' | head -20`,
3602
+ );
3603
+ const nftRules = capture(
3604
+ `sudo nft list ruleset 2>/dev/null | grep -nE '(dport|ports) .*(443|https)|reject' | head -20`,
3605
+ );
3606
+ // A hostPort claim outranks any process listening on the node: the CNI
3607
+ // plugin DNATs the port to the claiming pod first, so the packet never
3608
+ // reaches the gateway. This is the one failure that leaves every other
3609
+ // signal green.
3610
+ const hostPortHijack = capture(
3611
+ `sudo nft list ruleset 2>/dev/null | grep -E 'dport (80|443)\\b.*dnat to' | head -5`,
3612
+ );
3613
+ const hostPortClaims = capture(
3614
+ `kubectl get daemonset,deployment -A ` +
3615
+ `-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}={.spec.template.spec.containers[*].ports[*].hostPort}{"\\n"}{end}'`,
3616
+ )
3617
+ .split('\n')
3618
+ .filter((line) => /=.*\b(80|443)\b/.test(line));
3619
+ logger.warn(`[gateway-status] ${unreachable.length}/${probes.length} hostnames did not answer over HTTPS`, {
3620
+ listenerPid: listenerPid || '(unknown)',
3621
+ hostNetns: hostNetns || '(unknown)',
3622
+ listenerNetns: listenerNetns || '(unknown)',
3623
+ sameNetworkNamespace: !!hostNetns && hostNetns === listenerNetns,
3624
+ nodeIpReachable: probes.some((entry) => answered(entry.nodeIp)),
3625
+ endpointlessServices: capture(
3626
+ `kubectl get svc -A -o jsonpath=` + `'{range .items[*]}{.metadata.namespace}/{.metadata.name} {end}'`,
3627
+ )
3628
+ .split(/\s+/)
3629
+ .filter(Boolean)
3630
+ .filter((ref) => {
3631
+ const [ns, name] = ref.split('/');
3632
+ return !capture(
3633
+ `kubectl get endpointslice -n ${ns} -l kubernetes.io/service-name=${name} ` +
3634
+ `-o jsonpath='{.items[*].endpoints[*].addresses[*]}'`,
3635
+ ).trim();
3636
+ }),
3637
+ httpReachable: probes.some((entry) => answered(entry.http)),
3638
+ hostPortHijack: hostPortHijack || '(none)',
3639
+ hostPortClaims: hostPortClaims.length > 0 ? hostPortClaims : '(none)',
3640
+ rejectRules: rejectRules || '(none matching 443 in iptables)',
3641
+ nftRules: nftRules || '(none matching 443 in nftables)',
3642
+ });
3643
+ logger.warn(
3644
+ '[gateway-status] Read it as: hostPortHijack non-empty → another workload reserved host port ' +
3645
+ '80/443 and the CNI DNATs those ports to it before the gateway can see them; hostPortClaims ' +
3646
+ 'names the owner, and only one ingress stack can hold them. httpReachable=true → the same Envoy ' +
3647
+ 'answers on 80, so only 443 is rejected. sameNetworkNamespace=false → the listener is not on ' +
3648
+ 'this host despite hostNetwork. endpointlessServices naming a Service that publishes 80/443 → ' +
3649
+ 'kube-proxy REJECTs those ports on its behalf; delete it.',
2188
3650
  );
2189
3651
  }
3652
+
3653
+ return { programmed: programmed?.code === 0, servesHttps, dataPlane, services, hostListeners, probes };
2190
3654
  },
2191
3655
  /**
2192
3656
  * @method deploy
@@ -2197,11 +3661,11 @@ EOF`);
2197
3661
  */
2198
3662
  deploy: async (path, options = DEFAULT_OPTION) => {
2199
3663
  const deployId = path;
3664
+ const env = options.dev ? 'development' : 'production';
2200
3665
  const { validVersion } = Underpost.repo.privateConfUpdate(deployId);
2201
3666
  if (!validVersion) throw new Error('Version mismatch');
2202
- const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
3667
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env });
2203
3668
  const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
2204
- const env = options.dev ? 'development' : 'production';
2205
3669
  const ignorePods = Underpost.kubectl
2206
3670
  .get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace)
2207
3671
  .map((p) => p.NAME);
@@ -2388,7 +3852,11 @@ EOF`);
2388
3852
  }
2389
3853
  const success = await Underpost.test.statusMonitor(podToMonitor);
2390
3854
  if (success) {
2391
- const versions = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
3855
+ const versions =
3856
+ Underpost.deploy.getCurrentTraffic(deployId, {
3857
+ namespace: options.namespace,
3858
+ env: options.dev ? 'development' : 'production',
3859
+ }) || 'blue';
2392
3860
  if (!node) node = os.hostname();
2393
3861
  const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
2394
3862
  shellExec(
@@ -2419,10 +3887,7 @@ EOF`);
2419
3887
  */
2420
3888
  'etc-hosts': async (path = '', options = DEFAULT_OPTION) => {
2421
3889
  const hosts = path ? path.split(',') : [];
2422
- if (options.deployId) {
2423
- const confServer = loadConfServerJson(`./engine-private/conf/${options.deployId}/conf.server.json`);
2424
- hosts.push(...Object.keys(confServer));
2425
- }
3890
+ if (options.deployId) hosts.push(...deployHostsFactory(options.deployId));
2426
3891
  const hostListenResult = etcHostFactory(hosts);
2427
3892
  logger.info(hostListenResult.renderHosts);
2428
3893
  },
@@ -2775,31 +4240,314 @@ EOF`);
2775
4240
  * @memberof UnderpostRun
2776
4241
  */
2777
4242
  'generate-pass': (path, options = DEFAULT_OPTION) => {
2778
- const length = path && parseInt(path) > 0 ? parseInt(path) : 16;
2779
- const lower = 'abcdefghijklmnopqrstuvwxyz';
2780
- const upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
2781
- const digits = '0123456789';
2782
- const special = '@#$%^&*()_+';
2783
- const all = lower + upper + digits + special;
2784
- const buf = crypto.randomBytes(length + 4);
2785
- // Guarantee at least one character from each required class
2786
- const chars = [
2787
- lower[buf[0] % lower.length],
2788
- upper[buf[1] % upper.length],
2789
- digits[buf[2] % digits.length],
2790
- special[buf[3] % special.length],
2791
- ];
2792
- for (let i = 4; i < length; i++) chars.push(all[buf[i] % all.length]);
2793
- // Fisher-Yates shuffle using an independent random buffer
2794
- const shuf = crypto.randomBytes(length);
2795
- for (let i = chars.length - 1; i > 0; i--) {
2796
- const j = shuf[i % shuf.length] % (i + 1);
2797
- [chars[i], chars[j]] = [chars[j], chars[i]];
2798
- }
2799
- const password = chars.join('');
4243
+ const password = generateSecurePassword(path && parseInt(path) > 0 ? parseInt(path) : 16);
2800
4244
  if (options.copy) pbcopy(password);
2801
4245
  else console.log(password);
2802
4246
  },
4247
+ /**
4248
+ * @method sops-setup
4249
+ * @description End-to-end SOPS/Age onboarding for a host: installs tooling, generates the Age
4250
+ * keypair and creation rules, pins the key path for non-interactive runs, encrypts the
4251
+ * requested Secrets into the Git-tracked store, then validates and applies them.
4252
+ *
4253
+ * Every step is idempotent and re-runnable. Notably it delegates key generation to
4254
+ * `secret sops --init` rather than calling `age-keygen` directly: a bare `age-keygen -o`
4255
+ * overwrites an existing key, which would orphan every manifest already encrypted to the
4256
+ * previous recipient with no way to recover them.
4257
+ *
4258
+ * On a host that pulled a store created elsewhere, the freshly generated key is not a recipient
4259
+ * of the inherited manifests. `init()` registers this host in the creation rules so what it
4260
+ * encrypts from here on stays readable, but existing manifests can only be re-keyed from a host
4261
+ * that still holds a decrypting key. That case is reported per secret and then raised by the
4262
+ * apply pre-flight with the available remedies, rather than surfacing as a sops decrypt error.
4263
+ *
4264
+ * Onboards the whole self-hosted data tier by default — PostgreSQL, MariaDB, and MongoDB
4265
+ * (`postgres-secret`, `mariadb-secret`, `mongodb-secret`, `mongodb-keyfile`). The MongoDB
4266
+ * keyfile is included because the StatefulSet mounts it for intra-replica-set auth and will
4267
+ * not start without it. Pass an explicit comma-separated list to narrow the set.
4268
+ *
4269
+ * Secret values are resolved per data key, in order:
4270
+ * 1. the origin seed file, when one exists (`engine-private/postgresql-password`) — this is
4271
+ * the real onboarding path, carrying the credential the cluster already runs on;
4272
+ * 2. `--args` as `key=value` pairs, for a value supplied by the operator;
4273
+ * 3. a freshly generated value: a base64 keyfile for `mongodb-keyfile`, `admin` for a
4274
+ * `username`, otherwise a 24-character secure password.
4275
+ *
4276
+ * Plaintext manifests are written by Node under `/dev/shm` at mode 600 and shredded by
4277
+ * `encrypt()`. They are never emitted through a shell heredoc, which would place the
4278
+ * credential in the command string and therefore in the process table and the command log.
4279
+ *
4280
+ * Usage:
4281
+ * underpost run sops-setup # postgres + mariadb + mongo
4282
+ * underpost run sops-setup mongodb-secret,mongodb-keyfile --namespace prod
4283
+ * underpost run sops-setup postgres-secret --args "password=s3cr3t"
4284
+ * underpost run sops-setup --dry-run # stop before mutating cluster
4285
+ * underpost run sops-setup --force # replace stored manifests
4286
+ * @param {string} path - Comma-separated Secret names to onboard. Defaults to the full data
4287
+ * tier: postgres-secret, mariadb-secret, mongodb-secret, mongodb-keyfile.
4288
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
4289
+ * @param {string} options.namespace - Target namespace for the store and the apply (default: 'default').
4290
+ * @param {string} options.args - Comma-separated `key=value` overrides for Secret data keys.
4291
+ * @param {boolean} options.dryRun - Validate and server-dry-run only; never apply.
4292
+ * @param {boolean} options.force - Replace encrypted manifests that already exist.
4293
+ * @memberof UnderpostRun
4294
+ */
4295
+ 'sops-setup': (path = '', options = DEFAULT_OPTION) => {
4296
+ const namespace = options.namespace || 'default';
4297
+ const secretNames = (path || SOPS_SETUP_DEFAULT_SECRETS.join(','))
4298
+ .split(',')
4299
+ .map((name) => name.trim())
4300
+ .filter(Boolean);
4301
+
4302
+ // `--args key=value,key2=value2` overrides, applied to any secret that declares that key.
4303
+ const overrides = `${options.args || ''}`.split(',').reduce((acc, pair) => {
4304
+ const separator = pair.indexOf('=');
4305
+ if (separator > 0) acc[pair.slice(0, separator).trim()] = pair.slice(separator + 1).trim();
4306
+ return acc;
4307
+ }, {});
4308
+
4309
+ logger.info('sops-setup', { secretNames, namespace, dryRun: !!options.dryRun, force: !!options.force });
4310
+
4311
+ // 1. Host tooling, then keypair + creation rules. Both no-op when already present.
4312
+ Underpost.secret.sops.installTooling();
4313
+ Underpost.secret.sops.init();
4314
+
4315
+ // 2. Pin the resolved key path for non-interactive runs (systemd units, CronJobs, sudo).
4316
+ // Written with the concrete path rather than a guessed default, because `sudo` resets
4317
+ // HOME and a wrong guess surfaces later as an opaque decrypt failure.
4318
+ const keyFile = Underpost.secret.sops.keyFile();
4319
+ shellExec(
4320
+ `sudo tee /etc/profile.d/underpost-sops.sh >/dev/null <<'UNDERPOST_SOPS_ENV_EOF'
4321
+ export SOPS_AGE_KEY_FILE="\${SOPS_AGE_KEY_FILE:-${keyFile}}"
4322
+ UNDERPOST_SOPS_ENV_EOF`,
4323
+ );
4324
+ shellExec(`sudo chmod 644 /etc/profile.d/underpost-sops.sh`);
4325
+
4326
+ // 3. Build and encrypt each requested Secret.
4327
+ const stageDir = '/dev/shm/underpost-secrets';
4328
+ const held = Underpost.secret.sops.localRecipients();
4329
+ fs.ensureDirSync(stageDir);
4330
+ fs.chmodSync(stageDir, 0o700);
4331
+ try {
4332
+ for (const name of secretNames) {
4333
+ const stored = Underpost.secret.sops.has(name, namespace);
4334
+ if (stored && !options.force) {
4335
+ // A stored manifest this host cannot open is present but unusable here, so reporting it
4336
+ // as onboarded would send the operator on to an apply that is guaranteed to fail.
4337
+ if (Underpost.secret.sops.decryptable(Underpost.secret.sops.manifestPath(name, namespace), held))
4338
+ logger.info(`${name} is already onboarded in ns/${namespace}; skipping (use --force to replace)`);
4339
+ else
4340
+ logger.warn(
4341
+ `${name} is stored in ns/${namespace} but is sealed to an Age recipient this host does not hold; ` +
4342
+ `skipping. Adopt the store's key, re-key it from a host that holds one, or re-onboard from the ` +
4343
+ `origin seed files with --force.`,
4344
+ );
4345
+ continue;
4346
+ }
4347
+
4348
+ // Data keys come from the secret's origin seed contract, so an onboarded manifest
4349
+ // carries exactly the keys the workload's secretKeyRef already expects.
4350
+ const seedSources = Underpost.secret.sops.seedSources(name);
4351
+ const dataKeys = Object.keys(seedSources).length > 0 ? Object.keys(seedSources) : ['password'];
4352
+ const stringData = {};
4353
+ for (const key of dataKeys) {
4354
+ const seedPath = seedSources[key];
4355
+ if (seedPath && fs.existsSync(seedPath)) {
4356
+ stringData[key] = fs.readFileSync(seedPath, 'utf8').trim();
4357
+ logger.info(`${name}.${key} seeded from ${seedPath}`);
4358
+ } else if (overrides[key] !== undefined) {
4359
+ stringData[key] = overrides[key];
4360
+ logger.info(`${name}.${key} taken from --args`);
4361
+ } else {
4362
+ stringData[key] = generateSeedValue(key);
4363
+ // Replacing a stored manifest with a value nothing seeded means the credential the
4364
+ // running datastore still authenticates against is being thrown away.
4365
+ if (stored)
4366
+ logger.warn(
4367
+ `${name}.${key} generated while replacing the stored manifest — no seed file at ` +
4368
+ `${seedPath || '(unmapped)'} and no --args override. The running datastore keeps its old ` +
4369
+ `credential until this value is applied to it; pass --args "${key}=<value>" to keep the ` +
4370
+ `existing one.`,
4371
+ );
4372
+ else logger.info(`${name}.${key} generated`);
4373
+ }
4374
+ }
4375
+
4376
+ const stagePath = `${stageDir}/${name}.yaml`;
4377
+ fs.outputFileSync(
4378
+ stagePath,
4379
+ [
4380
+ 'apiVersion: v1',
4381
+ 'kind: Secret',
4382
+ 'metadata:',
4383
+ ` name: ${name}`,
4384
+ ` namespace: ${namespace}`,
4385
+ ' labels:',
4386
+ ' app.kubernetes.io/managed-by: underpost',
4387
+ 'type: Opaque',
4388
+ 'stringData:',
4389
+ // Single-quoted YAML scalars with doubled internal quotes: values are generated or
4390
+ // operator-supplied and may contain characters YAML would otherwise interpret.
4391
+ ...Object.entries(stringData).map(([key, value]) => ` ${key}: '${`${value}`.replace(/'/g, "''")}'`),
4392
+ '',
4393
+ ].join('\n'),
4394
+ 'utf8',
4395
+ );
4396
+ fs.chmodSync(stagePath, 0o600);
4397
+ // encrypt() stages, validates, moves into place, and shreds the plaintext source.
4398
+ Underpost.secret.sops.encrypt(stagePath, namespace, options);
4399
+ }
4400
+ } finally {
4401
+ // Defense in depth: encrypt() shreds each source, but a throw mid-loop must not leave a
4402
+ // plaintext manifest sitting in shared memory.
4403
+ fs.removeSync(stageDir);
4404
+ }
4405
+
4406
+ Underpost.secret.sops.list();
4407
+
4408
+ // 4. Validate every manifest in the namespace, then apply unless this is a dry run.
4409
+ Underpost.secret.sops.apply(namespace, { dryRun: true });
4410
+ if (options.dryRun) return logger.info('--dry-run: validated only, cluster left unchanged');
4411
+ Underpost.secret.sops.apply(namespace);
4412
+ },
4413
+ /**
4414
+ * @method sops-status
4415
+ * @description Reports the live state of the SOPS/Age secret system: host tooling, the Age
4416
+ * key and its recipient, the committed creation rules, every stored manifest with whether the
4417
+ * local key can open it and whether the cluster still matches, and which managed Secrets are
4418
+ * onboarded versus still seeding from their origin path.
4419
+ *
4420
+ * Read-only and safe to run anywhere. Decryption happens only for the drift check, only for
4421
+ * manifests the local key is a recipient of, and only into `kubectl diff` with its output
4422
+ * discarded — no secret value is ever printed or written to disk.
4423
+ *
4424
+ * Usage:
4425
+ * underpost run sops-status # every managed key, ns default
4426
+ * underpost run sops-status mongo # partial match: both mongo keys
4427
+ * underpost run sops-status --namespace prod # every managed key in ns prod
4428
+ * @param {string} path - Comma-separated managed Secret keys to report on; empty reports all.
4429
+ * Matched as case-insensitive substrings (`mongo` selects mongodb-secret and mongodb-keyfile).
4430
+ * Filters both the stored-manifest listing and the coverage table.
4431
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
4432
+ * @param {string} options.namespace - Namespace to inspect (DEFAULT_OPTION scheme, default 'default').
4433
+ * @memberof UnderpostRun
4434
+ */
4435
+ 'sops-status': (path = '', options = DEFAULT_OPTION) => {
4436
+ const sops = Underpost.secret.sops;
4437
+ // `--namespace` selects the namespace (DEFAULT_OPTION scheme); `path` narrows which managed
4438
+ // Secret keys to report on, so the two axes stay independent.
4439
+ const namespace = options.namespace || 'default';
4440
+ const manageSecretKeyFilter = path
4441
+ .split(',')
4442
+ .map((key) => key.trim().toLowerCase())
4443
+ .filter(Boolean);
4444
+ // Partial, case-insensitive substring match, so `mongo` reaches both `mongodb-secret` and
4445
+ // `mongodb-keyfile` without having to spell either out.
4446
+ const matchesKeyFilter = (name) =>
4447
+ manageSecretKeyFilter.length === 0 || manageSecretKeyFilter.some((key) => name.toLowerCase().includes(key));
4448
+ const mark = (ok) => (ok ? 'yes' : 'no');
4449
+
4450
+ // ── Tooling ────────────────────────────────────────────────────────────
4451
+ const version = (bin, flag) =>
4452
+ sops.hasBinary(bin)
4453
+ ? shellExec(`${bin} ${flag} 2>/dev/null | head -1`, { stdout: true, silent: true, disableLog: true }).trim()
4454
+ : '(not installed)';
4455
+ logger.info(
4456
+ '[sops-status] Tooling\n' +
4457
+ ` sops ${version('sops', '--version')}\n` +
4458
+ ` age ${version('age', '--version')}\n` +
4459
+ ` age-keygen ${sops.hasBinary('age-keygen') ? 'installed' : '(not installed)'}`,
4460
+ );
4461
+
4462
+ // ── Age key ────────────────────────────────────────────────────────────
4463
+ const keyFile = sops.keyFile();
4464
+ const keyExists = fs.existsSync(keyFile);
4465
+ // A key file may hold several identities — that is how a host joins a store it did not
4466
+ // create — so every check below works against the whole held set, not one recipient.
4467
+ const held = sops.localRecipients();
4468
+ const keyMode = keyExists ? (fs.statSync(keyFile).mode & 0o777).toString(8) : '';
4469
+ logger.info(
4470
+ '[sops-status] Age key\n' +
4471
+ ` path ${keyFile}\n` +
4472
+ ` present ${mark(keyExists)}${keyExists ? ` (mode ${keyMode}${keyMode === '600' || keyMode === '400' ? '' : ' — INSECURE, run chmod 600'})` : ''}\n` +
4473
+ ` recipients ${held.join(', ') || (keyExists ? '(none — unreadable key file)' : '(none)')}` +
4474
+ (keyExists ? '' : `\n searched ${sops.keyFileCandidates().join(', ')}`),
4475
+ );
4476
+
4477
+ // ── Creation rules ─────────────────────────────────────────────────────
4478
+ const confPath = './engine-private/secrets/.sops.yaml';
4479
+ const ruleRecipients = sops.creationRecipients();
4480
+ logger.info(
4481
+ '[sops-status] Creation rules\n' +
4482
+ ` config ${confPath} ${fs.existsSync(confPath) ? '' : '(missing — run: underpost secret sops --init)'}\n` +
4483
+ ` recipients ${ruleRecipients.length > 0 ? ruleRecipients.join(', ') : '(none)'}\n` +
4484
+ ` local key listed ${mark(held.some((recipient) => ruleRecipients.includes(recipient)))}`,
4485
+ );
4486
+
4487
+ // ── Stored manifests ───────────────────────────────────────────────────
4488
+ const manifests = sops.manifests(namespace).filter((manifest) => matchesKeyFilter(manifest.name));
4489
+ const onboarded = new Set();
4490
+ if (manifests.length === 0)
4491
+ logger.warn(
4492
+ `[sops-status] Store\n no encrypted manifests in ns/${namespace}` +
4493
+ (manageSecretKeyFilter.length > 0 ? ` matching ${manageSecretKeyFilter.join(', ')}` : ''),
4494
+ );
4495
+ else {
4496
+ const rows = manifests.map((manifest) => {
4497
+ onboarded.add(manifest.name);
4498
+ const recipients = sops.manifestRecipients(manifest.path);
4499
+ const decryptable = sops.decryptable(manifest.path, held);
4500
+ const live = shellExec(
4501
+ `kubectl get secret ${manifest.name} -n ${manifest.namespace} --ignore-not-found -o name 2>/dev/null || true`,
4502
+ { stdout: true, silent: true, silentOnError: true, disableLog: true },
4503
+ ).trim();
4504
+ // Drift is decided by kubectl's exit code; its stdout would contain the decrypted
4505
+ // values, so it is discarded rather than captured.
4506
+ let sync = 'n/a';
4507
+ if (live && decryptable) {
4508
+ const result = shellExec(
4509
+ `bash -c 'set -o pipefail; SOPS_AGE_KEY_FILE="${keyFile}" sops --decrypt "${manifest.path}" ` +
4510
+ `| kubectl diff -f - -n "${manifest.namespace}" >/dev/null 2>&1'`,
4511
+ { silentOnError: true, disableLog: true, stdout: false },
4512
+ );
4513
+ sync = result.code === 0 ? 'in-sync' : result.code === 1 ? 'DRIFT' : 'error';
4514
+ } else if (!live) sync = 'not applied';
4515
+ else if (!decryptable) sync = 'no local key';
4516
+ return (
4517
+ ` ${`${manifest.namespace}/${manifest.name}`.padEnd(34)} ` +
4518
+ `recipients=${String(recipients.length).padEnd(3)} ` +
4519
+ `decryptable=${mark(decryptable).padEnd(4)} ` +
4520
+ `live=${mark(!!live).padEnd(4)} ` +
4521
+ `${sync}`
4522
+ );
4523
+ });
4524
+ logger.info(`[sops-status] Store — ns/${namespace} (${manifests.length} manifest(s))\n` + rows.join('\n'));
4525
+ }
4526
+
4527
+ // ── Coverage ───────────────────────────────────────────────────────────
4528
+ const coverage = sops
4529
+ .managedSecrets()
4530
+ .filter(matchesKeyFilter)
4531
+ .map((name) => {
4532
+ const seeds = Object.values(sops.seedSources(name));
4533
+ const seedPresent = seeds.length > 0 && seeds.every((seed) => fs.existsSync(seed));
4534
+ const source = onboarded.has(name)
4535
+ ? 'sops'
4536
+ : seedPresent
4537
+ ? 'origin seed'
4538
+ : seeds.length
4539
+ ? 'MISSING'
4540
+ : 'unmapped';
4541
+ return ` ${name.padEnd(24)} ${source.padEnd(12)} ${seeds.length ? `seed=${mark(seedPresent)}` : ''}`;
4542
+ });
4543
+ if (coverage.length === 0)
4544
+ logger.warn(
4545
+ `[sops-status] Coverage\n no managed Secret matches ${manageSecretKeyFilter.join(', ')}\n` +
4546
+ ` known keys: ${sops.managedSecrets().join(', ')}`,
4547
+ );
4548
+ else
4549
+ logger.info('[sops-status] Coverage (which source each managed Secret deploys from)\n' + coverage.join('\n'));
4550
+ },
2803
4551
  /**
2804
4552
  * @method secret
2805
4553
  * @description Creates an Underpost secret named 'underpost' from a file, defaulting to `/home/dd/engine/engine-private/conf/dd-cron/.env.production` if no path is provided.
@@ -2920,7 +4668,7 @@ EOF`);
2920
4668
 
2921
4669
  const envs = Underpost.env.list();
2922
4670
 
2923
- const cmd = `kubectl apply -f - <<EOF
4671
+ const cmd = `kubectl apply -f - <<'EOF'
2924
4672
  apiVersion: ${apiVersion}
2925
4673
  kind: ${kindType}
2926
4674
  metadata:
@@ -3165,6 +4913,26 @@ EOF`;
3165
4913
  }
3166
4914
  },
3167
4915
 
4916
+ /**
4917
+ * @method kubeadm-wireguard
4918
+ * @description
4919
+ * Configures Calico to keep using the Kubernetes NodeInternalIP after WireGuard
4920
+ * is added to the node. This prevents the WireGuard interface (for example
4921
+ * 10.0.0.2) from being selected as the Calico/BGP node address.
4922
+ *
4923
+ * @param {string} path - Unused.
4924
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options.
4925
+ * @memberof UnderpostRun
4926
+ */
4927
+ 'kubeadm-wireguard': (path, options = DEFAULT_OPTION) => {
4928
+ shellExec(`kubectl patch installation.operator.tigera.io default --type='json' \
4929
+ -p='[
4930
+ {"op":"replace","path":"/spec/calicoNetwork/nodeAddressAutodetectionV4","value":{"kubernetes":"NodeInternalIP"}}
4931
+ ]'`);
4932
+
4933
+ shellExec(`kubectl rollout restart daemonset/calico-node -n calico-system`);
4934
+ },
4935
+
3168
4936
  /**
3169
4937
  * @method build-cluster-deployment-manifests
3170
4938
  * @description Builds deployment manifests for both production and development environments using `node bin deploy --build-manifest`, syncing them, and setting replicas to 1 for the `dd` deployment.