underpost 3.2.70 → 3.2.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/.github/workflows/publish.ci.yml +3 -3
  2. package/.github/workflows/release.cd.yml +1 -1
  3. package/CHANGELOG.md +1358 -1038
  4. package/CLI-HELP.md +39 -16
  5. package/README.md +3 -3
  6. package/bin/build.js +10 -4
  7. package/bin/deploy.js +18 -16
  8. package/docker-compose.yml +1 -1
  9. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
  10. package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
  11. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  12. package/manifests/deployment/playwright/deployment.yaml +1 -1
  13. package/manifests/mongodb/kustomization.yaml +4 -1
  14. package/manifests/mongodb/statefulset.yaml +4 -0
  15. package/manifests/mongodb/storage-class.yaml +9 -2
  16. package/package.json +20 -20
  17. package/scripts/nat-iptables.sh +10 -4
  18. package/scripts/test-monitor.sh +4 -3
  19. package/src/api/core/core.controller.js +4 -65
  20. package/src/api/core/core.router.js +8 -14
  21. package/src/api/default/default.controller.js +2 -70
  22. package/src/api/default/default.router.js +7 -17
  23. package/src/api/document/document.controller.js +5 -77
  24. package/src/api/document/document.router.js +9 -13
  25. package/src/api/file/file.controller.js +9 -53
  26. package/src/api/file/file.router.js +14 -6
  27. package/src/api/test/test.controller.js +8 -53
  28. package/src/api/test/test.router.js +1 -4
  29. package/src/cli/cluster.js +771 -66
  30. package/src/cli/db.js +6 -4
  31. package/src/cli/deploy.js +1715 -168
  32. package/src/cli/docker-compose.js +19 -24
  33. package/src/cli/fs.js +0 -1
  34. package/src/cli/image.js +40 -13
  35. package/src/cli/index.js +129 -35
  36. package/src/cli/ipfs.js +82 -11
  37. package/src/cli/monitor.js +1 -1
  38. package/src/cli/release.js +4 -0
  39. package/src/cli/repository.js +14 -3
  40. package/src/cli/run.js +2253 -439
  41. package/src/cli/secrets.js +969 -0
  42. package/src/cli/ssh.js +38 -39
  43. package/src/client/components/core/Modal.js +38 -4
  44. package/src/client-builder/client-build.js +94 -11
  45. package/src/client-builder/ssr.js +27 -73
  46. package/src/db/mongo/MongoBootstrap.js +295 -54
  47. package/src/db/mongo/MongooseDB.js +47 -32
  48. package/src/index.js +1 -1
  49. package/src/server/conf.js +1307 -6
  50. package/src/server/cri.js +70 -0
  51. package/src/server/downloader.js +3 -3
  52. package/src/server/middlewares.js +152 -0
  53. package/src/server/underpost-gateway.js +1073 -0
  54. package/src/server/underpost-ingress.js +364 -0
  55. package/test/cluster-instances.test.js +435 -0
  56. package/test/deploy-node-placement.test.js +45 -0
  57. package/test/instance-traffic-plan.test.js +710 -0
  58. package/test/sops-secret-store.test.js +612 -0
  59. package/test/underpost-gateway.test.js +469 -0
  60. package/test/underpost-ingress.test.js +253 -0
package/src/cli/run.js CHANGED
@@ -5,48 +5,72 @@
5
5
  */
6
6
 
7
7
  import { daemonProcess, getTerminalPid, pbcopy, shellCd, shellExec } from '../server/process.js';
8
- import crypto from 'crypto';
9
8
  import {
10
9
  awaitDeployMonitor,
11
10
  buildKindPorts,
12
- Config,
11
+ clusterContextFactory,
12
+ clusterTypeFactory,
13
13
  cronDeployIdResolve,
14
+ dispatchBuildInstanceEnv,
15
+ deployHostsFactory,
14
16
  etcHostFactory,
17
+ exposePartialMatchesFactory,
18
+ exposePathPartsFactory,
19
+ exposePortListFactory,
20
+ exposePortPlanFactory,
21
+ exposeTcpPortsFactory,
22
+ gatewayApiEnabledFactory,
23
+ generateSecurePassword,
15
24
  getNpmRootPath,
25
+ instanceHttpRouteRulesFactory,
26
+ instanceInterceptStatusesFactory,
27
+ instancePortFactory,
28
+ instanceProxyRoutesFactory,
29
+ instanceStatusPageEntriesFactory,
16
30
  isDeployRunnerContext,
31
+ loadConfInstances,
32
+ loadProjectInstanceEnvBuilder,
17
33
  loadConfServerJson,
18
34
  loadReplicas,
35
+ resolveDeployList,
36
+ resolveEnvScoped,
37
+ selectConfInstances,
38
+ waitForPort,
19
39
  writeEnv,
40
+ clusterInstancesFactory,
41
+ deployTrafficEntriesFactory,
42
+ curlStatusChainFactory,
43
+ hostIngressFactsFactory,
44
+ hostRenderInstancesFactory,
45
+ instanceTrafficPlanFactory,
46
+ trafficTableRowsFactory,
47
+ isTrafficServingFactory,
48
+ nextTrafficFactory,
49
+ stopPlanFactory,
50
+ trafficFromRoutingInfoFactory,
20
51
  } from '../server/conf.js';
21
52
  import { actionInitLog, loggerFactory } from '../server/logger.js';
22
53
 
23
54
  import fs from 'fs-extra';
24
- import net from 'net';
25
55
  import { range, s4, setPad, timer } from '../client/components/core/CommonJs.js';
26
56
 
27
57
  import os from 'os';
28
58
  import Underpost from '../index.js';
29
59
  import dotenv from 'dotenv';
30
60
  import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
31
-
32
- const waitForPort = (port, host = '127.0.0.1', { maxAttempts = 30, interval = 2000 } = {}) =>
33
- new Promise((resolve, reject) => {
34
- let attempts = 0;
35
- const tryConnect = () => {
36
- attempts++;
37
- const socket = net.createConnection({ port, host }, () => {
38
- socket.destroy();
39
- resolve();
40
- });
41
- socket.on('error', () => {
42
- socket.destroy();
43
- if (attempts >= maxAttempts) return reject(new Error(`Port ${port} not ready after ${maxAttempts} attempts`));
44
- setTimeout(tryConnect, interval);
45
- });
46
- };
47
- tryConnect();
48
- });
49
-
61
+ import {
62
+ UNDERPOST_GATEWAY,
63
+ assertStaticAssets,
64
+ gatewayFallbackProbeRunner,
65
+ hostServerConfFactory,
66
+ installGatewayConf,
67
+ instanceFallbackChecksFactory,
68
+ placeInstanceStaticAssets,
69
+ pwaFallbackChecksFactory,
70
+ readHostInstanceRegistry,
71
+ writeHostInstanceRegistry,
72
+ writeHostServerConf,
73
+ } from '../server/underpost-gateway.js';
50
74
  const logger = loggerFactory(import.meta);
51
75
 
52
76
  /**
@@ -57,8 +81,12 @@ const logger = loggerFactory(import.meta);
57
81
  * @property {boolean} dev - Whether to run in development mode.
58
82
  * @property {string} podName - The name of the pod to run.
59
83
  * @property {string} nodeName - The name of the node to run.
84
+ * @property {string} ingressNode - Dedicated node for the host-network public ingress; never inherited from nodeName.
60
85
  * @property {string} sshKeyPath - Private key path for node SSH operations, forwarded to volume shipping over SSH.
61
86
  * @property {number} port - Custom port to use.
87
+ * @property {string} exposeContainerPorts - Comma-separated Service/container destination ports.
88
+ * @property {string} exposeHostPorts - Comma-separated host listening ports.
89
+ * @property {boolean} localProxy - Start the development path proxy after exposing matched resources.
62
90
  * @property {string} volumeHostPath - The host path for the volume.
63
91
  * @property {string} volumeMountPath - The mount path for the volume.
64
92
  * @property {string} imageName - The name of the image to run.
@@ -73,6 +101,11 @@ const logger = loggerFactory(import.meta);
73
101
  * @property {boolean} force - Whether to force the operation.
74
102
  * @property {boolean} reset - Whether to reset the operation.
75
103
  * @property {boolean} tls - Whether to use TLS.
104
+ * @property {boolean} gatewayApi - Apply the Gateway API stack (Gateway + HTTPRoute) instead of the Contour HTTPProxy. Both manifest sets are always generated.
105
+ * @property {boolean} disableGatewayApi - Fall back to the Contour HTTPProxy stack in runners where the Gateway API is the default (`cluster`).
106
+ * @property {string} gatewayClass - GatewayClass name baked into generated Gateway manifests.
107
+ * @property {boolean} disableHttp3 - Omit QUIC/HTTP3 listener config and the Alt-Svc advertisement.
108
+ * @property {number} quicPort - UDP port advertised for QUIC/HTTP3.
76
109
  * @property {string} cmd - The command to run in the container.
77
110
  * @property {string} tty - The TTY option for the container.
78
111
  * @property {string} stdin - The stdin option for the container.
@@ -123,14 +156,31 @@ const logger = loggerFactory(import.meta);
123
156
  * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
124
157
  * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
125
158
  * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
159
+ * @property {string} hostAliases - Pod `/etc/hosts` entries, as semicolon-separated `ip=host1,host2` groups.
160
+ * @property {string} args - Comma-separated arguments forwarded to the runner's own command.
161
+ * @property {boolean} cert - Issue cert-manager certificates; set implicitly by `tls` in the promote workflow.
162
+ * @property {boolean} instanceOnly - Act on the one instance id given, without expanding its variant family.
163
+ * @property {string} labels - Comma-separated `key=value` pairs applied to the created resources.
164
+ * @property {string} npmRoot - Resolved npm global root, cached on the options once looked up.
165
+ * @property {object} on - Lifecycle hooks (`{ init }`) a programmatic caller supplies; unused from the CLI.
166
+ * @property {string} traffic - Blue/green colour to bake into generated manifests (default: `blue`).
167
+ * @property {boolean} gatewayBootstrapComplete - Internal marker: a parent orchestration already proved the static gateway fallback.
168
+ * @property {boolean} noBackendCheckpoint - Internal marker: this promote is the deliberate no-backend fallback checkpoint, so it must not wait for the target colour's endpoints.
169
+ * @property {Object<string,string>} targetTrafficById - Internal instance id → explicitly pre-routed traffic colour.
170
+ * @property {string} volumeType - hostPath volume type (`DirectoryOrCreate`, `FileOrCreate`, or `dev` for the latter).
171
+ * @property {string} branch - The Git branch to use for operations (e.g., for template-deploy, ssh-deploy).
126
172
  * @memberof UnderpostRun
127
173
  */
128
174
  const DEFAULT_OPTION = {
129
175
  dev: false,
130
176
  podName: '',
131
177
  nodeName: '',
178
+ ingressNode: '',
132
179
  sshKeyPath: '',
133
180
  port: 0,
181
+ exposeContainerPorts: '',
182
+ exposeHostPorts: '',
183
+ localProxy: false,
134
184
  volumeHostPath: '',
135
185
  volumeMountPath: '',
136
186
  imageName: '',
@@ -145,6 +195,11 @@ const DEFAULT_OPTION = {
145
195
  force: false,
146
196
  reset: false,
147
197
  tls: false,
198
+ gatewayApi: false,
199
+ disableGatewayApi: false,
200
+ gatewayClass: '',
201
+ disableHttp3: false,
202
+ quicPort: 0,
148
203
  cmd: '',
149
204
  tty: '',
150
205
  stdin: '',
@@ -193,6 +248,18 @@ const DEFAULT_OPTION = {
193
248
  pullBundle: false,
194
249
  remove: false,
195
250
  test: false,
251
+ branch: '',
252
+ args: '',
253
+ cert: false,
254
+ instanceOnly: false,
255
+ gatewayBootstrapComplete: false,
256
+ noBackendCheckpoint: false,
257
+ targetTrafficById: {},
258
+ labels: '',
259
+ npmRoot: '',
260
+ on: undefined,
261
+ traffic: '',
262
+ volumeType: '',
196
263
  };
197
264
 
198
265
  /**
@@ -204,6 +271,34 @@ const DEFAULT_OPTION = {
204
271
  * runners for executing specific commands.
205
272
  * @memberof UnderpostRun
206
273
  */
274
+
275
+ // Secrets `sops-setup` onboards when no explicit list is passed: the full self-hosted data tier.
276
+ // `mongodb-keyfile` is listed alongside `mongodb-secret` because the MongoDB StatefulSet mounts
277
+ // it as a volume for intra-replica-set auth and will not start without it, so onboarding the
278
+ // credentials alone would leave Mongo broken.
279
+ const SOPS_SETUP_DEFAULT_SECRETS = ['postgres-secret', 'mariadb-secret', 'mongodb-secret', 'mongodb-keyfile'];
280
+
281
+ /**
282
+ * Produces a value for a Secret data key that has no origin seed file and no `--args` override.
283
+ * Key-aware because the data tier does not want one shape of secret: a replica-set keyfile is a
284
+ * long base64 blob, a username is an identifier, and everything else is a password.
285
+ * @param {string} key - Secret data key (e.g. 'password', 'username', 'mongodb-keyfile').
286
+ * @returns {string} Generated value.
287
+ * @memberof UnderpostRun
288
+ */
289
+ const generateSeedValue = (key) => {
290
+ if (key === 'username') return 'admin';
291
+ // MongoDB keyfile: 6-1024 base64 characters shared by every replica-set member. Newlines are
292
+ // stripped so the value round-trips identically through YAML and through
293
+ // MongoBootstrap.readCredential, which strips them too.
294
+ if (key === 'mongodb-keyfile')
295
+ return shellExec(`openssl rand -base64 756`, { stdout: true, silent: true, disableLog: true }).replace(
296
+ /\r?\n/g,
297
+ '',
298
+ );
299
+ return generateSecurePassword(24);
300
+ };
301
+
207
302
  class UnderpostRun {
208
303
  /**
209
304
  * @static
@@ -212,6 +307,180 @@ class UnderpostRun {
212
307
  * @memberof UnderpostRun
213
308
  */
214
309
  static RUNNERS = {
310
+ /**
311
+ * @method status
312
+ * @description Reports deployment traffic, routing, Pods, expanded instances, and host capacity.
313
+ * @param {string} path - Deploy id, comma-separated ids, or `dd`; empty uses the router/configured projects.
314
+ * @param {UnderpostRunDefaultOptions} options - Namespace, environment (`--dev`), cluster, and node options.
315
+ * @returns {Promise<{deployments: object[], machine: object}>} Structured status report.
316
+ * @memberof UnderpostRun
317
+ */
318
+ status: async (path = '', options = DEFAULT_OPTION) => {
319
+ options = {
320
+ ...options,
321
+ gatewayApi: gatewayApiEnabledFactory(options),
322
+ namespace: options.namespace || 'default',
323
+ };
324
+ if (!/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(options.namespace))
325
+ throw new Error(`Invalid Kubernetes namespace: ${options.namespace}`);
326
+ if (options.nodeName && !/^[a-zA-Z0-9._-]+$/.test(options.nodeName))
327
+ throw new Error(`Invalid Kubernetes node name: ${options.nodeName}`);
328
+ const env = options.dev ? 'development' : 'production';
329
+ const requestedDeploys = `${path || options.deployId || ''}`.trim();
330
+ const routerPath = './engine-private/deploy/dd.router';
331
+ const confRoot = './engine-private/conf';
332
+ const deployIds = [
333
+ ...new Set(
334
+ requestedDeploys
335
+ ? resolveDeployList(requestedDeploys)
336
+ : fs.existsSync(routerPath)
337
+ ? resolveDeployList('dd')
338
+ : fs.existsSync(confRoot)
339
+ ? fs
340
+ .readdirSync(confRoot)
341
+ .filter(
342
+ (deployId) =>
343
+ fs.existsSync(`${confRoot}/${deployId}/conf.server.json`) ||
344
+ fs.existsSync(`${confRoot}/${deployId}/conf.instances.json`),
345
+ )
346
+ .sort()
347
+ : [],
348
+ ),
349
+ ];
350
+ if (deployIds.length === 0) throw new Error('No deployments found for status');
351
+ if (deployIds.some((deployId) => !/^[a-zA-Z0-9._-]+$/.test(deployId)))
352
+ throw new Error(`Invalid deployment status path: ${requestedDeploys}`);
353
+
354
+ const deployments = [];
355
+ for (const deployId of deployIds) {
356
+ const instances = [];
357
+ if (fs.existsSync(`${confRoot}/${deployId}/conf.instances.json`)) {
358
+ for (const instance of loadConfInstances(deployId)) {
359
+ const instanceDeployId = `${deployId}-${instance.id}`;
360
+ instances.push({
361
+ id: instance.id,
362
+ host: instance.host,
363
+ path: instance.path,
364
+ fromPort: instance.fromPort,
365
+ toPort: instance.toPort,
366
+ fromDebugPort: instance.fromDebugPort,
367
+ toDebugPort: instance.toDebugPort,
368
+ traffic: Underpost.deploy.getCurrentTraffic(instanceDeployId, {
369
+ namespace: options.namespace,
370
+ hostTest: instance.host,
371
+ env,
372
+ gatewayApi: options.gatewayApi,
373
+ }),
374
+ });
375
+ }
376
+ }
377
+ const deployment = {
378
+ deployId,
379
+ env,
380
+ traffic: Underpost.deploy.getCurrentTraffic(deployId, {
381
+ namespace: options.namespace,
382
+ env,
383
+ gatewayApi: options.gatewayApi,
384
+ }),
385
+ router: await Underpost.deploy.routerFactory(deployId, env),
386
+ pods: Underpost.kubectl.get(deployId, 'pods', options.namespace),
387
+ instances,
388
+ };
389
+ deployments.push(deployment);
390
+ logger.info('', deployment);
391
+ }
392
+
393
+ const interfaceName = Underpost.dns.getDefaultNetworkInterface();
394
+ const machine = {
395
+ hostname: os.hostname(),
396
+ arch: Underpost.baremetal.getHostArch(),
397
+ clusterType: clusterTypeFactory(options),
398
+ ipv4Public: await Underpost.dns.getPublicIp(),
399
+ ipv4Local: Underpost.dns.getLocalIPv4Address(),
400
+ resources: Underpost.cluster.getResourcesCapacity(options.nodeName),
401
+ defaultInterfaceName: interfaceName,
402
+ defaultInterfaceInfo: os.networkInterfaces()[interfaceName],
403
+ };
404
+ logger.info('Machine', machine);
405
+ return { deployments, machine };
406
+ },
407
+
408
+ /**
409
+ * @method expose
410
+ * @description Port-forwards every Service whose name partially matches path, falling back to matching Pods.
411
+ * Works through the active kubeconfig for Kind, k3s, and kubeadm clusters.
412
+ * Comma-separated path fragments determine resource index order; host and
413
+ * container port lists are paired against that same order.
414
+ * @param {string} path - One or more comma-separated literal Service/Pod name fragments.
415
+ * @param {UnderpostRunDefaultOptions} options - Namespace, cluster type, and optional port overrides.
416
+ * @returns {Array<{kindType: string, name: string, localPort: number, remotePort: number}>} Forward plan.
417
+ * @memberof UnderpostRun
418
+ */
419
+ expose: (path, options = DEFAULT_OPTION) => {
420
+ const namespace = options.namespace || 'default';
421
+ const clusterType = clusterTypeFactory(options);
422
+ const pathParts = exposePathPartsFactory(path || options.podName);
423
+ if (!/^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/.test(namespace))
424
+ throw new Error(`Invalid Kubernetes namespace: ${namespace}`);
425
+ let kindType = 'svc';
426
+ let resources = exposePartialMatchesFactory(Underpost.kubectl.get('', kindType, namespace), pathParts);
427
+
428
+ if (resources.length === 0) {
429
+ kindType = 'pod';
430
+ resources = exposePartialMatchesFactory(Underpost.kubectl.get('', 'pods', namespace), pathParts);
431
+ }
432
+ if (resources.length === 0)
433
+ throw new Error(`No Service or Pod partially matching '${pathParts.join(',')}' in namespace '${namespace}'`);
434
+
435
+ const containerPorts = exposePortListFactory(options.exposeContainerPorts, '--expose-container-ports');
436
+ const hostPorts = exposePortListFactory(options.exposeHostPorts, '--expose-host-ports');
437
+ const portsOf = (resource) => {
438
+ let declaredPorts = exposeTcpPortsFactory(resource);
439
+ if (kindType === 'pod' && declaredPorts.length === 0) {
440
+ const podJson = shellExec(`sudo kubectl get pod ${resource.NAME} -n ${namespace} -o json`, {
441
+ stdout: true,
442
+ silent: true,
443
+ });
444
+ const pod = JSON.parse(podJson);
445
+ declaredPorts = (pod.spec?.containers || [])
446
+ .flatMap((container) => container.ports || [])
447
+ .map(({ containerPort }) => parseInt(containerPort))
448
+ .filter((port) => Number.isInteger(port) && port > 0);
449
+ }
450
+ return declaredPorts;
451
+ };
452
+ const plan = exposePortPlanFactory({ resources, kindType, containerPorts, hostPorts, portsOf });
453
+
454
+ logger.info('[expose] Kubernetes port-forward plan', {
455
+ clusterType,
456
+ namespace,
457
+ matches: pathParts,
458
+ plan,
459
+ });
460
+ for (const { kindType, name, localPort, remotePort } of plan)
461
+ shellExec(`sudo kubectl port-forward -n ${namespace} ${kindType}/${name} ${localPort}:${remotePort}`, {
462
+ async: true,
463
+ });
464
+
465
+ if (options.localProxy) {
466
+ const deployId = options.deployId || pathParts[0];
467
+ const env = options.dev ? 'development' : 'production';
468
+ const envFile = `./engine-private/conf/${deployId}/.env.${env}`;
469
+ let basePort = plan[0].localPort - 1;
470
+ if (fs.existsSync(envFile)) {
471
+ const portMatch = fs.readFileSync(envFile, 'utf8').match(/^PORT=(\d+)/m);
472
+ if (portMatch) basePort = parseInt(portMatch[1]);
473
+ }
474
+ const tlsFlag = options.tls ? ' tls' : '';
475
+ shellExec(
476
+ `NODE_ENV=${env} PORT=${basePort} DEV_PROXY_PORT_OFFSET=0 node src/proxy proxy ${deployId} ${env}${tlsFlag}`,
477
+ { async: true },
478
+ );
479
+ }
480
+
481
+ return plan;
482
+ },
483
+
215
484
  /**
216
485
  * @method dev-cluster
217
486
  * @description Resets and deploys a full development cluster including MongoDB, Valkey, exposes services, and updates `/etc/hosts` for local access.
@@ -223,26 +492,21 @@ class UnderpostRun {
223
492
  const baseCommand = options.dev ? 'node bin' : 'underpost';
224
493
  const mongoHosts = ['mongodb-0.mongodb-service'];
225
494
  let primaryMongoHost = 'mongodb-0.mongodb-service';
226
- if (!options.expose) {
227
- shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --reset`);
228
- shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''}`);
495
+ const clusterType = clusterTypeFactory(options);
496
+ const clusterFlag = ` --${clusterType}`;
497
+ const clusterInitFlag = clusterType === 'kind' ? '' : clusterFlag;
498
+ const clusterOptions = `${options.dev ? ' --dev' : ''}${clusterInitFlag} --namespace ${options.namespace}`;
499
+ if (!options.expose && !options.remove) {
500
+ shellExec(`${baseCommand} cluster${clusterOptions} --reset`);
501
+ shellExec(`${baseCommand} cluster${clusterOptions}`);
229
502
 
230
503
  shellExec(
231
- `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --service-host ${mongoHosts.join(
232
- ',',
233
- )} --pull-image`,
504
+ `${baseCommand} cluster${clusterOptions} --mongodb --service-host ${mongoHosts.join(',')} --pull-image`,
234
505
  );
235
- shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --valkey --pull-image`);
506
+ shellExec(`${baseCommand} cluster${clusterOptions} --valkey --pull-image`);
236
507
  }
237
- if (options.k3s) {
238
- if (options.remove) {
239
- shellExec(`${baseCommand} lxd --delete-expose k3s-control:27017`);
240
- shellExec(`${baseCommand} lxd --delete-expose k3s-control:6379`);
241
- } else {
242
- shellExec(`${baseCommand} lxd --expose k3s-control:27017 --node-port 32017`);
243
- shellExec(`${baseCommand} lxd --expose k3s-control:6379 --node-port 32079`);
244
- }
245
- shellExec(`lxc config device show k3s-control`);
508
+ if (options.remove) {
509
+ shellExec(`${baseCommand} run kill '6379,27017'`);
246
510
  } else {
247
511
  try {
248
512
  const primaryPodName =
@@ -251,49 +515,26 @@ class UnderpostRun {
251
515
  podName: 'mongodb-0',
252
516
  disableAuth: options.dev,
253
517
  }) || 'mongodb-0';
254
- shellExec(
255
- `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config mongo`,
256
- { async: true },
257
- );
258
- shellExec(
259
- `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config valkey`,
260
- { async: true },
261
- );
518
+ primaryMongoHost = `${primaryPodName}.mongodb-service`;
262
519
  } catch (error) {
263
520
  logger.warn('Failed to detect MongoDB primary pod, using default', {
264
521
  error: error.message,
265
522
  default: primaryMongoHost,
266
523
  });
267
524
  }
525
+ shellExec(
526
+ `${baseCommand} run expose mongodb-service --namespace ${options.namespace}${clusterFlag} --expose-container-ports 27017 --expose-host-ports 27017`,
527
+ { async: true },
528
+ );
529
+ shellExec(
530
+ `${baseCommand} run expose valkey-service --namespace ${options.namespace}${clusterFlag} --expose-container-ports 6379 --expose-host-ports 6379`,
531
+ { async: true },
532
+ );
268
533
  }
269
534
  const hostListenResult = etcHostFactory([primaryMongoHost]);
270
535
  logger.info(hostListenResult.renderHosts);
271
536
  },
272
537
 
273
- /**
274
- * @method etc-hosts
275
- * @description Modifies the `/etc/hosts` file to add entries for local access to services,
276
- * based on the provided path input.
277
- * @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
278
- */
279
- 'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
280
- etcHostFactory(path.split(','));
281
- },
282
-
283
- /**
284
- * @method ipfs-expose
285
- * @description Exposes IPFS Cluster services on specified ports for local access.
286
- * @type {Function}
287
- * @memberof UnderpostRun
288
- */
289
- 'ipfs-expose': (path, options = DEFAULT_OPTION) => {
290
- const ports = [5001, 9094, 8080];
291
- for (const port of ports)
292
- shellExec(`node bin deploy --expose ipfs-cluster --expose-port ${port} --disable-update-underpost-config`, {
293
- async: true,
294
- });
295
- },
296
-
297
538
  /**
298
539
  * @method metadata
299
540
  * @description Generates metadata for the specified path after exposing the development cluster.
@@ -306,19 +547,30 @@ class UnderpostRun {
306
547
  shellExec(`node bin run kill '${ports}'`);
307
548
  shellExec(`node bin run dev-cluster --dev --expose --namespace ${options.namespace}`, { async: true });
308
549
  logger.info('Waiting for port-forward services to be ready...');
309
- try {
310
- await Promise.all([waitForPort(27017), waitForPort(6379)]);
311
- logger.info('Port-forward services are ready');
312
- } catch (err) {
313
- logger.error('Port-forward services failed to become ready', { error: err.message });
550
+ const ready = await Promise.all([27017, 6379].map((port) => waitForPort({ port })));
551
+ if (ready.some((reachable) => !reachable)) {
314
552
  shellExec(`node bin run kill '${ports}'`);
315
- throw err;
553
+ throw new Error('Port-forward services failed to become ready');
316
554
  }
555
+ logger.info('Port-forward services are ready');
317
556
  shellExec(`node bin metadata --generate ${path}`);
318
557
  shellExec(`node bin db --dev --clean-fs-collection dd`);
319
558
  shellExec(`node bin run kill '${ports}'`);
320
559
  },
321
560
 
561
+ /**
562
+ * @method ipfs-expose
563
+ * @description Exposes every declared TCP port on the matching IPFS Cluster Service.
564
+ * @type {Function}
565
+ * @memberof UnderpostRun
566
+ */
567
+ 'ipfs-expose': (path, options = DEFAULT_OPTION) => {
568
+ // 5001 Kubo RPC API / WebUI: Kubo/IPFS HTTP API + IPFS WebUI (e.g., http://localhost:5001/webui).
569
+ // 9094 IPFS Cluster HTTP API: IPFS Cluster REST API consumed by ipfs-cluster-ctl and WebUI clients (e.g., http://localhost:9094/).
570
+ // 8080 (or 8081) IPFS Gateway: Public HTTP Gateway for accessing pinned content by CID (e.g., http://localhost:8080/ipfs/Qm...).
571
+ shellExec(`node bin run expose ipfs --expose-host-ports 5001,9094,8080`);
572
+ },
573
+
322
574
  /**
323
575
  * @method svc-ls
324
576
  * @description Lists systemd services and installed packages, optionally filtering by the provided path.
@@ -354,28 +606,6 @@ class UnderpostRun {
354
606
  shellExec(`sudo rm -f /etc/yum.repos.d/${path}*.repo`);
355
607
  },
356
608
 
357
- /**
358
- * @method ssh-deploy-info
359
- * @description Retrieves deployment status and pod information from a remote server via SSH.
360
- * @param {string} path - The input value, identifier, or path for the operation.
361
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
362
- * @memberof UnderpostRun
363
- */
364
- 'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
365
- const env = options.dev ? 'development' : 'production';
366
- await Underpost.ssh.sshRemoteRunner(
367
- `node bin deploy ${path ? path : 'dd'} ${env} --status && kubectl get pods -A`,
368
- {
369
- deployId: options.deployId,
370
- user: options.user,
371
- dev: options.dev,
372
- remote: true,
373
- useSudo: true,
374
- cd: '/home/dd/engine',
375
- },
376
- );
377
- },
378
-
379
609
  /**
380
610
  * @method node-move
381
611
  * @description Abstract runner that relocates any schedulable Kubernetes workload
@@ -445,8 +675,8 @@ class UnderpostRun {
445
675
  services: 'service',
446
676
  })[k] || k;
447
677
 
448
- // Kinds that own a pod template we can patch; rolloutKinds additionally
449
- // support `kubectl rollout restart` to reschedule existing pods now.
678
+ // Kinds that own a pod template we can patch. Changing that template is
679
+ // itself the controller's rollout trigger.
450
680
  const templated = [
451
681
  'deployment',
452
682
  'statefulset',
@@ -456,7 +686,6 @@ class UnderpostRun {
456
686
  'cronjob',
457
687
  'replicationcontroller',
458
688
  ];
459
- const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
460
689
  const templateSelectorPath = (kind) =>
461
690
  kind === 'cronjob'
462
691
  ? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
@@ -545,10 +774,9 @@ class UnderpostRun {
545
774
  continue;
546
775
  }
547
776
 
548
- // Idempotency: skip the patch + rollout if the resource is already where
549
- // we want it. Compares the live pod-template nodeSelector against the
550
- // desired placement so a repeated run does not trigger an unnecessary
551
- // rollout restart.
777
+ // Idempotency: skip the patch if the resource is already where we want
778
+ // it. Compares the live pod-template nodeSelector against the desired
779
+ // placement so a repeated run does not trigger an unnecessary rollout.
552
780
  const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
553
781
  const jsonpath = (expr) =>
554
782
  shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
@@ -578,16 +806,17 @@ class UnderpostRun {
578
806
  }
579
807
 
580
808
  const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
581
- const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
582
809
  if (dryRun) {
583
810
  logger.info(`[dry-run] ${patchCmd}`);
584
- if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
585
811
  results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
586
812
  continue;
587
813
  }
588
814
 
589
815
  shellExec(patchCmd);
590
- if (rolloutKinds.includes(kind)) shellExec(restartCmd);
816
+ // nodeSelector is part of the pod template, so this patch already
817
+ // creates a new controller revision. A rollout restart here creates a
818
+ // second, immediately superseding revision and can strand the previous
819
+ // Ready replica in "pending termination" while the newest pod starts.
591
820
  logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
592
821
  namespace: ns,
593
822
  });
@@ -605,10 +834,12 @@ class UnderpostRun {
605
834
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
606
835
  * @memberof UnderpostRun
607
836
  */
608
- 'dev-hosts-expose': (path, options = DEFAULT_OPTION) => {
609
- shellExec(
610
- `node bin deploy ${path} development --disable-update-deployment --disable-update-proxy --kubeadm --etc-hosts`,
611
- );
837
+ 'dev-hosts-expose': async (path, options = DEFAULT_OPTION) => {
838
+ shellExec(`node bin deploy ${path} development --disable-update-deployment --disable-update-proxy --kubeadm`);
839
+ // /etc/hosts is written here, not by `deploy`: that command has no
840
+ // --etc-hosts option, and the `etc-hosts` runner already resolves a
841
+ // deploy's hosts from its conf.server.json.
842
+ await UnderpostRun.RUNNERS['etc-hosts']('', { ...options, deployId: path });
612
843
  },
613
844
 
614
845
  /**
@@ -619,7 +850,10 @@ class UnderpostRun {
619
850
  * @memberof UnderpostRun
620
851
  */
621
852
  'dev-hosts-restore': (path, options = DEFAULT_OPTION) => {
622
- shellExec(`node bin deploy --restore-hosts`);
853
+ // Rewrite /etc/hosts with the loopback block alone, dropping the deploy
854
+ // host entries `dev-hosts-expose` (and the `cluster` runner) added.
855
+ const hostListenResult = etcHostFactory([]);
856
+ logger.info(hostListenResult.renderHosts);
623
857
  },
624
858
 
625
859
  /**
@@ -714,10 +948,11 @@ class UnderpostRun {
714
948
  if (deployConfId) inputs.deploy_conf_id = deployConfId;
715
949
  if (deployType) inputs.deploy_type = deployType;
716
950
 
951
+ // Omit `ref` so dispatchWorkflow auto-detects the repo's default branch
952
+ // (a fork may default to `main` rather than the monorepo's `master`).
717
953
  Underpost.repo.dispatchWorkflow({
718
954
  repo,
719
955
  workflowFile: 'npmpkg.ci.yml',
720
- ref: 'master',
721
956
  inputs,
722
957
  });
723
958
  },
@@ -761,7 +996,7 @@ class UnderpostRun {
761
996
  * @memberof UnderpostRun
762
997
  */
763
998
  'docker-image': (path, options = DEFAULT_OPTION) => {
764
- const repo = Underpost.repo.resolveInstanceRepo(path, options.dev);
999
+ const repo = Underpost.repo.resolveInstanceRepo(path, !options.test);
765
1000
  Underpost.repo.dispatchWorkflow({
766
1001
  repo,
767
1002
  workflowFile: `docker-image${path ? `.${path}` : ''}${options.dev ? '.dev' : ''}.ci.yml`,
@@ -841,11 +1076,13 @@ class UnderpostRun {
841
1076
  job = 'init';
842
1077
  confId = path.replace(/^init-/, '');
843
1078
  }
844
- const repo = Underpost.repo.resolveInstanceRepo(confId, options.dev);
1079
+ const repo = Underpost.repo.resolveInstanceRepo(confId, !options.test);
1080
+ // Omit `ref` so dispatchWorkflow auto-detects the target repo's default
1081
+ // branch (getDefaultBranch): the monorepo is `master` but instance repos
1082
+ // like engine-cyberia default to `main` — hardcoding either 422s.
845
1083
  Underpost.repo.dispatchWorkflow({
846
1084
  repo,
847
1085
  workflowFile: `${confId}.cd.yml`,
848
- ref: 'master',
849
1086
  inputs: { job },
850
1087
  });
851
1088
  },
@@ -902,6 +1139,7 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
902
1139
  sync: async (path, options = DEFAULT_OPTION) => {
903
1140
  // Dev usage: node bin run --dev --build sync dd-default
904
1141
  const env = options.dev ? 'development' : 'production';
1142
+ options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
905
1143
  const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
906
1144
  const baseClusterCommand = options.dev ? ' --dev' : '';
907
1145
  const clusterFlag = options.k3s ? ' --k3s' : options.kind ? ' --kind' : ' --kubeadm';
@@ -939,11 +1177,28 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
939
1177
  }
940
1178
 
941
1179
  const currentTraffic = isDeployRunnerContext(path, options)
942
- ? Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace })
1180
+ ? Underpost.deploy.getCurrentTraffic(deployId, {
1181
+ namespace: options.namespace,
1182
+ env,
1183
+ gatewayApi: options.gatewayApi,
1184
+ })
943
1185
  : '';
944
1186
  let targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'green';
945
1187
  if (targetTraffic) versions = versions ? versions : targetTraffic;
946
1188
 
1189
+ // The routed colour is only live traffic while it still has a ready
1190
+ // endpoint. Everything downstream that would take the host offline to prove
1191
+ // the maintenance fallback is conditional on this being false: a first
1192
+ // bring-up has nothing to interrupt, a re-deploy does.
1193
+ const serving = isTrafficServingFactory({
1194
+ liveTraffic: currentTraffic,
1195
+ hasReadyEndpoints: (colour) =>
1196
+ Underpost.deploy.serviceHasReadyEndpoints({
1197
+ service: `${deployId}-${env}-${colour}-service`,
1198
+ namespace: options.namespace || 'default',
1199
+ }),
1200
+ });
1201
+
947
1202
  const ignorePods =
948
1203
  isDeployRunnerContext(path, options) && targetTraffic
949
1204
  ? Underpost.kubectl.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace).map((p) => p.NAME)
@@ -959,16 +1214,69 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
959
1214
  const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
960
1215
  const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
961
1216
  const sshKeyPathFlag = options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '';
1217
+ const gatewayApiFlags = Underpost.deploy.gatewayApiFlagsFactory(options);
1218
+
1219
+ // A direct sync owns the same gateway-first contract as the full cluster
1220
+ // runner. Build the host-side SSR documents before generating routes unless
1221
+ // the caller explicitly selected a pre-built bundle workflow.
1222
+ if (isDeployRunnerContext(path, options) && !options.skipFullBuild)
1223
+ shellExec(`${baseCommand} client ${deployId} ${env}`);
962
1224
 
963
1225
  shellExec(
964
1226
  `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
965
1227
  image ? ` --image ${image}` : ''
966
1228
  }${versions ? ` --versions ${versions}` : ''}${
967
1229
  options.namespace ? ` --namespace ${options.namespace}` : ''
968
- }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag} ${deployId} ${env}`,
1230
+ }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags} ${deployId} ${env}`,
969
1231
  );
970
1232
 
971
1233
  if (isDeployRunnerContext(path, options)) {
1234
+ if (options.gatewayApi) {
1235
+ const namespace = options.namespace || 'default';
1236
+ const gatewayRoot = Underpost.deploy.underpostGatewayRootFactory(options);
1237
+ shellExec(`kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${namespace} --timeout=5m`);
1238
+ const staticAssets = Underpost.deploy.syncStaticAssets(deployId, env, {
1239
+ ...options,
1240
+ namespace,
1241
+ // Prefer the currently serving workload as the document source. On
1242
+ // an initial deploy there is none, so sync falls back to the checkout.
1243
+ versions: currentTraffic || versions || targetTraffic,
1244
+ });
1245
+ assertStaticAssets({ records: staticAssets, hostRoot: gatewayRoot, label: 'sync' });
1246
+
1247
+ if (serving)
1248
+ logger.info('[sync] Live colour serving; holding traffic until the target colour is Ready', {
1249
+ deployId,
1250
+ live: currentTraffic,
1251
+ target: targetTraffic,
1252
+ });
1253
+ else {
1254
+ // Remove any inactive-colour workload left by an earlier cycle. Without
1255
+ // this cleanup the probe could hit a stale Ready pod and never exercise
1256
+ // the configured unavailable-backend fallback.
1257
+ shellExec(
1258
+ `kubectl delete service ${deployId}-${env}-${targetTraffic}-service -n ${namespace} --ignore-not-found`,
1259
+ );
1260
+ shellExec(
1261
+ `kubectl delete deployment ${deployId}-${env}-${targetTraffic} -n ${namespace} --ignore-not-found`,
1262
+ );
1263
+ // Publish the target-colour route while its Service is deliberately
1264
+ // absent. Site paths reach underpost-gateway and must return the
1265
+ // configured maintenance body before deployment.yaml is submitted.
1266
+ shellExec(
1267
+ `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-deployment ${deployId} ${env} --versions ${versions}${
1268
+ options.namespace ? ` --namespace ${options.namespace}` : ''
1269
+ }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags}`,
1270
+ );
1271
+ await gatewayFallbackProbeRunner({
1272
+ gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
1273
+ checks: pwaFallbackChecksFactory(deployId),
1274
+ options,
1275
+ label: 'sync',
1276
+ });
1277
+ }
1278
+ }
1279
+
972
1280
  // Backup app/services repositories with repo-backup configured
973
1281
  shellExec(
974
1282
  `${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
@@ -976,163 +1284,119 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
976
1284
  shellExec(
977
1285
  `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
978
1286
  options.namespace ? ` --namespace ${options.namespace}` : ''
979
- }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}`,
1287
+ }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}${gatewayApiFlags}`,
980
1288
  );
981
1289
  if (!targetTraffic)
982
- targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
1290
+ targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
1291
+ namespace: options.namespace,
1292
+ env,
1293
+ gatewayApi: options.gatewayApi,
1294
+ });
983
1295
  await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
984
1296
  Underpost.deploy.switchTraffic(deployId, env, targetTraffic, replicas, options.namespace, options);
985
1297
  } else
986
- logger.info('current traffic', Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }));
1298
+ logger.info(
1299
+ 'current traffic',
1300
+ Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }),
1301
+ );
987
1302
  },
988
1303
 
989
1304
  /**
990
1305
  * @method stop
991
- * @description Stops a deployment by deleting the corresponding Kubernetes deployment and service resources.
992
- * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
1306
+ * @description Deletes colour-suffixed Deployments and their Services, leaving routing untouched.
1307
+ *
1308
+ * Four ways to say what to stop, resolved by {@link ServerConfBuilder.stopPlanFactory}:
1309
+ *
1310
+ * ```bash
1311
+ * # Literal: exactly this Deployment, flags ignored
1312
+ * node bin run stop dd-cyberia-mmo-server-forest-development-blue
1313
+ *
1314
+ * # The deploy's PWA workload, inactive colour
1315
+ * node bin run stop --deploy-id dd-cyberia
1316
+ *
1317
+ * # ...plus every variant of each instance family, inactive colour
1318
+ * node bin run stop --deploy-id dd-cyberia --instance-id mmo-client,mmo-server
1319
+ *
1320
+ * # Explicit colours; both, where they exist
1321
+ * node bin run stop --deploy-id dd-cyberia --traffic blue,green
1322
+ * ```
1323
+ *
1324
+ * The default colour is the blue/green partner of whatever each target is
1325
+ * serving, so a stop is safe against a live host unless `--traffic` names the
1326
+ * serving colour outright — which is warned about, not refused.
1327
+ *
1328
+ * That colour is read through the same routing stack that published it: the
1329
+ * Gateway API HTTPRoute by default, the Contour HTTPProxy under
1330
+ * `--disable-gateway-api`. Reading the wrong kind finds no colour, degrades to
1331
+ * "blue", and stops whichever Deployment happens to carry that name. It is
1332
+ * resolved per target too — an instance's colour lives under its own
1333
+ * `<deployId>-<instanceId>` prefix, and on a shared host the parent's answer
1334
+ * belongs to a different variant.
1335
+ * @param {string} [path] - Literal comma-separated Deployment names; when set, every flag is ignored.
993
1336
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
994
1337
  * @memberof UnderpostRun
995
1338
  */
996
1339
  stop: async (path = '', options = DEFAULT_OPTION) => {
997
- let currentTraffic = Underpost.deploy.getCurrentTraffic(options.deployId, {
998
- namespace: options.namespace,
999
- hostTest: options.hosts,
1000
- });
1001
1340
  const env = options.dev ? 'development' : 'production';
1341
+ const namespace = options.namespace || 'default';
1342
+ const gatewayApi = gatewayApiEnabledFactory(options);
1343
+
1344
+ // Memoized because the plan and the serving check ask the same question,
1345
+ // and `--traffic` skips the plan's lookup entirely — which is exactly when
1346
+ // the check has to make its own.
1347
+ const liveTraffic = {};
1348
+ const liveTrafficOf = (target) => {
1349
+ if (liveTraffic[target.id] === undefined)
1350
+ liveTraffic[target.id] =
1351
+ Underpost.deploy.getCurrentTraffic(target.id, {
1352
+ hostTest: target.host || options.hosts,
1353
+ namespace,
1354
+ env,
1355
+ gatewayApi,
1356
+ }) || '';
1357
+ return liveTraffic[target.id];
1358
+ };
1002
1359
 
1003
- if (!path.match('current')) currentTraffic === 'blue' ? (currentTraffic = 'green') : (currentTraffic = 'blue');
1004
- const [_deployId] = path.split(',');
1005
- const deploymentId = `${_deployId ? _deployId : options.deployId}${
1006
- options.instanceId ? `-${options.instanceId}` : ''
1007
- }-${env}-${currentTraffic}`;
1008
-
1009
- shellExec(`kubectl delete deployment ${deploymentId} -n ${options.namespace}`);
1010
- shellExec(`kubectl delete svc ${deploymentId}-service -n ${options.namespace}`);
1011
- },
1012
-
1013
- /**
1014
- * @method ssh-deploy-stop
1015
- * @description Stops a remote deployment via SSH by executing the appropriate Underpost command on the remote server.
1016
- * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
1017
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1018
- * @memberof UnderpostRun
1019
- */
1020
- 'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
1021
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1022
- const baseClusterCommand = options.dev ? ' --dev' : '';
1023
-
1024
- const remoteCommand = [
1025
- `${baseCommand} run${baseClusterCommand} stop${path ? ` ${path}` : ''}`,
1026
- ` --deploy-id ${options.deployId}${options.instanceId ? ` --instance-id ${options.instanceId}` : ''}`,
1027
- ` --namespace ${options.namespace}${options.hosts ? ` --hosts ${options.hosts}` : ''}`,
1028
- ].join('');
1029
-
1030
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1031
- deployId: options.deployId,
1032
- user: options.user,
1033
- dev: options.dev,
1034
- remote: true,
1035
- useSudo: true,
1036
- cd: '/home/dd/engine',
1037
- });
1038
- },
1039
-
1040
- /**
1041
- * @method ssh-deploy-db-rollback
1042
- * @description Performs a database rollback on remote deployment via SSH.
1043
- * @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
1044
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1045
- * @param {string} options.deployId - The deployment identifier
1046
- * @param {string} options.user - The SSH user for credential lookup
1047
- * @param {boolean} options.dev - Development mode flag
1048
- * @memberof UnderpostRun
1049
- */
1050
- 'ssh-deploy-db-rollback': async (path = '', options = DEFAULT_OPTION) => {
1051
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1052
- let [deployId, nCommitsReset] = path.split(',');
1053
- if (!nCommitsReset) nCommitsReset = 1;
1054
-
1055
- const remoteCommand = `${baseCommand} db ${deployId} --git --kubeadm --primary-pod --force-clone --macro-rollback-export ${nCommitsReset}${options.namespace ? ` --ns ${options.namespace}` : ''}`;
1056
-
1057
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1058
- deployId: options.deployId,
1059
- user: options.user,
1060
- dev: options.dev,
1061
- remote: true,
1062
- useSudo: true,
1063
- cd: '/home/dd/engine',
1064
- });
1065
- },
1066
-
1067
- /**
1068
- * @method ssh-deploy-db
1069
- * @description Imports/restores a database on remote deployment via SSH.
1070
- * @param {string} path - The deployment ID for database import
1071
- * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1072
- * @param {string} options.deployId - The deployment identifier
1073
- * @param {string} options.user - The SSH user for credential lookup
1074
- * @param {boolean} options.dev - Development mode flag
1075
- * @memberof UnderpostRun
1076
- */
1077
- 'ssh-deploy-db': async (path, options = DEFAULT_OPTION) => {
1078
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1079
-
1080
- const remoteCommand = `${baseCommand} db ${path} --import --drop --preserveUUID --git --kubeadm --primary-pod --force-clone${options.namespace ? ` --ns ${options.namespace}` : ''}`;
1081
-
1082
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1360
+ const { deployments, error } = stopPlanFactory({
1361
+ path,
1083
1362
  deployId: options.deployId,
1084
- user: options.user,
1085
- dev: options.dev,
1086
- remote: true,
1087
- useSudo: true,
1088
- cd: '/home/dd/engine',
1363
+ instanceId: options.instanceId,
1364
+ traffic: options.traffic,
1365
+ env,
1366
+ instancesFor: (instanceId) => selectConfInstances(loadConfInstances(options.deployId), instanceId),
1367
+ liveTrafficOf,
1089
1368
  });
1090
- },
1091
-
1092
- /**
1093
- * @method ssh-deploy-db-status
1094
- * @description Retrieves database status/stats for a deployment (or all deployments from dd.router) via SSH.
1095
- * @param {string} path - Comma-separated deployId(s) or 'dd' to use the dd.router list.
1096
- * @param {UnderpostRunDefaultOptions} options - Runner options (uses options.deployId for SSH host lookup).
1097
- * @param {string} options.deployId - Deployment identifier used for SSH config lookup.
1098
- * @param {string} options.user - SSH user for credential lookup.
1099
- * @param {boolean} options.dev - Development mode flag.
1100
- * @param {string} [options.namespace] - Kubernetes namespace to pass to the db check.
1101
- * @memberof UnderpostRun
1102
- */
1103
- 'ssh-deploy-db-status': async (path = '', options = DEFAULT_OPTION) => {
1104
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1105
-
1106
- let deployList = [];
1107
- if (!path || path === 'dd') {
1108
- if (!fs.existsSync('./engine-private/deploy/dd.router')) {
1109
- logger.warn('dd.router not found; nothing to run');
1110
- return;
1111
- }
1112
- deployList = fs
1113
- .readFileSync('./engine-private/deploy/dd.router', 'utf8')
1114
- .split(',')
1115
- .map((d) => d.trim())
1116
- .filter(Boolean);
1117
- } else {
1118
- deployList = path
1119
- .split(',')
1120
- .map((d) => d.trim())
1121
- .filter(Boolean);
1369
+ if (error) {
1370
+ logger.error(error);
1371
+ return [];
1122
1372
  }
1123
1373
 
1124
- for (const deployId of deployList) {
1125
- const remoteCommand = `${baseCommand} db ${deployId} --stats --kubeadm --primary-pod${options.namespace ? ` --ns ${options.namespace}` : ''}`;
1374
+ // Read once: it decides what is reported as actually stopped, and a target
1375
+ // that is already gone is a no-op worth naming rather than a silent delete.
1376
+ const deployedNames = Underpost.kubectl.get('', 'deployment', namespace).map((entry) => entry.NAME);
1377
+ const stopped = [];
1378
+ for (const target of deployments) {
1379
+ const existed = deployedNames.includes(target.deployment);
1380
+ if (existed) stopped.push(target.deployment);
1381
+ // The Service is deleted either way: an orphan left by a half-finished
1382
+ // cycle outlives its Deployment and would keep resolving to no endpoints.
1383
+ shellExec(`kubectl delete deployment ${target.deployment} -n ${namespace} --ignore-not-found`);
1384
+ shellExec(`kubectl delete svc ${target.deployment}-service -n ${namespace} --ignore-not-found`);
1385
+ }
1126
1386
 
1127
- await Underpost.ssh.sshRemoteRunner(remoteCommand, {
1128
- deployId: options.deployId,
1129
- user: options.user,
1130
- dev: options.dev,
1131
- remote: true,
1132
- useSudo: true,
1133
- cd: '/home/dd/engine',
1387
+ const serving = deployments.filter(
1388
+ (target) => target.kind !== 'literal' && target.colour === liveTrafficOf(target),
1389
+ );
1390
+ if (serving.length > 0)
1391
+ logger.warn('Stopped the serving colour; those routes stay published and now have no backend', {
1392
+ deployments: serving.map((target) => target.deployment),
1134
1393
  });
1135
- }
1394
+ logger.info('Stop complete', {
1395
+ namespace,
1396
+ stopped,
1397
+ absent: deployments.filter((target) => !stopped.includes(target.deployment)).map((t) => t.deployment),
1398
+ });
1399
+ return stopped;
1136
1400
  },
1137
1401
 
1138
1402
  /**
@@ -1157,102 +1421,543 @@ echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com
1157
1421
  },
1158
1422
 
1159
1423
  /**
1160
- * @method get-proxy
1161
- * @description Retrieves and logs the HTTPProxy resources in the specified namespace using `kubectl get HTTPProxy`.
1162
- * @param {string} path - The input value, identifier, or path for the operation (used as an optional filter for the HTTPProxy resources).
1424
+ * @method get-traffic
1425
+ * @description Prints the live blue/green colour of every routable
1426
+ * deployment, of both kinds, as a table.
1427
+ * @param {string} [path] - Comma-separated hosts to report on; empty reports every host.
1163
1428
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1164
1429
  * @memberof UnderpostRun
1165
1430
  */
1166
- 'get-proxy': async (path = '', options = DEFAULT_OPTION) => {
1167
- console.log(
1168
- shellExec(`kubectl get HTTPProxy -n ${options.namespace} ${path} -o yaml`, {
1169
- silent: true,
1431
+ 'get-traffic': async (path = '', options = DEFAULT_OPTION) => {
1432
+ options = {
1433
+ ...options,
1434
+ gatewayApi: gatewayApiEnabledFactory(options),
1435
+ namespace: options.namespace || 'default',
1436
+ };
1437
+ // A report answers "what is live", so it cannot pick one environment from a
1438
+ // flag and call the other absent: a cluster running `development` would be
1439
+ // reported as entirely unrouted. Both are scanned and the environment is a
1440
+ // column; `--dev` narrows to development when only that is wanted.
1441
+ const envs = options.dev ? ['development'] : options.test ? ['development', 'production'] : ['production'];
1442
+ const hosts = `${path || ''}`
1443
+ .split(',')
1444
+ .map((host) => host.trim())
1445
+ .filter(Boolean);
1446
+
1447
+ const confRoot = './engine-private/conf';
1448
+ const deployList = options.deployId
1449
+ ? resolveDeployList(options.deployId)
1450
+ : fs.existsSync(confRoot)
1451
+ ? fs
1452
+ .readdirSync(confRoot)
1453
+ .filter((deployId) => fs.existsSync(`${confRoot}/${deployId}/conf.server.json`))
1454
+ .sort()
1455
+ : [];
1456
+
1457
+ const deployments = Underpost.kubectl.get('', 'deployment', options.namespace);
1458
+ const deployedNames = deployments.map((entry) => entry.NAME);
1459
+
1460
+ // Four cluster-wide reads, correlated once: which kind describes a host,
1461
+ // whether its listener terminates TLS, and whether QUIC is enabled on it.
1462
+ // A missing CRD is an empty list, so a single-stack cluster reads the same
1463
+ // way as one running both.
1464
+ const listResources = (kind) => {
1465
+ const raw = shellExec(`kubectl get ${kind} -A -o json`, {
1170
1466
  stdout: true,
1171
- })
1172
- .replaceAll(`blue`, `blue`.bgBlue.bold.black)
1173
- .replaceAll('green', 'green'.bgGreen.bold.black)
1174
- .replaceAll('Error', 'Error'.bold.red)
1175
- .replaceAll('error', 'error'.bold.red)
1176
- .replaceAll('ERROR', 'ERROR'.bold.red)
1177
- .replaceAll('Invalid', 'Invalid'.bold.red)
1178
- .replaceAll('invalid', 'invalid'.bold.red)
1179
- .replaceAll('INVALID', 'INVALID'.bold.red),
1467
+ silent: true,
1468
+ silentOnError: true,
1469
+ });
1470
+ try {
1471
+ const parsed = JSON.parse(`${raw || ''}`);
1472
+ return Array.isArray(parsed?.items) ? parsed.items : [];
1473
+ } catch {
1474
+ return [];
1475
+ }
1476
+ };
1477
+ const ingressFacts = hostIngressFactsFactory({
1478
+ httpRoutes: listResources('httproute'),
1479
+ httpProxies: listResources('httpproxy'),
1480
+ gateways: listResources('gateway'),
1481
+ clientTrafficPolicies: listResources('clienttrafficpolicy'),
1482
+ });
1483
+ const trafficServiceSelectors = Object.fromEntries(
1484
+ listResources('service')
1485
+ .filter((service) => service?.metadata?.labels?.['underpost.net/traffic-service'] === 'true')
1486
+ .map((service) => [
1487
+ `${service?.metadata?.namespace || 'default'}/${service?.metadata?.name}`,
1488
+ service?.spec?.selector?.app || '',
1489
+ ]),
1180
1490
  );
1491
+
1492
+ // One read per host, reused across every deployment and environment that
1493
+ // shares it — the colour match is pure, only the fetch is expensive.
1494
+ const routingInfo = {};
1495
+ const hostRoutingInfo = (host) => {
1496
+ if (routingInfo[host] === undefined)
1497
+ routingInfo[host] = Underpost.deploy.readHostRoutingInfo({ host, options });
1498
+ return routingInfo[host];
1499
+ };
1500
+ const trafficState = {};
1501
+ const liveTrafficStateOf = (entry, env) => {
1502
+ const key = `${entry.id}/${env}`;
1503
+ if (trafficState[key]) return trafficState[key];
1504
+ const stableService = Underpost.deploy.trafficServiceNameFactory({ deployId: entry.id, env });
1505
+ const selector = trafficServiceSelectors[`${options.namespace}/${stableService}`] || '';
1506
+ const stableTraffic = trafficFromRoutingInfoFactory({ info: selector, deployId: entry.id, env });
1507
+ if (stableTraffic)
1508
+ return (trafficState[key] = {
1509
+ colour: stableTraffic,
1510
+ service: stableService,
1511
+ });
1512
+ const legacyTraffic = trafficFromRoutingInfoFactory({
1513
+ info: hostRoutingInfo(entry.host),
1514
+ deployId: entry.id,
1515
+ env,
1516
+ });
1517
+ return (trafficState[key] = {
1518
+ colour: legacyTraffic,
1519
+ service: legacyTraffic ? `${entry.deployment}-${legacyTraffic}-service` : '',
1520
+ });
1521
+ };
1522
+ const servingState = {};
1523
+ const servesTraffic = (entry, env) => {
1524
+ const service = liveTrafficStateOf(entry, env).service;
1525
+ if (!service) return false;
1526
+ const key = `${options.namespace}/${service}`;
1527
+ if (servingState[key] === undefined)
1528
+ servingState[key] = Underpost.deploy.serviceHasReadyEndpoints({
1529
+ service,
1530
+ namespace: options.namespace,
1531
+ });
1532
+ return servingState[key];
1533
+ };
1534
+
1535
+ const rows = envs.flatMap((env) =>
1536
+ trafficTableRowsFactory({
1537
+ entries: deployList
1538
+ .flatMap((deployId) => deployTrafficEntriesFactory({ deployId, env }))
1539
+ .map((entry) => ({ ...entry, env })),
1540
+ hosts,
1541
+ liveTrafficOf: (entry) => liveTrafficStateOf(entry, env).colour,
1542
+ servesTraffic: (entry) => servesTraffic(entry, env),
1543
+ }),
1544
+ );
1545
+
1546
+ if (rows.length === 0) {
1547
+ logger.warn('No configured hosts matched the requested traffic report', {
1548
+ hosts,
1549
+ envs,
1550
+ deployList,
1551
+ namespace: options.namespace,
1552
+ });
1553
+ return rows;
1554
+ }
1555
+
1556
+ // Probe the exact public URL represented by each host/path row. PWA rows
1557
+ // can carry several configured paths in one cell, while instance rows
1558
+ // carry one; cache by URL so shared rows never repeat network work.
1559
+ const shellArg = (value) => `'${`${value}`.replaceAll("'", "'\\''")}'`;
1560
+ const probeCache = new Map();
1561
+ const probePath = (host, routePath, tls) => {
1562
+ const normalizedPath = `${routePath || '/'}`.startsWith('/') ? `${routePath || '/'}` : `/${routePath}`;
1563
+ let url;
1564
+ try {
1565
+ url = new URL(normalizedPath, `${tls ? 'https' : 'http'}://${host}`).href;
1566
+ } catch {
1567
+ return { path: normalizedPath, url: '', statuses: ['000'] };
1568
+ }
1569
+ if (!probeCache.has(url)) {
1570
+ // -L follows the real redirect chain, -v supplies one response line
1571
+ // per hop, -i keeps full response headers available, and -s removes
1572
+ // only the progress meter. The body is discarded to keep a status
1573
+ // report bounded even when a host returns a large application page.
1574
+ const raw = shellExec(
1575
+ `curl -L -v -i -s --connect-timeout 3 --max-time 12 --max-redirs 10 ` +
1576
+ `-o /dev/null -w '\nUNDERPOST_CURL_FINAL=%{http_code}\n' ${shellArg(url)} 2>&1 || true`,
1577
+ { stdout: true, silent: true, silentOnError: true, disableLog: true },
1578
+ );
1579
+ probeCache.set(url, curlStatusChainFactory(raw));
1580
+ }
1581
+ return { path: normalizedPath, url, statuses: probeCache.get(url) };
1582
+ };
1583
+
1584
+ const deploymentByName = new Map(deployments.map((deployment) => [deployment.NAME, deployment]));
1585
+ const readinessOf = (deployment) => {
1586
+ const replicas = deployment?.READY || '-';
1587
+ const [ready, desired] = `${replicas}`.split('/').map(Number);
1588
+ return {
1589
+ replicas,
1590
+ exists: Boolean(deployment),
1591
+ ready: Number.isFinite(ready) && Number.isFinite(desired) && desired > 0 && ready === desired,
1592
+ };
1593
+ };
1594
+ const deploymentStatusOf = (row, traffic) => {
1595
+ if (!traffic) return null;
1596
+ const deployment = `${row.deployment}-${traffic}`;
1597
+ return {
1598
+ deployment,
1599
+ traffic,
1600
+ ...readinessOf(deploymentByName.get(deployment)),
1601
+ };
1602
+ };
1603
+ const reportRows = rows.map((row) => {
1604
+ const facts = ingressFacts[row.host] || {};
1605
+ const probes = [...new Set(`${row.path || '/'}`.split(/\s+/).filter(Boolean))].map((routePath) =>
1606
+ probePath(row.host, routePath, facts.tls),
1607
+ );
1608
+ const oppositeTraffic = row.traffic ? nextTrafficFactory(row.traffic) : '';
1609
+ const current = deploymentStatusOf(row, row.traffic);
1610
+ const opposite = deploymentStatusOf(row, oppositeTraffic);
1611
+ return {
1612
+ ...row,
1613
+ probes,
1614
+ current: current ? { ...current, serving: row.serving } : null,
1615
+ opposite,
1616
+ };
1617
+ });
1618
+
1619
+ // Padded on the raw values, coloured afterwards: an ANSI escape counts
1620
+ // toward String.length and would skew every column right of it.
1621
+ const columns = ['HOST', 'PATH', 'KIND', 'ROUTE', 'TLS', 'HTTP3', 'CURRENT', 'OPPOSITE'];
1622
+ const deploymentStatus = (status, includeServing = false) => {
1623
+ if (!status) return `unrouted - missing${includeServing ? ' not-serving' : ''}`;
1624
+ return [
1625
+ status.deployment,
1626
+ status.replicas,
1627
+ status.exists ? (status.ready ? 'ready' : 'not-ready') : 'missing',
1628
+ ...(includeServing ? [status.serving ? 'serving' : 'not-serving'] : []),
1629
+ ].join(' ');
1630
+ };
1631
+ const cellOf = (row) => {
1632
+ const facts = ingressFacts[row.host] || {};
1633
+ const pathStatus = row.probes.map((probe) => `${probe.path} [${probe.statuses.join('→')}]`).join(' ');
1634
+ return [
1635
+ row.host,
1636
+ pathStatus,
1637
+ row.kind,
1638
+ facts.route || 'none',
1639
+ facts.tls ? 'yes' : 'no',
1640
+ facts.http3 ? 'yes' : 'no',
1641
+ deploymentStatus(row.current, true),
1642
+ deploymentStatus(row.opposite),
1643
+ ];
1644
+ };
1645
+ const paint = (value, i) => {
1646
+ if (columns[i] === 'PATH')
1647
+ return value.replace(/\b(?:000|[1-5][0-9]{2})\b/g, (status) => {
1648
+ if (/^1/.test(status)) return status.cyan;
1649
+ if (/^2/.test(status)) return status.green;
1650
+ if (/^3/.test(status)) return status.cyan;
1651
+ if (/^4/.test(status)) return status.yellow;
1652
+ return status.red;
1653
+ });
1654
+ if (columns[i] === 'CURRENT' || columns[i] === 'OPPOSITE') {
1655
+ if (value === '-') return value;
1656
+ const [deployment, replicas, status, serving] = value.split(' ');
1657
+ const deploymentDisplay = deployment.replace(
1658
+ /-(blue|green)$/,
1659
+ (_, traffic) => `-${traffic === 'blue' ? traffic.bgBlue.bold.black : traffic.bgGreen.bold.black}`,
1660
+ );
1661
+ const replicasDisplay = readinessOf({ READY: replicas }).ready ? replicas.green : replicas.red;
1662
+ const statusDisplay = status === 'ready' ? status.green : status.red;
1663
+ const servingDisplay = serving ? (serving === 'serving' ? serving.green : serving.red) : '';
1664
+ return [deploymentDisplay, replicasDisplay, statusDisplay, servingDisplay].filter(Boolean).join(' ');
1665
+ }
1666
+ // TLS and HTTP/3 being off is a normal development state, not a fault, so
1667
+ // only the affirmative is highlighted.
1668
+ if (columns[i] === 'TLS' || columns[i] === 'HTTP3') return value === 'yes' ? value.green : value;
1669
+ if (columns[i] === 'ROUTE') return value === 'none' ? value.red : value;
1670
+ return value;
1671
+ };
1672
+ // A shared table renderer, called once per environment when both are
1673
+ // scanned: mixing development and production rows into one table is what
1674
+ // made an unrouted production host look like a duplicate of the same,
1675
+ // live development host.
1676
+ const printTable = (cells, heading) => {
1677
+ const widths = columns.map((column, i) => Math.max(column.length, ...cells.map((cell) => `${cell[i]}`.length)));
1678
+ const line = (values, painted) =>
1679
+ values
1680
+ .map(
1681
+ (value, i) => (painted ? paint(`${value}`, i) : `${value}`) + ' '.repeat(widths[i] - `${value}`.length),
1682
+ )
1683
+ .join(' ');
1684
+ console.log(heading ? `\n${heading.bold}\n${line(columns, false).bold}` : `\n${line(columns, false).bold}`);
1685
+ console.log(widths.map((width) => '-'.repeat(width)).join(' '));
1686
+ for (const cell of cells) console.log(line(cell, true));
1687
+ };
1688
+ for (const env of envs) {
1689
+ const envCells = reportRows.filter((row) => row.env === env).map(cellOf);
1690
+ if (envCells.length > 0) printTable(envCells, `[${env.toUpperCase()}]`);
1691
+ }
1692
+ console.log('');
1693
+ return reportRows;
1694
+ },
1695
+
1696
+ /**
1697
+ * @method restore-mongo
1698
+ * @description Initializes a MongoDB replica set in the cluster without resetting existing data.
1699
+ * @param {string} path - The input value, identifier, or path for the operation.
1700
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1701
+ * @memberof UnderpostRun
1702
+ */
1703
+ 'restore-mongo': async (path, options = DEFAULT_OPTION) => {
1704
+ await MongoBootstrap.initReplicaSet({
1705
+ namespace: options.namespace || 'default',
1706
+ reset: options.reset || false,
1707
+ clusterType: options.kubeadm ? 'kubeadm' : options.k3s ? 'k3s' : 'kind', // o 'k3s' / 'kubeadm' según corresponda
1708
+ underpostRoot: '.',
1709
+ });
1710
+ },
1711
+
1712
+ /**
1713
+ * @method ingress-refresh
1714
+ * @description Rebuilds the shared HTTPProxy/HTTPRoute host map without
1715
+ * inheriting application workload placement. Supplying a path or
1716
+ * `--ingress-node` is the explicit recovery mechanism for relocating the
1717
+ * public 80/443 listener.
1718
+ * @param {string} [path] - Optional ingress node name.
1719
+ * @param {UnderpostRunDefaultOptions} options - Runner options.
1720
+ * @returns {boolean} True after the ingress is Ready with the refreshed map.
1721
+ * @memberof UnderpostRun
1722
+ */
1723
+ 'ingress-refresh': (path = '', options = DEFAULT_OPTION) => {
1724
+ const namespace = options.namespace || 'default';
1725
+ const ingressNode = options.ingressNode || `${path || ''}`.trim();
1726
+ const updated = Underpost.cluster.refreshUnderpostIngress({
1727
+ namespace,
1728
+ options: { ...options, ingressNode },
1729
+ });
1730
+ if (!updated)
1731
+ throw new Error(
1732
+ `[ingress-refresh] ${UNDERPOST_INGRESS.name} is not installed; install both ingress stacks first`,
1733
+ );
1734
+ logger.info('[ingress-refresh] Shared ingress is operational', {
1735
+ namespace,
1736
+ node: ingressNode || '(preserved)',
1737
+ });
1738
+ return true;
1181
1739
  },
1182
1740
 
1183
1741
  'instance-promote': async (path, options = DEFAULT_OPTION) => {
1184
1742
  const env = options.dev ? 'development' : 'production';
1185
- const baseCommand = options.dev ? 'node bin' : 'underpost';
1186
- const baseClusterCommand = options.dev ? ' --dev' : '';
1743
+ options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
1187
1744
  let [deployId, id] = path.split(',');
1188
- const confInstances = JSON.parse(
1189
- fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
1190
- );
1745
+ const confInstances = loadConfInstances(deployId);
1746
+ const promoted = selectConfInstances(confInstances, id);
1191
1747
  let promotedTraffic = '';
1192
- for (const instance of confInstances) {
1193
- let {
1194
- id: _id,
1195
- host: _host,
1196
- path: _path,
1197
- image: _image,
1198
- fromPort: _fromPort,
1199
- toPort: _toPort,
1200
- fromDebugPort: _fromDebugPort,
1201
- toDebugPort: _toDebugPort,
1202
- cmd: _cmd,
1203
- volumes: _volumes,
1204
- metadata: _metadata,
1205
- } = instance;
1206
- if (id !== _id) continue;
1207
- const _deployId = `${deployId}-${_id}`;
1208
- // Use debug ports in development when defined, fall back to production ports.
1209
- if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
1210
- if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
1211
- const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
1212
- hostTest: _host,
1748
+
1749
+ // A Contour HTTPProxy is named after its host, so every instance sharing a
1750
+ // host shares one object. Rebuilding it from the promoted instance alone
1751
+ // would drop its siblings' routes, so each host is rendered from the full
1752
+ // set: promoted instances flip colour, the rest keep their live colour.
1753
+ //
1754
+ // "Full set" is not what the conf currently declares. Each variant sub-path
1755
+ // is its own deployment, and one that was dropped from the conf while its
1756
+ // workload is still up must keep its route — otherwise editing the variant
1757
+ // list to scope a deploy takes the untouched variants offline. The
1758
+ // descriptors last published for the host cover exactly that gap, and a
1759
+ // variant leaves the render only once its Deployment is gone.
1760
+ const promotedIds = new Set(promoted.map((instance) => instance.id));
1761
+ const hosts = [...new Set(promoted.map((instance) => instance.host))];
1762
+ const namespace = options.namespace || 'default';
1763
+ const gatewayConfDir = Underpost.deploy.gatewayConfDirFactory({ deployId, env });
1764
+ const deployedNames = Underpost.kubectl.get('', 'deployment', namespace).map((entry) => entry.NAME);
1765
+ const instancesByHost = Object.fromEntries(
1766
+ hosts.map((host) => [
1767
+ host,
1768
+ hostRenderInstancesFactory({
1769
+ declared: confInstances.filter((instance) => instance.host === host),
1770
+ preserved: readHostInstanceRegistry({ confDir: gatewayConfDir, host }),
1771
+ isDeployed: (instance) =>
1772
+ deployedNames.some((name) => name.startsWith(`${deployId}-${instance.id}-${env}-`)),
1773
+ }),
1774
+ ]),
1775
+ );
1776
+ const affected = hosts.flatMap((host) => instancesByHost[host]);
1777
+ const trafficById = {};
1778
+ const currentTrafficById = {};
1779
+ const bootstrapTrafficById = {};
1780
+ for (const instance of affected) {
1781
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
1782
+ hostTest: instance.host,
1213
1783
  namespace: options.namespace,
1784
+ env,
1785
+ gatewayApi: options.gatewayApi,
1214
1786
  });
1215
- const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
1216
- promotedTraffic = targetTraffic;
1217
- let proxyYaml =
1218
- Underpost.deploy.baseProxyYamlFactory({ host: _host, env: options.tls ? 'production' : env, options }) +
1219
- Underpost.deploy.deploymentYamlServiceFactory({
1220
- path: _path,
1221
- port: _fromPort,
1222
- // serviceId: deployId,
1223
- deployId: _deployId,
1224
- env,
1225
- deploymentVersions: [targetTraffic],
1226
- // pathRewritePolicy,
1787
+ currentTrafficById[instance.id] = currentTraffic;
1788
+ if (!promotedIds.has(instance.id)) {
1789
+ trafficById[instance.id] = currentTraffic || 'blue';
1790
+ continue;
1791
+ }
1792
+ trafficById[instance.id] = nextTrafficFactory(currentTraffic, options.targetTrafficById?.[instance.id]);
1793
+ promotedTraffic = trafficById[instance.id];
1794
+ }
1795
+
1796
+ // Readiness is mandatory for an actual promotion. The only exception is
1797
+ // the explicit no-backend checkpoint, whose purpose is to prove that an
1798
+ // endpointless selector returns the configured fallback before a workload
1799
+ // exists.
1800
+ if (!options.noBackendCheckpoint)
1801
+ for (const instance of affected.filter((entry) => promotedIds.has(entry.id))) {
1802
+ const podId = `${deployId}-${instance.id}-${env}-${trafficById[instance.id]}`;
1803
+ if (
1804
+ !Underpost.deploy.awaitDeploymentReady({ deployment: podId, namespace }) ||
1805
+ !Underpost.deploy.awaitServiceEndpoints({ service: `${podId}-service`, namespace })
1806
+ )
1807
+ throw new Error(`Refusing to promote ${instance.id} to unready colour ${trafficById[instance.id]}`);
1808
+ }
1809
+
1810
+ // Bootstrap one stable Service per instance on its current ready colour.
1811
+ // Routes and fallback blocks can now migrate without changing traffic;
1812
+ // promoted selectors move to their targets only after the whole host has
1813
+ // converged below.
1814
+ for (const instance of affected) {
1815
+ const instanceDeployId = `${deployId}-${instance.id}`;
1816
+ const currentTraffic = currentTrafficById[instance.id];
1817
+ const currentReady =
1818
+ !!currentTraffic &&
1819
+ Underpost.deploy.serviceHasReadyEndpoints({
1820
+ service: `${instanceDeployId}-${env}-${currentTraffic}-service`,
1821
+ namespace,
1822
+ });
1823
+ const bootstrapTraffic = currentReady ? currentTraffic : trafficById[instance.id];
1824
+ bootstrapTrafficById[instance.id] = bootstrapTraffic;
1825
+ Underpost.deploy.applyTrafficService({
1826
+ deployId: instanceDeployId,
1827
+ env,
1828
+ traffic: bootstrapTraffic,
1829
+ namespace,
1830
+ fromPort: instancePortFactory({ instance, env }),
1831
+ toPort: instancePortFactory({ instance, env, container: true }),
1832
+ });
1833
+ }
1834
+
1835
+ // Instance routes attach to the Gateway the parent deploy owns. A Gateway
1836
+ // per instance host cannot work beside it: `mergeGateways` collapses every
1837
+ // Gateway of the class onto one listener per (port, protocol), so a
1838
+ // hostname-scoped listener and the deploy's hostname-less one contend for
1839
+ // the same port — on 80 the hostname-scoped one is dropped outright, and
1840
+ // on 443 it keeps an SNI filter chain whose route table is left empty.
1841
+ // Either way every path on the instance host answers 404 while the
1842
+ // Gateway reports Programmed and the route reports Accepted.
1843
+ const gatewayName = Underpost.deploy.gatewayNameFactory({ deployId, env });
1844
+ for (const host of hosts) {
1845
+ const hostInstances = instancesByHost[host];
1846
+ const routingEnv = options.tls ? 'production' : env;
1847
+ // Every variant of this host contributes one proxied sub-path to the
1848
+ // shared gateway, whose interception turns each workload's own error
1849
+ // into that variant's declared document.
1850
+ if (options.gatewayApi)
1851
+ writeHostServerConf({
1852
+ confDir: gatewayConfDir,
1853
+ host,
1854
+ conf: hostServerConfFactory({
1855
+ host,
1856
+ namespace: options.namespace || 'default',
1857
+ routes: hostInstances.map((instance) => ({
1858
+ path: instance.path,
1859
+ upstream: `${Underpost.deploy.trafficServiceNameFactory({
1860
+ deployId: `${deployId}-${instance.id}`,
1861
+ env,
1862
+ })}:${instancePortFactory({ instance, env })}`,
1863
+ statuses: instanceInterceptStatusesFactory(instance),
1864
+ stripPrefix: Array.isArray(instance.pathRewritePolicy) && instance.pathRewritePolicy.length > 0,
1865
+ })),
1866
+ }),
1867
+ });
1868
+ // Recorded before the route is published, so a variant dropped from the
1869
+ // conf keeps its descriptor from the render that still carried it.
1870
+ writeHostInstanceRegistry({ confDir: gatewayConfDir, host, instances: hostInstances });
1871
+ // The route below sends every intercepted path to this shared Nginx
1872
+ // service. Load the host block first so an Accepted route can never race
1873
+ // a gateway that still has only its default server configuration.
1874
+ if (options.gatewayApi)
1875
+ installGatewayConf({
1876
+ hostRoot: Underpost.deploy.underpostGatewayRootFactory(options),
1877
+ confSourceDir: gatewayConfDir,
1878
+ namespace: options.namespace,
1227
1879
  });
1880
+ let proxyYaml = options.gatewayApi
1881
+ ? Underpost.deploy.httpRouteYamlFactory({
1882
+ host,
1883
+ options,
1884
+ parentName: gatewayName,
1885
+ rules: instanceHttpRouteRulesFactory({ deployId, instances: hostInstances, env, trafficById, options }),
1886
+ })
1887
+ : Underpost.deploy.baseProxyYamlFactory({ host, env: routingEnv, options }) +
1888
+ instanceProxyRoutesFactory({ deployId, instances: hostInstances, env, trafficById });
1228
1889
  if (options.tls) {
1229
1890
  if (options.test) {
1230
- const sslDir = `./engine-private/ssl/${_host}`;
1231
- const nameSafe = _host.replace(/[^a-zA-Z0-9_.-]/g, '_');
1232
- fs.mkdirpSync(sslDir);
1233
- shellExec(`bash ./scripts/ssl.sh "${sslDir}" "${_host}"`);
1234
- shellExec(`kubectl delete secret ${_host} -n ${options.namespace} --ignore-not-found`);
1235
- shellExec(
1236
- `kubectl create secret tls ${_host} --cert="${sslDir}/${nameSafe}.pem" --key="${sslDir}/${nameSafe}-key.pem" -n ${options.namespace}`,
1237
- );
1891
+ Underpost.deploy.selfSignedTlsSecretFactory({
1892
+ host,
1893
+ namespace: options.namespace,
1894
+ underpostRoot: options.underpostRoot || '.',
1895
+ });
1238
1896
  } else {
1239
- shellExec(`sudo kubectl delete Certificate ${_host} -n ${options.namespace} --ignore-not-found`);
1240
- proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host: _host });
1897
+ shellExec(`sudo kubectl delete Certificate ${host} -n ${options.namespace} --ignore-not-found`);
1898
+ proxyYaml += Underpost.deploy.buildCertManagerCertificate({ ...options, host });
1241
1899
  }
1242
1900
  }
1243
- // console.log(proxyYaml);
1244
- shellExec(`kubectl delete HTTPProxy ${_host} --namespace ${options.namespace} --ignore-not-found`);
1901
+ if (options.gatewayApi) {
1902
+ // Left by the per-host model this replaces. Both outlive the route
1903
+ // that referenced them and keep contending for the merged listener.
1904
+ for (const name of [`Gateway ${host}`, `ClientTrafficPolicy ${host}-http3`])
1905
+ shellExec(`kubectl delete ${name} --namespace ${options.namespace} --ignore-not-found`, { silent: true });
1906
+ }
1907
+ // The host's route object is replaced in place, never deleted first:
1908
+ // `apply` moves the whole spec in one transition, so the hostname always
1909
+ // has a route. Delete-then-apply left a window with none, and Envoy
1910
+ // answered every request that landed in it 404 — an outage on each
1911
+ // promote, independent of which colour was healthy.
1245
1912
  shellExec(
1246
- `kubectl apply -f - -n ${options.namespace} <<EOF
1913
+ `kubectl apply -f - -n ${options.namespace} <<'EOF'
1247
1914
  ${proxyYaml}
1248
1915
  EOF
1249
1916
  `,
1250
1917
  { disableLog: true },
1251
1918
  );
1252
1919
  }
1920
+ // These hostnames may have just moved between stacks, or appeared for the
1921
+ // first time. A shared edge that has not been told answers them from the
1922
+ // other data plane, which has no route for them — a 404 in front of a
1923
+ // healthy workload. No-op when no shared edge is installed.
1924
+ const sharedIngressUpdated = Underpost.cluster.refreshUnderpostIngress({ namespace, options });
1925
+ if (sharedIngressUpdated) {
1926
+ Underpost.deploy.removeInactiveHostRoutes({ hosts, gatewayApi: options.gatewayApi, namespace });
1927
+ Underpost.cluster.refreshUnderpostIngress({ namespace, options });
1928
+ }
1929
+ if (!options.noBackendCheckpoint)
1930
+ for (const instance of affected.filter((entry) => promotedIds.has(entry.id))) {
1931
+ const instanceDeployId = `${deployId}-${instance.id}`;
1932
+ const targetTraffic = trafficById[instance.id];
1933
+ const bootstrapTraffic = bootstrapTrafficById[instance.id];
1934
+ if (targetTraffic !== bootstrapTraffic)
1935
+ Underpost.deploy.applyTrafficService({
1936
+ deployId: instanceDeployId,
1937
+ env,
1938
+ traffic: targetTraffic,
1939
+ namespace,
1940
+ fromPort: instancePortFactory({ instance, env }),
1941
+ toPort: instancePortFactory({ instance, env, container: true }),
1942
+ });
1943
+ const trafficService = Underpost.deploy.trafficServiceNameFactory({ deployId: instanceDeployId, env });
1944
+ if (!Underpost.deploy.awaitServiceEndpoints({ service: trafficService, namespace })) {
1945
+ if (targetTraffic !== bootstrapTraffic)
1946
+ Underpost.deploy.applyTrafficService({
1947
+ deployId: instanceDeployId,
1948
+ env,
1949
+ traffic: bootstrapTraffic,
1950
+ namespace,
1951
+ fromPort: instancePortFactory({ instance, env }),
1952
+ toPort: instancePortFactory({ instance, env, container: true }),
1953
+ });
1954
+ throw new Error(`Traffic Service ${trafficService} never became ready on ${targetTraffic}`);
1955
+ }
1956
+ }
1253
1957
  // Refresh the gRPC service to ensure it points to the parent deploy's current traffic.
1254
1958
  if (promotedTraffic) {
1255
- const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
1959
+ const parentTraffic =
1960
+ Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }) || 'blue';
1256
1961
  const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
1257
1962
  deployId,
1258
1963
  env,
@@ -1272,13 +1977,75 @@ EOF
1272
1977
  */
1273
1978
  instance: async (path = '', options = DEFAULT_OPTION) => {
1274
1979
  const env = options.dev ? 'development' : 'production';
1980
+ options = {
1981
+ ...options,
1982
+ gatewayApi: gatewayApiEnabledFactory(options),
1983
+ namespace: options.namespace || 'default',
1984
+ };
1275
1985
  const baseCommand = options.dev ? 'node bin' : 'underpost';
1276
1986
  const baseClusterCommand = options.dev ? ' --dev' : '';
1277
1987
  let [deployId, id, replicas] = path.split(',');
1278
1988
  if (!replicas) replicas = options.replicas;
1279
- const confInstances = JSON.parse(
1280
- fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
1281
- );
1989
+ const confInstances = selectConfInstances(loadConfInstances(deployId), id);
1990
+ const { liveTrafficById, targetTrafficById, serving } = instanceTrafficPlanFactory({
1991
+ instances: confInstances,
1992
+ requestedTraffic: options.traffic,
1993
+ liveTrafficOf: (instance) =>
1994
+ Underpost.deploy.getCurrentTraffic(`${deployId}-${instance.id}`, {
1995
+ hostTest: instance.host,
1996
+ namespace: options.namespace,
1997
+ env,
1998
+ gatewayApi: options.gatewayApi,
1999
+ }),
2000
+ servesTraffic: (instance, colour) =>
2001
+ Underpost.deploy.serviceHasReadyEndpoints({
2002
+ service: `${deployId}-${instance.id}-${env}-${colour}-service`,
2003
+ namespace: options.namespace,
2004
+ }),
2005
+ });
2006
+
2007
+ let prePromoted = false;
2008
+ const fallbackChecks = instanceFallbackChecksFactory(confInstances);
2009
+ // The promote points intercepted statuses at these documents on either
2010
+ // path, so they are placed regardless of which one runs.
2011
+ if (options.gatewayApi && fallbackChecks.length > 0 && !options.expose)
2012
+ placeInstanceStaticAssets({ instances: confInstances, options, label: 'instance' });
2013
+ if (
2014
+ options.gatewayApi &&
2015
+ !options.gatewayBootstrapComplete &&
2016
+ fallbackChecks.length > 0 &&
2017
+ !options.expose &&
2018
+ serving.length === 0
2019
+ ) {
2020
+ // Clear the target colour before routing to it. A previous blue/green
2021
+ // cycle may have left that inactive Deployment Ready, which would turn
2022
+ // this into a stale-app probe instead of a no-backend fallback probe.
2023
+ for (const instance of confInstances) {
2024
+ const podId = `${deployId}-${instance.id}-${env}-${targetTrafficById[instance.id]}`;
2025
+ shellExec(`kubectl delete service ${podId}-service --namespace ${options.namespace} --ignore-not-found`);
2026
+ shellExec(`kubectl delete deployment ${podId} --namespace ${options.namespace} --ignore-not-found`);
2027
+ }
2028
+ // A direct `run instance` must publish and prove its configured static
2029
+ // fallback before it submits the first instance Deployment document.
2030
+ await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, {
2031
+ ...options,
2032
+ targetTrafficById,
2033
+ noBackendCheckpoint: true,
2034
+ });
2035
+ await gatewayFallbackProbeRunner({
2036
+ gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
2037
+ checks: fallbackChecks,
2038
+ options,
2039
+ label: 'instance',
2040
+ });
2041
+ prePromoted = true;
2042
+ } else if (serving.length > 0)
2043
+ logger.info('[instance] Live colour serving; holding traffic until the target colour is Ready', {
2044
+ hosts: [...new Set(serving.map((instance) => instance.host))],
2045
+ live: serving.map((instance) => `${instance.id}:${liveTrafficById[instance.id]}`),
2046
+ target: serving.map((instance) => `${instance.id}:${targetTrafficById[instance.id]}`),
2047
+ });
2048
+
1282
2049
  const etcHosts = [];
1283
2050
  for (const instance of confInstances) {
1284
2051
  let {
@@ -1286,10 +2053,6 @@ EOF
1286
2053
  host: _host,
1287
2054
  path: _path,
1288
2055
  image: _image,
1289
- fromPort: _fromPort,
1290
- toPort: _toPort,
1291
- fromDebugPort: _fromDebugPort,
1292
- toDebugPort: _toDebugPort,
1293
2056
  cmd: _cmd,
1294
2057
  volumes: _volumes,
1295
2058
  metadata: _metadata,
@@ -1297,11 +2060,9 @@ EOF
1297
2060
  readinessProbe: _readinessProbe,
1298
2061
  livenessProbe: _livenessProbe,
1299
2062
  } = instance;
1300
- if (id !== _id) continue;
1301
2063
  const _deployId = `${deployId}-${_id}`;
1302
- // Use debug ports in development when defined, fall back to production ports.
1303
- if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
1304
- if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
2064
+ const _fromPort = instancePortFactory({ instance, env });
2065
+ const _toPort = instancePortFactory({ instance, env, container: true });
1305
2066
  etcHosts.push(_host);
1306
2067
  if (options.expose) continue;
1307
2068
  // Examples images:
@@ -1318,12 +2079,7 @@ EOF
1318
2079
  k3s: options.k3s,
1319
2080
  });
1320
2081
 
1321
- const currentTraffic = Underpost.deploy.getCurrentTraffic(_deployId, {
1322
- hostTest: _host,
1323
- namespace: options.namespace,
1324
- });
1325
-
1326
- const targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'blue';
2082
+ const targetTraffic = targetTrafficById[instance.id];
1327
2083
  const podId = `${_deployId}-${env}-${targetTraffic}`;
1328
2084
  const ignorePods = Underpost.kubectl.get(podId, 'pods', options.namespace).map((p) => p.NAME);
1329
2085
  Underpost.deploy.configMap(env, options.namespace);
@@ -1343,20 +2099,21 @@ EOF
1343
2099
  k3s: options.k3s,
1344
2100
  env,
1345
2101
  }),
1346
- clusterContext: options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind',
2102
+ clusterContext: clusterTypeFactory(options),
1347
2103
  gitClean: options.gitClean || false,
1348
2104
  sshKeyPath: options.sshKeyPath || '',
1349
2105
  });
1350
2106
  // Regenerate the parent deploy's gRPC ClusterIP service pointing to the
1351
2107
  // parent's current traffic colour and apply it before the instance pod starts so
1352
2108
  // DNS is resolvable the moment the pod boots.
1353
- const parentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
2109
+ const parentTraffic =
2110
+ Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env }) || 'blue';
1354
2111
  const grpcServicePath = Underpost.deploy.buildGrpcServiceManifest({
1355
2112
  deployId,
1356
2113
  env,
1357
2114
  confServer: loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`),
1358
2115
  namespace: options.namespace,
1359
- traffic: [targetTraffic],
2116
+ traffic: [parentTraffic],
1360
2117
  host: _host,
1361
2118
  });
1362
2119
  if (grpcServicePath) shellExec(`kubectl apply -f ${grpcServicePath} -n ${options.namespace}`);
@@ -1371,7 +2128,6 @@ EOF
1371
2128
  // Resolve env-scoped lifecycle/probe blocks: each can be either
1372
2129
  // { ...envObj } // shared shape
1373
2130
  // { development: {...}, production: {...} } // env-specific
1374
- const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
1375
2131
 
1376
2132
  // Convention: an instance config may place `imagePullPolicy` inside
1377
2133
  // the env-scoped lifecycle block (alongside postStart/preStop).
@@ -1379,7 +2135,7 @@ EOF
1379
2135
  // strip it from the lifecycle hash so the rendered YAML stays valid.
1380
2136
  // CLI override (`--image-pull-policy`) wins over the conf value.
1381
2137
  const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
1382
- Underpost.deploy.extractInstanceImagePullPolicy(pickEnv(_lifecycle));
2138
+ Underpost.deploy.extractInstanceImagePullPolicy(resolveEnvScoped(_lifecycle, env));
1383
2139
  const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
1384
2140
 
1385
2141
  let deploymentYaml = `---
@@ -1395,16 +2151,31 @@ ${Underpost.deploy
1395
2151
  volumes: _volumes,
1396
2152
  cmd: resolvedCmd,
1397
2153
  lifecycle: lifecycleForManifest,
1398
- readinessProbe: pickEnv(_readinessProbe),
1399
- livenessProbe: pickEnv(_livenessProbe),
2154
+ readinessProbe: Underpost.deploy.requiredReadinessProbeFactory({
2155
+ probe: resolveEnvScoped(_readinessProbe, env),
2156
+ port: _toPort,
2157
+ }),
2158
+ livenessProbe: resolveEnvScoped(_livenessProbe, env),
1400
2159
  containerPort: _toPort,
1401
2160
  imagePullPolicy: instanceImagePullPolicy,
2161
+ // Pin the pod in the manifest submitted for its only rollout. Volumes were
2162
+ // already resolved against this node; leaving the pod unconstrained could
2163
+ // schedule it away from its data and require a second, post-ready move.
2164
+ nodeName: options.nodeName
2165
+ ? Underpost.deploy.resolveDeployNode({
2166
+ node: options.nodeName,
2167
+ kind: options.kind,
2168
+ kubeadm: options.kubeadm,
2169
+ k3s: options.k3s,
2170
+ env,
2171
+ })
2172
+ : '',
1402
2173
  })
1403
2174
  .replace('{{ports}}', buildKindPorts(_fromPort, _toPort))}
1404
2175
  `;
1405
2176
  // console.log(deploymentYaml);
1406
2177
  shellExec(
1407
- `kubectl apply -f - -n ${options.namespace} <<EOF
2178
+ `kubectl apply -f - -n ${options.namespace} <<'EOF'
1408
2179
  ${deploymentYaml}
1409
2180
  EOF
1410
2181
  `,
@@ -1426,13 +2197,13 @@ EOF
1426
2197
  logger.error(`Deployment ${deployId} did not become ready in time.`);
1427
2198
  return;
1428
2199
  }
1429
- shellExec(
1430
- `${baseCommand} run${baseClusterCommand} --namespace ${options.namespace}` +
1431
- `${options.nodeName ? ` --node-name ${options.nodeName}` : ''}` +
1432
- `${options.tls ? ` --tls ${options.test ? '--test' : ''}` : ''}` +
1433
- ` instance-promote '${path}'`,
1434
- );
1435
2200
  }
2201
+ // Cluster-invoked instances inherit a fallback route for the old
2202
+ // colour, so they still promote the family atomically after every variant
2203
+ // is Ready. A direct run already routed the exact target colour before the
2204
+ // Deployment and proved its fallback; that route simply starts proxying the
2205
+ // new endpoints and must not be toggled a second time.
2206
+ if (!options.expose && !prePromoted) await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${id}`, options);
1436
2207
  if (options.etcHosts) {
1437
2208
  const hostListenResult = etcHostFactory(etcHosts);
1438
2209
  logger.info(hostListenResult.renderHosts);
@@ -1485,35 +2256,48 @@ EOF
1485
2256
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
1486
2257
  * @memberof UnderpostRun
1487
2258
  */
1488
- 'instance-build-manifest': (path, options = DEFAULT_OPTION) => {
2259
+ 'instance-build-manifest': async (path, options = DEFAULT_OPTION) => {
1489
2260
  const env = options.dev ? 'development' : 'production';
1490
2261
  let [deployId, id, projectPath] = path.split(',');
1491
2262
  const rootPath = projectPath ? projectPath : '.';
2263
+
2264
+ const confInstances = loadConfInstances(deployId);
2265
+ // Targeting a template id builds every world in the family. The fan-out
2266
+ // re-enters this runner with `instanceOnly` because the default world
2267
+ // keeps the template id verbatim — without it, selecting `mmo-server`
2268
+ // would match the family again and recurse forever. Only that default
2269
+ // world publishes to the project root, so the repo keeps exactly one
2270
+ // canonical Dockerfile/deployment.yaml pair.
2271
+ const selected = options.instanceOnly
2272
+ ? confInstances.filter((instance) => instance.id === id)
2273
+ : selectConfInstances(confInstances, id);
2274
+ if (selected.length === 0) {
2275
+ logger.error(`Instance with id '${id}' not found in conf.instances.json for deployId '${deployId}'`);
2276
+ return;
2277
+ }
2278
+ if (!options.instanceOnly && (selected.length > 1 || selected[0].id !== id)) {
2279
+ for (const instance of selected)
2280
+ await UnderpostRun.RUNNERS['instance-build-manifest'](
2281
+ [deployId, instance.id, projectPath].filter((v) => v !== undefined).join(','),
2282
+ { ...options, instanceOnly: true },
2283
+ );
2284
+ return;
2285
+ }
2286
+
1492
2287
  const envManifestPath = `${rootPath}/manifests/deployments/${id}-${env}`;
1493
2288
  const outputPath = `${envManifestPath}/deployment.yaml`;
1494
2289
  const dockerfileManifestPath = `${envManifestPath}/Dockerfile`;
1495
2290
 
1496
2291
  fs.mkdirpSync(envManifestPath);
1497
2292
 
1498
- const confInstances = JSON.parse(
1499
- fs.readFileSync(`./engine-private/conf/${deployId}/conf.instances.json`, 'utf8'),
1500
- );
1501
-
1502
- const instance = confInstances.find((i) => i.id === id);
1503
- if (!instance) {
1504
- logger.error(`Instance with id '${id}' not found in conf.instances.json for deployId '${deployId}'`);
1505
- return;
1506
- }
2293
+ const instance = selected[0];
2294
+ const isDefaultInstance = instance.id === instance.templateId || !instance.templateId;
2295
+ const instanceEnvBuilder = await loadProjectInstanceEnvBuilder(deployId);
1507
2296
 
1508
2297
  let {
1509
2298
  id: _id,
1510
2299
  host: _host,
1511
- path: _path,
1512
2300
  image: _image,
1513
- fromPort: _fromPort,
1514
- toPort: _toPort,
1515
- fromDebugPort: _fromDebugPort,
1516
- toDebugPort: _toDebugPort,
1517
2301
  cmd: _cmd,
1518
2302
  volumes: _volumes,
1519
2303
  metadata: _metadata,
@@ -1550,9 +2334,8 @@ EOF
1550
2334
 
1551
2335
  const _deployId = `${deployId}-${_id}`;
1552
2336
  if (!_image) _image = `underpost/underpost-engine:${Underpost.version}`;
1553
- // Use debug ports in development when defined, fall back to production ports.
1554
- if (env === 'development' && _fromDebugPort) _fromPort = _fromDebugPort;
1555
- if (env === 'development' && _toDebugPort) _toPort = _toDebugPort;
2337
+ const _fromPort = instancePortFactory({ instance, env });
2338
+ const _toPort = instancePortFactory({ instance, env, container: true });
1556
2339
 
1557
2340
  // Build image from projectPath Dockerfile and load into cluster when --build is set.
1558
2341
  if (options.build && projectPath) {
@@ -1585,13 +2368,12 @@ EOF
1585
2368
 
1586
2369
  // Env-aware lifecycle / probe selection. Each block may either be
1587
2370
  // a single object (shared across envs) or `{ development, production }`.
1588
- const pickEnv = (v) => (v && (v.development || v.production) ? v[env] : v);
1589
2371
 
1590
2372
  // Convention: an instance config may place `imagePullPolicy` inside
1591
2373
  // the env-scoped lifecycle block (alongside postStart/preStop).
1592
2374
  // Extract it onto the container spec and strip it from the lifecycle hash.
1593
2375
  const { lifecycle: lifecycleForManifest, imagePullPolicy: lifecycleImagePullPolicy } =
1594
- Underpost.deploy.extractInstanceImagePullPolicy(pickEnv(_lifecycle));
2376
+ Underpost.deploy.extractInstanceImagePullPolicy(resolveEnvScoped(_lifecycle, env));
1595
2377
  const instanceImagePullPolicy = options.imagePullPolicy || lifecycleImagePullPolicy;
1596
2378
 
1597
2379
  const deploymentYaml =
@@ -1608,10 +2390,22 @@ EOF
1608
2390
  volumes: _volumes,
1609
2391
  cmd: resolvedCmd,
1610
2392
  lifecycle: lifecycleForManifest,
1611
- readinessProbe: pickEnv(_readinessProbe),
1612
- livenessProbe: pickEnv(_livenessProbe),
2393
+ readinessProbe: Underpost.deploy.requiredReadinessProbeFactory({
2394
+ probe: resolveEnvScoped(_readinessProbe, env),
2395
+ port: _toPort,
2396
+ }),
2397
+ livenessProbe: resolveEnvScoped(_livenessProbe, env),
1613
2398
  containerPort: _toPort,
1614
2399
  imagePullPolicy: instanceImagePullPolicy,
2400
+ nodeName: options.nodeName
2401
+ ? Underpost.deploy.resolveDeployNode({
2402
+ node: options.nodeName,
2403
+ kind: options.kind,
2404
+ kubeadm: options.kubeadm,
2405
+ k3s: options.k3s,
2406
+ env,
2407
+ })
2408
+ : '',
1615
2409
  })
1616
2410
  .replace('{{ports}}', buildKindPorts(_fromPort, _toPort));
1617
2411
 
@@ -1651,17 +2445,51 @@ EOF
1651
2445
  })}\n`;
1652
2446
  }
1653
2447
 
1654
- // proxy.yaml — HTTPProxy for the instance host (mirrors instance-promote).
2448
+ // proxy.yaml — this instance's OWN route only (its sub-path → its own
2449
+ // service), so each variant's build dir carries a distinct, instance-scoped
2450
+ // fragment rather than an identical copy of the whole host proxy. The
2451
+ // complete host HTTPProxy — every variant's route aggregated onto the one
2452
+ // fqdn, each pointing at its live colour — is assembled and applied by
2453
+ // `instance-promote` at deploy time, and only once EVERY variant is ready.
1655
2454
  const proxyYaml =
1656
2455
  Underpost.deploy.baseProxyYamlFactory({ host: _host, env, options }) +
1657
- Underpost.deploy.deploymentYamlServiceFactory({
1658
- path: _path,
1659
- port: _fromPort,
1660
- deployId: _deployId,
2456
+ instanceProxyRoutesFactory({
2457
+ deployId,
2458
+ instances: [instance],
1661
2459
  env,
1662
- deploymentVersions: [targetTraffic],
2460
+ trafficById: { [instance.id]: targetTraffic },
1663
2461
  });
1664
2462
 
2463
+ // A status route is only emitted for a page this project actually ships, so
2464
+ // a rewrite never points at a document that cannot exist. The check is a
2465
+ // read: placing the document into the gateway volume is `deploy
2466
+ // --sync-static`'s job at apply time, and a build must not mutate the host.
2467
+ // `projectPath` is passed through because this runner is given one
2468
+ // explicitly; the sync derives the same root from the instance itself.
2469
+ const statusPageEntries = instanceStatusPageEntriesFactory({
2470
+ instances: [instance],
2471
+ projectPath: rootPath,
2472
+ }).filter((entry) => fs.existsSync(entry.sourcePath));
2473
+
2474
+ // httproute.yaml — this instance's own routes, including the status routes
2475
+ // that reach the static utility instead of this workload. No Gateway is
2476
+ // emitted beside it: the parent deploy owns the one Gateway that
2477
+ // terminates every hostname it serves, and a second, hostname-scoped one
2478
+ // would contend with it for the merged listener rather than add to it.
2479
+ const httpRouteYaml = Underpost.deploy.httpRouteYamlFactory({
2480
+ host: _host,
2481
+ options,
2482
+ parentName: Underpost.deploy.gatewayNameFactory({ deployId, env }),
2483
+ rules: instanceHttpRouteRulesFactory({
2484
+ deployId,
2485
+ instances: [instance],
2486
+ env,
2487
+ trafficById: { [instance.id]: targetTraffic },
2488
+ options,
2489
+ servedStatuses: statusPageEntries.map((page) => `${page.status}`),
2490
+ }),
2491
+ });
2492
+
1665
2493
  // grpc-service.yaml — the parent deploy's gRPC ClusterIP (shared; the
1666
2494
  // instance cmd resolves {{grpc-service-dns}} to it). Reuse the parent's
1667
2495
  // generated manifest when present rather than regenerating it here.
@@ -1677,31 +2505,108 @@ EOF
1677
2505
  fs.writeFileSync(`${instanceBuildDir}/deployment.yaml`, deploymentYaml, 'utf8');
1678
2506
  const siblingManifests = {
1679
2507
  'pv-pvc.yaml': pvPvcYaml,
2508
+ 'traffic-service.yaml': Underpost.deploy.trafficServiceYamlFactory({
2509
+ deployId: _deployId,
2510
+ env,
2511
+ traffic: targetTraffic,
2512
+ namespace: options.namespace,
2513
+ fromPort: _fromPort,
2514
+ toPort: _toPort,
2515
+ }),
1680
2516
  'proxy.yaml': proxyYaml,
2517
+ // No gateway.yaml: the parent deploy owns the Gateway. `writeManifest`
2518
+ // removes the file a previous per-host build left behind.
2519
+ 'gateway.yaml': '',
2520
+ 'httproute.yaml': httpRouteYaml,
1681
2521
  'grpc-service.yaml': grpcServiceYaml,
1682
2522
  };
1683
- for (const [name, content] of Object.entries(siblingManifests)) {
1684
- if (!content) continue;
1685
- fs.writeFileSync(`${envManifestPath}/${name}`, content, 'utf8');
1686
- fs.writeFileSync(`${instanceBuildDir}/${name}`, content, 'utf8');
1687
- }
2523
+ // Written only when they carry objects, and removed otherwise: an empty
2524
+ // manifest makes `kubectl apply` fail with "no objects passed to apply",
2525
+ // so a build that produces none must leave none behind either.
2526
+ for (const [name, content] of Object.entries(siblingManifests))
2527
+ for (const dir of [envManifestPath, instanceBuildDir])
2528
+ Underpost.deploy.writeManifest({ filePath: `${dir}/${name}`, content });
1688
2529
  logger.info('[instance-build-manifest] Sibling manifests written', {
1689
2530
  project: envManifestPath,
1690
2531
  enginePrivate: instanceBuildDir,
1691
2532
  pvPvc: !!pvPvcYaml,
1692
2533
  proxy: !!proxyYaml,
2534
+ httpRoute: !!httpRouteYaml,
2535
+ statusPages: statusPageEntries.length,
1693
2536
  grpcService: !!grpcServiceYaml,
1694
2537
  });
2538
+ const { gatewayClassName, http3, quicPort, altSvc } = Underpost.deploy.gatewayApiConfigFactory(options);
2539
+ logger.info('[instance-build-manifest] Gateway API manifests written', {
2540
+ host: _host,
2541
+ gatewayClass: gatewayClassName,
2542
+ http3,
2543
+ quicPort,
2544
+ altSvc: http3 ? altSvc : null,
2545
+ statusPages: (instance.customStatusPages || []).map((page) => ({
2546
+ status: page.status,
2547
+ route: `${instance.path === '/' ? '' : instance.path}/${page.status}`,
2548
+ hostPath: page.hostPath,
2549
+ })),
2550
+ statusPageResources: statusPageEntries,
2551
+ });
1695
2552
 
1696
- if (env === 'production') {
2553
+ // --- Per-instance env files -----------------------------------------
2554
+ // Each env file starts from the template instance's canonical file for the
2555
+ // same mode. Operator-owned keys remain private and are copied verbatim;
2556
+ // deploy-specific builders may then derive application env from the
2557
+ // normalized instance path/code.
2558
+ //
2559
+ // A derived instance's env dir is generated in full: both development.env
2560
+ // and production.env are written on every build, so a deploy in either
2561
+ // environment always finds the env file its `cmd` sources, no matter which
2562
+ // mode this build ran. The default/template instance owns the committed
2563
+ // source files, so only its current-mode file is idempotently refreshed.
2564
+ if (instance.templateId) {
2565
+ const instanceEnvDir = `./engine-private/conf/${deployId}/instances/${_id}/env`;
2566
+ fs.mkdirpSync(instanceEnvDir);
2567
+ const envsToWrite = isDefaultInstance ? [env] : ['development', 'production'];
2568
+ for (const targetEnv of envsToWrite) {
2569
+ const templateEnvPath = `./engine-private/conf/${deployId}/instances/${instance.templateId}/env/${targetEnv}.env`;
2570
+ if (!fs.existsSync(templateEnvPath))
2571
+ throw new Error(`[instance-build-manifest] Missing canonical env file: ${templateEnvPath}`);
2572
+ const baseEnv = dotenv.parse(fs.readFileSync(templateEnvPath, 'utf8'));
2573
+ const builtEnv = dispatchBuildInstanceEnv({
2574
+ deployId,
2575
+ instance,
2576
+ environment: targetEnv,
2577
+ baseEnv,
2578
+ containerDeployId: `${_deployId}-${targetEnv}`,
2579
+ builders: instanceEnvBuilder ? { [deployId]: instanceEnvBuilder } : {},
2580
+ });
2581
+ writeEnv(`${instanceEnvDir}/${targetEnv}.env`, builtEnv);
2582
+ }
2583
+ logger.info('[instance-build-manifest] Instance env written', {
2584
+ dir: instanceEnvDir,
2585
+ instanceCode: instance.instanceCode,
2586
+ envs: envsToWrite,
2587
+ builder: instanceEnvBuilder?.name || 'canonical-copy',
2588
+ });
2589
+ }
2590
+
2591
+ if (env === 'production' && isDefaultInstance) {
1697
2592
  if (fs.existsSync(dockerfileManifestPath)) {
1698
2593
  fs.copyFileSync(dockerfileManifestPath, `${rootPath}/Dockerfile`);
1699
2594
  }
1700
2595
  fs.copyFileSync(outputPath, `${rootPath}/deployment.yaml`);
1701
2596
  // Sibling manifests alongside deployment.yaml at the project root.
1702
- for (const name of ['pv-pvc.yaml', 'proxy.yaml', 'grpc-service.yaml']) {
2597
+ for (const name of [
2598
+ 'pv-pvc.yaml',
2599
+ 'traffic-service.yaml',
2600
+ 'proxy.yaml',
2601
+ 'gateway.yaml',
2602
+ 'httproute.yaml',
2603
+ 'grpc-service.yaml',
2604
+ ]) {
1703
2605
  const src = `${envManifestPath}/${name}`;
2606
+ // Absence is mirrored too, so the repo never ships a manifest this
2607
+ // build stopped producing.
1704
2608
  if (fs.existsSync(src)) fs.copyFileSync(src, `${rootPath}/${name}`);
2609
+ else fs.removeSync(`${rootPath}/${name}`);
1705
2610
  }
1706
2611
  logger.info('[instance-build-manifest] Production artifacts copied to project root', {
1707
2612
  rootPath,
@@ -1908,9 +2813,7 @@ EOF`);
1908
2813
  shellExec(`kubectl apply -k ${underpostRoot}/manifests/deployment/adminer/. -n ${options.namespace}`);
1909
2814
  const successInstance = await Underpost.test.statusMonitor('adminer', 'Running', 'pods', 1000, 60 * 10);
1910
2815
 
1911
- if (successInstance) {
1912
- shellExec(`underpost deploy --expose adminer --namespace ${options.namespace}`);
1913
- }
2816
+ if (successInstance) return UnderpostRun.RUNNERS.expose(path || 'adminer', options);
1914
2817
  },
1915
2818
 
1916
2819
  /**
@@ -1969,6 +2872,7 @@ EOF`);
1969
2872
  * @memberof UnderpostRun
1970
2873
  */
1971
2874
  promote: async (path, options = DEFAULT_OPTION) => {
2875
+ options = { ...options, gatewayApi: gatewayApiEnabledFactory(options) };
1972
2876
  let [inputDeployId, inputEnv, inputReplicas] = path.split(',');
1973
2877
  if (!inputEnv) inputEnv = 'production';
1974
2878
  if (!inputReplicas) inputReplicas = 1;
@@ -1996,13 +2900,19 @@ EOF`);
1996
2900
 
1997
2901
  if (inputDeployId === 'dd') {
1998
2902
  for (const deployId of fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',')) {
1999
- const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
2903
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, {
2904
+ namespace: options.namespace,
2905
+ env: inputEnv,
2906
+ });
2000
2907
  const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
2001
2908
  Underpost.deploy.switchTraffic(deployId, inputEnv, targetTraffic, inputReplicas, options.namespace, options);
2002
2909
  applyCerts(deployId, targetTraffic);
2003
2910
  }
2004
2911
  } else {
2005
- const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, { namespace: options.namespace });
2912
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(inputDeployId, {
2913
+ namespace: options.namespace,
2914
+ env: inputEnv,
2915
+ });
2006
2916
  const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
2007
2917
  Underpost.deploy.switchTraffic(
2008
2918
  inputDeployId,
@@ -2055,8 +2965,24 @@ EOF`);
2055
2965
  },
2056
2966
  /**
2057
2967
  * @method cluster
2058
- * @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB, MariaDB, Valkey, Contour (Ingress), and Cert-Manager, and deploys all services.
2059
- * @param {string} path - The input value, identifier, or path for the operation.
2968
+ * @description Deploys a full production/development ready Kubernetes cluster environment including MongoDB,
2969
+ * MariaDB, Valkey, the Gateway API data plane (Envoy Gateway), Contour, and Cert-Manager, and deploys all services.
2970
+ *
2971
+ * Ingress is served by the Gateway API stack (Gateway + HTTPRoute) with QUIC/HTTP3 in **both** environments;
2972
+ * `--disable-gateway-api` falls back to the Contour HTTPProxy stack. Because HTTP/3 has no cleartext transport,
2973
+ * development terminates TLS too: a self-signed certificate per host (mkcert via `scripts/ssl.sh`, whose root CA
2974
+ * the script installs into the system + NSS trust stores) is issued into the secret the Gateway listener
2975
+ * references, and every host is written to `/etc/hosts` so the operator's browser reaches the PWA at
2976
+ * `https://<host>` on the local machine.
2977
+ *
2978
+ * Custom instances are the optional third segment of `path`. They are resolved per deploy against that deploy's
2979
+ * own `conf.instances.json`, so an id only runs where its deploy declares it, and each one is deployed after its
2980
+ * deploy's default workload is serving — an instance reads the parent's world configuration over the parent's
2981
+ * gRPC ClusterIP at boot. Instance hosts share the deploy's environment: the same self-signed certificates and
2982
+ * `/etc/hosts` pass in development, the same cert-manager issuance in production.
2983
+ * @param {string} path - `<runtime-image>,<deploy-list>[,<instance-list>]` — `+`-separated lists, e.g.
2984
+ * `express,dd-cyberia,mmo-server` or `express,dd-cyberia+dd-core,mmo-server+mmo-client`. An instance list entry
2985
+ * may be a template id (`mmo-server`), which selects its whole variant family.
2060
2986
  * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
2061
2987
  * @memberof UnderpostRun
2062
2988
  */
@@ -2065,30 +2991,39 @@ EOF`);
2065
2991
  const env = options.dev ? 'development' : 'production';
2066
2992
  const baseCommand = options.dev ? 'node bin' : 'underpost';
2067
2993
  const baseClusterCommand = options.dev ? ' --dev' : '';
2068
- const clusterType = options.k3s ? 'k3s' : 'kubeadm';
2994
+ const clusterType = clusterTypeFactory(options, 'kubeadm');
2069
2995
  shellCd(`/home/dd/engine`);
2070
2996
  shellExec(`${baseCommand} cluster${baseClusterCommand} --reset --${clusterType}`);
2071
2997
  await timer(5000);
2072
2998
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType}`);
2073
2999
  await timer(5000);
2074
- let [runtimeImage, deployList] =
3000
+ let [runtimeImage, deployList, instanceListId] =
2075
3001
  path && path.trim() && path.split(',')
2076
3002
  ? path.split(',')
2077
3003
  : [
2078
3004
  'express',
2079
3005
  fs.readFileSync(`${underpostRoot}/engine-private/deploy/dd.router`, 'utf8').replaceAll(',', '+'),
3006
+ '',
2080
3007
  ];
2081
- shellExec(
2082
- `${baseCommand} image${baseClusterCommand} --build ${
2083
- runtimeImage ? ` --pull-base --path ${underpostRoot}/src/runtime/${runtimeImage}` : ''
2084
- } --${clusterType}`,
2085
- );
3008
+ // shellExec(
3009
+ // `${baseCommand} image${baseClusterCommand} --build ${
3010
+ // runtimeImage ? ` --pull-base --path ${underpostRoot}/src/runtime/${runtimeImage}` : ''
3011
+ // } --${clusterType}`,
3012
+ // );
2086
3013
  if (!deployList) {
2087
3014
  deployList = [];
2088
3015
  logger.warn('No deploy list provided');
2089
3016
  } else deployList = deployList.split('+');
2090
3017
  await timer(5000);
2091
- shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mongodb`);
3018
+ // --reset-mongodb wipes the retained hostPath volumes before the rollout.
3019
+ // This workflow already tore the node down and re-imports every database
3020
+ // from its git backup a few lines below, so inheriting the previous
3021
+ // cluster's replica set config is never wanted — that state leaves mongod
3022
+ // parked outside its own config and the bootstrap fighting to recover it.
3023
+ shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mongodb --reset-mongodb`);
3024
+ if (deployList.includes('dd-cyberia'))
3025
+ shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --ipfs --replicas 1`);
3026
+
2092
3027
  if (runtimeImage === 'lampp') {
2093
3028
  await timer(5000);
2094
3029
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --mariadb`);
@@ -2101,19 +3036,614 @@ EOF`);
2101
3036
  }
2102
3037
  await timer(5000);
2103
3038
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --pull-image --valkey`);
3039
+ // Exactly one ingress stack is installed, because the two cannot coexist
3040
+ // on this node: Contour's Envoy DaemonSet declares hostPort 80/443, so the
3041
+ // CNI hostport plugin DNATs everything arriving on those ports straight to
3042
+ // it — before the Gateway API data plane's own listener can see them. With
3043
+ // no HTTPProxy objects to program (this workflow applies HTTPRoutes),
3044
+ // Contour's Envoy has no listeners and refuses the redirected connection,
3045
+ // which looks exactly like a gateway that is not listening at all.
3046
+ const gatewayApi = gatewayApiEnabledFactory(options);
3047
+ const gatewayApiFlags = Underpost.deploy.gatewayApiFlagsFactory({ ...options, gatewayApi });
2104
3048
  await timer(5000);
2105
- shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --contour`);
3049
+ if (gatewayApi) {
3050
+ shellExec(
3051
+ `${baseCommand} cluster${baseClusterCommand} --${clusterType} --gateway-api${
3052
+ options.gatewayClass ? ` --gateway-class ${options.gatewayClass}` : ''
3053
+ }`,
3054
+ );
3055
+ } else shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --contour`);
3056
+ if (gatewayApi)
3057
+ shellExec(
3058
+ `kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${options.namespace || 'default'} --timeout=5m`,
3059
+ );
2106
3060
  if (env === 'production') {
2107
3061
  await timer(5000);
2108
3062
  shellExec(`${baseCommand} cluster${baseClusterCommand} --${clusterType} --cert-manager`);
2109
3063
  }
3064
+
3065
+ const { byDeployId: instancesByDeployId, unmatched: unmatchedInstanceIds } = clusterInstancesFactory(
3066
+ deployList,
3067
+ instanceListId,
3068
+ );
3069
+ if (unmatchedInstanceIds.length > 0)
3070
+ logger.warn('[cluster] No deploy declares these instances; they will not be deployed', {
3071
+ instances: unmatchedInstanceIds,
3072
+ deployList,
3073
+ });
3074
+
3075
+ // Development terminates TLS with a locally trusted certificate instead of
3076
+ // cert-manager: QUIC/HTTP3 has no cleartext transport, so without it the
3077
+ // dev gateway would fall back to an HTTP-only listener. The hosts are
3078
+ // written to /etc/hosts in a single pass — etcHostFactory rewrites the
3079
+ // file, so one call per deploy would drop the previous deploy's entries.
3080
+ // Instance hosts come through the same resolver the Gateway's certificate
3081
+ // list uses, so the two can never disagree about what the deploy serves.
3082
+ const hosts = [...new Set(deployList.flatMap((deployId) => deployHostsFactory(deployId)))];
3083
+ if (env === 'development') {
3084
+ for (const host of hosts)
3085
+ Underpost.deploy.selfSignedTlsSecretFactory({
3086
+ host,
3087
+ namespace: options.namespace || 'default',
3088
+ underpostRoot,
3089
+ });
3090
+ const hostListenResult = etcHostFactory(hosts);
3091
+ logger.info(hostListenResult.renderHosts);
3092
+ }
3093
+ const version = 'v3.2.90';
3094
+ const instanceOptionsFactory = (deployId, instanceId) => ({
3095
+ ...options,
3096
+ ...clusterContextFactory(clusterType),
3097
+ gatewayApi,
3098
+ gatewayBootstrapComplete: true,
3099
+ tls: true,
3100
+ test: env === 'development',
3101
+ etcHosts: false,
3102
+ namespace: options.namespace || 'default',
3103
+ imageName:
3104
+ deployId === 'dd-cyberia' && env === 'development' && instanceId === 'mmo-server'
3105
+ ? `underpost/cyberia-server-dev:${version}`
3106
+ : deployId === 'dd-cyberia' && env === 'development' && instanceId === 'mmo-client'
3107
+ ? `underpost/cyberia-client-dev:${version}`
3108
+ : undefined,
3109
+ });
3110
+ const deployFlagsById = {};
3111
+ const fallbackChecks = new Map();
3112
+
3113
+ // Regenerating the manifests is required, not incidental: the TLS listener
3114
+ // — and with it the QUIC policy and the HTTPRoute set — is only emitted
3115
+ // when the TLS and gateway flags are known at generation time. It takes two
3116
+ // passes because `--build-manifest` returns after writing the manifests, so
3117
+ // the same flags have to be repeated on the apply call.
2110
3118
  for (const deployId of deployList) {
2111
- shellExec(
2112
- `${baseCommand} deploy ${deployId} ${env} --${clusterType}${env === 'production' ? ' --cert' : ''}${
2113
- env === 'development' ? ' --etc-hosts' : ''
2114
- }${options.namespace ? ` --namespace ${options.namespace}` : ''}`,
3119
+ const deployFlags =
3120
+ `--${clusterType}${env === 'production' ? ' --cert' : ' --self-signed'}${gatewayApiFlags}` +
3121
+ `${options.namespace ? ` --namespace ${options.namespace}` : ''}` +
3122
+ (deployId === 'dd-cyberia'
3123
+ ? ` --image 'underpost/engine-cyberia:${version}' \
3124
+ --versions blue \
3125
+ --image-pull-policy Always \
3126
+ --cmd 'cd /home/dd/engine, \
3127
+ underpost clone underpostnet/engine-cyberia, \
3128
+ mkdir -p /home/dd/engine/src/client/public/itemledger \
3129
+ /home/dd/engine/src/client/public/cryptokoyn \
3130
+ /home/dd/engine/src/client/components/cryptokoyn \
3131
+ /home/dd/engine/src/client/components/itemledger \
3132
+ /home/dd/engine/hardhat, \
3133
+ cp -a ./engine-cyberia/src/client/public/itemledger/. /home/dd/engine/src/client/public/itemledger/, \
3134
+ cp -a ./engine-cyberia/src/client/public/cryptokoyn/. /home/dd/engine/src/client/public/cryptokoyn/, \
3135
+ cp -a ./engine-cyberia/src/client/components/cryptokoyn/. /home/dd/engine/src/client/components/cryptokoyn/, \
3136
+ cp -a ./engine-cyberia/src/client/components/itemledger/. /home/dd/engine/src/client/components/itemledger/, \
3137
+ cp -a ./engine-cyberia/src/client/Itemledger.index.js /home/dd/engine/src/client/Itemledger.index.js, \
3138
+ cp -a ./engine-cyberia/src/client/Cryptokoyn.index.js /home/dd/engine/src/client/Cryptokoyn.index.js, \
3139
+ rm -rf ./engine-cyberia, \
3140
+ sudo rm -rf ./engine-private/, \
3141
+ node bin clone underpostnet/engine-cyberia-private, \
3142
+ sudo mv ./engine-cyberia-private ./engine-private, \
3143
+ node bin env dd-cyberia ${env}, \
3144
+ node ./engine-private/itc-scripts/dd-cyberia-0.js, \
3145
+ sudo chown -R dd:dd /home/dd/engine/src/client/public/cyberia, \
3146
+ node bin env dd-cyberia ${env}, \
3147
+ node bin client dd-cyberia ${env}, \
3148
+ node bin start dd-cyberia ${env} --run'`
3149
+ : '');
3150
+ deployFlagsById[deployId] = deployFlags;
3151
+ // SSR status and context documents belong to the ingress bootstrap, so
3152
+ // build them on the host before any workload Deployment is submitted.
3153
+ shellExec(`${baseCommand} client ${deployId} ${env}`);
3154
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --build-manifest ${deployFlags}`);
3155
+ // Seed the static tree before the routes exist, so every status page and
3156
+ // intercepted context the manifests just pointed at resolves from the
3157
+ // first request. This pass places what this checkout built; the pass
3158
+ // after the rollout replaces each document with the container's own,
3159
+ // which is the only place clients built from private sources exist.
3160
+ if (gatewayApi) {
3161
+ const staticAssets = Underpost.deploy.syncStaticAssets(deployId, env, {
3162
+ ...options,
3163
+ ...clusterContextFactory(clusterType),
3164
+ gatewayApi,
3165
+ namespace: options.namespace || 'default',
3166
+ versions: /--versions\s+([^\s]+)/.exec(deployFlags)?.[1] || options.versions || 'blue',
3167
+ });
3168
+ const missingAssets = staticAssets.filter((entry) => !entry.source);
3169
+ if (missingAssets.length > 0)
3170
+ throw new Error(
3171
+ `[cluster] Static gateway bootstrap is missing configured assets for ${deployId}: ` +
3172
+ missingAssets.map((entry) => entry.assetPath).join(', '),
3173
+ );
3174
+
3175
+ // Record the exact documents the no-backend checkpoint must return.
3176
+ // PWA paths use the SSR maintenance view; custom instances have no
3177
+ // maintenance view and reuse their first declared status document.
3178
+ const confServer = loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`);
3179
+ const confSSRPath = `./engine-private/conf/${deployId}/conf.ssr.json`;
3180
+ const confSSR = fs.existsSync(confSSRPath) ? JSON.parse(fs.readFileSync(confSSRPath, 'utf8')) : {};
3181
+ for (const host of Object.keys(confServer))
3182
+ for (const path of Object.keys(confServer[host])) {
3183
+ const maintenance = Underpost.deploy
3184
+ .edgeRouteEntriesFactory({ confServer, confSSR, host, path })
3185
+ .find((entry) => entry.context === 'maintenance');
3186
+ if (maintenance)
3187
+ fallbackChecks.set(`${host}${path}`, {
3188
+ host,
3189
+ path,
3190
+ assetPath: maintenance.assetPath,
3191
+ kind: maintenance.kind,
3192
+ });
3193
+ }
3194
+ const selectedInstances = instancesByDeployId[deployId].ids.flatMap((instanceId) =>
3195
+ selectConfInstances(loadConfInstances(deployId), instanceId),
3196
+ );
3197
+ for (const entry of instanceStatusPageEntriesFactory({ instances: selectedInstances }))
3198
+ if (!fallbackChecks.has(`${entry.host}${entry.path}`))
3199
+ fallbackChecks.set(`${entry.host}${entry.path}`, {
3200
+ host: entry.host,
3201
+ path: entry.path,
3202
+ assetPath: entry.assetPath,
3203
+ kind: `status:${entry.status}`,
3204
+ });
3205
+ }
3206
+ // Apply only the gateway tier. Application Services and Deployments are
3207
+ // deliberately absent so the status fallback can be observed first.
3208
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --disable-update-deployment ${deployFlags}`);
3209
+ // Instance host routes and their custom status pages are part of
3210
+ // the same ingress bootstrap. `instance-promote` is safe here: it only
3211
+ // writes the Nginx host block and routing objects; no instance Deployment
3212
+ // is created until the second phase below.
3213
+ for (const instanceId of instancesByDeployId[deployId].ids) {
3214
+ logger.info('[cluster] Bootstrapping custom instance gateway', { deployId, instanceId, env });
3215
+ await UnderpostRun.RUNNERS['instance-promote'](`${deployId},${instanceId}`, {
3216
+ ...instanceOptionsFactory(deployId, instanceId),
3217
+ noBackendCheckpoint: true,
3218
+ });
3219
+ }
3220
+ }
3221
+
3222
+ // This is the deliberate no-backend checkpoint. The static Nginx pod, all
3223
+ // Gateway listeners, parent HTTPRoutes and selected instance HTTPRoutes
3224
+ // must be live before the first application Deployment YAML is submitted.
3225
+ if (gatewayApi && fallbackChecks.size > 0) {
3226
+ const fallbackResults = await gatewayFallbackProbeRunner({
3227
+ gatewayStatusRunner: UnderpostRun.RUNNERS['gateway-status'],
3228
+ checks: [...fallbackChecks.values()],
3229
+ options: { ...options, gatewayApi, namespace: options.namespace || 'default' },
3230
+ label: 'cluster',
3231
+ });
3232
+ logger.info('[cluster] Gateway fallback checkpoint passed; starting application deployments', {
3233
+ hosts,
3234
+ fallbacks: fallbackResults,
3235
+ });
3236
+ }
3237
+
3238
+ for (const deployId of deployList) {
3239
+ const deployFlags = deployFlagsById[deployId];
3240
+ // Preserve the already-operational ingress objects and apply only the
3241
+ // workload manifests. EndpointSlices will update as pods become Ready;
3242
+ // the site route continues to reach underpost-gateway throughout.
3243
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --disable-update-proxy ${deployFlags}`);
3244
+ if (gatewayApi) {
3245
+ const namespace = options.namespace || 'default';
3246
+ const version = /--versions\s+([^\s,]+)/.exec(deployFlags)?.[1] || 'blue';
3247
+ shellExec(`kubectl rollout status deployment/${deployId}-${env}-${version} -n ${namespace} --timeout=15m`);
3248
+ shellExec(`${baseCommand} deploy ${deployId} ${env} --sync-static ${deployFlags}`);
3249
+ }
3250
+
3251
+ // Custom instance pods depend on the parent's gRPC service, so they are
3252
+ // still started after the parent is Ready. Their routes already exist and
3253
+ // keep serving the custom fallback until the atomic promotion completes.
3254
+ for (const instanceId of instancesByDeployId[deployId].ids) {
3255
+ logger.info('[cluster] Deploying custom instance', { deployId, instanceId, env, clusterType });
3256
+ await UnderpostRun.RUNNERS.instance(
3257
+ `${deployId},${instanceId},${options.replicas || 1}`,
3258
+ instanceOptionsFactory(deployId, instanceId),
3259
+ );
3260
+ }
3261
+ }
3262
+ logger.info('[cluster] Ingress stack deployed', {
3263
+ env,
3264
+ stack: gatewayApi ? 'gateway-api' : 'httpproxy',
3265
+ gatewayClass: gatewayApi ? Underpost.deploy.gatewayApiConfigFactory(options).gatewayClassName : null,
3266
+ http3: gatewayApi && options.disableHttp3 !== true,
3267
+ tls: env === 'production' ? 'cert-manager' : 'self-signed',
3268
+ hosts,
3269
+ instances: Object.fromEntries(
3270
+ deployList
3271
+ .filter((deployId) => instancesByDeployId[deployId].ids.length > 0)
3272
+ .map((deployId) => [deployId, instancesByDeployId[deployId].ids]),
3273
+ ),
3274
+ });
3275
+ if (gatewayApi) await UnderpostRun.RUNNERS['gateway-status']('', options);
3276
+ },
3277
+
3278
+ /**
3279
+ * @method gateway-status
3280
+ * @description Reports whether the Gateway API data plane is actually
3281
+ * serving. Applying a Gateway only records intent: the controller
3282
+ * provisions Envoy asynchronously, so a deploy can finish cleanly while
3283
+ * nothing listens on the node — the failure then surfaces much later as a
3284
+ * bare "connection refused" from the browser. This waits for the Gateways to
3285
+ * be Programmed and prints the data plane's pods and services.
3286
+ * @param {string} path - Unused.
3287
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
3288
+ * @memberof UnderpostRun
3289
+ */
3290
+ 'gateway-status': async (path = '', options = DEFAULT_OPTION) => {
3291
+ const namespace = options.namespace || 'default';
3292
+ const dataPlaneNamespace = 'envoy-gateway-system';
3293
+ const capture = (cmd) => shellExec(cmd, { stdout: true, silent: true, silentOnError: true })?.trim?.() || '';
3294
+
3295
+ const programmed = shellExec(
3296
+ `kubectl wait --for=condition=Programmed --timeout=180s gateway --all -n ${namespace}`,
3297
+ { silentOnError: true },
3298
+ );
3299
+ if (programmed?.code !== 0)
3300
+ logger.warn(
3301
+ '[gateway-status] Gateways are not Programmed. The listeners are not being served; ' +
3302
+ `check 'kubectl describe gateway -n ${namespace}' and the controller logs in ${dataPlaneNamespace}.`,
3303
+ );
3304
+ shellExec(`kubectl get gateway -n ${namespace} -o wide`, { silentOnError: true });
3305
+
3306
+ // Per-listener truth. A Gateway reports Programmed while individual
3307
+ // listeners are rejected, so the aggregate condition hides exactly the
3308
+ // case where some hostnames serve and others do not. Prints each
3309
+ // listener's attached route count and any failing condition with its
3310
+ // reason — an unresolved TLS secret or a rejected listener names itself.
3311
+ const listenerStatus = capture(
3312
+ `kubectl get gateway -n ${namespace} -o jsonpath=` +
3313
+ `'{range .items[*]}{.metadata.name}{" "}` +
3314
+ `{range .status.listeners[*]}{.name}=routes:{.attachedRoutes}` +
3315
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}{end}{" "}{end}{"\\n"}{end}'`,
3316
+ );
3317
+ logger.info('[gateway-status] Listeners (name=routes + failing conditions)\n' + (listenerStatus || '(none)'));
3318
+
3319
+ // Envoy Gateway policies attach to a Gateway but configure the merged
3320
+ // listener, so two of them competing for the same listener is resolved by
3321
+ // rejecting one — and the rejection is recorded here, not on the Gateway.
3322
+ const policyStatus = capture(
3323
+ `kubectl get clienttrafficpolicy -n ${namespace} -o jsonpath=` +
3324
+ `'{range .items[*]}{.metadata.name}{range .status.ancestors[*]}` +
3325
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\\n"}{end}'`,
3326
+ );
3327
+ logger.info(
3328
+ '[gateway-status] ClientTrafficPolicies (failing conditions)\n' +
3329
+ (policyStatus
3330
+ .split('\n')
3331
+ .filter((line) => line.includes(' '))
3332
+ .join('\n') || '(none — all policies accepted)'),
3333
+ );
3334
+
3335
+ // Envoy Gateway policies that fail to translate are not visible on the
3336
+ // Gateway or the route: the resource is admitted, its status carries the
3337
+ // rejection, and the xDS snapshot it belonged to can go with it — which
3338
+ // reads downstream as every hostname answering route_not_found.
3339
+ const backendPolicyStatus = capture(
3340
+ `kubectl get backendtrafficpolicy -n ${namespace} -o jsonpath=` +
3341
+ `'{range .items[*]}{.metadata.name}{range .status.ancestors[*]}` +
3342
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\n"}{end}' ` +
3343
+ `2>/dev/null`,
3344
+ );
3345
+ logger.info(
3346
+ '[gateway-status] BackendTrafficPolicies (failing conditions)\n' +
3347
+ (backendPolicyStatus
3348
+ .split('\n')
3349
+ .filter((line) => line.includes(' '))
3350
+ .join('\n') || '(none — all policies accepted)'),
3351
+ );
3352
+
3353
+ // Route-level conditions. A rule that references something unresolvable
3354
+ // can cost the whole route, and then every path on that hostname answers
3355
+ // with a bare gateway 404 while the Gateway and its listeners stay green.
3356
+ const routeStatus = capture(
3357
+ `kubectl get httproute -n ${namespace} -o jsonpath=` +
3358
+ `'{range .items[*]}{.metadata.name}{range .status.parents[*]}` +
3359
+ `{range .conditions[?(@.status=="False")]}{" "}{.type}/{.reason}: {.message}{end}{end}{"\\n"}{end}'`,
3360
+ )
3361
+ .split('\n')
3362
+ .filter((line) => line.includes(' '));
3363
+ logger.info(
3364
+ '[gateway-status] HTTPRoutes (failing conditions)\n' +
3365
+ (routeStatus.join('\n') || '(none — all routes accepted)'),
3366
+ );
3367
+
3368
+ // The workloads the routes point at. A status code that moves between runs
3369
+ // (500 here, 404 there) is the application's, not the gateway's, and the
3370
+ // gateway config cannot explain it — restarts and unready containers can.
3371
+ const backends = capture(
3372
+ `kubectl get pods -n ${namespace} -o custom-columns=` +
3373
+ `'NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount,STATUS:.status.phase'`,
3374
+ );
3375
+ logger.info('[gateway-status] Workloads behind the routes\n' + (backends || '(none)'));
3376
+
3377
+ // A Programmed Gateway only means Envoy was provisioned — not that it is
3378
+ // reachable from this machine. Which of the two is false decides the fix,
3379
+ // so report the pod's network mode and container ports, the service type
3380
+ // and node ports, and what the host is actually listening on.
3381
+ const dataPlane = capture(
3382
+ `kubectl get pods -n ${dataPlaneNamespace} -o custom-columns=` +
3383
+ `'NAME:.metadata.name,HOST_NETWORK:.spec.hostNetwork,PORTS:.spec.containers[*].ports[*].containerPort'`,
3384
+ );
3385
+ const services = capture(
3386
+ `kubectl get svc -n ${dataPlaneNamespace} -o custom-columns=` +
3387
+ `'NAME:.metadata.name,TYPE:.spec.type,PORTS:.spec.ports[*].port,NODEPORTS:.spec.ports[*].nodePort,TARGETS:.spec.ports[*].targetPort'`,
3388
+ );
3389
+ logger.info('[gateway-status] Data plane\n' + dataPlane + '\n\n' + services);
3390
+
3391
+ // Envoy creates its listener sockets only once the control plane has
3392
+ // pushed a config with routes attached, which lands a little after the
3393
+ // pod reports Ready. Polling that window is the difference between
3394
+ // "connection refused" and a working gateway, so wait for the socket
3395
+ // rather than sampling it once.
3396
+ const listenerFilter = `grep -E ':(80|443|10080|10443) '`;
3397
+ let hostListeners = '';
3398
+ for (let attempt = 0; attempt < 30; attempt++) {
3399
+ hostListeners = capture(`sudo ss -lntupH 2>/dev/null | ${listenerFilter}`);
3400
+ if (/:443\s/.test(hostListeners)) break;
3401
+ await timer(2000);
3402
+ }
3403
+ const servesHttps = /:443\s/.test(hostListeners);
3404
+ logger.info('[gateway-status] Host listeners on 80/443/10080/10443\n' + (hostListeners || '(none)'));
3405
+
3406
+ if (!servesHttps) {
3407
+ logger.warn(
3408
+ '[gateway-status] Nothing is listening on this host port 443, so a browser reaching the ' +
3409
+ 'hostnames through /etc/hosts gets "connection refused". Compare the two tables above: ' +
3410
+ 'HOST_NETWORK=false means the pod is on the pod network (only the ClusterIP/NodePort is ' +
3411
+ 'reachable); HOST_NETWORK=true with PORTS 10080/10443 means the privileged-port remap is ' +
3412
+ 'still active. Until it is resolved, forward the merged service to expose it locally:\n' +
3413
+ ` kubectl port-forward -n ${dataPlaneNamespace} svc/<envoy-service> 443:443 80:80 --address 127.0.0.1`,
3414
+ );
3415
+ return { programmed: programmed?.code === 0, servesHttps, dataPlane, services, hostListeners, probes: [] };
3416
+ }
3417
+
3418
+ // An open socket still is not proof the hostname routes anywhere. Probe
3419
+ // each Gateway hostname exactly as the browser would — through /etc/hosts,
3420
+ // validating the certificate against the trust store `scripts/ssl.sh`
3421
+ // populated — so the workflow ends on an observed response, not an
3422
+ // inference. Each host is probed twice: over loopback, and pinned to the
3423
+ // node IP. The two answers separate a loopback-specific block from a data
3424
+ // plane that is not reachable at all.
3425
+ const nodeIp = capture(
3426
+ `kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'`,
3427
+ );
3428
+ // The `server` header is reported alongside the status because the code
3429
+ // alone cannot say who produced it: `500 (envoy)` is a gateway or upstream
3430
+ // failure, while `500` from the workload's own server is an application
3431
+ // error, and the two lead to completely different fixes.
3432
+ const probeUrl = (url, resolveArgs = '') => {
3433
+ const raw = capture(
3434
+ `curl -sS -o /dev/null -D - -w 'HTTP_CODE=%{http_code}' --max-time 10 ${resolveArgs}${url} 2>&1 | tr -d '\\r'`,
3435
+ );
3436
+ const status = /HTTP_CODE=([0-9]{3})/.exec(raw)?.[1];
3437
+ if (!status) return raw.split('\n').find((line) => line.startsWith('curl:')) || 'no-response';
3438
+ const server = /^server:\s*(.+)$/im.exec(raw)?.[1]?.trim();
3439
+ return server ? `${status} (${server})` : status;
3440
+ };
3441
+ const probe = (host, resolveTo) =>
3442
+ probeUrl(`https://${host}`, resolveTo ? `--resolve ${host}:443:${resolveTo} ` : '');
3443
+ // Port 80 is the same Envoy process, same host, same listener set — only
3444
+ // the port differs. It separates "this gateway is unreachable" from
3445
+ // "something specifically rejects 443".
3446
+ const probeHttp = (host) => probeUrl(`http://${host}`);
3447
+ // curl writes `000` when it never got a response, so a bare three-digit
3448
+ // match would read a failed connection as success.
3449
+ const answered = (status) => /^[1-5][0-9]{2}\b/.test(status);
3450
+ // The routes, not the listeners, are where the hostnames live: a
3451
+ // consolidated Gateway serves every hostname from one hostname-less
3452
+ // listener and picks the certificate by SNI, so reading the listeners
3453
+ // would leave nothing to probe.
3454
+ const hosts = [
3455
+ ...new Set(
3456
+ (path
3457
+ ? path.split(',')
3458
+ : capture(`kubectl get httproute -n ${namespace} -o jsonpath='{.items[*].spec.hostnames[*]}'`).split(/\s+/)
3459
+ )
3460
+ .map((host) => host.trim())
3461
+ .filter(Boolean),
3462
+ ),
3463
+ ];
3464
+ const probes = hosts.map((host) => ({
3465
+ host,
3466
+ http: probeHttp(host),
3467
+ loopback: probe(host),
3468
+ ...(nodeIp ? { nodeIp: probe(host, nodeIp) } : {}),
3469
+ }));
3470
+ logger.info('[gateway-status] HTTPS probe', { nodeIp: nodeIp || '(unknown)', probes });
3471
+
3472
+ // A gateway answer alone cannot say whether the gateway or the workload
3473
+ // produced the code: Envoy relays an upstream response unchanged. So when
3474
+ // a hostname fails, ask its backend the *same* question from inside the
3475
+ // cluster, bypassing Envoy.
3476
+ //
3477
+ // Same question is the whole point: these workloads route by virtual host,
3478
+ // so a probe carrying `Host: <service-name>` exercises a different branch
3479
+ // than the browser did and its answer means nothing. Each probe therefore
3480
+ // replays the route it came from — the Gateway hostname, the rule path,
3481
+ // and the rewrite the rule would have applied — against the rule's own
3482
+ // backend. Only then do the two columns compare.
3483
+ const failing = probes.filter((entry) => /^[45]/.test(entry.loopback) || /^[45]/.test(entry.http));
3484
+ if (failing.length > 0) {
3485
+ const routes = JSON.parse(
3486
+ capture(`kubectl get httproute -n ${namespace} -o json`) || '{"items":[]}',
3487
+ ).items.filter((route) => failing.some((entry) => route.spec?.hostnames?.includes(entry.host)));
3488
+ // Exec into the static utility rather than spawning a probe pod: it is
3489
+ // installed with the gateway stack, sits in this namespace, and its
3490
+ // BusyBox shell already carries wget — no image pull, no pod churn.
3491
+ const probePod = capture(
3492
+ `kubectl get pods -n ${namespace} -l app=${UNDERPOST_GATEWAY.name} -o jsonpath='{.items[0].metadata.name}'`,
3493
+ );
3494
+ if (!probePod || routes.length === 0) {
3495
+ logger.warn(
3496
+ '[gateway-status] Cannot probe backends directly: ' +
3497
+ (probePod
3498
+ ? 'no HTTPRoute matched a failing hostname'
3499
+ : `no ${UNDERPOST_GATEWAY.name} pod in ${namespace}`),
3500
+ );
3501
+ } else {
3502
+ const backendProbes = [];
3503
+ for (const route of routes) {
3504
+ const host = route.spec.hostnames[0];
3505
+ for (const rule of route.spec.rules || []) {
3506
+ const backend = (rule.backendRefs || [])[0];
3507
+ if (!backend) continue;
3508
+ const match = rule.matches?.[0]?.path?.value || '/';
3509
+ const rewrite = (rule.filters || []).find((filter) => filter.type === 'URLRewrite')?.urlRewrite?.path;
3510
+ const target =
3511
+ rewrite?.type === 'ReplaceFullPath'
3512
+ ? rewrite.replaceFullPath
3513
+ : rewrite?.type === 'ReplacePrefixMatch'
3514
+ ? rewrite.replacePrefixMatch
3515
+ : match;
3516
+ const raw = capture(
3517
+ `kubectl exec -n ${namespace} ${probePod} -- wget -S -O /dev/null -T 5 ` +
3518
+ `--header 'Host: ${host}' http://${backend.name}:${backend.port}${target} 2>&1 || true`,
3519
+ );
3520
+ const code = /HTTP\/[0-9.]+\s+([0-9]{3})/.exec(raw)?.[1];
3521
+ const failure = raw.split('\n').find((line) => line.includes('wget:'));
3522
+ backendProbes.push({
3523
+ request: `${host}${match}`,
3524
+ backend: `${backend.name}:${backend.port}${target}`,
3525
+ direct: code || failure?.trim() || 'no-response',
3526
+ gateway: failing.find((entry) => entry.host === host)?.http,
3527
+ });
3528
+ }
3529
+ }
3530
+ logger.info('[gateway-status] Backend probe (same Host and path, bypassing Envoy)', { backendProbes });
3531
+ // Only the rule the browser actually hit is comparable, so judge on
3532
+ // the root rule rather than on every rule of the route.
3533
+ const rootProbes = backendProbes.filter((entry) => entry.request.endsWith('/'));
3534
+ const appFault = rootProbes.filter((entry) => `${entry.direct}` === `${entry.gateway}`);
3535
+ const gatewayFault = rootProbes.filter(
3536
+ (entry) => /^[0-9]{3}$/.test(entry.direct) && `${entry.direct}` !== `${entry.gateway}`,
3537
+ );
3538
+ if (gatewayFault.length > 0) {
3539
+ logger.warn(
3540
+ '[gateway-status] These backends answer differently without Envoy in the path, so the gateway is ' +
3541
+ 'not relaying the workload response — the routing layer is the fault:\n ' +
3542
+ gatewayFault
3543
+ .map((entry) => `${entry.request} -> backend ${entry.direct}, gateway ${entry.gateway}`)
3544
+ .join('\n '),
3545
+ );
3546
+ // The access log is the only artifact that says *why*. Its
3547
+ // response_flags column separates an Envoy local reply (NR no route,
3548
+ // UF upstream connect failure, UH no healthy upstream, DPE protocol
3549
+ // error) from a relayed upstream status, which no amount of probing
3550
+ // from outside can distinguish.
3551
+ // Selector, not owning-gateway labels: under `mergeGateways` the data
3552
+ // plane belongs to the GatewayClass, so per-Gateway labels are absent
3553
+ // and a selector built from them silently matches nothing.
3554
+ const dataPlaneLog = capture(
3555
+ `kubectl logs -n ${dataPlaneNamespace} ` +
3556
+ `-l app.kubernetes.io/name=envoy,app.kubernetes.io/component=proxy ` +
3557
+ `-c envoy --tail=60 --prefix 2>/dev/null | tail -60`,
3558
+ );
3559
+ logger.info(
3560
+ '[gateway-status] Data plane access log (response_flags names the reason)\n' +
3561
+ (dataPlaneLog || `(none — check: kubectl get pods -n ${dataPlaneNamespace} --show-labels)`),
3562
+ );
3563
+ // A translation the control plane rejected after admitting the
3564
+ // resource surfaces only here, never on the object's own status.
3565
+ const controlPlaneLog = capture(
3566
+ `kubectl logs -n ${dataPlaneNamespace} deployment/envoy-gateway --tail=40 2>/dev/null ` +
3567
+ `| grep -Ei 'error|warn|reject|invalid' | tail -20`,
3568
+ );
3569
+ logger.info('[gateway-status] Control plane errors\n' + (controlPlaneLog || '(none in the last 40 lines)'));
3570
+ }
3571
+ if (appFault.length > 0)
3572
+ logger.warn(
3573
+ '[gateway-status] These backends return the same code without Envoy in the path, so the gateway is ' +
3574
+ 'relaying an application response — the routing layer is not the fault:\n ' +
3575
+ appFault.map((entry) => `${entry.request} -> ${entry.direct}`).join('\n ') +
3576
+ `\n Read the workload log: kubectl logs -n ${namespace} <pod>`,
3577
+ );
3578
+ }
3579
+ }
3580
+
3581
+ const unreachable = probes.filter((entry) => !answered(entry.loopback));
3582
+ if (unreachable.length > 0) {
3583
+ // Narrow it here rather than leaving the operator with a bare refusal.
3584
+ // A socket bound on 0.0.0.0 that resets a loopback connection is either
3585
+ // not in this network namespace, or something is rejecting the packet.
3586
+ const listenerPid = /pid=(\d+)/.exec(hostListeners)?.[1];
3587
+ const hostNetns = capture(`sudo readlink /proc/1/ns/net`);
3588
+ const listenerNetns = listenerPid ? capture(`sudo readlink /proc/${listenerPid}/ns/net`) : '';
3589
+ // Anything that can answer a SYN with ICMP port-unreachable, from either
3590
+ // rule engine. firewalld on RHEL 9 keeps its rules in its own `inet
3591
+ // firewalld` nftables table, which `iptables-save` cannot see at all —
3592
+ // grepping only iptables hides half the packet path.
3593
+ const rejectRules = capture(
3594
+ `sudo iptables-save 2>/dev/null | grep -E '(REJECT|DROP)' | grep -E '(dport|dpt:) ?(443|https)\\b' | head -20`,
3595
+ );
3596
+ const nftRules = capture(
3597
+ `sudo nft list ruleset 2>/dev/null | grep -nE '(dport|ports) .*(443|https)|reject' | head -20`,
3598
+ );
3599
+ // A hostPort claim outranks any process listening on the node: the CNI
3600
+ // plugin DNATs the port to the claiming pod first, so the packet never
3601
+ // reaches the gateway. This is the one failure that leaves every other
3602
+ // signal green.
3603
+ const hostPortHijack = capture(
3604
+ `sudo nft list ruleset 2>/dev/null | grep -E 'dport (80|443)\\b.*dnat to' | head -5`,
3605
+ );
3606
+ const hostPortClaims = capture(
3607
+ `kubectl get daemonset,deployment -A ` +
3608
+ `-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}={.spec.template.spec.containers[*].ports[*].hostPort}{"\\n"}{end}'`,
3609
+ )
3610
+ .split('\n')
3611
+ .filter((line) => /=.*\b(80|443)\b/.test(line));
3612
+ logger.warn(`[gateway-status] ${unreachable.length}/${probes.length} hostnames did not answer over HTTPS`, {
3613
+ listenerPid: listenerPid || '(unknown)',
3614
+ hostNetns: hostNetns || '(unknown)',
3615
+ listenerNetns: listenerNetns || '(unknown)',
3616
+ sameNetworkNamespace: !!hostNetns && hostNetns === listenerNetns,
3617
+ nodeIpReachable: probes.some((entry) => answered(entry.nodeIp)),
3618
+ endpointlessServices: capture(
3619
+ `kubectl get svc -A -o jsonpath=` + `'{range .items[*]}{.metadata.namespace}/{.metadata.name} {end}'`,
3620
+ )
3621
+ .split(/\s+/)
3622
+ .filter(Boolean)
3623
+ .filter((ref) => {
3624
+ const [ns, name] = ref.split('/');
3625
+ return !capture(
3626
+ `kubectl get endpointslice -n ${ns} -l kubernetes.io/service-name=${name} ` +
3627
+ `-o jsonpath='{.items[*].endpoints[*].addresses[*]}'`,
3628
+ ).trim();
3629
+ }),
3630
+ httpReachable: probes.some((entry) => answered(entry.http)),
3631
+ hostPortHijack: hostPortHijack || '(none)',
3632
+ hostPortClaims: hostPortClaims.length > 0 ? hostPortClaims : '(none)',
3633
+ rejectRules: rejectRules || '(none matching 443 in iptables)',
3634
+ nftRules: nftRules || '(none matching 443 in nftables)',
3635
+ });
3636
+ logger.warn(
3637
+ '[gateway-status] Read it as: hostPortHijack non-empty → another workload reserved host port ' +
3638
+ '80/443 and the CNI DNATs those ports to it before the gateway can see them; hostPortClaims ' +
3639
+ 'names the owner, and only one ingress stack can hold them. httpReachable=true → the same Envoy ' +
3640
+ 'answers on 80, so only 443 is rejected. sameNetworkNamespace=false → the listener is not on ' +
3641
+ 'this host despite hostNetwork. endpointlessServices naming a Service that publishes 80/443 → ' +
3642
+ 'kube-proxy REJECTs those ports on its behalf; delete it.',
2115
3643
  );
2116
3644
  }
3645
+
3646
+ return { programmed: programmed?.code === 0, servesHttps, dataPlane, services, hostListeners, probes };
2117
3647
  },
2118
3648
  /**
2119
3649
  * @method deploy
@@ -2124,11 +3654,11 @@ EOF`);
2124
3654
  */
2125
3655
  deploy: async (path, options = DEFAULT_OPTION) => {
2126
3656
  const deployId = path;
3657
+ const env = options.dev ? 'development' : 'production';
2127
3658
  const { validVersion } = Underpost.repo.privateConfUpdate(deployId);
2128
3659
  if (!validVersion) throw new Error('Version mismatch');
2129
- const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
3660
+ const currentTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace, env });
2130
3661
  const targetTraffic = currentTraffic === 'blue' ? 'green' : 'blue';
2131
- const env = options.dev ? 'development' : 'production';
2132
3662
  const ignorePods = Underpost.kubectl
2133
3663
  .get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace)
2134
3664
  .map((p) => p.NAME);
@@ -2315,7 +3845,11 @@ EOF`);
2315
3845
  }
2316
3846
  const success = await Underpost.test.statusMonitor(podToMonitor);
2317
3847
  if (success) {
2318
- const versions = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }) || 'blue';
3848
+ const versions =
3849
+ Underpost.deploy.getCurrentTraffic(deployId, {
3850
+ namespace: options.namespace,
3851
+ env: options.dev ? 'development' : 'production',
3852
+ }) || 'blue';
2319
3853
  if (!node) node = os.hostname();
2320
3854
  const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
2321
3855
  shellExec(
@@ -2346,10 +3880,7 @@ EOF`);
2346
3880
  */
2347
3881
  'etc-hosts': async (path = '', options = DEFAULT_OPTION) => {
2348
3882
  const hosts = path ? path.split(',') : [];
2349
- if (options.deployId) {
2350
- const confServer = loadConfServerJson(`./engine-private/conf/${options.deployId}/conf.server.json`);
2351
- hosts.push(...Object.keys(confServer));
2352
- }
3883
+ if (options.deployId) hosts.push(...deployHostsFactory(options.deployId));
2353
3884
  const hostListenResult = etcHostFactory(hosts);
2354
3885
  logger.info(hostListenResult.renderHosts);
2355
3886
  },
@@ -2702,31 +4233,314 @@ EOF`);
2702
4233
  * @memberof UnderpostRun
2703
4234
  */
2704
4235
  'generate-pass': (path, options = DEFAULT_OPTION) => {
2705
- const length = path && parseInt(path) > 0 ? parseInt(path) : 16;
2706
- const lower = 'abcdefghijklmnopqrstuvwxyz';
2707
- const upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
2708
- const digits = '0123456789';
2709
- const special = '@#$%^&*()_+';
2710
- const all = lower + upper + digits + special;
2711
- const buf = crypto.randomBytes(length + 4);
2712
- // Guarantee at least one character from each required class
2713
- const chars = [
2714
- lower[buf[0] % lower.length],
2715
- upper[buf[1] % upper.length],
2716
- digits[buf[2] % digits.length],
2717
- special[buf[3] % special.length],
2718
- ];
2719
- for (let i = 4; i < length; i++) chars.push(all[buf[i] % all.length]);
2720
- // Fisher-Yates shuffle using an independent random buffer
2721
- const shuf = crypto.randomBytes(length);
2722
- for (let i = chars.length - 1; i > 0; i--) {
2723
- const j = shuf[i % shuf.length] % (i + 1);
2724
- [chars[i], chars[j]] = [chars[j], chars[i]];
2725
- }
2726
- const password = chars.join('');
4236
+ const password = generateSecurePassword(path && parseInt(path) > 0 ? parseInt(path) : 16);
2727
4237
  if (options.copy) pbcopy(password);
2728
4238
  else console.log(password);
2729
4239
  },
4240
+ /**
4241
+ * @method sops-setup
4242
+ * @description End-to-end SOPS/Age onboarding for a host: installs tooling, generates the Age
4243
+ * keypair and creation rules, pins the key path for non-interactive runs, encrypts the
4244
+ * requested Secrets into the Git-tracked store, then validates and applies them.
4245
+ *
4246
+ * Every step is idempotent and re-runnable. Notably it delegates key generation to
4247
+ * `secret sops --init` rather than calling `age-keygen` directly: a bare `age-keygen -o`
4248
+ * overwrites an existing key, which would orphan every manifest already encrypted to the
4249
+ * previous recipient with no way to recover them.
4250
+ *
4251
+ * On a host that pulled a store created elsewhere, the freshly generated key is not a recipient
4252
+ * of the inherited manifests. `init()` registers this host in the creation rules so what it
4253
+ * encrypts from here on stays readable, but existing manifests can only be re-keyed from a host
4254
+ * that still holds a decrypting key. That case is reported per secret and then raised by the
4255
+ * apply pre-flight with the available remedies, rather than surfacing as a sops decrypt error.
4256
+ *
4257
+ * Onboards the whole self-hosted data tier by default — PostgreSQL, MariaDB, and MongoDB
4258
+ * (`postgres-secret`, `mariadb-secret`, `mongodb-secret`, `mongodb-keyfile`). The MongoDB
4259
+ * keyfile is included because the StatefulSet mounts it for intra-replica-set auth and will
4260
+ * not start without it. Pass an explicit comma-separated list to narrow the set.
4261
+ *
4262
+ * Secret values are resolved per data key, in order:
4263
+ * 1. the origin seed file, when one exists (`engine-private/postgresql-password`) — this is
4264
+ * the real onboarding path, carrying the credential the cluster already runs on;
4265
+ * 2. `--args` as `key=value` pairs, for a value supplied by the operator;
4266
+ * 3. a freshly generated value: a base64 keyfile for `mongodb-keyfile`, `admin` for a
4267
+ * `username`, otherwise a 24-character secure password.
4268
+ *
4269
+ * Plaintext manifests are written by Node under `/dev/shm` at mode 600 and shredded by
4270
+ * `encrypt()`. They are never emitted through a shell heredoc, which would place the
4271
+ * credential in the command string and therefore in the process table and the command log.
4272
+ *
4273
+ * Usage:
4274
+ * underpost run sops-setup # postgres + mariadb + mongo
4275
+ * underpost run sops-setup mongodb-secret,mongodb-keyfile --namespace prod
4276
+ * underpost run sops-setup postgres-secret --args "password=s3cr3t"
4277
+ * underpost run sops-setup --dry-run # stop before mutating cluster
4278
+ * underpost run sops-setup --force # replace stored manifests
4279
+ * @param {string} path - Comma-separated Secret names to onboard. Defaults to the full data
4280
+ * tier: postgres-secret, mariadb-secret, mongodb-secret, mongodb-keyfile.
4281
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
4282
+ * @param {string} options.namespace - Target namespace for the store and the apply (default: 'default').
4283
+ * @param {string} options.args - Comma-separated `key=value` overrides for Secret data keys.
4284
+ * @param {boolean} options.dryRun - Validate and server-dry-run only; never apply.
4285
+ * @param {boolean} options.force - Replace encrypted manifests that already exist.
4286
+ * @memberof UnderpostRun
4287
+ */
4288
+ 'sops-setup': (path = '', options = DEFAULT_OPTION) => {
4289
+ const namespace = options.namespace || 'default';
4290
+ const secretNames = (path || SOPS_SETUP_DEFAULT_SECRETS.join(','))
4291
+ .split(',')
4292
+ .map((name) => name.trim())
4293
+ .filter(Boolean);
4294
+
4295
+ // `--args key=value,key2=value2` overrides, applied to any secret that declares that key.
4296
+ const overrides = `${options.args || ''}`.split(',').reduce((acc, pair) => {
4297
+ const separator = pair.indexOf('=');
4298
+ if (separator > 0) acc[pair.slice(0, separator).trim()] = pair.slice(separator + 1).trim();
4299
+ return acc;
4300
+ }, {});
4301
+
4302
+ logger.info('sops-setup', { secretNames, namespace, dryRun: !!options.dryRun, force: !!options.force });
4303
+
4304
+ // 1. Host tooling, then keypair + creation rules. Both no-op when already present.
4305
+ Underpost.secret.sops.installTooling();
4306
+ Underpost.secret.sops.init();
4307
+
4308
+ // 2. Pin the resolved key path for non-interactive runs (systemd units, CronJobs, sudo).
4309
+ // Written with the concrete path rather than a guessed default, because `sudo` resets
4310
+ // HOME and a wrong guess surfaces later as an opaque decrypt failure.
4311
+ const keyFile = Underpost.secret.sops.keyFile();
4312
+ shellExec(
4313
+ `sudo tee /etc/profile.d/underpost-sops.sh >/dev/null <<'UNDERPOST_SOPS_ENV_EOF'
4314
+ export SOPS_AGE_KEY_FILE="\${SOPS_AGE_KEY_FILE:-${keyFile}}"
4315
+ UNDERPOST_SOPS_ENV_EOF`,
4316
+ );
4317
+ shellExec(`sudo chmod 644 /etc/profile.d/underpost-sops.sh`);
4318
+
4319
+ // 3. Build and encrypt each requested Secret.
4320
+ const stageDir = '/dev/shm/underpost-secrets';
4321
+ const held = Underpost.secret.sops.localRecipients();
4322
+ fs.ensureDirSync(stageDir);
4323
+ fs.chmodSync(stageDir, 0o700);
4324
+ try {
4325
+ for (const name of secretNames) {
4326
+ const stored = Underpost.secret.sops.has(name, namespace);
4327
+ if (stored && !options.force) {
4328
+ // A stored manifest this host cannot open is present but unusable here, so reporting it
4329
+ // as onboarded would send the operator on to an apply that is guaranteed to fail.
4330
+ if (Underpost.secret.sops.decryptable(Underpost.secret.sops.manifestPath(name, namespace), held))
4331
+ logger.info(`${name} is already onboarded in ns/${namespace}; skipping (use --force to replace)`);
4332
+ else
4333
+ logger.warn(
4334
+ `${name} is stored in ns/${namespace} but is sealed to an Age recipient this host does not hold; ` +
4335
+ `skipping. Adopt the store's key, re-key it from a host that holds one, or re-onboard from the ` +
4336
+ `origin seed files with --force.`,
4337
+ );
4338
+ continue;
4339
+ }
4340
+
4341
+ // Data keys come from the secret's origin seed contract, so an onboarded manifest
4342
+ // carries exactly the keys the workload's secretKeyRef already expects.
4343
+ const seedSources = Underpost.secret.sops.seedSources(name);
4344
+ const dataKeys = Object.keys(seedSources).length > 0 ? Object.keys(seedSources) : ['password'];
4345
+ const stringData = {};
4346
+ for (const key of dataKeys) {
4347
+ const seedPath = seedSources[key];
4348
+ if (seedPath && fs.existsSync(seedPath)) {
4349
+ stringData[key] = fs.readFileSync(seedPath, 'utf8').trim();
4350
+ logger.info(`${name}.${key} seeded from ${seedPath}`);
4351
+ } else if (overrides[key] !== undefined) {
4352
+ stringData[key] = overrides[key];
4353
+ logger.info(`${name}.${key} taken from --args`);
4354
+ } else {
4355
+ stringData[key] = generateSeedValue(key);
4356
+ // Replacing a stored manifest with a value nothing seeded means the credential the
4357
+ // running datastore still authenticates against is being thrown away.
4358
+ if (stored)
4359
+ logger.warn(
4360
+ `${name}.${key} generated while replacing the stored manifest — no seed file at ` +
4361
+ `${seedPath || '(unmapped)'} and no --args override. The running datastore keeps its old ` +
4362
+ `credential until this value is applied to it; pass --args "${key}=<value>" to keep the ` +
4363
+ `existing one.`,
4364
+ );
4365
+ else logger.info(`${name}.${key} generated`);
4366
+ }
4367
+ }
4368
+
4369
+ const stagePath = `${stageDir}/${name}.yaml`;
4370
+ fs.outputFileSync(
4371
+ stagePath,
4372
+ [
4373
+ 'apiVersion: v1',
4374
+ 'kind: Secret',
4375
+ 'metadata:',
4376
+ ` name: ${name}`,
4377
+ ` namespace: ${namespace}`,
4378
+ ' labels:',
4379
+ ' app.kubernetes.io/managed-by: underpost',
4380
+ 'type: Opaque',
4381
+ 'stringData:',
4382
+ // Single-quoted YAML scalars with doubled internal quotes: values are generated or
4383
+ // operator-supplied and may contain characters YAML would otherwise interpret.
4384
+ ...Object.entries(stringData).map(([key, value]) => ` ${key}: '${`${value}`.replace(/'/g, "''")}'`),
4385
+ '',
4386
+ ].join('\n'),
4387
+ 'utf8',
4388
+ );
4389
+ fs.chmodSync(stagePath, 0o600);
4390
+ // encrypt() stages, validates, moves into place, and shreds the plaintext source.
4391
+ Underpost.secret.sops.encrypt(stagePath, namespace, options);
4392
+ }
4393
+ } finally {
4394
+ // Defense in depth: encrypt() shreds each source, but a throw mid-loop must not leave a
4395
+ // plaintext manifest sitting in shared memory.
4396
+ fs.removeSync(stageDir);
4397
+ }
4398
+
4399
+ Underpost.secret.sops.list();
4400
+
4401
+ // 4. Validate every manifest in the namespace, then apply unless this is a dry run.
4402
+ Underpost.secret.sops.apply(namespace, { dryRun: true });
4403
+ if (options.dryRun) return logger.info('--dry-run: validated only, cluster left unchanged');
4404
+ Underpost.secret.sops.apply(namespace);
4405
+ },
4406
+ /**
4407
+ * @method sops-status
4408
+ * @description Reports the live state of the SOPS/Age secret system: host tooling, the Age
4409
+ * key and its recipient, the committed creation rules, every stored manifest with whether the
4410
+ * local key can open it and whether the cluster still matches, and which managed Secrets are
4411
+ * onboarded versus still seeding from their origin path.
4412
+ *
4413
+ * Read-only and safe to run anywhere. Decryption happens only for the drift check, only for
4414
+ * manifests the local key is a recipient of, and only into `kubectl diff` with its output
4415
+ * discarded — no secret value is ever printed or written to disk.
4416
+ *
4417
+ * Usage:
4418
+ * underpost run sops-status # every managed key, ns default
4419
+ * underpost run sops-status mongo # partial match: both mongo keys
4420
+ * underpost run sops-status --namespace prod # every managed key in ns prod
4421
+ * @param {string} path - Comma-separated managed Secret keys to report on; empty reports all.
4422
+ * Matched as case-insensitive substrings (`mongo` selects mongodb-secret and mongodb-keyfile).
4423
+ * Filters both the stored-manifest listing and the coverage table.
4424
+ * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
4425
+ * @param {string} options.namespace - Namespace to inspect (DEFAULT_OPTION scheme, default 'default').
4426
+ * @memberof UnderpostRun
4427
+ */
4428
+ 'sops-status': (path = '', options = DEFAULT_OPTION) => {
4429
+ const sops = Underpost.secret.sops;
4430
+ // `--namespace` selects the namespace (DEFAULT_OPTION scheme); `path` narrows which managed
4431
+ // Secret keys to report on, so the two axes stay independent.
4432
+ const namespace = options.namespace || 'default';
4433
+ const manageSecretKeyFilter = path
4434
+ .split(',')
4435
+ .map((key) => key.trim().toLowerCase())
4436
+ .filter(Boolean);
4437
+ // Partial, case-insensitive substring match, so `mongo` reaches both `mongodb-secret` and
4438
+ // `mongodb-keyfile` without having to spell either out.
4439
+ const matchesKeyFilter = (name) =>
4440
+ manageSecretKeyFilter.length === 0 || manageSecretKeyFilter.some((key) => name.toLowerCase().includes(key));
4441
+ const mark = (ok) => (ok ? 'yes' : 'no');
4442
+
4443
+ // ── Tooling ────────────────────────────────────────────────────────────
4444
+ const version = (bin, flag) =>
4445
+ sops.hasBinary(bin)
4446
+ ? shellExec(`${bin} ${flag} 2>/dev/null | head -1`, { stdout: true, silent: true, disableLog: true }).trim()
4447
+ : '(not installed)';
4448
+ logger.info(
4449
+ '[sops-status] Tooling\n' +
4450
+ ` sops ${version('sops', '--version')}\n` +
4451
+ ` age ${version('age', '--version')}\n` +
4452
+ ` age-keygen ${sops.hasBinary('age-keygen') ? 'installed' : '(not installed)'}`,
4453
+ );
4454
+
4455
+ // ── Age key ────────────────────────────────────────────────────────────
4456
+ const keyFile = sops.keyFile();
4457
+ const keyExists = fs.existsSync(keyFile);
4458
+ // A key file may hold several identities — that is how a host joins a store it did not
4459
+ // create — so every check below works against the whole held set, not one recipient.
4460
+ const held = sops.localRecipients();
4461
+ const keyMode = keyExists ? (fs.statSync(keyFile).mode & 0o777).toString(8) : '';
4462
+ logger.info(
4463
+ '[sops-status] Age key\n' +
4464
+ ` path ${keyFile}\n` +
4465
+ ` present ${mark(keyExists)}${keyExists ? ` (mode ${keyMode}${keyMode === '600' || keyMode === '400' ? '' : ' — INSECURE, run chmod 600'})` : ''}\n` +
4466
+ ` recipients ${held.join(', ') || (keyExists ? '(none — unreadable key file)' : '(none)')}` +
4467
+ (keyExists ? '' : `\n searched ${sops.keyFileCandidates().join(', ')}`),
4468
+ );
4469
+
4470
+ // ── Creation rules ─────────────────────────────────────────────────────
4471
+ const confPath = './engine-private/secrets/.sops.yaml';
4472
+ const ruleRecipients = sops.creationRecipients();
4473
+ logger.info(
4474
+ '[sops-status] Creation rules\n' +
4475
+ ` config ${confPath} ${fs.existsSync(confPath) ? '' : '(missing — run: underpost secret sops --init)'}\n` +
4476
+ ` recipients ${ruleRecipients.length > 0 ? ruleRecipients.join(', ') : '(none)'}\n` +
4477
+ ` local key listed ${mark(held.some((recipient) => ruleRecipients.includes(recipient)))}`,
4478
+ );
4479
+
4480
+ // ── Stored manifests ───────────────────────────────────────────────────
4481
+ const manifests = sops.manifests(namespace).filter((manifest) => matchesKeyFilter(manifest.name));
4482
+ const onboarded = new Set();
4483
+ if (manifests.length === 0)
4484
+ logger.warn(
4485
+ `[sops-status] Store\n no encrypted manifests in ns/${namespace}` +
4486
+ (manageSecretKeyFilter.length > 0 ? ` matching ${manageSecretKeyFilter.join(', ')}` : ''),
4487
+ );
4488
+ else {
4489
+ const rows = manifests.map((manifest) => {
4490
+ onboarded.add(manifest.name);
4491
+ const recipients = sops.manifestRecipients(manifest.path);
4492
+ const decryptable = sops.decryptable(manifest.path, held);
4493
+ const live = shellExec(
4494
+ `kubectl get secret ${manifest.name} -n ${manifest.namespace} --ignore-not-found -o name 2>/dev/null || true`,
4495
+ { stdout: true, silent: true, silentOnError: true, disableLog: true },
4496
+ ).trim();
4497
+ // Drift is decided by kubectl's exit code; its stdout would contain the decrypted
4498
+ // values, so it is discarded rather than captured.
4499
+ let sync = 'n/a';
4500
+ if (live && decryptable) {
4501
+ const result = shellExec(
4502
+ `bash -c 'set -o pipefail; SOPS_AGE_KEY_FILE="${keyFile}" sops --decrypt "${manifest.path}" ` +
4503
+ `| kubectl diff -f - -n "${manifest.namespace}" >/dev/null 2>&1'`,
4504
+ { silentOnError: true, disableLog: true, stdout: false },
4505
+ );
4506
+ sync = result.code === 0 ? 'in-sync' : result.code === 1 ? 'DRIFT' : 'error';
4507
+ } else if (!live) sync = 'not applied';
4508
+ else if (!decryptable) sync = 'no local key';
4509
+ return (
4510
+ ` ${`${manifest.namespace}/${manifest.name}`.padEnd(34)} ` +
4511
+ `recipients=${String(recipients.length).padEnd(3)} ` +
4512
+ `decryptable=${mark(decryptable).padEnd(4)} ` +
4513
+ `live=${mark(!!live).padEnd(4)} ` +
4514
+ `${sync}`
4515
+ );
4516
+ });
4517
+ logger.info(`[sops-status] Store — ns/${namespace} (${manifests.length} manifest(s))\n` + rows.join('\n'));
4518
+ }
4519
+
4520
+ // ── Coverage ───────────────────────────────────────────────────────────
4521
+ const coverage = sops
4522
+ .managedSecrets()
4523
+ .filter(matchesKeyFilter)
4524
+ .map((name) => {
4525
+ const seeds = Object.values(sops.seedSources(name));
4526
+ const seedPresent = seeds.length > 0 && seeds.every((seed) => fs.existsSync(seed));
4527
+ const source = onboarded.has(name)
4528
+ ? 'sops'
4529
+ : seedPresent
4530
+ ? 'origin seed'
4531
+ : seeds.length
4532
+ ? 'MISSING'
4533
+ : 'unmapped';
4534
+ return ` ${name.padEnd(24)} ${source.padEnd(12)} ${seeds.length ? `seed=${mark(seedPresent)}` : ''}`;
4535
+ });
4536
+ if (coverage.length === 0)
4537
+ logger.warn(
4538
+ `[sops-status] Coverage\n no managed Secret matches ${manageSecretKeyFilter.join(', ')}\n` +
4539
+ ` known keys: ${sops.managedSecrets().join(', ')}`,
4540
+ );
4541
+ else
4542
+ logger.info('[sops-status] Coverage (which source each managed Secret deploys from)\n' + coverage.join('\n'));
4543
+ },
2730
4544
  /**
2731
4545
  * @method secret
2732
4546
  * @description Creates an Underpost secret named 'underpost' from a file, defaulting to `/home/dd/engine/engine-private/conf/dd-cron/.env.production` if no path is provided.
@@ -2847,7 +4661,7 @@ EOF`);
2847
4661
 
2848
4662
  const envs = Underpost.env.list();
2849
4663
 
2850
- const cmd = `kubectl apply -f - <<EOF
4664
+ const cmd = `kubectl apply -f - <<'EOF'
2851
4665
  apiVersion: ${apiVersion}
2852
4666
  kind: ${kindType}
2853
4667
  metadata: